0.8.64

Changelog.yaml

@@ -19,6 +19,59 @@
 #   new recipes:
 #     - title: 
 
+- version: 0.8.64
+  date: 2012-08-09
+
+  new features:
+    - title: "E-book viewer: Allow viewing images in the book in a separate pop-up window by right clicking on the image. Useful if you want to keep some image, like a map to the side while reading the book."
+
+    - title: "Catalogs: Allow generation of catalogs in AZW3 format. Also add more powerful configuration options to exclude books and set prefixes. See http://www.mobileread.com/forums/showthread.php?t=187298 for details."
+
+    - title: "Generate a PDF version of the User Manual"
+ 
+  bug fixes:
+    - title: "News download: Fix broken handling of nesting for HTML 5 tags when parsing with BeautifulSoup"
+
+    - title: "EPUB: Handle files in the EPUB that have semi-colons in their file names. This means in particular using URL escaping when creating the NCX as ADE cannot handle unescaped semi-colons in the NCX."
+      tickets: [1033665]
+
+    - title: "Conversion pipeline: Ignore unparseable CSS instead of erroring out on it."
+      tickets: [1034074]
+
+    - title: "When setting up a column coloring rule based on the languages column, allow entry of localized language names instead of only ISO codes"
+
+    - title: "Catalogs: Generate cover for mobi/azw3 catalogs"
+
+    - title: "Update the last modified column record of a book, whenever a format is added to the book."
+
+    - title: "E-book viewer: Fix line scrolling stops at breaks option not working in paged mode"
+      tickets: [1033430]
+
+    - title: "MOBI Output: Fix ToC at start option having no effect when converting some input documents that have an out-of-spine ToC."
+      tickets: [1033656]
+
+    - title: "Catalog Generation: When generating EPUB/MOBI catalogs add more flexible rules for excluding books. Also add rules to customize the prefix characters used."
+
+    - title: "Make setting published date using metadata search/replace more robust."
+
+    - title: "Tag Browser: Flatten the display of sub-groups when sort by is not set to 'name'."
+      tickets: [1032746]
+
+    - title: "Fix isbn:false not matching if other identifiers are attached to the book."
+
+  improved recipes:
+    - The New Republic
+    - ZDNet
+    - Metro UK
+    - FHM UK
+
+  new recipes:
+    - title: eKundelek.pl 
+      author: Artur Stachecki
+
+    - title: Sueddeutsche Mobil 
+      author: Andreas Zeiser
+
 - version: 0.8.63
   date: 2012-08-02
 

recipes/the_new_republic.recipe

@@ -44,6 +44,10 @@ class TNR(BasicNewsRecipe):
             em=post.find('em')
             b=post.find('b')
             a=post.find('a',href=True)
+            p=post.find('img', src=True)
+            #Find cover
+            if p is not None:
+                self.cover_url = p['src'].strip()
             if em is not None:
                 section_title = self.tag_to_string(em).strip()
                 subsection_title = ''

recipes/zdnet.recipe

@@ -12,7 +12,7 @@ class cdnet(BasicNewsRecipe):
 
     title = 'zdnet'
     description = 'zdnet security'
-    __author__ = 'Oliver Niesner'
+    __author__ = 'Oliver Niesner, Krittika Goyal'
     language = 'en'
 
     use_embedded_content   = False
@@ -20,41 +20,42 @@ class cdnet(BasicNewsRecipe):
     max_articles_per_feed = 40
     no_stylesheets = True
     encoding = 'latin1'
+    auto_cleanup = True
 
 
 
-    remove_tags = [dict(id='eyebrows'),
+    #remove_tags = [dict(id='eyebrows'),
-		   dict(id='header'),
+           #dict(id='header'),
-		   dict(id='search'),
+           #dict(id='search'),
-		   dict(id='nav'),
+           #dict(id='nav'),
-		   dict(id='blog-author-info'),
+           #dict(id='blog-author-info'),
-		   dict(id='post-tags'),
+           #dict(id='post-tags'),
-		   dict(id='bio-naraine'),
+           #dict(id='bio-naraine'),
-		   dict(id='bio-kennedy'),
+           #dict(id='bio-kennedy'),
-		   dict(id='author-short-disclosure-kennedy'),
+           #dict(id='author-short-disclosure-kennedy'),
-		   dict(id=''),
+           #dict(id=''),
-		   dict(name='div', attrs={'class':'banner'}),
+           #dict(name='div', attrs={'class':'banner'}),
-		   dict(name='div', attrs={'class':'int'}),
+           #dict(name='div', attrs={'class':'int'}),
-		   dict(name='div', attrs={'class':'talkback clear space-2'}),
+           #dict(name='div', attrs={'class':'talkback clear space-2'}),
-		   dict(name='div', attrs={'class':'content-1 clear'}),
+           #dict(name='div', attrs={'class':'content-1 clear'}),
-		   dict(name='div', attrs={'class':'space-2'}),
+           #dict(name='div', attrs={'class':'space-2'}),
-		   dict(name='div', attrs={'class':'space-3'}),
+           #dict(name='div', attrs={'class':'space-3'}),
-		   dict(name='div', attrs={'class':'thumb-2 left'}),
+           #dict(name='div', attrs={'class':'thumb-2 left'}),
-		   dict(name='div', attrs={'class':'hotspot'}),
+           #dict(name='div', attrs={'class':'hotspot'}),
-		   dict(name='div', attrs={'class':'hed hed-1 space-1'}),
+           #dict(name='div', attrs={'class':'hed hed-1 space-1'}),
-		   dict(name='div', attrs={'class':'view-1 clear content-3 space-2'}),
+           #dict(name='div', attrs={'class':'view-1 clear content-3 space-2'}),
-		   dict(name='div', attrs={'class':'hed hed-1 space-1'}),
+           #dict(name='div', attrs={'class':'hed hed-1 space-1'}),
-		   dict(name='div', attrs={'class':'hed hed-1'}),
+           #dict(name='div', attrs={'class':'hed hed-1'}),
-		   dict(name='div', attrs={'class':'post-header'}),
+           #dict(name='div', attrs={'class':'post-header'}),
-		   dict(name='div', attrs={'class':'lvl-nav clear'}),
+           #dict(name='div', attrs={'class':'lvl-nav clear'}),
-		   dict(name='div', attrs={'class':'t-share-overlay overlay-pop contain-overlay-4'}),
+           #dict(name='div', attrs={'class':'t-share-overlay overlay-pop contain-overlay-4'}),
-		   dict(name='p', attrs={'class':'tags'}),
+           #dict(name='p', attrs={'class':'tags'}),
-		   dict(name='span', attrs={'class':'follow'}),
+           #dict(name='span', attrs={'class':'follow'}),
-		   dict(name='span', attrs={'class':'int'}),
+           #dict(name='span', attrs={'class':'int'}),
-		   dict(name='h4', attrs={'class':'h s-4'}),
+           #dict(name='h4', attrs={'class':'h s-4'}),
-		   dict(name='a', attrs={'href':'http://www.twitter.com/ryanaraine'}),
+           #dict(name='a', attrs={'href':'http://www.twitter.com/ryanaraine'}),
-		   dict(name='div', attrs={'class':'special1'})]
+           #dict(name='div', attrs={'class':'special1'})]
-    remove_tags_after = [dict(name='div', attrs={'class':'clear'})]
+    #remove_tags_after = [dict(name='div', attrs={'class':'clear'})]
 
     feeds =  [ ('zdnet', 'http://feeds.feedburner.com/zdnet/security') ]
 
@@ -63,6 +64,3 @@ class cdnet(BasicNewsRecipe):
         for item in soup.findAll(style=True):
             del item['style']
         return soup
-
-
-

setup/build_environment.py

@@ -17,7 +17,7 @@ OSX_SDK = '/Developer/SDKs/MacOSX10.5.sdk'
 
 os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.5'
 
-NMAKE = RC = msvc = MT = win_inc = win_lib = win_ddk = None
+NMAKE = RC = msvc = MT = win_inc = win_lib = win_ddk = win_ddk_lib_dirs = None
 if iswindows:
     from distutils import msvc9compiler
     msvc = msvc9compiler.MSVCCompiler()
@@ -26,7 +26,8 @@ if iswindows:
     RC = msvc.find_exe('rc.exe')
     SDK = os.environ.get('WINSDK', r'C:\Program Files\Microsoft SDKs\Windows\v6.0A')
     DDK = os.environ.get('WINDDK', r'Q:\WinDDK\7600.16385.0')
-    win_ddk = [DDK+'\\inc\\'+x for x in ('api',)]
+    win_ddk = [DDK+'\\inc\\'+x for x in ('atl71',)]
+    win_ddk_lib_dirs = [DDK+'\\lib\\ATL\\i386']
     win_inc = os.environ['include'].split(';')
     win_lib = os.environ['lib'].split(';')
     for p in win_inc:

setup/extensions.py

@@ -17,7 +17,7 @@ from setup.build_environment import (fc_inc, fc_lib, chmlib_inc_dirs, fc_error,
         podofo_inc, podofo_lib, podofo_error, pyqt, OSX_SDK, NMAKE, QMAKE,
         msvc, MT, win_inc, win_lib, win_ddk, magick_inc_dirs, magick_lib_dirs,
         magick_libs, chmlib_lib_dirs, sqlite_inc_dirs, icu_inc_dirs,
-        icu_lib_dirs)
+        icu_lib_dirs, win_ddk_lib_dirs)
 MT
 isunix = islinux or isosx or isbsd
 
@@ -162,11 +162,19 @@ extensions = [
 
 
 if iswindows:
-    extensions.append(Extension('winutil',
+    extensions.extend([
+        Extension('winutil',
                 ['calibre/utils/windows/winutil.c'],
                 libraries=['shell32', 'setupapi', 'wininet'],
                 cflags=['/X']
-                ))
+                ),
+        Extension('wpd',
+            ['calibre/devices/mtp/windows/wpd.cpp'],
+            libraries=['ole32', 'portabledeviceguids'],
+            # needs_ddk=True,
+            cflags=['/X']
+            ),
+        ])
 
 if isosx:
     extensions.append(Extension('usbobserver',
@@ -325,8 +333,8 @@ class Build(Command):
         obj_dir = self.j(self.obj_dir, ext.name)
         if ext.needs_ddk:
             ddk_flags = ['-I'+x for x in win_ddk]
-            i = [i for i in range(len(cflags)) if 'VC\\INCLUDE' in cflags[i]][0]
+            cflags.extend(ddk_flags)
-            cflags[i+1:i+2] = ddk_flags
+            ldflags.extend(['/LIBPATH:'+x for x in win_ddk_lib_dirs])
         if not os.path.exists(obj_dir):
             os.makedirs(obj_dir)
         for src in ext.sources:

setup/installer/linux/freeze2.py

@@ -23,6 +23,8 @@ MAGICK_PREFIX = '/usr'
 binary_includes = [
                 '/usr/bin/pdftohtml',
                 '/usr/bin/pdfinfo',
+                '/usr/lib/libusb-1.0.so.0',
+                '/usr/lib/libmtp.so.9',
                 '/usr/lib/libglib-2.0.so.0',
                 '/usr/bin/pdftoppm',
                 '/usr/lib/libwmflite-0.2.so.7',

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 8, 63)
+numeric_version = (0, 8, 64)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 
@@ -90,7 +90,7 @@ class Plugins(collections.Mapping):
                 'speedup',
             ]
         if iswindows:
-            plugins.append('winutil')
+            plugins.extend(['winutil', 'wpd'])
         if isosx:
             plugins.append('usbobserver')
         if islinux:

src/calibre/devices/__init__.py

@@ -55,7 +55,13 @@ def get_connected_device():
             break
     return dev
 
-def debug(ioreg_to_tmp=False, buf=None):
+def debug(ioreg_to_tmp=False, buf=None, plugins=None):
+    '''
+    If plugins is None, then this method calls startup and shutdown on the
+    device plugins. So if you are using it in a context where startup could
+    already have been called (for example in the main GUI), pass in the list of
+    device plugins as the plugins parameter.
+    '''
     import textwrap
     from calibre.customize.ui import device_plugins
     from calibre.devices.scanner import DeviceScanner, win_pnp_drives
@@ -66,9 +72,19 @@ def debug(ioreg_to_tmp=False, buf=None):
     if buf is None:
         buf = StringIO()
     sys.stdout = sys.stderr = buf
+    out = partial(prints, file=buf)
+
+    devplugins = device_plugins() if plugins is None else plugins
+    devplugins = list(sorted(devplugins, cmp=lambda
+            x,y:cmp(x.__class__.__name__, y.__class__.__name__)))
+    if plugins is None:
+        for d in devplugins:
+            try:
+                d.startup()
+            except:
+                out('Startup failed for device plugin: %s'%d)
 
     try:
-        out = partial(prints, file=buf)
         out('Version:', __version__)
         s = DeviceScanner()
         s.scan()
@@ -96,8 +112,6 @@ def debug(ioreg_to_tmp=False, buf=None):
             ioreg += 'Output from osx_get_usb_drives:\n'+drives+'\n\n'
             ioreg += Device.run_ioreg()
         connected_devices = []
-        devplugins = list(sorted(device_plugins(), cmp=lambda
-                x,y:cmp(x.__class__.__name__, y.__class__.__name__)))
         out('Available plugins:', textwrap.fill(' '.join([x.__class__.__name__ for x in
             devplugins])))
         out(' ')
@@ -155,6 +169,12 @@ def debug(ioreg_to_tmp=False, buf=None):
     finally:
         sys.stdout = oldo
         sys.stderr = olde
+        if plugins is None:
+            for d in devplugins:
+                try:
+                    d.shutdown()
+                except:
+                    pass
 
 def device_info(ioreg_to_tmp=False, buf=None):
     from calibre.devices.scanner import DeviceScanner, win_pnp_drives

src/calibre/devices/interface.py

@@ -503,14 +503,16 @@ class DevicePlugin(Plugin):
         Called when calibre is is starting the device. Do any initialization
         required. Note that multiple instances of the class can be instantiated,
         and thus __init__ can be called multiple times, but only one instance
-        will have this method called.
+        will have this method called. This method is called on the device
+        thread, not the GUI thread.
         '''
         pass
 
     def shutdown(self):
         '''
         Called when calibre is shutting down, either for good or in preparation
-        to restart. Do any cleanup required.
+        to restart. Do any cleanup required. This method is called on the
+        device thread, not the GUI thread.
         '''
         pass
 

src/calibre/devices/mtp/base.py

@@ -7,8 +7,17 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
+from functools import wraps
+
 from calibre.devices.interface import DevicePlugin
 
+def synchronous(func):
+    @wraps(func)
+    def synchronizer(self, *args, **kwargs):
+        with self.lock:
+            return func(self, *args, **kwargs)
+    return synchronizer
+
 class MTPDeviceBase(DevicePlugin):
     name = 'SmartDevice App Interface'
     gui_name = _('MTP Device')

src/calibre/devices/mtp/unix/driver.py

@@ -9,18 +9,76 @@ __docformat__ = 'restructuredtext en'
 
 import time, operator
 from threading import RLock
-from functools import wraps
+from itertools import chain
+from collections import deque, OrderedDict
+from io import BytesIO
 
+from calibre import prints
 from calibre.devices.errors import OpenFailed
-from calibre.devices.mtp.base import MTPDeviceBase
+from calibre.devices.mtp.base import MTPDeviceBase, synchronous
 from calibre.devices.mtp.unix.detect import MTPDetect
 
-def synchronous(func):
+class FilesystemCache(object):
-    @wraps(func)
+
-    def synchronizer(self, *args, **kwargs):
+    def __init__(self, files, folders):
-        with self.lock:
+        self.files = files
-            return func(self, *args, **kwargs)
+        self.folders = folders
-    return synchronizer
+        self.file_id_map = {f['id']:f for f in files}
+        self.folder_id_map = {f['id']:f for f in self.iterfolders(set_level=0)}
+
+        # Set the parents of each file
+        self.files_in_root = OrderedDict()
+        for f in files:
+            parents = deque()
+            pid = f['parent_id']
+            while pid is not None and pid > 0:
+                try:
+                    parent = self.folder_id_map[pid]
+                except KeyError:
+                    break
+                parents.appendleft(pid)
+                pid = parent['parent_id']
+            f['parents'] = parents
+            if not parents:
+                self.files_in_root[f['id']] = f
+
+        # Set the files in each folder
+        for f in self.iterfolders():
+            f['files'] = [i for i in files if i['parent_id'] ==
+                    f['id']]
+
+        # Decode the file and folder names
+        for f in chain(files, folders):
+            try:
+                name = f['name'].decode('utf-8')
+            except UnicodeDecodeError:
+                name = 'undecodable_%d'%f['id']
+            f['name'] = name
+
+    def iterfolders(self, folders=None, set_level=None):
+        clevel = None if set_level is None else set_level + 1
+        if folders is None:
+            folders = self.folders
+        for f in folders:
+            if set_level is not None:
+                f['level'] = set_level
+            yield f
+            for c in f['children']:
+                for child in self.iterfolders([c], set_level=clevel):
+                    yield child
+
+    def dump_filesystem(self):
+        indent = 2
+        for f in self.iterfolders():
+            prefix = ' '*(indent*f['level'])
+            prints(prefix, '+', f['name'], 'id=%s'%f['id'])
+            for leaf in f['files']:
+                prints(prefix, ' '*indent, '-', leaf['name'],
+                        'id=%d'%leaf['id'], 'size=%d'%leaf['size'],
+                        'modtime=%d'%leaf['modtime'])
+        for leaf in self.files_in_root.itervalues():
+            prints('-', leaf['name'], 'id=%d'%leaf['id'],
+                    'size=%d'%leaf['size'], 'modtime=%d'%leaf['modtime'])
 
 class MTP_DEVICE(MTPDeviceBase):
 
@@ -28,11 +86,14 @@ class MTP_DEVICE(MTPDeviceBase):
 
     def __init__(self, *args, **kwargs):
         MTPDeviceBase.__init__(self, *args, **kwargs)
-        self.detect = MTPDetect()
         self.dev = None
+        self.filesystem_cache = None
         self.lock = RLock()
         self.blacklisted_devices = set()
 
+    def set_debug_level(self, lvl):
+        self.detect.libmtp.set_debug_level(lvl)
+
     def report_progress(self, sent, total):
         try:
             p = int(sent/total * 100)
@@ -73,14 +134,26 @@ class MTP_DEVICE(MTPDeviceBase):
 
     @synchronous
     def post_yank_cleanup(self):
-        self.dev = None
+        self.dev = self.filesystem_cache = None
+
+    @synchronous
+    def startup(self):
+        self.detect = MTPDetect()
+        for x in vars(self.detect.libmtp):
+            if x.startswith('LIBMTP'):
+                setattr(self, x, getattr(self.detect.libmtp, x))
 
     @synchronous
     def shutdown(self):
-        self.dev = None
+        self.dev = self.filesystem_cache = None
+
+    def format_errorstack(self, errs):
+        return '\n'.join(['%d:%s'%(code, msg.decode('utf-8', 'replace')) for
+            code, msg in errs])
 
     @synchronous
     def open(self, connected_device, library_uuid):
+        self.dev = self.filesystem_cache = None
         def blacklist_device():
             d = connected_device
             self.blacklisted_devices.add((d.busnum, d.devnum, d.vendor_id,
@@ -112,6 +185,20 @@ class MTP_DEVICE(MTPDeviceBase):
         if len(storage) > 2:
             self._cardb_id = storage[2]['id']
 
+        try:
+            files, errs = self.dev.get_filelist(self)
+            if errs and not files:
+                raise OpenFailed('Failed to read files from device. Underlying errors:\n'
+                        +self.format_errorstack(errs))
+            folders, errs = self.dev.get_folderlist()
+            if errs and not folders:
+                raise OpenFailed('Failed to read folders from device. Underlying errors:\n'
+                        +self.format_errorstack(errs))
+            self.filesystem_cache = FilesystemCache(files, folders)
+        except:
+            self.dev = self._main_id = self._carda_id = self._cardb_id = None
+            raise
+
     @synchronous
     def get_device_information(self, end_session=True):
         d = self.dev
@@ -149,8 +236,14 @@ class MTP_DEVICE(MTPDeviceBase):
 
 
 if __name__ == '__main__':
+    BytesIO
+    class PR:
+        def report_progress(self, sent, total):
+            print (sent, total, end=', ')
+
     from pprint import pprint
     dev = MTP_DEVICE(None)
+    dev.startup()
     from calibre.devices.scanner import linux_scanner
     devs = linux_scanner()
     mtp_devs = dev.detect(devs)
@@ -160,8 +253,19 @@ if __name__ == '__main__':
     print ("Storage info:")
     pprint(d.storage_info)
     print("Free space:", dev.free_space())
-    files, errs = d.get_filelist(dev)
+    # print (d.create_folder(dev._main_id, 0, 'testf'))
-    pprint((len(files), errs))
+    # raw = b'test'
-    folders, errs = d.get_folderlist()
+    # fname = b'moose.txt'
-    pprint((len(folders), errs))
+    # src = BytesIO(raw)
+    # print (d.put_file(dev._main_id, 0, fname, src, len(raw), PR()))
+    dev.filesystem_cache.dump_filesystem()
+    # with open('/tmp/flint.epub', 'wb') as f:
+    #     print(d.get_file(786, f, PR()))
+    # print()
+    # with open('/tmp/bleak.epub', 'wb') as f:
+    #     print(d.get_file(601, f, PR()))
+    # print()
+    dev.set_debug_level(dev.LIBMTP_DEBUG_ALL)
+    del d
+    dev.shutdown()
 

src/calibre/devices/mtp/unix/libmtp.c

@@ -7,15 +7,17 @@
 #include "devices.h"
 
 // Macros and utilities
+static PyObject *MTPError = NULL;
+
 #define ENSURE_DEV(rval) \
     if (self->device == NULL) { \
-        PyErr_SetString(PyExc_ValueError, "This device has not been initialized."); \
+        PyErr_SetString(MTPError, "This device has not been initialized."); \
         return rval; \
     }
 
 #define ENSURE_STORAGE(rval) \
     if (self->device->storage == NULL) { \
-        PyErr_SetString(PyExc_RuntimeError, "The device has no storage information."); \
+        PyErr_SetString(MTPError, "The device has no storage information."); \
         return rval; \
     }
 
@@ -31,8 +33,10 @@
 #define AC_ReadOnly             0x0001
 #define AC_ReadOnly_with_Object_Deletion    0x0002
 
+
 typedef struct {
     PyObject *obj;
+    PyObject *extra;
     PyThreadState *state;
 } ProgressCallback;
 
@@ -64,6 +68,48 @@ static void dump_errorstack(LIBMTP_mtpdevice_t *dev, PyObject *list) {
     LIBMTP_Clear_Errorstack(dev);
 }
 
+static uint16_t data_to_python(void *params, void *priv, uint32_t sendlen, unsigned char *data, uint32_t *putlen) {
+    PyObject *res;
+    ProgressCallback *cb;
+    uint16_t ret = LIBMTP_HANDLER_RETURN_OK;
+
+    cb = (ProgressCallback *)priv;
+    *putlen = sendlen;
+    PyEval_RestoreThread(cb->state);
+    res = PyObject_CallMethod(cb->extra, "write", "s#", data, sendlen);
+    if (res == NULL) {
+        ret = LIBMTP_HANDLER_RETURN_ERROR;
+        *putlen = 0;
+        PyErr_Print();
+    } else Py_DECREF(res);
+
+    cb->state = PyEval_SaveThread();
+    return ret;
+}
+
+static uint16_t data_from_python(void *params, void *priv, uint32_t wantlen, unsigned char *data, uint32_t *gotlen) {
+    PyObject *res;
+    ProgressCallback *cb;
+    char *buf = NULL;
+    Py_ssize_t len = 0;
+    uint16_t ret = LIBMTP_HANDLER_RETURN_ERROR;
+
+    *gotlen = 0;
+
+    cb = (ProgressCallback *)priv;
+    PyEval_RestoreThread(cb->state);
+    res = PyObject_CallMethod(cb->extra, "read", "k", wantlen);
+    if (res != NULL && PyBytes_AsStringAndSize(res, &buf, &len) != -1 && len <= wantlen) {
+        memcpy(data, buf, len);
+        *gotlen = len;
+        ret = LIBMTP_HANDLER_RETURN_OK;
+    } else PyErr_Print();
+
+    Py_XDECREF(res);
+    cb->state = PyEval_SaveThread();
+    return ret;
+}
+
 // }}}
 
 // Device object definition {{{
@@ -84,7 +130,11 @@ typedef struct {
 static void
 libmtp_Device_dealloc(libmtp_Device* self)
 {
-    if (self->device != NULL) LIBMTP_Release_Device(self->device);
+    if (self->device != NULL) {
+        Py_BEGIN_ALLOW_THREADS;
+        LIBMTP_Release_Device(self->device);
+        Py_END_ALLOW_THREADS;
+    }
     self->device = NULL;
 
     Py_XDECREF(self->ids); self->ids = NULL;
@@ -136,7 +186,7 @@ libmtp_Device_init(libmtp_Device *self, PyObject *args, PyObject *kwds)
     Py_END_ALLOW_THREADS;
 
     if (dev == NULL) { 
-        PyErr_SetString(PyExc_ValueError, "Unable to open raw device."); 
+        PyErr_SetString(MTPError, "Unable to open raw device."); 
         return -1;
     }
 
@@ -225,7 +275,7 @@ static PyObject*
 libmtp_Device_update_storage_info(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
     ENSURE_DEV(NULL);
     if (LIBMTP_Get_Storage(self->device, LIBMTP_STORAGE_SORTBY_NOTSORTED) < 0) {
-        PyErr_SetString(PyExc_RuntimeError, "Failed to get storage infor for device.");
+        PyErr_SetString(MTPError, "Failed to get storage infor for device.");
         return NULL;
     }
     Py_RETURN_NONE;
@@ -287,9 +337,11 @@ libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs
     errs = PyList_New(0);
     if (ans == NULL || errs == NULL) { PyErr_NoMemory(); return NULL; }
 
+    Py_XINCREF(callback);
     cb.state = PyEval_SaveThread();
     tf = LIBMTP_Get_Filelisting_With_Callback(self->device, report_progress, &cb);
     PyEval_RestoreThread(cb.state);
+    Py_XDECREF(callback);
 
     if (tf == NULL) { 
         dump_errorstack(self->device, errs);
@@ -301,7 +353,7 @@ libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs
                 "id", f->item_id,
                 "parent_id", f->parent_id,
                 "storage_id", f->storage_id,
-                "filename", f->filename,
+                "name", f->filename,
                 "size", f->filesize,
                 "modtime", f->modificationdate
         );
@@ -334,7 +386,7 @@ int folderiter(LIBMTP_folder_t *f, PyObject *parent) {
 
     folder = Py_BuildValue("{s:k,s:k,s:k,s:s,s:N}",
             "id", f->folder_id,
-            "parent_d", f->parent_id,
+            "parent_id", f->parent_id,
             "storage_id", f->storage_id,
             "name", f->name,
             "children", children);
@@ -380,6 +432,159 @@ libmtp_Device_get_folderlist(libmtp_Device *self, PyObject *args, PyObject *kwar
 
 } // }}}
 
+// Device.get_file {{{
+static PyObject *
+libmtp_Device_get_file(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
+    PyObject *stream, *callback = NULL, *errs;
+    ProgressCallback cb;
+    uint32_t fileid;
+    int ret;
+
+    ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
+
+
+    if (!PyArg_ParseTuple(args, "kO|O", &fileid, &stream, &callback)) return NULL; 
+    errs = PyList_New(0);
+    if (errs == NULL) { PyErr_NoMemory(); return NULL; }
+
+    cb.obj = callback; cb.extra = stream;
+    Py_XINCREF(callback); Py_INCREF(stream);
+    cb.state = PyEval_SaveThread();
+    ret = LIBMTP_Get_File_To_Handler(self->device, fileid, data_to_python, &cb, report_progress, &cb);
+    PyEval_RestoreThread(cb.state);
+    Py_XDECREF(callback); Py_DECREF(stream);
+
+    if (ret != 0) { 
+        dump_errorstack(self->device, errs);
+    }
+    Py_XDECREF(PyObject_CallMethod(stream, "flush", NULL));
+    return Py_BuildValue("ON", (ret == 0) ? Py_True : Py_False, errs);
+
+} // }}}
+
+// Device.put_file {{{
+static PyObject *
+libmtp_Device_put_file(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
+    PyObject *stream, *callback = NULL, *errs, *fo;
+    ProgressCallback cb;
+    uint32_t parent_id, storage_id;
+    uint64_t filesize;
+    int ret;
+    char *name;
+    LIBMTP_file_t f, *nf;
+
+    ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
+
+    if (!PyArg_ParseTuple(args, "kksOK|O", &storage_id, &parent_id, &name, &stream, &filesize, &callback)) return NULL; 
+    errs = PyList_New(0);
+    if (errs == NULL) { PyErr_NoMemory(); return NULL; }
+
+    cb.obj = callback; cb.extra = stream;
+    f.parent_id = parent_id; f.storage_id = storage_id; f.item_id = 0; f.filename = name; f.filetype = LIBMTP_FILETYPE_UNKNOWN; f.filesize = filesize;
+    Py_XINCREF(callback); Py_INCREF(stream);
+    cb.state = PyEval_SaveThread();
+    ret = LIBMTP_Send_File_From_Handler(self->device, data_from_python, &cb, &f, report_progress, &cb);
+    PyEval_RestoreThread(cb.state);
+    Py_XDECREF(callback); Py_DECREF(stream);
+
+    fo = Py_None; Py_INCREF(fo);
+    if (ret != 0) dump_errorstack(self->device, errs);
+    else {
+        Py_BEGIN_ALLOW_THREADS;
+        nf = LIBMTP_Get_Filemetadata(self->device, f.item_id);
+        Py_END_ALLOW_THREADS;
+        if (nf == NULL) dump_errorstack(self->device, errs);
+        else {
+            Py_DECREF(fo);
+            fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:K,s:k}",
+                    "id", nf->item_id,
+                    "parent_id", nf->parent_id,
+                    "storage_id", nf->storage_id,
+                    "name", nf->filename,
+                    "size", nf->filesize,
+                    "modtime", nf->modificationdate
+            );
+            LIBMTP_destroy_file_t(nf);
+        }
+    }
+
+    return Py_BuildValue("NN", fo, errs);
+
+} // }}}
+
+// Device.delete_object {{{
+static PyObject *
+libmtp_Device_delete_object(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
+    PyObject *errs;
+    uint32_t id;
+    int res;
+
+    ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
+
+    if (!PyArg_ParseTuple(args, "k", &id)) return NULL;
+    errs = PyList_New(0);
+    if (errs == NULL) { PyErr_NoMemory(); return NULL; }
+
+    Py_BEGIN_ALLOW_THREADS;
+    res = LIBMTP_Delete_Object(self->device, id);
+    Py_END_ALLOW_THREADS;
+    if (res != 0) dump_errorstack(self->device, errs);
+
+    return Py_BuildValue("ON", (res == 0) ? Py_True : Py_False, errs);
+} // }}}
+
+// Device.create_folder {{{
+static PyObject *
+libmtp_Device_create_folder(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
+    PyObject *errs, *fo, *children, *temp;
+    uint32_t parent_id, storage_id;
+    char *name;
+    uint32_t folder_id;
+    LIBMTP_folder_t *f = NULL, *cf;
+
+    ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
+
+    if (!PyArg_ParseTuple(args, "kks", &storage_id, &parent_id, &name)) return NULL;
+    errs = PyList_New(0);
+    if (errs == NULL) { PyErr_NoMemory(); return NULL; }
+
+    fo = Py_None; Py_INCREF(fo);
+
+    Py_BEGIN_ALLOW_THREADS;
+    folder_id = LIBMTP_Create_Folder(self->device, name, parent_id, storage_id);
+    Py_END_ALLOW_THREADS;
+    if (folder_id == 0) dump_errorstack(self->device, errs);
+    else {
+        Py_BEGIN_ALLOW_THREADS;
+        // Cannot use Get_Folder_List_For_Storage as it fails
+        f = LIBMTP_Get_Folder_List(self->device);
+        Py_END_ALLOW_THREADS;
+        if (f == NULL) dump_errorstack(self->device, errs);
+        else {
+            cf = LIBMTP_Find_Folder(f, folder_id);
+            if (cf == NULL) {
+                temp = Py_BuildValue("is", 1, "Newly created folder not present on device!");
+                if (temp == NULL) { PyErr_NoMemory(); return NULL;}
+                PyList_Append(errs, temp);
+                Py_DECREF(temp);
+            } else {
+                Py_DECREF(fo);
+                children = PyList_New(0);
+                if (children == NULL) { PyErr_NoMemory(); return NULL; }
+                fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:N}",
+                        "id", cf->folder_id,
+                        "parent_id", cf->parent_id,
+                        "storage_id", cf->storage_id,
+                        "name", cf->name,
+                        "children", children);
+            }
+            LIBMTP_destroy_folder_t(f);
+        }
+    }
+
+    return Py_BuildValue("NN", fo, errs);
+} // }}}
+
 static PyMethodDef libmtp_Device_methods[] = {
     {"update_storage_info", (PyCFunction)libmtp_Device_update_storage_info, METH_VARARGS,
      "update_storage_info() -> Reread the storage info from the device (total, space, free space, storage locations, etc.)"
@@ -390,9 +595,26 @@ static PyMethodDef libmtp_Device_methods[] = {
     },
 
     {"get_folderlist", (PyCFunction)libmtp_Device_get_folderlist, METH_VARARGS,
-     "get_folderlist() -> Get the list of folders on the device. Returns files, erros."
+     "get_folderlist() -> Get the list of folders on the device. Returns files, errors."
     },
 
+    {"get_file", (PyCFunction)libmtp_Device_get_file, METH_VARARGS,
+     "get_file(fileid, stream, callback=None) -> Get the file specified by fileid from the device. stream must be a file-like object. The file will be written to it. callback works the same as in get_filelist(). Returns ok, errs, where errs is a list of errors (if any)."
+    },
+
+    {"put_file", (PyCFunction)libmtp_Device_put_file, METH_VARARGS,
+     "put_file(storage_id, parent_id, filename, stream, size, callback=None) -> Put a file on the device. The file is read from stream. It is put inside the folder identified by parent_id on the storage identified by storage_id. Use parent_id=0 to put it in the root. stream must be a file-like object. size is the size in bytes of the data in stream. callback works the same as in get_filelist(). Returns fileinfo, errs, where errs is a list of errors (if any), and fileinfo is a file information dictionary, as returned by get_filelist(). fileinfo will be None if case or errors."
+    },
+
+    {"create_folder", (PyCFunction)libmtp_Device_create_folder, METH_VARARGS,
+     "create_folder(storage_id, parent_id, name) -> Create a folder named name under parent parent_id (use 0 for root) in the storage identified by storage_id. Returns folderinfo, errors, where folderinfo is the same dict as returned by get_folderlist(), it will be None if there are errors."
+    },
+
+    {"delete_object", (PyCFunction)libmtp_Device_delete_object, METH_VARARGS,
+     "delete_object(id) -> Delete the object identified by id from the device. Can be used to delete files, folders, etc. Returns ok, errs."
+    },
+
+
     {NULL}  /* Sentinel */
 };
 
@@ -542,6 +764,8 @@ initlibmtp(void) {
     
     m = Py_InitModule3("libmtp", libmtp_methods, "Interface to libmtp.");
     if (m == NULL) return;
+    MTPError = PyErr_NewException("libmtp.MTPError", NULL, NULL);
+    if (MTPError == NULL) return;
 
     LIBMTP_Init();
     LIBMTP_Set_Debug(LIBMTP_DEBUG_NONE);

src/calibre/devices/mtp/windows/wpd.cpp

@@ -0,0 +1,89 @@
+/*
+ * mtp.c
+ * Copyright (C) 2012 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the MIT license.
+ */
+
+
+#define UNICODE
+#include <Windows.h>
+#include <Python.h>
+
+#include <Objbase.h>
+#include <PortableDeviceApi.h>
+
+static int _com_initialized = 0;
+static PyObject *WPDError = NULL;
+static PyObject *NoWPD = NULL;
+static IPortableDeviceManager *portable_device_manager = NULL;
+
+static PyObject *
+wpd_init(PyObject *self, PyObject *args) {
+    HRESULT hr;
+
+    if (!_com_initialized) {
+        hr = CoInitializeEx(NULL, COINIT_APARTMENTTHREADED);
+        if (SUCCEEDED(hr)) _com_initialized = 1;
+        else {PyErr_SetString(WPDError, "Failed to initialize COM"); return NULL;}
+    }
+
+    if (portable_device_manager == NULL) {
+        hr = CoCreateInstance(CLSID_PortableDeviceManager, NULL,
+                CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&portable_device_manager));
+
+        if (FAILED(hr)) {
+            portable_device_manager = NULL;
+            PyErr_SetString((hr == REGDB_E_CLASSNOTREG) ? NoWPD : WPDError, (hr == REGDB_E_CLASSNOTREG) ? 
+                "This computer is not running the Windows Portable Device framework. You may need to install Windows Media Player 11 or newer." : 
+                "Failed to create the WPD device manager interface");
+            return NULL;
+        }
+    }
+
+    Py_RETURN_NONE;
+}
+
+static PyObject *
+wpd_uninit(PyObject *self, PyObject *args) {
+    if (portable_device_manager != NULL) {
+        portable_device_manager->Release();
+        portable_device_manager = NULL;
+    }
+
+    if (_com_initialized) {
+        CoUninitialize();
+        _com_initialized = 0;
+    }
+
+    Py_RETURN_NONE;
+}
+
+static PyMethodDef wpd_methods[] = {
+    {"init", wpd_init, METH_VARARGS,
+        "init()\n\n Initializes this module. Call this method *only* in the thread in which you intend to use this module. Also remember to call uninit before the thread exits."
+    },
+
+    {"uninit", wpd_uninit, METH_VARARGS,
+        "uninit()\n\n Uninitialize this module. Must be called in the same thread as init(). Do not use any function/objects from this module after uninit has been called."
+    },
+
+    {NULL, NULL, 0, NULL}
+};
+
+
+PyMODINIT_FUNC
+initwpd(void) {
+    PyObject *m;
+
+    m = Py_InitModule3("wpd", wpd_methods, "Interface to the WPD windows service.");
+    if (m == NULL) return;
+
+    WPDError = PyErr_NewException("wpd.WPDError", NULL, NULL);
+    if (WPDError == NULL) return;
+
+    NoWPD = PyErr_NewException("wpd.NoWPD", NULL, NULL);
+    if (NoWPD == NULL) return;
+}
+
+

src/calibre/devices/prs500/cli/main.py

@@ -174,6 +174,13 @@ def ls(dev, path, term, recurse=False, color=False, human_readable_size=False, l
     output.close()
     return listing
 
+def shutdown_plugins():
+    for d in device_plugins():
+        try:
+            d.shutdown()
+        except:
+            pass
+
 def main():
     term = TerminalController()
     cols = term.COLS
@@ -201,6 +208,10 @@ def main():
     scanner.scan()
     connected_devices = []
     for d in device_plugins():
+        try:
+            d.startup()
+        except:
+            print ('Startup failed for device plugin: %s'%d)
         ok, det = scanner.is_device_connected(d)
         if ok:
             dev = d
@@ -209,6 +220,7 @@ def main():
 
     if dev is None:
         print >>sys.stderr, 'Unable to find a connected ebook reader.'
+        shutdown_plugins()
         return 1
 
     for det, d in connected_devices:
@@ -358,6 +370,9 @@ def main():
     except (ArgumentError, DeviceError) as e:
         print >>sys.stderr, e
         return 1
+    finally:
+        shutdown_plugins()
+
     return 0
 
 if __name__ == '__main__':

src/calibre/devices/smart_device_app/driver.py

@@ -576,7 +576,9 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                 self._debug('Protocol error - bogus accepted extensions')
                 self._close_device_socket()
                 return False
-            self.FORMATS = exts
+            config = self._configProxy()
+            config['format_map'] = exts
+            self._debug('selected formats', config['format_map']);
             if password:
                 returned_hash = result.get('passwordHash', None)
                 if result.get('passwordHash', None) is None:
@@ -678,7 +680,6 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         if opcode == 'OK':
             count = result['count']
             for i in range(0, count):
-                self._debug('retrieve metadata book', i)
                 opcode, result = self._call_client('GET_BOOK_METADATA', {'index': i},
                                                   print_debug_info=False)
                 if opcode == 'OK':

src/calibre/ebooks/BeautifulSoup.py

@@ -1454,7 +1454,9 @@ class BeautifulSoup(BeautifulStoneSoup):
     #According to the HTML standard, these block tags can contain
     #another tag of the same type. Furthermore, it's common
     #to actually use these tags this way.
-    NESTABLE_BLOCK_TAGS = ['blockquote', 'div', 'fieldset', 'ins', 'del']
+    # Changed by Kovid: Added HTML 5 block tags
+    NESTABLE_BLOCK_TAGS = ['blockquote', 'div', 'fieldset', 'ins', 'del',
+            'article', 'aside', 'header', 'footer', 'nav', 'figcaption', 'figure', 'section']
 
     #Lists can contain other lists, but there are restrictions.
     NESTABLE_LIST_TAGS = { 'ol' : [],

src/calibre/ebooks/metadata/toc.py

@@ -196,8 +196,8 @@ class TOC(list):
                     content = content[0]
                     src = get_attr(content, attr='src')
                     if src:
-                        purl = urlparse(unquote(content.get('src')))
+                        purl = urlparse(content.get('src'))
-                        href, fragment = purl[2], purl[5]
+                        href, fragment = unquote(purl[2]), unquote(purl[5])
                         nd = dest.add_item(href, fragment, text)
                         nd.play_order = play_order
 

src/calibre/ebooks/oeb/base.py

@@ -186,8 +186,9 @@ def rewrite_links(root, link_repl_func, resolve_base_href=False):
     If the ``link_repl_func`` returns None, the attribute or
     tag text will be removed completely.
     '''
-    from cssutils import parseString, parseStyle, replaceUrls, log
+    from cssutils import replaceUrls, log, CSSParser
     log.setLevel(logging.WARN)
+    log.raiseExceptions = False
 
     if resolve_base_href:
         resolve_base_href(root)
@@ -214,6 +215,8 @@ def rewrite_links(root, link_repl_func, resolve_base_href=False):
                 new = cur[:pos] + new_link + cur[pos+len(link):]
                 el.attrib[attrib] = new
 
+    parser = CSSParser(raiseExceptions=False, log=_css_logger,
+            fetcher=lambda x:(None, None))
     for el in root.iter(etree.Element):
         try:
             tag = el.tag
@@ -223,7 +226,7 @@ def rewrite_links(root, link_repl_func, resolve_base_href=False):
         if tag == XHTML('style') and el.text and \
                 (_css_url_re.search(el.text) is not None or '@import' in
                         el.text):
-            stylesheet = parseString(el.text, validate=False)
+            stylesheet = parser.parseString(el.text, validate=False)
             replaceUrls(stylesheet, link_repl_func)
             repl = stylesheet.cssText
             if isbytestring(repl):
@@ -234,7 +237,7 @@ def rewrite_links(root, link_repl_func, resolve_base_href=False):
             text = el.attrib['style']
             if _css_url_re.search(text) is not None:
                 try:
-                    stext = parseStyle(text, validate=False)
+                    stext = parser.parseStyle(text, validate=False)
                 except:
                     # Parsing errors are raised by cssutils
                     continue
@@ -862,6 +865,7 @@ class Manifest(object):
         def _parse_css(self, data):
             from cssutils import CSSParser, log, resolveImports
             log.setLevel(logging.WARN)
+            log.raiseExceptions = False
             self.oeb.log.debug('Parsing', self.href, '...')
             data = self.oeb.decode(data)
             data = self.oeb.css_preprocessor(data, add_namespace=True)
@@ -1537,7 +1541,9 @@ class TOC(object):
             if title:
                 title = re.sub(r'\s+', ' ', title)
             element(label, NCX('text')).text = title
-            element(point, NCX('content'), src=urlunquote(node.href))
+            # Do not unescape this URL as ADE requires it to be escaped to
+            # handle semi colons and other special characters in the file names
+            element(point, NCX('content'), src=node.href)
             node.to_ncx(point)
         return parent
 

src/calibre/ebooks/oeb/transforms/jacket.py

@@ -194,9 +194,17 @@ def render_jacket(mi, output_profile,
                 args[key] = escape(val)
                 args[key+'_label'] = escape(display_name)
             except:
+                # if the val (custom column contents) is None, don't add to args
                 pass
 
+        if False:
+            print("Custom column values available in jacket template:")
+            for key in args.keys():
+                if key.startswith('_') and not key.endswith('_label'):
+                    print(" %s: %s" % ('#' + key[1:], args[key]))
+
         # Used in the comment describing use of custom columns in templates
+        # Don't change this unless you also change it in template.xhtml
         args['_genre_label'] = args.get('_genre_label', '{_genre_label}')
         args['_genre'] = args.get('_genre', '{_genre}')
 

src/calibre/gui2/actions/catalog.py

@@ -85,7 +85,7 @@ class GenerateCatalogAction(InterfaceAction):
             dynamic.set('catalogs_to_be_synced', sync)
         self.gui.status_bar.show_message(_('Catalog generated.'), 3000)
         self.gui.sync_catalogs()
-        if job.fmt not in ['EPUB','MOBI']:
+        if job.fmt not in {'EPUB','MOBI', 'AZW3'}:
             export_dir = choose_dir(self.gui, _('Export Catalog Directory'),
                     _('Select destination for %(title)s.%(fmt)s') % dict(
                         title=job.catalog_title, fmt=job.fmt.lower()))

src/calibre/gui2/catalog/catalog_epub_mobi.py

@@ -14,24 +14,20 @@ from calibre.gui2 import gprefs, question_dialog
 from calibre.utils.icu import sort_key
 
 from catalog_epub_mobi_ui import Ui_Form
-from PyQt4 import QtGui
+from PyQt4.Qt import (Qt, QAbstractItemView, QCheckBox, QComboBox,
-from PyQt4.Qt import (Qt, QAbstractItemView, QCheckBox, QComboBox, QDialog,
+        QDoubleSpinBox, QIcon, QLineEdit, QRadioButton, QSize, QSizePolicy,
-                      QDialogButtonBox, QDoubleSpinBox,
+        QTableWidget, QTableWidgetItem, QToolButton, QVBoxLayout, QWidget)
-                      QHBoxLayout, QIcon, QLabel, QLineEdit,
-                      QPlainTextEdit, QRadioButton, QSize, QSizePolicy,
-                      QTableWidget, QTableWidgetItem, QTimer,
-                      QToolButton, QVBoxLayout, QWidget)
 
 class PluginWidget(QWidget,Ui_Form):
 
     TITLE = _('E-book options')
-    HELP  = _('Options specific to')+' EPUB/MOBI '+_('output')
+    HELP  = _('Options specific to')+' AZW3/EPUB/MOBI '+_('output')
 
     # Output synced to the connected device?
     sync_enabled = True
 
     # Formats supported by this plugin
-    formats = set(['epub','mobi'])
+    formats = set(['azw3','epub','mobi'])
 
     def __init__(self, parent=None):
         QWidget.__init__(self, parent)
@@ -402,7 +398,6 @@ class PluginWidget(QWidget,Ui_Form):
                 self.exclude_genre.setText(default[1])
                 break
 
-
 class CheckableTableWidgetItem(QTableWidgetItem):
     '''
     Borrowed from kiwidude
@@ -466,8 +461,7 @@ class GenericRulesTable(QTableWidget):
         self.db = db
         QTableWidget.__init__(self)
         self.setObjectName(object_name)
-        self.layout = QHBoxLayout()
+        self.layout = parent_gb.layout()
-        parent_gb.setLayout(self.layout)
 
         # Add ourselves to the layout
         #print("verticalHeader: %s" % dir(self.verticalHeader()))
@@ -538,7 +532,7 @@ class GenericRulesTable(QTableWidget):
 
     def create_blank_row_data(self):
         '''
-        ovverride
+        override
         '''
         pass
 
@@ -571,6 +565,9 @@ class GenericRulesTable(QTableWidget):
         self.clearSelection()
 
     def get_data(self):
+        '''
+        override
+        '''
         pass
 
     def move_row_down(self):
@@ -598,6 +595,7 @@ class GenericRulesTable(QTableWidget):
 
             # Populate it with the saved data
             self.populate_table_row(src_row, saved_data)
+
         self.blockSignals(False)
         scroll_to_row = last_sel_row + 1
         if scroll_to_row < self.rowCount() - 1:
@@ -637,8 +635,9 @@ class GenericRulesTable(QTableWidget):
         pass
 
     def resize_name(self, scale):
-        current_width = self.columnWidth(1)
+        #current_width = self.columnWidth(1)
-        self.setColumnWidth(1, min(225,int(current_width * scale)))
+        #self.setColumnWidth(1, min(225,int(current_width * scale)))
+        self.setColumnWidth(1, 225)
 
     def rule_name_edited(self):
         current_row = self.currentRow()
@@ -650,23 +649,21 @@ class GenericRulesTable(QTableWidget):
         self.selectRow(row)
         self.scrollToItem(self.currentItem())
 
-    def tweak_height(self, height=4):
-        for i in range(min(3,self.rowCount())):
-            height += self.rowHeight(i)
-        height += self.verticalHeader().sizeHint().height()
-        print("computed table height for %d rows: %d" % (self.rowCount(),height, ))
-        self.setMinimumSize(QSize(16777215, height))
-        self.setMaximumSize(QSize(16777215, height))
-
 class ExclusionRules(GenericRulesTable):
 
+    COLUMNS = { 'ENABLED':{'ordinal': 0, 'name': ''},
+                'NAME':   {'ordinal': 1, 'name': 'Name'},
+                'FIELD':  {'ordinal': 2, 'name': 'Field'},
+                'PATTERN':  {'ordinal': 3, 'name': 'Value'},}
+
     def __init__(self, parent_gb_hl, object_name, rules, eligible_custom_fields, db):
         super(ExclusionRules, self).__init__(parent_gb_hl, object_name, rules, eligible_custom_fields, db)
         self._init_table_widget()
         self._initialize()
 
     def _init_table_widget(self):
-        header_labels = ['','Name','Field','Value']
+        header_labels = [self.COLUMNS[index]['name'] \
+                         for index in sorted(self.COLUMNS.keys(), key=lambda c: self.COLUMNS[c]['ordinal'])]
         self.setColumnCount(len(header_labels))
         self.setHorizontalHeaderLabels(header_labels)
         self.setSortingEnabled(False)
@@ -682,10 +679,10 @@ class ExclusionRules(GenericRulesTable):
     def convert_row_to_data(self, row):
         data = self.create_blank_row_data()
         data['ordinal'] = row
-        data['enabled'] = self.item(row,0).checkState() == Qt.Checked
+        data['enabled'] = self.item(row,self.COLUMNS['ENABLED']['ordinal']).checkState() == Qt.Checked
-        data['name'] = unicode(self.cellWidget(row,1).text()).strip()
+        data['name'] = unicode(self.cellWidget(row,self.COLUMNS['NAME']['ordinal']).text()).strip()
-        data['field'] = unicode(self.cellWidget(row,2).currentText()).strip()
+        data['field'] = unicode(self.cellWidget(row,self.COLUMNS['FIELD']['ordinal']).currentText()).strip()
-        data['pattern'] = unicode(self.cellWidget(row,3).currentText()).strip()
+        data['pattern'] = unicode(self.cellWidget(row,self.COLUMNS['PATTERN']['ordinal']).currentText()).strip()
         return data
 
     def create_blank_row_data(self):
@@ -740,18 +737,18 @@ class ExclusionRules(GenericRulesTable):
         # Entry point
         self.blockSignals(True)
 
-        # Column 0: Enabled
+        # Enabled
-        self.setItem(row, 0, CheckableTableWidgetItem(data['enabled']))
+        self.setItem(row, self.COLUMNS['ENABLED']['ordinal'], CheckableTableWidgetItem(data['enabled']))
 
-        # Column 1: Rule name
+        # Rule name
-        set_rule_name_in_row(row, 1, name=data['name'])
+        set_rule_name_in_row(row, self.COLUMNS['NAME']['ordinal'], name=data['name'])
 
-        # Column 2: Source field
+        # Source field
-        source_combo = set_source_field_in_row(row, 2, field=data['field'])
+        source_combo = set_source_field_in_row(row, self.COLUMNS['FIELD']['ordinal'], field=data['field'])
 
-        # Column 3: Pattern
+        # Pattern
         # The contents of the Pattern field is driven by the Source field
-        self.source_index_changed(source_combo, row, 3, pattern=data['pattern'])
+        self.source_index_changed(source_combo, row, self.COLUMNS['PATTERN']['ordinal'], pattern=data['pattern'])
 
         self.blockSignals(False)
 
@@ -775,17 +772,24 @@ class ExclusionRules(GenericRulesTable):
                 values = ['any date','unspecified']
 
         values_combo = ComboBox(self, values, pattern)
-        self.setCellWidget(row, 3, values_combo)
+        self.setCellWidget(row, self.COLUMNS['PATTERN']['ordinal'], values_combo)
 
 class PrefixRules(GenericRulesTable):
 
+    COLUMNS = { 'ENABLED':{'ordinal': 0, 'name': ''},
+                'NAME':   {'ordinal': 1, 'name': 'Name'},
+                'PREFIX': {'ordinal': 2, 'name': 'Prefix'},
+                'FIELD':  {'ordinal': 3, 'name': 'Field'},
+                'PATTERN':{'ordinal': 4, 'name': 'Value'},}
+
     def __init__(self, parent_gb_hl, object_name, rules, eligible_custom_fields, db):
         super(PrefixRules, self).__init__(parent_gb_hl, object_name, rules, eligible_custom_fields, db)
         self._init_table_widget()
         self._initialize()
 
     def _init_table_widget(self):
-        header_labels = ['','Name','Prefix','Field','Value']
+        header_labels = [self.COLUMNS[index]['name'] \
+                         for index in sorted(self.COLUMNS.keys(), key=lambda c: self.COLUMNS[c]['ordinal'])]
         self.setColumnCount(len(header_labels))
         self.setHorizontalHeaderLabels(header_labels)
         self.setSortingEnabled(False)
@@ -803,10 +807,10 @@ class PrefixRules(GenericRulesTable):
         data = self.create_blank_row_data()
         data['ordinal'] = row
         data['enabled'] = self.item(row,0).checkState() == Qt.Checked
-        data['name'] = unicode(self.cellWidget(row,1).text()).strip()
+        data['name'] = unicode(self.cellWidget(row,self.COLUMNS['NAME']['ordinal']).text()).strip()
-        data['prefix'] = unicode(self.cellWidget(row,2).currentText()).strip()
+        data['prefix'] = unicode(self.cellWidget(row,self.COLUMNS['PREFIX']['ordinal']).currentText()).strip()
-        data['field'] = unicode(self.cellWidget(row,3).currentText()).strip()
+        data['field'] = unicode(self.cellWidget(row,self.COLUMNS['FIELD']['ordinal']).currentText()).strip()
-        data['pattern'] = unicode(self.cellWidget(row,4).currentText()).strip()
+        data['pattern'] = unicode(self.cellWidget(row,self.COLUMNS['PATTERN']['ordinal']).currentText()).strip()
         return data
 
     def create_blank_row_data(self):
@@ -872,7 +876,6 @@ class PrefixRules(GenericRulesTable):
             ('Math plus circled',u'\u2295'),
             ('Math times circled',u'\u2297'),
             ('Math times',u'\u00d7'),
-            ('O slash',u'\u00d8'),
             ('Paragraph',u'\u00b6'),
             ('Percent',u'%'),
             ('Plus-or-minus',u'\u00b1'),
@@ -1004,22 +1007,21 @@ class PrefixRules(GenericRulesTable):
         self.blockSignals(True)
         #print("prefix_rules_populate_table_row processing rule:\n%s\n" % data)
 
-        # Column 0: Enabled
+        # Enabled
-        self.setItem(row, 0, CheckableTableWidgetItem(data['enabled']))
+        self.setItem(row, self.COLUMNS['ENABLED']['ordinal'], CheckableTableWidgetItem(data['enabled']))
 
-        # Column 1: Rule name
+        # Rule name
-        #rule_name = QTableWidgetItem(data['name'])
+        set_rule_name_in_row(row, self.COLUMNS['NAME']['ordinal'], name=data['name'])
-        set_rule_name_in_row(row, 1, name=data['name'])
 
-        # Column 2: Prefix
+        # Prefix
-        set_prefix_field_in_row(row, 2, field=data['prefix'])
+        set_prefix_field_in_row(row, self.COLUMNS['PREFIX']['ordinal'], field=data['prefix'])
 
-        # Column 3: Source field
+        # Source field
-        source_combo = set_source_field_in_row(row, 3, field=data['field'])
+        source_combo = set_source_field_in_row(row, self.COLUMNS['FIELD']['ordinal'], field=data['field'])
 
-        # Column 4: Pattern
+        # Pattern
         # The contents of the Pattern field is driven by the Source field
-        self.source_index_changed(source_combo, row, 4, pattern=data['pattern'])
+        self.source_index_changed(source_combo, row, self.COLUMNS['PATTERN']['ordinal'], pattern=data['pattern'])
 
         self.blockSignals(False)
 
@@ -1045,5 +1047,5 @@ class PrefixRules(GenericRulesTable):
                 values = ['any date','unspecified']
 
         values_combo = ComboBox(self, values, pattern)
-        self.setCellWidget(row, 4, values_combo)
+        self.setCellWidget(row, self.COLUMNS['PATTERN']['ordinal'], values_combo)
 

src/calibre/gui2/catalog/catalog_epub_mobi.ui

@@ -210,6 +210,11 @@ The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book]
      <property name="title">
       <string>Excluded books</string>
      </property>
+     <layout class="QHBoxLayout" name="horizontalLayout_4">
+      <item>
+       <layout class="QHBoxLayout" name="horizontalLayout_3"/>
+      </item>
+     </layout>
     </widget>
    </item>
    <item>
@@ -221,11 +226,16 @@ The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book]
       </sizepolicy>
      </property>
      <property name="toolTip">
-      <string>The first enabled matching rule will be used to add a prefix to book listings in the generated catalog.</string>
+      <string>The first matching prefix rule applies a prefix to book listings in the generated catalog.</string>
      </property>
      <property name="title">
-      <string>Prefix rules</string>
+      <string>Prefixes</string>
      </property>
+     <layout class="QHBoxLayout" name="horizontalLayout_8">
+      <item>
+       <layout class="QHBoxLayout" name="horizontalLayout_6"/>
+      </item>
+     </layout>
     </widget>
    </item>
    <item>

src/calibre/gui2/device.py

@@ -3,7 +3,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
 # Imports {{{
-import os, traceback, Queue, time, cStringIO, re, sys
+import os, traceback, Queue, time, cStringIO, re, sys, weakref
 from threading import Thread, Event
 
 from PyQt4.Qt import (QMenu, QAction, QActionGroup, QIcon, SIGNAL,
@@ -369,6 +369,18 @@ class DeviceManager(Thread): # {{{
         except:
             return False
 
+    def _debug_detection(self):
+        from calibre.devices import debug
+        raw = debug(plugins=self.devices)
+        return raw
+
+    def debug_detection(self, done):
+        if self.is_device_connected:
+            raise ValueError('Device is currently detected in calibre, cannot'
+                    ' debug device detection')
+        self.create_job(self._debug_detection, done,
+                _('Debug device detection'))
+
     def _get_device_information(self):
         info = self.device.get_device_information(end_session=False)
         if len(info) < 5:
@@ -771,6 +783,15 @@ class DeviceMixin(object): # {{{
         if tweaks['auto_connect_to_folder']:
             self.connect_to_folder_named(tweaks['auto_connect_to_folder'])
 
+    def debug_detection(self, done):
+        self.debug_detection_callback = weakref.ref(done)
+        self.device_manager.debug_detection(FunctionDispatcher(self.debug_detection_done))
+
+    def debug_detection_done(self, job):
+        d = self.debug_detection_callback()
+        if d is not None:
+            d(job)
+
     def show_open_feedback(self, devname, e):
         try:
             self.__of_dev_mem__ = d = e.custom_dialog(self)

src/calibre/gui2/preferences/coloring.py

@@ -20,6 +20,8 @@ from calibre.gui2.dialogs.template_dialog import TemplateDialog
 from calibre.gui2.metadata.single_download import RichTextDelegate
 from calibre.library.coloring import (Rule, conditionable_columns,
     displayable_columns, rule_from_template)
+from calibre.utils.localization import lang_map
+from calibre.utils.icu import lower
 
 class ConditionEditor(QWidget): # {{{
 
@@ -143,7 +145,11 @@ class ConditionEditor(QWidget): # {{{
 
     @property
     def current_val(self):
-        return unicode(self.value_box.text()).strip()
+        ans = unicode(self.value_box.text()).strip()
+        if self.current_col == 'languages':
+            rmap = {lower(v):k for k, v in lang_map().iteritems()}
+            ans = rmap.get(lower(ans), ans)
+        return ans
 
     @dynamic_property
     def condition(self):
@@ -210,6 +216,11 @@ class ConditionEditor(QWidget): # {{{
         if col == 'identifiers':
             tt = _('Enter either an identifier type or an '
                     'identifier type and value of the form identifier:value')
+        elif col == 'languages':
+            tt = _('Enter a 3 letter ISO language code, like fra for French'
+                    ' or deu for German or eng for English. You can also use'
+                    ' the full language name, in which case calibre will try to'
+                    ' automatically convert it to the language code.')
         elif dt in ('int', 'float', 'rating'):
             tt = _('Enter a number')
             v = QIntValidator if dt == 'int' else QDoubleValidator

src/calibre/gui2/preferences/device_debug.py

@@ -10,15 +10,18 @@ __docformat__ = 'restructuredtext en'
 from PyQt4.Qt import QDialog, QVBoxLayout, QPlainTextEdit, QTimer, \
     QDialogButtonBox, QPushButton, QApplication, QIcon
 
+from calibre.gui2 import error_dialog
+
 class DebugDevice(QDialog):
 
-    def __init__(self, parent=None):
+    def __init__(self, gui, parent=None):
         QDialog.__init__(self, parent)
+        self.gui = gui
         self._layout = QVBoxLayout(self)
         self.setLayout(self._layout)
         self.log = QPlainTextEdit(self)
         self._layout.addWidget(self.log)
-        self.log.setPlainText(_('Getting debug information')+'...')
+        self.log.setPlainText(_('Getting debug information, please wait')+'...')
         self.copy = QPushButton(_('Copy to &clipboard'))
         self.copy.setDefault(True)
         self.setWindowTitle(_('Debug device detection'))
@@ -36,12 +39,26 @@ class DebugDevice(QDialog):
         QTimer.singleShot(1000, self.debug)
 
     def debug(self):
-        try:
+        if self.gui.device_manager.is_device_connected:
-            from calibre.devices import debug
+            error_dialog(self, _('Device already detected'),
-            raw = debug()
+                    _('A device (%s) is already detected by calibre.'
-            self.log.setPlainText(raw)
+                        ' If you wish to debug the detection of another device'
-        finally:
+                        ', first disconnect this device.')%
+                    self.gui.device_manager.connected_device.get_gui_name(),
+                    show=True)
             self.bbox.setEnabled(True)
+            return
+        self.gui.debug_detection(self)
+
+    def __call__(self, job):
+        if not self.isVisible(): return
+        self.bbox.setEnabled(True)
+        if job.failed:
+            return error_dialog(self, _('Debugging failed'),
+                    _('Running debug device detection failed. Click Show '
+                        'Details for more information.'), det_msg=job.details,
+                    show=True)
+        self.log.setPlainText(job.result)
 
     def copy_to_clipboard(self):
         QApplication.clipboard().setText(self.log.toPlainText())

src/calibre/gui2/preferences/misc.py

@@ -52,7 +52,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
     def debug_device_detection(self, *args):
         from calibre.gui2.preferences.device_debug import DebugDevice
-        d = DebugDevice(self)
+        d = DebugDevice(self.gui, self)
         d.exec_()
 
     def user_defined_device(self, *args):

src/calibre/gui2/viewer/documentview.py

@@ -21,6 +21,7 @@ from calibre.gui2.viewer.keys import SHORTCUTS
 from calibre.gui2.viewer.javascript import JavaScriptLoader
 from calibre.gui2.viewer.position import PagePosition
 from calibre.gui2.viewer.config import config, ConfigDialog
+from calibre.gui2.viewer.image_popup import ImagePopup
 from calibre.ebooks.oeb.display.webview import load_html
 from calibre.constants import isxp
 # }}}
@@ -470,6 +471,9 @@ class DocumentView(QWebView): # {{{
         self.dictionary_action.setShortcut(Qt.CTRL+Qt.Key_L)
         self.dictionary_action.triggered.connect(self.lookup)
         self.addAction(self.dictionary_action)
+        self.image_popup = ImagePopup(self)
+        self.view_image_action = QAction(_('View &image...'), self)
+        self.view_image_action.triggered.connect(self.image_popup)
         self.search_action = QAction(QIcon(I('dictionary.png')),
                 _('&Search for next occurrence'), self)
         self.search_action.setShortcut(Qt.CTRL+Qt.Key_S)
@@ -554,6 +558,11 @@ class DocumentView(QWebView): # {{{
             self.manager.selection_changed(unicode(self.document.selectedText()))
 
     def contextMenuEvent(self, ev):
+        mf = self.document.mainFrame()
+        r = mf.hitTestContent(ev.pos())
+        img = r.pixmap()
+        self.image_popup.current_img = img
+        self.image_popup.current_url = r.imageUrl()
         menu = self.document.createStandardContextMenu()
         for action in self.unimplemented_actions:
             menu.removeAction(action)
@@ -561,6 +570,8 @@ class DocumentView(QWebView): # {{{
         if text:
             menu.insertAction(list(menu.actions())[0], self.dictionary_action)
             menu.insertAction(list(menu.actions())[0], self.search_action)
+        if not img.isNull():
+            menu.addAction(self.view_image_action)
         menu.addSeparator()
         menu.addAction(self.goto_location_action)
         if self.document.in_fullscreen_mode and self.manager is not None:

src/calibre/gui2/viewer/image_popup.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from PyQt4.Qt import (QDialog, QPixmap, QUrl, QScrollArea, QLabel, QSizePolicy,
+        QDialogButtonBox, QVBoxLayout, QPalette, QApplication, QSize, QIcon, Qt)
+
+from calibre.gui2 import choose_save_file, gprefs
+
+class ImageView(QDialog):
+
+    def __init__(self, parent, current_img, current_url):
+        QDialog.__init__(self)
+        dw = QApplication.instance().desktop()
+        self.avail_geom = dw.availableGeometry(parent)
+        self.current_img = current_img
+        self.current_url = current_url
+        self.factor = 1.0
+
+        self.label = l = QLabel()
+        l.setBackgroundRole(QPalette.Base);
+        l.setSizePolicy(QSizePolicy.Ignored, QSizePolicy.Ignored)
+        l.setScaledContents(True)
+
+        self.scrollarea = sa = QScrollArea()
+        sa.setBackgroundRole(QPalette.Dark)
+        sa.setWidget(l)
+
+        self.bb = bb = QDialogButtonBox(QDialogButtonBox.Close)
+        bb.accepted.connect(self.accept)
+        bb.rejected.connect(self.reject)
+        self.zi_button = zi = bb.addButton(_('Zoom &in'), bb.ActionRole)
+        self.zo_button = zo = bb.addButton(_('Zoom &out'), bb.ActionRole)
+        self.save_button = so = bb.addButton(_('&Save as'), bb.ActionRole)
+        zi.setIcon(QIcon(I('plus.png')))
+        zo.setIcon(QIcon(I('minus.png')))
+        so.setIcon(QIcon(I('save.png')))
+        zi.clicked.connect(self.zoom_in)
+        zo.clicked.connect(self.zoom_out)
+        so.clicked.connect(self.save_image)
+
+        self.l = l = QVBoxLayout()
+        self.setLayout(l)
+        l.addWidget(sa)
+        l.addWidget(bb)
+
+    def zoom_in(self):
+        self.factor *= 1.25
+        self.adjust_image(1.25)
+
+    def zoom_out(self):
+        self.factor *= 0.8
+        self.adjust_image(0.8)
+
+    def save_image(self):
+        filters=[('Images', ['png', 'jpeg', 'jpg'])]
+        f = choose_save_file(self, 'viewer image view save dialog',
+                _('Choose a file to save to'), filters=filters,
+                all_files=False)
+        if f:
+            self.current_img.save(f)
+
+    def adjust_image(self, factor):
+        self.label.resize(self.factor * self.current_img.size())
+        self.zi_button.setEnabled(self.factor <= 3)
+        self.zo_button.setEnabled(self.factor >= 0.3333)
+        self.adjust_scrollbars(factor)
+
+    def adjust_scrollbars(self, factor):
+        for sb in (self.scrollarea.horizontalScrollBar(),
+                self.scrollarea.verticalScrollBar()):
+            sb.setValue(int(factor*sb.value()) + ((factor - 1) * sb.pageStep()/2))
+
+    def __call__(self):
+        geom = self.avail_geom
+        self.label.setPixmap(self.current_img)
+        self.label.adjustSize()
+        self.resize(QSize(int(geom.width()/2.5), geom.height()-50))
+        geom = gprefs.get('viewer_image_popup_geometry', None)
+        if geom is not None:
+            self.restoreGeometry(geom)
+        self.current_image_name = unicode(self.current_url.toString()).rpartition('/')[-1]
+        title = _('View Image: %s')%self.current_image_name
+        self.setWindowTitle(title)
+        self.show()
+
+    def done(self, e):
+        gprefs['viewer_image_popup_geometry'] = bytearray(self.saveGeometry())
+        return QDialog.done(self, e)
+
+class ImagePopup(object):
+
+    def __init__(self, parent):
+        self.current_img = QPixmap()
+        self.current_url = QUrl()
+        self.parent = parent
+        self.dialogs = []
+
+    def __call__(self):
+        if self.current_img.isNull():
+            return
+        d = ImageView(self.parent, self.current_img, self.current_url)
+        self.dialogs.append(d)
+        d.finished.connect(self.cleanup, type=Qt.QueuedConnection)
+        d()
+
+    def cleanup(self):
+        for d in tuple(self.dialogs):
+            if not d.isVisible():
+                self.dialogs.remove(d)
+

src/calibre/gui2/viewer/main.py

@@ -577,7 +577,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
                 if not os.path.exists(item.abspath):
                     return error_dialog(self, _('No such location'),
                             _('The location pointed to by this item'
-                                ' does not exist.'), show=True)
+                                ' does not exist.'), det_msg=item.abspath, show=True)
                 url = QUrl.fromLocalFile(item.abspath)
                 if item.fragment:
                     url.setFragment(item.fragment)

src/calibre/library/catalogs/epub_mobi.py

@@ -13,6 +13,8 @@ from collections import namedtuple
 from calibre import strftime
 from calibre.customize import CatalogPlugin
 from calibre.customize.conversion import OptionRecommendation, DummyReporter
+from calibre.ebooks import calibre_cover
+from calibre.ptempfile import PersistentTemporaryFile
 
 Option = namedtuple('Option', 'option, default, dest, action, help')
 
@@ -20,12 +22,12 @@ class EPUB_MOBI(CatalogPlugin):
     'ePub catalog generator'
 
     name = 'Catalog_EPUB_MOBI'
-    description = 'EPUB/MOBI catalog generator'
+    description = 'AZW3/EPUB/MOBI catalog generator'
     supported_platforms = ['windows', 'osx', 'linux']
     minimum_calibre_version = (0, 7, 40)
     author = 'Greg Riker'
     version = (1, 0, 0)
-    file_types = set(['epub','mobi'])
+    file_types = set(['azw3','epub','mobi'])
 
     THUMB_SMALLEST = "1.0"
     THUMB_LARGEST = "2.0"
@@ -36,7 +38,7 @@ class EPUB_MOBI(CatalogPlugin):
                           action = None,
                           help = _('Title of generated catalog used as title in metadata.\n'
                           "Default: '%default'\n"
-                          "Applies to: ePub, MOBI output formats")),
+                          "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--debug-pipeline',
                            default=None,
                            dest='debug_pipeline',
@@ -46,17 +48,17 @@ class EPUB_MOBI(CatalogPlugin):
                            "directory. Useful if you are unsure at which stage "
                            "of the conversion process a bug is occurring.\n"
                            "Default: '%default'\n"
-                           "Applies to: ePub, MOBI output formats")),
+                           "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--exclude-genre',
                           default='\[.+\]|\+',
                           dest='exclude_genre',
                           action = None,
                           help=_("Regex describing tags to exclude as genres.\n"
                           "Default: '%default' excludes bracketed tags, e.g. '[Project Gutenberg]', and '+', the default tag for read books.\n"
-                          "Applies to: ePub, MOBI output formats")),
+                          "Applies to: AZW3, ePub, MOBI output formats")),
 
                    Option('--exclusion-rules',
-                          default="(('Excluded tags','Tags','~,Catalog'),)",
+                          default="(('Excluded tags','Tags','Catalog'),)",
                           dest='exclusion_rules',
                           action=None,
                           help=_("Specifies the rules used to exclude books from the generated catalog.\n"
@@ -67,7 +69,7 @@ class EPUB_MOBI(CatalogPlugin):
                           "will exclude a book with a value of 'Archived' in the custom column 'status'.\n"
                           "When multiple rules are defined, all rules will be applied.\n"
                           "Default: \n" + '"' + '%default' + '"' + "\n"
-                          "Applies to ePub, MOBI output formats")),
+                          "Applies to AZW3, ePub, MOBI output formats")),
 
                    Option('--generate-authors',
                           default=False,
@@ -75,49 +77,49 @@ class EPUB_MOBI(CatalogPlugin):
                           action = 'store_true',
                           help=_("Include 'Authors' section in catalog.\n"
                           "Default: '%default'\n"
-                          "Applies to: ePub, MOBI output formats")),
+                          "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--generate-descriptions',
                           default=False,
                           dest='generate_descriptions',
                           action = 'store_true',
                           help=_("Include 'Descriptions' section in catalog.\n"
                           "Default: '%default'\n"
-                          "Applies to: ePub, MOBI output formats")),
+                          "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--generate-genres',
                           default=False,
                           dest='generate_genres',
                           action = 'store_true',
                           help=_("Include 'Genres' section in catalog.\n"
                           "Default: '%default'\n"
-                          "Applies to: ePub, MOBI output formats")),
+                          "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--generate-titles',
                           default=False,
                           dest='generate_titles',
                           action = 'store_true',
                           help=_("Include 'Titles' section in catalog.\n"
                           "Default: '%default'\n"
-                          "Applies to: ePub, MOBI output formats")),
+                          "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--generate-series',
                           default=False,
                           dest='generate_series',
                           action = 'store_true',
                           help=_("Include 'Series' section in catalog.\n"
                           "Default: '%default'\n"
-                          "Applies to: ePub, MOBI output formats")),
+                          "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--generate-recently-added',
                           default=False,
                           dest='generate_recently_added',
                           action = 'store_true',
                           help=_("Include 'Recently Added' section in catalog.\n"
                           "Default: '%default'\n"
-                          "Applies to: ePub, MOBI output formats")),
+                          "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--header-note-source-field',
                           default='',
                           dest='header_note_source_field',
                           action = None,
                           help=_("Custom field containing note text to insert in Description header.\n"
                           "Default: '%default'\n"
-                          "Applies to: ePub, MOBI output formats")),
+                          "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--merge-comments',
                           default='::',
                           dest='merge_comments',
@@ -127,23 +129,23 @@ class EPUB_MOBI(CatalogPlugin):
                           " [before|after] Placement of notes with respect to Comments\n"
                           " [True|False] - A horizontal rule is inserted between notes and Comments\n"
                           "Default: '%default'\n"
-                          "Applies to ePub, MOBI output formats")),
+                          "Applies to AZW3, ePub, MOBI output formats")),
                    Option('--output-profile',
                           default=None,
                           dest='output_profile',
                           action = None,
                           help=_("Specifies the output profile.  In some cases, an output profile is required to optimize the catalog for the device.  For example, 'kindle' or 'kindle_dx' creates a structured Table of Contents with Sections and Articles.\n"
                           "Default: '%default'\n"
-                          "Applies to: ePub, MOBI output formats")),
+                          "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--prefix-rules',
                           default="(('Read books','tags','+','\u2713'),('Wishlist items','tags','Wishlist','\u00d7'))",
                           dest='prefix_rules',
                           action=None,
-                          help=_("Specifies the rules used to include prefixes indicating read books, wishlist items and other user-specifed prefixes.\n"
+                          help=_("Specifies the rules used to include prefixes indicating read books, wishlist items and other user-specified prefixes.\n"
                           "The model for a prefix rule is ('<rule name>','<source field>','<pattern>','<prefix>').\n"
                           "When multiple rules are defined, the first matching rule will be used.\n"
                           "Default:\n" + '"' + '%default' + '"' + "\n"
-                          "Applies to ePub, MOBI output formats")),
+                          "Applies to AZW3, ePub, MOBI output formats")),
                    Option('--thumb-width',
                           default='1.0',
                           dest='thumb_width',
@@ -151,7 +153,7 @@ class EPUB_MOBI(CatalogPlugin):
                           help=_("Size hint (in inches) for book covers in catalog.\n"
                           "Range: 1.0 - 2.0\n"
                           "Default: '%default'\n"
-                          "Applies to ePub, MOBI output formats")),
+                          "Applies to AZW3, ePub, MOBI output formats")),
                           ]
     # }}}
 
@@ -172,7 +174,8 @@ class EPUB_MOBI(CatalogPlugin):
         if op is None:
             op = 'default'
 
-        if opts.connected_device['name'] and 'kindle' in opts.connected_device['name'].lower():
+        if opts.connected_device['name'] and \
+                opts.connected_device['short_name'] in ['kindle','kindle dx']:
             opts.connected_kindle = True
             if opts.connected_device['serial'] and \
                opts.connected_device['serial'][:4] in ['B004','B005']:
@@ -337,7 +340,7 @@ class EPUB_MOBI(CatalogPlugin):
                 OptionRecommendation.HIGH))
             recommendations.append(('comments', '', OptionRecommendation.HIGH))
 
-            # Use to debug generated catalog code before conversion
+            # >>> Use to debug generated catalog code before conversion <<<
             #setattr(opts,'debug_pipeline',os.path.expanduser("~/Desktop/Catalog debug"))
 
             dp = getattr(opts, 'debug_pipeline', None)
@@ -355,6 +358,7 @@ class EPUB_MOBI(CatalogPlugin):
 
             # If cover exists, use it
             cpath = None
+            generate_new_cover = False
             try:
                 search_text = 'title:"%s" author:%s' % (
                         opts.catalog_title.replace('"', '\\"'), 'calibre')
@@ -364,9 +368,26 @@ class EPUB_MOBI(CatalogPlugin):
                     if cpath and os.path.exists(cpath):
                         recommendations.append(('cover', cpath,
                             OptionRecommendation.HIGH))
+                        log.info("using existing cover")
+                    else:
+                        log.info("no existing cover, generating new cover")
+                        generate_new_cover = True
+                else:
+                    log.info("no existing cover, generating new cover")
+                    generate_new_cover = True
             except:
                 pass
 
+            if generate_new_cover:
+                new_cover_path = PersistentTemporaryFile(suffix='.jpg')
+                new_cover = calibre_cover(opts.catalog_title.replace('"', '\\"'), 'calibre')
+                new_cover_path.write(new_cover)
+                new_cover_path.close()
+                recommendations.append(('cover', new_cover_path.name, OptionRecommendation.HIGH))
+
+            if opts.verbose:
+                log.info("Invoking Plumber with recommendations:\n %s" % recommendations)
+
             # Run ebook-convert
             from calibre.ebooks.conversion.plumber import Plumber
             plumber = Plumber(os.path.join(catalog.catalogPath,

src/calibre/library/catalogs/epub_mobi_builder.py

@@ -1126,7 +1126,7 @@ Author '{0}':
                 aTag = Tag(soup, "a")
                 current_letter = self.letter_or_symbol(book['author_sort'][0].upper())
                 if current_letter == self.SYMBOLS:
-                    aTag['id'] = self.SYMBOLS
+                    aTag['id'] = self.SYMBOLS + '_authors'
                 else:
                     aTag['id'] = "%s_authors" % self.generateUnicodeName(current_letter)
                 pIndexTag.insert(0,aTag)
@@ -1337,7 +1337,7 @@ Author '{0}':
                     pBookTag['class'] = "line_item"
                     ptc = 0
 
-                    pBookTag.insert(ptc, self.formatPrefix(book['prefix'],soup))
+                    pBookTag.insert(ptc, self.formatPrefix(new_entry['prefix'],soup))
                     ptc += 1
 
                     spanTag = Tag(soup, "span")
@@ -3252,17 +3252,22 @@ Author '{0}':
 
     def formatPrefix(self,prefix_char,soup):
         # Generate the HTML for the prefix portion of the listing
-        spanTag = Tag(soup, "span")
+        # Kindle Previewer doesn't properly handle style=color:white
-        if prefix_char is None:
+        # MOBI does a better job allocating blank space with <code>
-            spanTag['style'] = "color:white"
+        if self.opts.fmt == 'mobi':
-            spanTag.insert(0,NavigableString(self.defaultPrefix))
+            codeTag = Tag(soup, "code")
-            # 2e3a is 'two-em dash', which matches width in Kindle Previewer
+            if prefix_char is None:
-            # too wide in calibre viewer
+                codeTag.insert(0,NavigableString('&nbsp;'))
-            # minimal visual distraction
+            else:
-            # spanTag.insert(0,NavigableString(u'\u2e3a'))
+                codeTag.insert(0,NavigableString(prefix_char))
+            return codeTag
         else:
+            spanTag = Tag(soup, "span")
+            if prefix_char is None:
+                spanTag['style'] = "color:white"
+                prefix_char = self.defaultPrefix
             spanTag.insert(0,NavigableString(prefix_char))
-        return spanTag
+            return spanTag
 
     def generateAuthorAnchor(self, author):
         # Strip white space to ''

src/calibre/library/cli.py

@@ -165,7 +165,7 @@ List the books available in the calibre database.
 
 def command_list(args, dbpath):
     pre = get_parser('')
-    pargs = [x for x in args if x in ('--with-library', '--library-path')
+    pargs = [x for x in args if x.startswith('--with-library') or x.startswith('--library-path')
         or not x.startswith('-')]
     opts = pre.parse_args(sys.argv[:1] + pargs)[0]
     db = get_db(dbpath, opts)

src/calibre/library/database2.py

@@ -1425,6 +1425,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         size=stream.tell()
         self.conn.execute('INSERT OR REPLACE INTO data (book,format,uncompressed_size,name) VALUES (?,?,?,?)',
                           (id, format.upper(), size, name))
+        self.update_last_modified([id], commit=False)
         self.conn.commit()
         self.format_filename_cache[id][format.upper()] = name
         self.refresh_ids([id])

src/calibre/test_build.py

@@ -100,6 +100,15 @@ def test_icu():
         raise RuntimeError('ICU module not loaded/valid')
     print ('ICU OK!')
 
+def test_wpd():
+    wpd = plugins['wpd'][0]
+    try:
+        wpd.init()
+    except wpd.NoWPD:
+        print ('This computer does not have WPD')
+    else:
+        wpd.uninit()
+
 def test():
     test_plugins()
     test_lxml()
@@ -112,6 +121,7 @@ def test():
     if iswindows:
         test_win32()
         test_winutil()
+        test_wpd()
 
 if __name__ == '__main__':
     test()