Merge from trunk

manual/Makefile

@@ -60,7 +60,7 @@ htmlhelp:
 
 latex:
 	mkdir -p .build/latex .build/doctrees
-	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) .build/latex
+	$(SPHINXBUILD) -b mylatex $(ALLSPHINXOPTS) .build/latex
 	@echo
 	@echo "Build finished; the LaTeX files are in .build/latex."
 	@echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \

manual/conf.py

@@ -14,10 +14,10 @@
 import sys, os
 
 # If your extensions are in another directory, add it here.
-sys.path.append(os.path.abspath('../src'))
 sys.path.append(os.path.abspath('.'))
-__appname__ = os.environ.get('__appname__', 'calibre')
+import init_calibre
-__version__ = os.environ.get('__version__', '0.0.0')
+init_calibre
+from calibre.constants import __appname__, __version__
 import custom
 custom
 # General configuration
@@ -154,7 +154,8 @@ latex_font_size = '10pt'
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, document class [howto/manual]).
-#latex_documents = []
+latex_documents = [('index', 'calibre.tex', 'calibre User Manual',
+    'Kovid Goyal', 'manual', False)]
 
 # Additional stuff for the LaTeX preamble.
 #latex_preamble = ''
@@ -164,3 +165,11 @@ latex_font_size = '10pt'
 
 # If false, no module index is generated.
 #latex_use_modindex = True
+
+latex_logo = 'resources/logo.png'
+latex_show_pagerefs = True
+latex_show_urls = 'footnote'
+latex_elements = {
+'papersize':'letterpaper',
+'fontenc':r'\usepackage[T2A,T1]{fontenc}'
+}

manual/custom.py

@@ -14,6 +14,7 @@ from sphinx.util.console import bold
 sys.path.append(os.path.abspath('../../../'))
 from calibre.linux import entry_points
 from epub import EPUBHelpBuilder
+from latex import LaTeXHelpBuilder
 
 def substitute(app, doctree):
     pass
@@ -251,6 +252,7 @@ def template_docs(app):
 def setup(app):
     app.add_config_value('kovid_epub_cover', None, False)
     app.add_builder(EPUBHelpBuilder)
+    app.add_builder(LaTeXHelpBuilder)
     app.connect('doctree-read', substitute)
     app.connect('builder-inited', generate_docs)
     app.connect('build-finished', finished)

manual/index.rst

@@ -17,7 +17,7 @@ To get started with more advanced usage, you should read about the :ref:`Graphic
 
 .. only:: online
     
-    **An ebook version of this user manual is available in** `EPUB format <calibre.epub>`_ and `AZW3 (Kindle Fire) format <calibre.azw3>`_.
+    **An ebook version of this user manual is available in** `EPUB format <calibre.epub>`_,  `AZW3 (Kindle Fire) format <calibre.azw3>`_ and `PDF format <calibre.pdf>`_.
     
 Sections
 ------------

manual/latex.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os
+
+
+from sphinx.builders.latex import LaTeXBuilder
+
+class LaTeXHelpBuilder(LaTeXBuilder):
+    name = 'mylatex'
+
+    def finish(self):
+        LaTeXBuilder.finish(self)
+        self.info('Fixing Cyrillic characters...')
+        tex = os.path.join(self.outdir, 'calibre.tex')
+        with open(tex, 'r+b') as f:
+            raw = f.read().replace(b'Михаил Горбачёв',
+                br'{\fontencoding{T2A}\selectfont Михаил Горбачёв}')
+            f.seek(0)
+            f.write(raw)

recipes/the_new_republic.recipe

@@ -44,6 +44,10 @@ class TNR(BasicNewsRecipe):
             em=post.find('em')
             b=post.find('b')
             a=post.find('a',href=True)
+            p=post.find('img', src=True)
+            #Find cover
+            if p is not None:
+                self.cover_url = p['src'].strip()
             if em is not None:
                 section_title = self.tag_to_string(em).strip()
                 subsection_title = ''

recipes/zdnet.recipe

@@ -12,7 +12,7 @@ class cdnet(BasicNewsRecipe):
 
     title = 'zdnet'
     description = 'zdnet security'
-    __author__ = 'Oliver Niesner'
+    __author__ = 'Oliver Niesner, Krittika Goyal'
     language = 'en'
 
     use_embedded_content   = False
@@ -20,41 +20,42 @@ class cdnet(BasicNewsRecipe):
     max_articles_per_feed = 40
     no_stylesheets = True
     encoding = 'latin1'
+    auto_cleanup = True
 
 
 
-    remove_tags = [dict(id='eyebrows'),
+    #remove_tags = [dict(id='eyebrows'),
-		   dict(id='header'),
+           #dict(id='header'),
-		   dict(id='search'),
+           #dict(id='search'),
-		   dict(id='nav'),
+           #dict(id='nav'),
-		   dict(id='blog-author-info'),
+           #dict(id='blog-author-info'),
-		   dict(id='post-tags'),
+           #dict(id='post-tags'),
-		   dict(id='bio-naraine'),
+           #dict(id='bio-naraine'),
-		   dict(id='bio-kennedy'),
+           #dict(id='bio-kennedy'),
-		   dict(id='author-short-disclosure-kennedy'),
+           #dict(id='author-short-disclosure-kennedy'),
-		   dict(id=''),
+           #dict(id=''),
-		   dict(name='div', attrs={'class':'banner'}),
+           #dict(name='div', attrs={'class':'banner'}),
-		   dict(name='div', attrs={'class':'int'}),
+           #dict(name='div', attrs={'class':'int'}),
-		   dict(name='div', attrs={'class':'talkback clear space-2'}),
+           #dict(name='div', attrs={'class':'talkback clear space-2'}),
-		   dict(name='div', attrs={'class':'content-1 clear'}),
+           #dict(name='div', attrs={'class':'content-1 clear'}),
-		   dict(name='div', attrs={'class':'space-2'}),
+           #dict(name='div', attrs={'class':'space-2'}),
-		   dict(name='div', attrs={'class':'space-3'}),
+           #dict(name='div', attrs={'class':'space-3'}),
-		   dict(name='div', attrs={'class':'thumb-2 left'}),
+           #dict(name='div', attrs={'class':'thumb-2 left'}),
-		   dict(name='div', attrs={'class':'hotspot'}),
+           #dict(name='div', attrs={'class':'hotspot'}),
-		   dict(name='div', attrs={'class':'hed hed-1 space-1'}),
+           #dict(name='div', attrs={'class':'hed hed-1 space-1'}),
-		   dict(name='div', attrs={'class':'view-1 clear content-3 space-2'}),
+           #dict(name='div', attrs={'class':'view-1 clear content-3 space-2'}),
-		   dict(name='div', attrs={'class':'hed hed-1 space-1'}),
+           #dict(name='div', attrs={'class':'hed hed-1 space-1'}),
-		   dict(name='div', attrs={'class':'hed hed-1'}),
+           #dict(name='div', attrs={'class':'hed hed-1'}),
-		   dict(name='div', attrs={'class':'post-header'}),
+           #dict(name='div', attrs={'class':'post-header'}),
-		   dict(name='div', attrs={'class':'lvl-nav clear'}),
+           #dict(name='div', attrs={'class':'lvl-nav clear'}),
-		   dict(name='div', attrs={'class':'t-share-overlay overlay-pop contain-overlay-4'}),
+           #dict(name='div', attrs={'class':'t-share-overlay overlay-pop contain-overlay-4'}),
-		   dict(name='p', attrs={'class':'tags'}),
+           #dict(name='p', attrs={'class':'tags'}),
-		   dict(name='span', attrs={'class':'follow'}),
+           #dict(name='span', attrs={'class':'follow'}),
-		   dict(name='span', attrs={'class':'int'}),
+           #dict(name='span', attrs={'class':'int'}),
-		   dict(name='h4', attrs={'class':'h s-4'}),
+           #dict(name='h4', attrs={'class':'h s-4'}),
-		   dict(name='a', attrs={'href':'http://www.twitter.com/ryanaraine'}),
+           #dict(name='a', attrs={'href':'http://www.twitter.com/ryanaraine'}),
-		   dict(name='div', attrs={'class':'special1'})]
+           #dict(name='div', attrs={'class':'special1'})]
-    remove_tags_after = [dict(name='div', attrs={'class':'clear'})]
+    #remove_tags_after = [dict(name='div', attrs={'class':'clear'})]
 
     feeds =  [ ('zdnet', 'http://feeds.feedburner.com/zdnet/security') ]
 
@@ -63,6 +64,3 @@ class cdnet(BasicNewsRecipe):
         for item in soup.findAll(style=True):
             del item['style']
         return soup
-
-
-

setup/installer/linux/freeze2.py

@@ -23,6 +23,8 @@ MAGICK_PREFIX = '/usr'
 binary_includes = [
                 '/usr/bin/pdftohtml',
                 '/usr/bin/pdfinfo',
+                '/usr/lib/libusb-1.0.so.0',
+                '/usr/lib/libmtp.so.9',
                 '/usr/lib/libglib-2.0.so.0',
                 '/usr/bin/pdftoppm',
                 '/usr/lib/libwmflite-0.2.so.7',

setup/publish.py

@@ -80,8 +80,17 @@ class Manual(Command):
                                    '-d', '.build/doctrees', '.', '.build/html'])
             subprocess.check_call(['sphinx-build', '-b', 'myepub', '-d',
                                    '.build/doctrees', '.', '.build/epub'])
+            subprocess.check_call(['sphinx-build', '-b', 'mylatex', '-d',
+                                   '.build/doctrees', '.', '.build/latex'])
+            pwd = os.getcwdu()
+            os.chdir('.build/latex')
+            subprocess.check_call(['make', 'all-pdf'], stdout=open(os.devnull,
+                'wb'))
+            os.chdir(pwd)
             epub_dest = self.j('.build', 'html', 'calibre.epub')
+            pdf_dest = self.j('.build', 'html', 'calibre.pdf')
             shutil.copyfile(self.j('.build', 'epub', 'calibre.epub'), epub_dest)
+            shutil.copyfile(self.j('.build', 'latex', 'calibre.pdf'), pdf_dest)
             subprocess.check_call(['ebook-convert', epub_dest,
                 epub_dest.rpartition('.')[0] + '.azw3',
                 '--page-breaks-before=/', '--disable-font-rescaling',

src/calibre/devices/__init__.py

@@ -55,7 +55,7 @@ def get_connected_device():
             break
     return dev
 
-def debug(ioreg_to_tmp=False, buf=None):
+def debug(ioreg_to_tmp=False, buf=None, plugins=None):
     import textwrap
     from calibre.customize.ui import device_plugins
     from calibre.devices.scanner import DeviceScanner, win_pnp_drives
@@ -66,9 +66,19 @@ def debug(ioreg_to_tmp=False, buf=None):
     if buf is None:
         buf = StringIO()
     sys.stdout = sys.stderr = buf
+    out = partial(prints, file=buf)
+
+    devplugins = device_plugins() if plugins is None else plugins
+    devplugins = list(sorted(devplugins, cmp=lambda
+            x,y:cmp(x.__class__.__name__, y.__class__.__name__)))
+    if plugins is None:
+        for d in devplugins:
+            try:
+                d.startup()
+            except:
+                out('Startup failed for device plugin: %s'%d)
 
     try:
-        out = partial(prints, file=buf)
         out('Version:', __version__)
         s = DeviceScanner()
         s.scan()
@@ -96,8 +106,6 @@ def debug(ioreg_to_tmp=False, buf=None):
             ioreg += 'Output from osx_get_usb_drives:\n'+drives+'\n\n'
             ioreg += Device.run_ioreg()
         connected_devices = []
-        devplugins = list(sorted(device_plugins(), cmp=lambda
-                x,y:cmp(x.__class__.__name__, y.__class__.__name__)))
         out('Available plugins:', textwrap.fill(' '.join([x.__class__.__name__ for x in
             devplugins])))
         out(' ')
@@ -155,6 +163,12 @@ def debug(ioreg_to_tmp=False, buf=None):
     finally:
         sys.stdout = oldo
         sys.stderr = olde
+        if plugins is None:
+            for d in devplugins:
+                try:
+                    d.shutdown()
+                except:
+                    pass
 
 def device_info(ioreg_to_tmp=False, buf=None):
     from calibre.devices.scanner import DeviceScanner, win_pnp_drives

src/calibre/devices/interface.py

@@ -503,14 +503,16 @@ class DevicePlugin(Plugin):
         Called when calibre is is starting the device. Do any initialization
         required. Note that multiple instances of the class can be instantiated,
         and thus __init__ can be called multiple times, but only one instance
-        will have this method called.
+        will have this method called. This method is called on the device
+        thread, not the GUI thread.
         '''
         pass
 
     def shutdown(self):
         '''
         Called when calibre is shutting down, either for good or in preparation
-        to restart. Do any cleanup required.
+        to restart. Do any cleanup required. This method is called on the
+        device thread, not the GUI thread.
         '''
         pass
 

src/calibre/devices/mtp/unix/driver.py

@@ -10,7 +10,11 @@ __docformat__ = 'restructuredtext en'
 import time, operator
 from threading import RLock
 from functools import wraps
+from itertools import chain
+from collections import deque, OrderedDict
+from io import BytesIO
 
+from calibre import prints
 from calibre.devices.errors import OpenFailed
 from calibre.devices.mtp.base import MTPDeviceBase
 from calibre.devices.mtp.unix.detect import MTPDetect
@@ -22,6 +26,68 @@ def synchronous(func):
             return func(self, *args, **kwargs)
     return synchronizer
 
+class FilesystemCache(object):
+
+    def __init__(self, files, folders):
+        self.files = files
+        self.folders = folders
+        self.file_id_map = {f['id']:f for f in files}
+        self.folder_id_map = {f['id']:f for f in self.iterfolders(set_level=0)}
+
+        # Set the parents of each file
+        self.files_in_root = OrderedDict()
+        for f in files:
+            parents = deque()
+            pid = f['parent_id']
+            while pid is not None and pid > 0:
+                try:
+                    parent = self.folder_id_map[pid]
+                except KeyError:
+                    break
+                parents.appendleft(pid)
+                pid = parent['parent_id']
+            f['parents'] = parents
+            if not parents:
+                self.files_in_root[f['id']] = f
+
+        # Set the files in each folder
+        for f in self.iterfolders():
+            f['files'] = [i for i in files if i['parent_id'] ==
+                    f['id']]
+
+        # Decode the file and folder names
+        for f in chain(files, folders):
+            try:
+                name = f['name'].decode('utf-8')
+            except UnicodeDecodeError:
+                name = 'undecodable_%d'%f['id']
+            f['name'] = name
+
+    def iterfolders(self, folders=None, set_level=None):
+        clevel = None if set_level is None else set_level + 1
+        if folders is None:
+            folders = self.folders
+        for f in folders:
+            if set_level is not None:
+                f['level'] = set_level
+            yield f
+            for c in f['children']:
+                for child in self.iterfolders([c], set_level=clevel):
+                    yield child
+
+    def dump_filesystem(self):
+        indent = 2
+        for f in self.iterfolders():
+            prefix = ' '*(indent*f['level'])
+            prints(prefix, '+', f['name'], 'id=%s'%f['id'])
+            for leaf in f['files']:
+                prints(prefix, ' '*indent, '-', leaf['name'],
+                        'id=%d'%leaf['id'], 'size=%d'%leaf['size'],
+                        'modtime=%d'%leaf['modtime'])
+        for leaf in self.files_in_root.itervalues():
+            prints('-', leaf['name'], 'id=%d'%leaf['id'],
+                    'size=%d'%leaf['size'], 'modtime=%d'%leaf['modtime'])
+
 class MTP_DEVICE(MTPDeviceBase):
 
     supported_platforms = ['linux']
@@ -30,8 +96,15 @@ class MTP_DEVICE(MTPDeviceBase):
         MTPDeviceBase.__init__(self, *args, **kwargs)
         self.detect = MTPDetect()
         self.dev = None
+        self.filesystem_cache = None
         self.lock = RLock()
         self.blacklisted_devices = set()
+        for x in vars(self.detect.libmtp):
+            if x.startswith('LIBMTP'):
+                setattr(self, x, getattr(self.detect.libmtp, x))
+
+    def set_debug_level(self, lvl):
+        self.detect.libmtp.set_debug_level(lvl)
 
     def report_progress(self, sent, total):
         try:
@@ -73,14 +146,19 @@ class MTP_DEVICE(MTPDeviceBase):
 
     @synchronous
     def post_yank_cleanup(self):
-        self.dev = None
+        self.dev = self.filesystem_cache = None
 
     @synchronous
     def shutdown(self):
-        self.dev = None
+        self.dev = self.filesystem_cache = None
+
+    def format_errorstack(self, errs):
+        return '\n'.join(['%d:%s'%(code, msg.decode('utf-8', 'replace')) for
+            code, msg in errs])
 
     @synchronous
     def open(self, connected_device, library_uuid):
+        self.dev = self.filesystem_cache = None
         def blacklist_device():
             d = connected_device
             self.blacklisted_devices.add((d.busnum, d.devnum, d.vendor_id,
@@ -112,6 +190,16 @@ class MTP_DEVICE(MTPDeviceBase):
         if len(storage) > 2:
             self._cardb_id = storage[2]['id']
 
+        files, errs = self.dev.get_filelist(self)
+        if errs and not files:
+            raise OpenFailed('Failed to read files from device. Underlying errors:\n'
+                    +self.format_errorstack(errs))
+        folders, errs = self.dev.get_folderlist()
+        if errs and not folders:
+            raise OpenFailed('Failed to read folders from device. Underlying errors:\n'
+                    +self.format_errorstack(errs))
+        self.filesystem_cache = FilesystemCache(files, folders)
+
     @synchronous
     def get_device_information(self, end_session=True):
         d = self.dev
@@ -149,6 +237,11 @@ class MTP_DEVICE(MTPDeviceBase):
 
 
 if __name__ == '__main__':
+    BytesIO
+    class PR:
+        def report_progress(self, sent, total):
+            print (sent, total, end=', ')
+
     from pprint import pprint
     dev = MTP_DEVICE(None)
     from calibre.devices.scanner import linux_scanner
@@ -160,8 +253,19 @@ if __name__ == '__main__':
     print ("Storage info:")
     pprint(d.storage_info)
     print("Free space:", dev.free_space())
-    files, errs = d.get_filelist(dev)
+    # print (d.create_folder(dev._main_id, 0, 'testf'))
-    pprint((len(files), errs))
+    # raw = b'test'
-    folders, errs = d.get_folderlist()
+    # fname = b'moose.txt'
-    pprint((len(folders), errs))
+    # src = BytesIO(raw)
+    # print (d.put_file(dev._main_id, 0, fname, src, len(raw), PR()))
+    dev.filesystem_cache.dump_filesystem()
+    # with open('/tmp/flint.epub', 'wb') as f:
+    #     print(d.get_file(786, f, PR()))
+    # print()
+    # with open('/tmp/bleak.epub', 'wb') as f:
+    #     print(d.get_file(601, f, PR()))
+    # print()
+    dev.set_debug_level(dev.LIBMTP_DEBUG_ALL)
+    del d
+    dev.shutdown()
 

src/calibre/devices/mtp/unix/libmtp.c

@@ -33,6 +33,7 @@
 
 typedef struct {
     PyObject *obj;
+    PyObject *extra;
     PyThreadState *state;
 } ProgressCallback;
 
@@ -64,6 +65,48 @@ static void dump_errorstack(LIBMTP_mtpdevice_t *dev, PyObject *list) {
     LIBMTP_Clear_Errorstack(dev);
 }
 
+static uint16_t data_to_python(void *params, void *priv, uint32_t sendlen, unsigned char *data, uint32_t *putlen) {
+    PyObject *res;
+    ProgressCallback *cb;
+    uint16_t ret = LIBMTP_HANDLER_RETURN_OK;
+
+    cb = (ProgressCallback *)priv;
+    *putlen = sendlen;
+    PyEval_RestoreThread(cb->state);
+    res = PyObject_CallMethod(cb->extra, "write", "s#", data, sendlen);
+    if (res == NULL) {
+        ret = LIBMTP_HANDLER_RETURN_ERROR;
+        *putlen = 0;
+        PyErr_Print();
+    } else Py_DECREF(res);
+
+    cb->state = PyEval_SaveThread();
+    return ret;
+}
+
+static uint16_t data_from_python(void *params, void *priv, uint32_t wantlen, unsigned char *data, uint32_t *gotlen) {
+    PyObject *res;
+    ProgressCallback *cb;
+    char *buf = NULL;
+    Py_ssize_t len = 0;
+    uint16_t ret = LIBMTP_HANDLER_RETURN_ERROR;
+
+    *gotlen = 0;
+
+    cb = (ProgressCallback *)priv;
+    PyEval_RestoreThread(cb->state);
+    res = PyObject_CallMethod(cb->extra, "read", "k", wantlen);
+    if (res != NULL && PyBytes_AsStringAndSize(res, &buf, &len) != -1 && len <= wantlen) {
+        memcpy(data, buf, len);
+        *gotlen = len;
+        ret = LIBMTP_HANDLER_RETURN_OK;
+    } else PyErr_Print();
+
+    Py_XDECREF(res);
+    cb->state = PyEval_SaveThread();
+    return ret;
+}
+
 // }}}
 
 // Device object definition {{{
@@ -84,7 +127,11 @@ typedef struct {
 static void
 libmtp_Device_dealloc(libmtp_Device* self)
 {
-    if (self->device != NULL) LIBMTP_Release_Device(self->device);
+    if (self->device != NULL) {
+        Py_BEGIN_ALLOW_THREADS;
+        LIBMTP_Release_Device(self->device);
+        Py_END_ALLOW_THREADS;
+    }
     self->device = NULL;
 
     Py_XDECREF(self->ids); self->ids = NULL;
@@ -287,9 +334,11 @@ libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs
     errs = PyList_New(0);
     if (ans == NULL || errs == NULL) { PyErr_NoMemory(); return NULL; }
 
+    Py_XINCREF(callback);
     cb.state = PyEval_SaveThread();
     tf = LIBMTP_Get_Filelisting_With_Callback(self->device, report_progress, &cb);
     PyEval_RestoreThread(cb.state);
+    Py_XDECREF(callback);
 
     if (tf == NULL) { 
         dump_errorstack(self->device, errs);
@@ -301,7 +350,7 @@ libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs
                 "id", f->item_id,
                 "parent_id", f->parent_id,
                 "storage_id", f->storage_id,
-                "filename", f->filename,
+                "name", f->filename,
                 "size", f->filesize,
                 "modtime", f->modificationdate
         );
@@ -334,7 +383,7 @@ int folderiter(LIBMTP_folder_t *f, PyObject *parent) {
 
     folder = Py_BuildValue("{s:k,s:k,s:k,s:s,s:N}",
             "id", f->folder_id,
-            "parent_d", f->parent_id,
+            "parent_id", f->parent_id,
             "storage_id", f->storage_id,
             "name", f->name,
             "children", children);
@@ -380,6 +429,159 @@ libmtp_Device_get_folderlist(libmtp_Device *self, PyObject *args, PyObject *kwar
 
 } // }}}
 
+// Device.get_file {{{
+static PyObject *
+libmtp_Device_get_file(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
+    PyObject *stream, *callback = NULL, *errs;
+    ProgressCallback cb;
+    uint32_t fileid;
+    int ret;
+
+    ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
+
+
+    if (!PyArg_ParseTuple(args, "kO|O", &fileid, &stream, &callback)) return NULL; 
+    errs = PyList_New(0);
+    if (errs == NULL) { PyErr_NoMemory(); return NULL; }
+
+    cb.obj = callback; cb.extra = stream;
+    Py_XINCREF(callback); Py_INCREF(stream);
+    cb.state = PyEval_SaveThread();
+    ret = LIBMTP_Get_File_To_Handler(self->device, fileid, data_to_python, &cb, report_progress, &cb);
+    PyEval_RestoreThread(cb.state);
+    Py_XDECREF(callback); Py_DECREF(stream);
+
+    if (ret != 0) { 
+        dump_errorstack(self->device, errs);
+    }
+    Py_XDECREF(PyObject_CallMethod(stream, "flush", NULL));
+    return Py_BuildValue("ON", (ret == 0) ? Py_True : Py_False, errs);
+
+} // }}}
+
+// Device.put_file {{{
+static PyObject *
+libmtp_Device_put_file(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
+    PyObject *stream, *callback = NULL, *errs, *fo;
+    ProgressCallback cb;
+    uint32_t parent_id, storage_id;
+    uint64_t filesize;
+    int ret;
+    char *name;
+    LIBMTP_file_t f, *nf;
+
+    ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
+
+    if (!PyArg_ParseTuple(args, "kksOK|O", &storage_id, &parent_id, &name, &stream, &filesize, &callback)) return NULL; 
+    errs = PyList_New(0);
+    if (errs == NULL) { PyErr_NoMemory(); return NULL; }
+
+    cb.obj = callback; cb.extra = stream;
+    f.parent_id = parent_id; f.storage_id = storage_id; f.item_id = 0; f.filename = name; f.filetype = LIBMTP_FILETYPE_UNKNOWN; f.filesize = filesize;
+    Py_XINCREF(callback); Py_INCREF(stream);
+    cb.state = PyEval_SaveThread();
+    ret = LIBMTP_Send_File_From_Handler(self->device, data_from_python, &cb, &f, report_progress, &cb);
+    PyEval_RestoreThread(cb.state);
+    Py_XDECREF(callback); Py_DECREF(stream);
+
+    fo = Py_None; Py_INCREF(fo);
+    if (ret != 0) dump_errorstack(self->device, errs);
+    else {
+        Py_BEGIN_ALLOW_THREADS;
+        nf = LIBMTP_Get_Filemetadata(self->device, f.item_id);
+        Py_END_ALLOW_THREADS;
+        if (nf == NULL) dump_errorstack(self->device, errs);
+        else {
+            Py_DECREF(fo);
+            fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:K,s:k}",
+                    "id", nf->item_id,
+                    "parent_id", nf->parent_id,
+                    "storage_id", nf->storage_id,
+                    "name", nf->filename,
+                    "size", nf->filesize,
+                    "modtime", nf->modificationdate
+            );
+            LIBMTP_destroy_file_t(nf);
+        }
+    }
+
+    return Py_BuildValue("NN", fo, errs);
+
+} // }}}
+
+// Device.delete_object {{{
+static PyObject *
+libmtp_Device_delete_object(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
+    PyObject *errs;
+    uint32_t id;
+    int res;
+
+    ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
+
+    if (!PyArg_ParseTuple(args, "k", &id)) return NULL;
+    errs = PyList_New(0);
+    if (errs == NULL) { PyErr_NoMemory(); return NULL; }
+
+    Py_BEGIN_ALLOW_THREADS;
+    res = LIBMTP_Delete_Object(self->device, id);
+    Py_END_ALLOW_THREADS;
+    if (res != 0) dump_errorstack(self->device, errs);
+
+    return Py_BuildValue("ON", (res == 0) ? Py_True : Py_False, errs);
+} // }}}
+
+// Device.create_folder {{{
+static PyObject *
+libmtp_Device_create_folder(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
+    PyObject *errs, *fo, *children, *temp;
+    uint32_t parent_id, storage_id;
+    char *name;
+    uint32_t folder_id;
+    LIBMTP_folder_t *f = NULL, *cf;
+
+    ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
+
+    if (!PyArg_ParseTuple(args, "kks", &storage_id, &parent_id, &name)) return NULL;
+    errs = PyList_New(0);
+    if (errs == NULL) { PyErr_NoMemory(); return NULL; }
+
+    fo = Py_None; Py_INCREF(fo);
+
+    Py_BEGIN_ALLOW_THREADS;
+    folder_id = LIBMTP_Create_Folder(self->device, name, parent_id, storage_id);
+    Py_END_ALLOW_THREADS;
+    if (folder_id == 0) dump_errorstack(self->device, errs);
+    else {
+        Py_BEGIN_ALLOW_THREADS;
+        // Cannot use Get_Folder_List_For_Storage as it fails
+        f = LIBMTP_Get_Folder_List(self->device);
+        Py_END_ALLOW_THREADS;
+        if (f == NULL) dump_errorstack(self->device, errs);
+        else {
+            cf = LIBMTP_Find_Folder(f, folder_id);
+            if (cf == NULL) {
+                temp = Py_BuildValue("is", 1, "Newly created folder not present on device!");
+                if (temp == NULL) { PyErr_NoMemory(); return NULL;}
+                PyList_Append(errs, temp);
+                Py_DECREF(temp);
+            } else {
+                Py_DECREF(fo);
+                children = PyList_New(0);
+                if (children == NULL) { PyErr_NoMemory(); return NULL; }
+                fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:N}",
+                        "id", cf->folder_id,
+                        "parent_id", cf->parent_id,
+                        "storage_id", cf->storage_id,
+                        "name", cf->name,
+                        "children", children);
+            }
+            LIBMTP_destroy_folder_t(f);
+        }
+    }
+
+    return Py_BuildValue("NN", fo, errs);
+} // }}}
+
 static PyMethodDef libmtp_Device_methods[] = {
     {"update_storage_info", (PyCFunction)libmtp_Device_update_storage_info, METH_VARARGS,
      "update_storage_info() -> Reread the storage info from the device (total, space, free space, storage locations, etc.)"
@@ -390,9 +592,26 @@ static PyMethodDef libmtp_Device_methods[] = {
     },
 
     {"get_folderlist", (PyCFunction)libmtp_Device_get_folderlist, METH_VARARGS,
-     "get_folderlist() -> Get the list of folders on the device. Returns files, erros."
+     "get_folderlist() -> Get the list of folders on the device. Returns files, errors."
     },
 
+    {"get_file", (PyCFunction)libmtp_Device_get_file, METH_VARARGS,
+     "get_file(fileid, stream, callback=None) -> Get the file specified by fileid from the device. stream must be a file-like object. The file will be written to it. callback works the same as in get_filelist(). Returns ok, errs, where errs is a list of errors (if any)."
+    },
+
+    {"put_file", (PyCFunction)libmtp_Device_put_file, METH_VARARGS,
+     "put_file(storage_id, parent_id, filename, stream, size, callback=None) -> Put a file on the device. The file is read from stream. It is put inside the folder identified by parent_id on the storage identified by storage_id. Use parent_id=0 to put it in the root. stream must be a file-like object. size is the size in bytes of the data in stream. callback works the same as in get_filelist(). Returns fileinfo, errs, where errs is a list of errors (if any), and fileinfo is a file information dictionary, as returned by get_filelist(). fileinfo will be None if case or errors."
+    },
+
+    {"create_folder", (PyCFunction)libmtp_Device_create_folder, METH_VARARGS,
+     "create_folder(storage_id, parent_id, name) -> Create a folder named name under parent parent_id (use 0 for root) in the storage identified by storage_id. Returns folderinfo, errors, where folderinfo is the same dict as returned by get_folderlist(), it will be None if there are errors."
+    },
+
+    {"delete_object", (PyCFunction)libmtp_Device_delete_object, METH_VARARGS,
+     "delete_object(id) -> Delete the object identified by id from the device. Can be used to delete files, folders, etc. Returns ok, errs."
+    },
+
+
     {NULL}  /* Sentinel */
 };
 

src/calibre/devices/prs500/cli/main.py

@@ -174,6 +174,13 @@ def ls(dev, path, term, recurse=False, color=False, human_readable_size=False, l
     output.close()
     return listing
 
+def shutdown_plugins():
+    for d in device_plugins():
+        try:
+            d.shutdown()
+        except:
+            pass
+
 def main():
     term = TerminalController()
     cols = term.COLS
@@ -201,6 +208,10 @@ def main():
     scanner.scan()
     connected_devices = []
     for d in device_plugins():
+        try:
+            d.startup()
+        except:
+            print ('Startup failed for device plugin: %s'%d)
         ok, det = scanner.is_device_connected(d)
         if ok:
             dev = d
@@ -209,6 +220,7 @@ def main():
 
     if dev is None:
         print >>sys.stderr, 'Unable to find a connected ebook reader.'
+        shutdown_plugins()
         return 1
 
     for det, d in connected_devices:
@@ -358,6 +370,9 @@ def main():
     except (ArgumentError, DeviceError) as e:
         print >>sys.stderr, e
         return 1
+    finally:
+        shutdown_plugins()
+
     return 0
 
 if __name__ == '__main__':

src/calibre/ebooks/BeautifulSoup.py

@@ -1454,7 +1454,9 @@ class BeautifulSoup(BeautifulStoneSoup):
     #According to the HTML standard, these block tags can contain
     #another tag of the same type. Furthermore, it's common
     #to actually use these tags this way.
-    NESTABLE_BLOCK_TAGS = ['blockquote', 'div', 'fieldset', 'ins', 'del']
+    # Changed by Kovid: Added HTML 5 block tags
+    NESTABLE_BLOCK_TAGS = ['blockquote', 'div', 'fieldset', 'ins', 'del',
+            'article', 'aside', 'header', 'footer', 'nav', 'figcaption', 'figure', 'section']
 
     #Lists can contain other lists, but there are restrictions.
     NESTABLE_LIST_TAGS = { 'ol' : [],

src/calibre/ebooks/metadata/toc.py

@@ -196,8 +196,8 @@ class TOC(list):
                     content = content[0]
                     src = get_attr(content, attr='src')
                     if src:
-                        purl = urlparse(unquote(content.get('src')))
+                        purl = urlparse(content.get('src'))
-                        href, fragment = purl[2], purl[5]
+                        href, fragment = unquote(purl[2]), unquote(purl[5])
                         nd = dest.add_item(href, fragment, text)
                         nd.play_order = play_order
 

src/calibre/ebooks/oeb/base.py

@@ -186,8 +186,9 @@ def rewrite_links(root, link_repl_func, resolve_base_href=False):
     If the ``link_repl_func`` returns None, the attribute or
     tag text will be removed completely.
     '''
-    from cssutils import parseString, parseStyle, replaceUrls, log
+    from cssutils import replaceUrls, log, CSSParser
     log.setLevel(logging.WARN)
+    log.raiseExceptions = False
 
     if resolve_base_href:
         resolve_base_href(root)
@@ -214,6 +215,8 @@ def rewrite_links(root, link_repl_func, resolve_base_href=False):
                 new = cur[:pos] + new_link + cur[pos+len(link):]
                 el.attrib[attrib] = new
 
+    parser = CSSParser(raiseExceptions=False, log=_css_logger,
+            fetcher=lambda x:(None, None))
     for el in root.iter(etree.Element):
         try:
             tag = el.tag
@@ -223,7 +226,7 @@ def rewrite_links(root, link_repl_func, resolve_base_href=False):
         if tag == XHTML('style') and el.text and \
                 (_css_url_re.search(el.text) is not None or '@import' in
                         el.text):
-            stylesheet = parseString(el.text, validate=False)
+            stylesheet = parser.parseString(el.text, validate=False)
             replaceUrls(stylesheet, link_repl_func)
             repl = stylesheet.cssText
             if isbytestring(repl):
@@ -234,7 +237,7 @@ def rewrite_links(root, link_repl_func, resolve_base_href=False):
             text = el.attrib['style']
             if _css_url_re.search(text) is not None:
                 try:
-                    stext = parseStyle(text, validate=False)
+                    stext = parser.parseStyle(text, validate=False)
                 except:
                     # Parsing errors are raised by cssutils
                     continue
@@ -862,6 +865,7 @@ class Manifest(object):
         def _parse_css(self, data):
             from cssutils import CSSParser, log, resolveImports
             log.setLevel(logging.WARN)
+            log.raiseExceptions = False
             self.oeb.log.debug('Parsing', self.href, '...')
             data = self.oeb.decode(data)
             data = self.oeb.css_preprocessor(data, add_namespace=True)
@@ -1537,7 +1541,7 @@ class TOC(object):
             if title:
                 title = re.sub(r'\s+', ' ', title)
             element(label, NCX('text')).text = title
-            element(point, NCX('content'), src=urlunquote(node.href))
+            element(point, NCX('content'), src=node.href)
             node.to_ncx(point)
         return parent
 

src/calibre/gui2/actions/catalog.py

@@ -85,7 +85,7 @@ class GenerateCatalogAction(InterfaceAction):
             dynamic.set('catalogs_to_be_synced', sync)
         self.gui.status_bar.show_message(_('Catalog generated.'), 3000)
         self.gui.sync_catalogs()
-        if job.fmt not in ['EPUB','MOBI']:
+        if job.fmt not in {'EPUB','MOBI', 'AZW3'}:
             export_dir = choose_dir(self.gui, _('Export Catalog Directory'),
                     _('Select destination for %(title)s.%(fmt)s') % dict(
                         title=job.catalog_title, fmt=job.fmt.lower()))

src/calibre/gui2/catalog/catalog_epub_mobi.py

@@ -25,13 +25,13 @@ from PyQt4.Qt import (Qt, QAbstractItemView, QCheckBox, QComboBox, QDialog,
 class PluginWidget(QWidget,Ui_Form):
 
     TITLE = _('E-book options')
-    HELP  = _('Options specific to')+' EPUB/MOBI '+_('output')
+    HELP  = _('Options specific to')+' AZW3/EPUB/MOBI '+_('output')
 
     # Output synced to the connected device?
     sync_enabled = True
 
     # Formats supported by this plugin
-    formats = set(['epub','mobi'])
+    formats = set(['azw3','epub','mobi'])
 
     def __init__(self, parent=None):
         QWidget.__init__(self, parent)
@@ -402,7 +402,6 @@ class PluginWidget(QWidget,Ui_Form):
                 self.exclude_genre.setText(default[1])
                 break
 
-
 class CheckableTableWidgetItem(QTableWidgetItem):
     '''
     Borrowed from kiwidude
@@ -466,8 +465,7 @@ class GenericRulesTable(QTableWidget):
         self.db = db
         QTableWidget.__init__(self)
         self.setObjectName(object_name)
-        self.layout = QHBoxLayout()
+        self.layout = parent_gb.layout()
-        parent_gb.setLayout(self.layout)
 
         # Add ourselves to the layout
         #print("verticalHeader: %s" % dir(self.verticalHeader()))
@@ -538,7 +536,7 @@ class GenericRulesTable(QTableWidget):
 
     def create_blank_row_data(self):
         '''
-        ovverride
+        override
         '''
         pass
 
@@ -571,6 +569,9 @@ class GenericRulesTable(QTableWidget):
         self.clearSelection()
 
     def get_data(self):
+        '''
+        override
+        '''
         pass
 
     def move_row_down(self):
@@ -598,6 +599,7 @@ class GenericRulesTable(QTableWidget):
 
             # Populate it with the saved data
             self.populate_table_row(src_row, saved_data)
+
         self.blockSignals(False)
         scroll_to_row = last_sel_row + 1
         if scroll_to_row < self.rowCount() - 1:
@@ -637,8 +639,9 @@ class GenericRulesTable(QTableWidget):
         pass
 
     def resize_name(self, scale):
-        current_width = self.columnWidth(1)
+        #current_width = self.columnWidth(1)
-        self.setColumnWidth(1, min(225,int(current_width * scale)))
+        #self.setColumnWidth(1, min(225,int(current_width * scale)))
+        self.setColumnWidth(1, 225)
 
     def rule_name_edited(self):
         current_row = self.currentRow()
@@ -650,23 +653,21 @@ class GenericRulesTable(QTableWidget):
         self.selectRow(row)
         self.scrollToItem(self.currentItem())
 
-    def tweak_height(self, height=4):
-        for i in range(min(3,self.rowCount())):
-            height += self.rowHeight(i)
-        height += self.verticalHeader().sizeHint().height()
-        print("computed table height for %d rows: %d" % (self.rowCount(),height, ))
-        self.setMinimumSize(QSize(16777215, height))
-        self.setMaximumSize(QSize(16777215, height))
-
 class ExclusionRules(GenericRulesTable):
 
+    COLUMNS = { 'ENABLED':{'ordinal': 0, 'name': ''},
+                'NAME':   {'ordinal': 1, 'name': 'Name'},
+                'FIELD':  {'ordinal': 2, 'name': 'Field'},
+                'PATTERN':  {'ordinal': 3, 'name': 'Value'},}
+
     def __init__(self, parent_gb_hl, object_name, rules, eligible_custom_fields, db):
         super(ExclusionRules, self).__init__(parent_gb_hl, object_name, rules, eligible_custom_fields, db)
         self._init_table_widget()
         self._initialize()
 
     def _init_table_widget(self):
-        header_labels = ['','Name','Field','Value']
+        header_labels = [self.COLUMNS[index]['name'] \
+                         for index in sorted(self.COLUMNS.keys(), key=lambda c: self.COLUMNS[c]['ordinal'])]
         self.setColumnCount(len(header_labels))
         self.setHorizontalHeaderLabels(header_labels)
         self.setSortingEnabled(False)
@@ -682,10 +683,10 @@ class ExclusionRules(GenericRulesTable):
     def convert_row_to_data(self, row):
         data = self.create_blank_row_data()
         data['ordinal'] = row
-        data['enabled'] = self.item(row,0).checkState() == Qt.Checked
+        data['enabled'] = self.item(row,self.COLUMNS['ENABLED']['ordinal']).checkState() == Qt.Checked
-        data['name'] = unicode(self.cellWidget(row,1).text()).strip()
+        data['name'] = unicode(self.cellWidget(row,self.COLUMNS['NAME']['ordinal']).text()).strip()
-        data['field'] = unicode(self.cellWidget(row,2).currentText()).strip()
+        data['field'] = unicode(self.cellWidget(row,self.COLUMNS['FIELD']['ordinal']).currentText()).strip()
-        data['pattern'] = unicode(self.cellWidget(row,3).currentText()).strip()
+        data['pattern'] = unicode(self.cellWidget(row,self.COLUMNS['PATTERN']['ordinal']).currentText()).strip()
         return data
 
     def create_blank_row_data(self):
@@ -740,18 +741,18 @@ class ExclusionRules(GenericRulesTable):
         # Entry point
         self.blockSignals(True)
 
-        # Column 0: Enabled
+        # Enabled
-        self.setItem(row, 0, CheckableTableWidgetItem(data['enabled']))
+        self.setItem(row, self.COLUMNS['ENABLED']['ordinal'], CheckableTableWidgetItem(data['enabled']))
 
-        # Column 1: Rule name
+        # Rule name
-        set_rule_name_in_row(row, 1, name=data['name'])
+        set_rule_name_in_row(row, self.COLUMNS['NAME']['ordinal'], name=data['name'])
 
-        # Column 2: Source field
+        # Source field
-        source_combo = set_source_field_in_row(row, 2, field=data['field'])
+        source_combo = set_source_field_in_row(row, self.COLUMNS['FIELD']['ordinal'], field=data['field'])
 
-        # Column 3: Pattern
+        # Pattern
         # The contents of the Pattern field is driven by the Source field
-        self.source_index_changed(source_combo, row, 3, pattern=data['pattern'])
+        self.source_index_changed(source_combo, row, self.COLUMNS['PATTERN']['ordinal'], pattern=data['pattern'])
 
         self.blockSignals(False)
 
@@ -775,17 +776,24 @@ class ExclusionRules(GenericRulesTable):
                 values = ['any date','unspecified']
 
         values_combo = ComboBox(self, values, pattern)
-        self.setCellWidget(row, 3, values_combo)
+        self.setCellWidget(row, self.COLUMNS['PATTERN']['ordinal'], values_combo)
 
 class PrefixRules(GenericRulesTable):
 
+    COLUMNS = { 'ENABLED':{'ordinal': 0, 'name': ''},
+                'NAME':   {'ordinal': 1, 'name': 'Name'},
+                'PREFIX': {'ordinal': 2, 'name': 'Prefix'},
+                'FIELD':  {'ordinal': 3, 'name': 'Field'},
+                'PATTERN':{'ordinal': 4, 'name': 'Value'},}
+
     def __init__(self, parent_gb_hl, object_name, rules, eligible_custom_fields, db):
         super(PrefixRules, self).__init__(parent_gb_hl, object_name, rules, eligible_custom_fields, db)
         self._init_table_widget()
         self._initialize()
 
     def _init_table_widget(self):
-        header_labels = ['','Name','Prefix','Field','Value']
+        header_labels = [self.COLUMNS[index]['name'] \
+                         for index in sorted(self.COLUMNS.keys(), key=lambda c: self.COLUMNS[c]['ordinal'])]
         self.setColumnCount(len(header_labels))
         self.setHorizontalHeaderLabels(header_labels)
         self.setSortingEnabled(False)
@@ -803,10 +811,10 @@ class PrefixRules(GenericRulesTable):
         data = self.create_blank_row_data()
         data['ordinal'] = row
         data['enabled'] = self.item(row,0).checkState() == Qt.Checked
-        data['name'] = unicode(self.cellWidget(row,1).text()).strip()
+        data['name'] = unicode(self.cellWidget(row,self.COLUMNS['NAME']['ordinal']).text()).strip()
-        data['prefix'] = unicode(self.cellWidget(row,2).currentText()).strip()
+        data['prefix'] = unicode(self.cellWidget(row,self.COLUMNS['PREFIX']['ordinal']).currentText()).strip()
-        data['field'] = unicode(self.cellWidget(row,3).currentText()).strip()
+        data['field'] = unicode(self.cellWidget(row,self.COLUMNS['FIELD']['ordinal']).currentText()).strip()
-        data['pattern'] = unicode(self.cellWidget(row,4).currentText()).strip()
+        data['pattern'] = unicode(self.cellWidget(row,self.COLUMNS['PATTERN']['ordinal']).currentText()).strip()
         return data
 
     def create_blank_row_data(self):
@@ -872,7 +880,6 @@ class PrefixRules(GenericRulesTable):
             ('Math plus circled',u'\u2295'),
             ('Math times circled',u'\u2297'),
             ('Math times',u'\u00d7'),
-            ('O slash',u'\u00d8'),
             ('Paragraph',u'\u00b6'),
             ('Percent',u'%'),
             ('Plus-or-minus',u'\u00b1'),
@@ -1004,22 +1011,21 @@ class PrefixRules(GenericRulesTable):
         self.blockSignals(True)
         #print("prefix_rules_populate_table_row processing rule:\n%s\n" % data)
 
-        # Column 0: Enabled
+        # Enabled
-        self.setItem(row, 0, CheckableTableWidgetItem(data['enabled']))
+        self.setItem(row, self.COLUMNS['ENABLED']['ordinal'], CheckableTableWidgetItem(data['enabled']))
 
-        # Column 1: Rule name
+        # Rule name
-        #rule_name = QTableWidgetItem(data['name'])
+        set_rule_name_in_row(row, self.COLUMNS['NAME']['ordinal'], name=data['name'])
-        set_rule_name_in_row(row, 1, name=data['name'])
 
-        # Column 2: Prefix
+        # Prefix
-        set_prefix_field_in_row(row, 2, field=data['prefix'])
+        set_prefix_field_in_row(row, self.COLUMNS['PREFIX']['ordinal'], field=data['prefix'])
 
-        # Column 3: Source field
+        # Source field
-        source_combo = set_source_field_in_row(row, 3, field=data['field'])
+        source_combo = set_source_field_in_row(row, self.COLUMNS['FIELD']['ordinal'], field=data['field'])
 
-        # Column 4: Pattern
+        # Pattern
         # The contents of the Pattern field is driven by the Source field
-        self.source_index_changed(source_combo, row, 4, pattern=data['pattern'])
+        self.source_index_changed(source_combo, row, self.COLUMNS['PATTERN']['ordinal'], pattern=data['pattern'])
 
         self.blockSignals(False)
 
@@ -1045,5 +1051,5 @@ class PrefixRules(GenericRulesTable):
                 values = ['any date','unspecified']
 
         values_combo = ComboBox(self, values, pattern)
-        self.setCellWidget(row, 4, values_combo)
+        self.setCellWidget(row, self.COLUMNS['PATTERN']['ordinal'], values_combo)
 

src/calibre/gui2/catalog/catalog_epub_mobi.ui

@@ -210,6 +210,11 @@ The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book]
      <property name="title">
       <string>Excluded books</string>
      </property>
+     <layout class="QHBoxLayout" name="horizontalLayout_4">
+      <item>
+       <layout class="QHBoxLayout" name="horizontalLayout_3"/>
+      </item>
+     </layout>
     </widget>
    </item>
    <item>
@@ -221,11 +226,16 @@ The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book]
       </sizepolicy>
      </property>
      <property name="toolTip">
-      <string>The first enabled matching rule will be used to add a prefix to book listings in the generated catalog.</string>
+      <string>The first matching prefix rule applies a prefix to book listings in the generated catalog.</string>
      </property>
      <property name="title">
-      <string>Prefix rules</string>
+      <string>Prefixes</string>
      </property>
+     <layout class="QHBoxLayout" name="horizontalLayout_8">
+      <item>
+       <layout class="QHBoxLayout" name="horizontalLayout_6"/>
+      </item>
+     </layout>
     </widget>
    </item>
    <item>

src/calibre/gui2/device.py

@@ -3,7 +3,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
 # Imports {{{
-import os, traceback, Queue, time, cStringIO, re, sys
+import os, traceback, Queue, time, cStringIO, re, sys, weakref
 from threading import Thread, Event
 
 from PyQt4.Qt import (QMenu, QAction, QActionGroup, QIcon, SIGNAL,
@@ -369,6 +369,18 @@ class DeviceManager(Thread): # {{{
         except:
             return False
 
+    def _debug_detection(self):
+        from calibre.devices import debug
+        raw = debug(plugins=self.devices)
+        return raw
+
+    def debug_detection(self, done):
+        if self.is_device_connected:
+            raise ValueError('Device is currently detected in calibre, cannot'
+                    ' debug device detection')
+        self.create_job(self._debug_detection, done,
+                _('Debug device detection'))
+
     def _get_device_information(self):
         info = self.device.get_device_information(end_session=False)
         if len(info) < 5:
@@ -771,6 +783,15 @@ class DeviceMixin(object): # {{{
         if tweaks['auto_connect_to_folder']:
             self.connect_to_folder_named(tweaks['auto_connect_to_folder'])
 
+    def debug_detection(self, done):
+        self.debug_detection_callback = weakref.ref(done)
+        self.device_manager.debug_detection(FunctionDispatcher(self.debug_detection_done))
+
+    def debug_detection_done(self, job):
+        d = self.debug_detection_callback()
+        if d is not None:
+            d(job)
+
     def show_open_feedback(self, devname, e):
         try:
             self.__of_dev_mem__ = d = e.custom_dialog(self)

src/calibre/gui2/preferences/coloring.py

@@ -20,6 +20,8 @@ from calibre.gui2.dialogs.template_dialog import TemplateDialog
 from calibre.gui2.metadata.single_download import RichTextDelegate
 from calibre.library.coloring import (Rule, conditionable_columns,
     displayable_columns, rule_from_template)
+from calibre.utils.localization import lang_map
+from calibre.utils.icu import lower
 
 class ConditionEditor(QWidget): # {{{
 
@@ -143,7 +145,11 @@ class ConditionEditor(QWidget): # {{{
 
     @property
     def current_val(self):
-        return unicode(self.value_box.text()).strip()
+        ans = unicode(self.value_box.text()).strip()
+        if self.current_col == 'languages':
+            rmap = {lower(v):k for k, v in lang_map().iteritems()}
+            ans = rmap.get(lower(ans), ans)
+        return ans
 
     @dynamic_property
     def condition(self):
@@ -210,6 +216,11 @@ class ConditionEditor(QWidget): # {{{
         if col == 'identifiers':
             tt = _('Enter either an identifier type or an '
                     'identifier type and value of the form identifier:value')
+        elif col == 'languages':
+            tt = _('Enter a 3 letter ISO language code, like fra for French'
+                    ' or deu for German or eng for English. You can also use'
+                    ' the full language name, in which case calibre will try to'
+                    ' automatically convert it to the language code.')
         elif dt in ('int', 'float', 'rating'):
             tt = _('Enter a number')
             v = QIntValidator if dt == 'int' else QDoubleValidator

src/calibre/gui2/preferences/device_debug.py

@@ -10,15 +10,18 @@ __docformat__ = 'restructuredtext en'
 from PyQt4.Qt import QDialog, QVBoxLayout, QPlainTextEdit, QTimer, \
     QDialogButtonBox, QPushButton, QApplication, QIcon
 
+from calibre.gui2 import error_dialog
+
 class DebugDevice(QDialog):
 
-    def __init__(self, parent=None):
+    def __init__(self, gui, parent=None):
         QDialog.__init__(self, parent)
+        self.gui = gui
         self._layout = QVBoxLayout(self)
         self.setLayout(self._layout)
         self.log = QPlainTextEdit(self)
         self._layout.addWidget(self.log)
-        self.log.setPlainText(_('Getting debug information')+'...')
+        self.log.setPlainText(_('Getting debug information, please wait')+'...')
         self.copy = QPushButton(_('Copy to &clipboard'))
         self.copy.setDefault(True)
         self.setWindowTitle(_('Debug device detection'))
@@ -36,12 +39,26 @@ class DebugDevice(QDialog):
         QTimer.singleShot(1000, self.debug)
 
     def debug(self):
-        try:
+        if self.gui.device_manager.is_device_connected:
-            from calibre.devices import debug
+            error_dialog(self, _('Device already detected'),
-            raw = debug()
+                    _('A device (%s) is already detected by calibre.'
-            self.log.setPlainText(raw)
+                        ' If you wish to debug the detection of another device'
-        finally:
+                        ', first disconnect this device.')%
+                    self.gui.device_manager.connected_device.get_gui_name(),
+                    show=True)
             self.bbox.setEnabled(True)
+            return
+        self.gui.debug_detection(self)
+
+    def __call__(self, job):
+        if not self.isVisible(): return
+        self.bbox.setEnabled(True)
+        if job.failed:
+            return error_dialog(self, _('Debugging failed'),
+                    _('Running debug device detection failed. Click Show '
+                        'Details for more information.'), det_msg=job.details,
+                    show=True)
+        self.log.setPlainText(job.result)
 
     def copy_to_clipboard(self):
         QApplication.clipboard().setText(self.log.toPlainText())

src/calibre/gui2/preferences/misc.py

@@ -52,7 +52,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
     def debug_device_detection(self, *args):
         from calibre.gui2.preferences.device_debug import DebugDevice
-        d = DebugDevice(self)
+        d = DebugDevice(self.gui, self)
         d.exec_()
 
     def user_defined_device(self, *args):

src/calibre/gui2/viewer/documentview.py

@@ -21,6 +21,7 @@ from calibre.gui2.viewer.keys import SHORTCUTS
 from calibre.gui2.viewer.javascript import JavaScriptLoader
 from calibre.gui2.viewer.position import PagePosition
 from calibre.gui2.viewer.config import config, ConfigDialog
+from calibre.gui2.viewer.image_popup import ImagePopup
 from calibre.ebooks.oeb.display.webview import load_html
 from calibre.constants import isxp
 # }}}
@@ -470,6 +471,9 @@ class DocumentView(QWebView): # {{{
         self.dictionary_action.setShortcut(Qt.CTRL+Qt.Key_L)
         self.dictionary_action.triggered.connect(self.lookup)
         self.addAction(self.dictionary_action)
+        self.image_popup = ImagePopup(self)
+        self.view_image_action = QAction(_('View &image...'), self)
+        self.view_image_action.triggered.connect(self.image_popup)
         self.search_action = QAction(QIcon(I('dictionary.png')),
                 _('&Search for next occurrence'), self)
         self.search_action.setShortcut(Qt.CTRL+Qt.Key_S)
@@ -554,6 +558,11 @@ class DocumentView(QWebView): # {{{
             self.manager.selection_changed(unicode(self.document.selectedText()))
 
     def contextMenuEvent(self, ev):
+        mf = self.document.mainFrame()
+        r = mf.hitTestContent(ev.pos())
+        img = r.pixmap()
+        self.image_popup.current_img = img
+        self.image_popup.current_url = r.imageUrl()
         menu = self.document.createStandardContextMenu()
         for action in self.unimplemented_actions:
             menu.removeAction(action)
@@ -561,6 +570,8 @@ class DocumentView(QWebView): # {{{
         if text:
             menu.insertAction(list(menu.actions())[0], self.dictionary_action)
             menu.insertAction(list(menu.actions())[0], self.search_action)
+        if not img.isNull():
+            menu.addAction(self.view_image_action)
         menu.addSeparator()
         menu.addAction(self.goto_location_action)
         if self.document.in_fullscreen_mode and self.manager is not None:

src/calibre/gui2/viewer/image_popup.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from PyQt4.Qt import (QDialog, QPixmap, QUrl, QScrollArea, QLabel, QSizePolicy,
+        QDialogButtonBox, QVBoxLayout, QPalette, QApplication, QSize, QIcon, Qt)
+
+from calibre.gui2 import choose_save_file, gprefs
+
+class ImageView(QDialog):
+
+    def __init__(self, parent, current_img, current_url):
+        QDialog.__init__(self)
+        dw = QApplication.instance().desktop()
+        self.avail_geom = dw.availableGeometry(parent)
+        self.current_img = current_img
+        self.current_url = current_url
+        self.factor = 1.0
+
+        self.label = l = QLabel()
+        l.setBackgroundRole(QPalette.Base);
+        l.setSizePolicy(QSizePolicy.Ignored, QSizePolicy.Ignored)
+        l.setScaledContents(True)
+
+        self.scrollarea = sa = QScrollArea()
+        sa.setBackgroundRole(QPalette.Dark)
+        sa.setWidget(l)
+
+        self.bb = bb = QDialogButtonBox(QDialogButtonBox.Close)
+        bb.accepted.connect(self.accept)
+        bb.rejected.connect(self.reject)
+        self.zi_button = zi = bb.addButton(_('Zoom &in'), bb.ActionRole)
+        self.zo_button = zo = bb.addButton(_('Zoom &out'), bb.ActionRole)
+        self.save_button = so = bb.addButton(_('&Save as'), bb.ActionRole)
+        zi.setIcon(QIcon(I('plus.png')))
+        zo.setIcon(QIcon(I('minus.png')))
+        so.setIcon(QIcon(I('save.png')))
+        zi.clicked.connect(self.zoom_in)
+        zo.clicked.connect(self.zoom_out)
+        so.clicked.connect(self.save_image)
+
+        self.l = l = QVBoxLayout()
+        self.setLayout(l)
+        l.addWidget(sa)
+        l.addWidget(bb)
+
+    def zoom_in(self):
+        self.factor *= 1.25
+        self.adjust_image(1.25)
+
+    def zoom_out(self):
+        self.factor *= 0.8
+        self.adjust_image(0.8)
+
+    def save_image(self):
+        filters=[('Images', ['png', 'jpeg', 'jpg'])]
+        f = choose_save_file(self, 'viewer image view save dialog',
+                _('Choose a file to save to'), filters=filters,
+                all_files=False)
+        if f:
+            self.current_img.save(f)
+
+    def adjust_image(self, factor):
+        self.label.resize(self.factor * self.current_img.size())
+        self.zi_button.setEnabled(self.factor <= 3)
+        self.zo_button.setEnabled(self.factor >= 0.3333)
+        self.adjust_scrollbars(factor)
+
+    def adjust_scrollbars(self, factor):
+        for sb in (self.scrollarea.horizontalScrollBar(),
+                self.scrollarea.verticalScrollBar()):
+            sb.setValue(int(factor*sb.value()) + ((factor - 1) * sb.pageStep()/2))
+
+    def __call__(self):
+        geom = self.avail_geom
+        self.label.setPixmap(self.current_img)
+        self.label.adjustSize()
+        self.resize(QSize(int(geom.width()/2.5), geom.height()-50))
+        geom = gprefs.get('viewer_image_popup_geometry', None)
+        if geom is not None:
+            self.restoreGeometry(geom)
+        self.current_image_name = unicode(self.current_url.toString()).rpartition('/')[-1]
+        title = _('View Image: %s')%self.current_image_name
+        self.setWindowTitle(title)
+        self.show()
+
+    def done(self, e):
+        gprefs['viewer_image_popup_geometry'] = bytearray(self.saveGeometry())
+        return QDialog.done(self, e)
+
+class ImagePopup(object):
+
+    def __init__(self, parent):
+        self.current_img = QPixmap()
+        self.current_url = QUrl()
+        self.parent = parent
+        self.dialogs = []
+
+    def __call__(self):
+        if self.current_img.isNull():
+            return
+        d = ImageView(self.parent, self.current_img, self.current_url)
+        self.dialogs.append(d)
+        d.finished.connect(self.cleanup, type=Qt.QueuedConnection)
+        d()
+
+    def cleanup(self):
+        for d in tuple(self.dialogs):
+            if not d.isVisible():
+                self.dialogs.remove(d)
+

src/calibre/gui2/viewer/main.py

@@ -577,7 +577,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
                 if not os.path.exists(item.abspath):
                     return error_dialog(self, _('No such location'),
                             _('The location pointed to by this item'
-                                ' does not exist.'), show=True)
+                                ' does not exist.'), det_msg=item.abspath, show=True)
                 url = QUrl.fromLocalFile(item.abspath)
                 if item.fragment:
                     url.setFragment(item.fragment)

src/calibre/library/catalogs/epub_mobi.py

@@ -13,6 +13,8 @@ from collections import namedtuple
 from calibre import strftime
 from calibre.customize import CatalogPlugin
 from calibre.customize.conversion import OptionRecommendation, DummyReporter
+from calibre.ebooks import calibre_cover
+from calibre.ptempfile import PersistentTemporaryFile
 
 Option = namedtuple('Option', 'option, default, dest, action, help')
 
@@ -20,12 +22,12 @@ class EPUB_MOBI(CatalogPlugin):
     'ePub catalog generator'
 
     name = 'Catalog_EPUB_MOBI'
-    description = 'EPUB/MOBI catalog generator'
+    description = 'AZW3/EPUB/MOBI catalog generator'
     supported_platforms = ['windows', 'osx', 'linux']
     minimum_calibre_version = (0, 7, 40)
     author = 'Greg Riker'
     version = (1, 0, 0)
-    file_types = set(['epub','mobi'])
+    file_types = set(['azw3','epub','mobi'])
 
     THUMB_SMALLEST = "1.0"
     THUMB_LARGEST = "2.0"
@@ -36,7 +38,7 @@ class EPUB_MOBI(CatalogPlugin):
                           action = None,
                           help = _('Title of generated catalog used as title in metadata.\n'
                           "Default: '%default'\n"
-                          "Applies to: ePub, MOBI output formats")),
+                          "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--debug-pipeline',
                            default=None,
                            dest='debug_pipeline',
@@ -46,17 +48,17 @@ class EPUB_MOBI(CatalogPlugin):
                            "directory. Useful if you are unsure at which stage "
                            "of the conversion process a bug is occurring.\n"
                            "Default: '%default'\n"
-                           "Applies to: ePub, MOBI output formats")),
+                           "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--exclude-genre',
                           default='\[.+\]|\+',
                           dest='exclude_genre',
                           action = None,
                           help=_("Regex describing tags to exclude as genres.\n"
                           "Default: '%default' excludes bracketed tags, e.g. '[Project Gutenberg]', and '+', the default tag for read books.\n"
-                          "Applies to: ePub, MOBI output formats")),
+                          "Applies to: AZW3, ePub, MOBI output formats")),
 
                    Option('--exclusion-rules',
-                          default="(('Excluded tags','Tags','~,Catalog'),)",
+                          default="(('Excluded tags','Tags','Catalog'),)",
                           dest='exclusion_rules',
                           action=None,
                           help=_("Specifies the rules used to exclude books from the generated catalog.\n"
@@ -67,7 +69,7 @@ class EPUB_MOBI(CatalogPlugin):
                           "will exclude a book with a value of 'Archived' in the custom column 'status'.\n"
                           "When multiple rules are defined, all rules will be applied.\n"
                           "Default: \n" + '"' + '%default' + '"' + "\n"
-                          "Applies to ePub, MOBI output formats")),
+                          "Applies to AZW3, ePub, MOBI output formats")),
 
                    Option('--generate-authors',
                           default=False,
@@ -75,49 +77,49 @@ class EPUB_MOBI(CatalogPlugin):
                           action = 'store_true',
                           help=_("Include 'Authors' section in catalog.\n"
                           "Default: '%default'\n"
-                          "Applies to: ePub, MOBI output formats")),
+                          "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--generate-descriptions',
                           default=False,
                           dest='generate_descriptions',
                           action = 'store_true',
                           help=_("Include 'Descriptions' section in catalog.\n"
                           "Default: '%default'\n"
-                          "Applies to: ePub, MOBI output formats")),
+                          "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--generate-genres',
                           default=False,
                           dest='generate_genres',
                           action = 'store_true',
                           help=_("Include 'Genres' section in catalog.\n"
                           "Default: '%default'\n"
-                          "Applies to: ePub, MOBI output formats")),
+                          "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--generate-titles',
                           default=False,
                           dest='generate_titles',
                           action = 'store_true',
                           help=_("Include 'Titles' section in catalog.\n"
                           "Default: '%default'\n"
-                          "Applies to: ePub, MOBI output formats")),
+                          "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--generate-series',
                           default=False,
                           dest='generate_series',
                           action = 'store_true',
                           help=_("Include 'Series' section in catalog.\n"
                           "Default: '%default'\n"
-                          "Applies to: ePub, MOBI output formats")),
+                          "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--generate-recently-added',
                           default=False,
                           dest='generate_recently_added',
                           action = 'store_true',
                           help=_("Include 'Recently Added' section in catalog.\n"
                           "Default: '%default'\n"
-                          "Applies to: ePub, MOBI output formats")),
+                          "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--header-note-source-field',
                           default='',
                           dest='header_note_source_field',
                           action = None,
                           help=_("Custom field containing note text to insert in Description header.\n"
                           "Default: '%default'\n"
-                          "Applies to: ePub, MOBI output formats")),
+                          "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--merge-comments',
                           default='::',
                           dest='merge_comments',
@@ -127,14 +129,14 @@ class EPUB_MOBI(CatalogPlugin):
                           " [before|after] Placement of notes with respect to Comments\n"
                           " [True|False] - A horizontal rule is inserted between notes and Comments\n"
                           "Default: '%default'\n"
-                          "Applies to ePub, MOBI output formats")),
+                          "Applies to AZW3, ePub, MOBI output formats")),
                    Option('--output-profile',
                           default=None,
                           dest='output_profile',
                           action = None,
                           help=_("Specifies the output profile.  In some cases, an output profile is required to optimize the catalog for the device.  For example, 'kindle' or 'kindle_dx' creates a structured Table of Contents with Sections and Articles.\n"
                           "Default: '%default'\n"
-                          "Applies to: ePub, MOBI output formats")),
+                          "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--prefix-rules',
                           default="(('Read books','tags','+','\u2713'),('Wishlist items','tags','Wishlist','\u00d7'))",
                           dest='prefix_rules',
@@ -143,7 +145,7 @@ class EPUB_MOBI(CatalogPlugin):
                           "The model for a prefix rule is ('<rule name>','<source field>','<pattern>','<prefix>').\n"
                           "When multiple rules are defined, the first matching rule will be used.\n"
                           "Default:\n" + '"' + '%default' + '"' + "\n"
-                          "Applies to ePub, MOBI output formats")),
+                          "Applies to AZW3, ePub, MOBI output formats")),
                    Option('--thumb-width',
                           default='1.0',
                           dest='thumb_width',
@@ -151,7 +153,7 @@ class EPUB_MOBI(CatalogPlugin):
                           help=_("Size hint (in inches) for book covers in catalog.\n"
                           "Range: 1.0 - 2.0\n"
                           "Default: '%default'\n"
-                          "Applies to ePub, MOBI output formats")),
+                          "Applies to AZW3, ePub, MOBI output formats")),
                           ]
     # }}}
 
@@ -172,7 +174,8 @@ class EPUB_MOBI(CatalogPlugin):
         if op is None:
             op = 'default'
 
-        if opts.connected_device['name'] and 'kindle' in opts.connected_device['name'].lower():
+        if opts.connected_device['name'] and \
+                opts.connected_device['short_name'] in ['kindle','kindle dx']:
             opts.connected_kindle = True
             if opts.connected_device['serial'] and \
                opts.connected_device['serial'][:4] in ['B004','B005']:
@@ -337,7 +340,7 @@ class EPUB_MOBI(CatalogPlugin):
                 OptionRecommendation.HIGH))
             recommendations.append(('comments', '', OptionRecommendation.HIGH))
 
-            # Use to debug generated catalog code before conversion
+            # >>> Use to debug generated catalog code before conversion <<<
             #setattr(opts,'debug_pipeline',os.path.expanduser("~/Desktop/Catalog debug"))
 
             dp = getattr(opts, 'debug_pipeline', None)
@@ -355,6 +358,7 @@ class EPUB_MOBI(CatalogPlugin):
 
             # If cover exists, use it
             cpath = None
+            generate_new_cover = False
             try:
                 search_text = 'title:"%s" author:%s' % (
                         opts.catalog_title.replace('"', '\\"'), 'calibre')
@@ -364,9 +368,26 @@ class EPUB_MOBI(CatalogPlugin):
                     if cpath and os.path.exists(cpath):
                         recommendations.append(('cover', cpath,
                             OptionRecommendation.HIGH))
+                        log.info("using existing cover")
+                    else:
+                        log.info("no existing cover, generating new cover")
+                        generate_new_cover = True
+                else:
+                    log.info("no existing cover, generating new cover")
+                    generate_new_cover = True
             except:
                 pass
 
+            if generate_new_cover:
+                new_cover_path = PersistentTemporaryFile(suffix='.jpg')
+                new_cover = calibre_cover(opts.catalog_title.replace('"', '\\"'), 'calibre')
+                new_cover_path.write(new_cover)
+                new_cover_path.close()
+                recommendations.append(('cover', new_cover_path.name, OptionRecommendation.HIGH))
+
+            if opts.verbose:
+                log.info("Invoking Plumber with recommendations:\n %s" % recommendations)
+
             # Run ebook-convert
             from calibre.ebooks.conversion.plumber import Plumber
             plumber = Plumber(os.path.join(catalog.catalogPath,

src/calibre/library/catalogs/epub_mobi_builder.py

@@ -1126,7 +1126,7 @@ Author '{0}':
                 aTag = Tag(soup, "a")
                 current_letter = self.letter_or_symbol(book['author_sort'][0].upper())
                 if current_letter == self.SYMBOLS:
-                    aTag['id'] = self.SYMBOLS
+                    aTag['id'] = self.SYMBOLS + '_authors'
                 else:
                     aTag['id'] = "%s_authors" % self.generateUnicodeName(current_letter)
                 pIndexTag.insert(0,aTag)
@@ -1337,7 +1337,7 @@ Author '{0}':
                     pBookTag['class'] = "line_item"
                     ptc = 0
 
-                    pBookTag.insert(ptc, self.formatPrefix(book['prefix'],soup))
+                    pBookTag.insert(ptc, self.formatPrefix(new_entry['prefix'],soup))
                     ptc += 1
 
                     spanTag = Tag(soup, "span")

src/calibre/library/database2.py

@@ -1425,6 +1425,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         size=stream.tell()
         self.conn.execute('INSERT OR REPLACE INTO data (book,format,uncompressed_size,name) VALUES (?,?,?,?)',
                           (id, format.upper(), size, name))
+        self.update_last_modified([id], commit=False)
         self.conn.commit()
         self.format_filename_cache[id][format.upper()] = name
         self.refresh_ids([id])