Merge from trunk

src/calibre/__init__.py

@@ -455,6 +455,24 @@ def prepare_string_for_xml(raw, attribute=False):
 def isbytestring(obj):
     return isinstance(obj, (str, bytes))
 
+def force_unicode(obj, enc=preferred_encoding):
+    if isbytestring(obj):
+        try:
+            obj = obj.decode(enc)
+        except:
+            try:
+                obj = obj.decode(filesystem_encoding if enc ==
+                        preferred_encoding else preferred_encoding)
+            except:
+                try:
+                    obj = obj.decode('utf-8')
+                except:
+                    obj = repr(obj)
+                    if isbytestring(obj):
+                        obj = obj.decode('utf-8')
+    return obj
+
+
 def human_readable(size):
     """ Convert a size in bytes into a human readable form """
     divisor, suffix = 1, "B"

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.7.20'
+__version__   = '0.7.23'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 
 import re

src/calibre/customize/__init__.py

@@ -218,7 +218,7 @@ class MetadataReaderPlugin(Plugin): # {{{
         with the input data.
         :param type: The type of file. Guaranteed to be one of the entries
         in :attr:`file_types`.
-        :return: A :class:`calibre.ebooks.metadata.MetaInformation` object
+        :return: A :class:`calibre.ebooks.metadata.book.Metadata` object
         '''
         return None
 # }}}
@@ -248,7 +248,7 @@ class MetadataWriterPlugin(Plugin): # {{{
         with the input data.
         :param type: The type of file. Guaranteed to be one of the entries
         in :attr:`file_types`.
-        :param mi: A :class:`calibre.ebooks.metadata.MetaInformation` object
+        :param mi: A :class:`calibre.ebooks.metadata.book.Metadata` object
         '''
         pass
 

src/calibre/customize/builtins.py

@@ -226,8 +226,7 @@ class OPFMetadataReader(MetadataReaderPlugin):
 
     def get_metadata(self, stream, ftype):
         from calibre.ebooks.metadata.opf2 import OPF
-        from calibre.ebooks.metadata import MetaInformation
-        return MetaInformation(OPF(stream, os.getcwd()))
+        return OPF(stream, os.getcwd()).to_book_metadata()
 
 class PDBMetadataReader(MetadataReaderPlugin):
 
@@ -447,7 +446,7 @@ from calibre.devices.eb600.driver import EB600, COOL_ER, SHINEBOOK, \
                 BOOQ, ELONEX, POCKETBOOK301, MENTOR
 from calibre.devices.iliad.driver import ILIAD
 from calibre.devices.irexdr.driver import IREXDR1000, IREXDR800
-from calibre.devices.jetbook.driver import JETBOOK, MIBUK
+from calibre.devices.jetbook.driver import JETBOOK, MIBUK, JETBOOK_MINI
 from calibre.devices.kindle.driver import KINDLE, KINDLE2, KINDLE_DX
 from calibre.devices.nook.driver import NOOK
 from calibre.devices.prs505.driver import PRS505
@@ -461,7 +460,8 @@ from calibre.devices.hanvon.driver import N516, EB511, ALEX, AZBOOKA, THEBOOK
 from calibre.devices.edge.driver import EDGE
 from calibre.devices.teclast.driver import TECLAST_K3, NEWSMY, IPAPYRUS, SOVOS
 from calibre.devices.sne.driver import SNE
-from calibre.devices.misc import PALMPRE, AVANT, SWEEX, PDNOVEL, KOGAN, GEMEI
+from calibre.devices.misc import PALMPRE, AVANT, SWEEX, PDNOVEL, KOGAN, \
+        GEMEI, VELOCITYMICRO, PDNOVEL_KOBO
 from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG
 from calibre.devices.kobo.driver import KOBO
 
@@ -469,14 +469,14 @@ from calibre.ebooks.metadata.fetch import GoogleBooks, ISBNDB, Amazon, \
     LibraryThing
 from calibre.ebooks.metadata.douban import DoubanBooks
 from calibre.ebooks.metadata.covers import OpenLibraryCovers, \
-        LibraryThingCovers
+        LibraryThingCovers, DoubanCovers
 from calibre.library.catalog import CSV_XML, EPUB_MOBI, BIBTEX
 from calibre.ebooks.epub.fix.unmanifested import Unmanifested
 from calibre.ebooks.epub.fix.epubcheck import Epubcheck
 
 plugins = [HTML2ZIP, PML2PMLZ, ArchiveExtract, GoogleBooks, ISBNDB, Amazon,
         LibraryThing, DoubanBooks, CSV_XML, EPUB_MOBI, BIBTEX, Unmanifested,
-        Epubcheck, OpenLibraryCovers, LibraryThingCovers]
+        Epubcheck, OpenLibraryCovers, LibraryThingCovers, DoubanCovers]
 plugins += [
     ComicInput,
     EPUBInput,
@@ -521,6 +521,7 @@ plugins += [
     IREXDR1000,
     IREXDR800,
     JETBOOK,
+    JETBOOK_MINI,
     MIBUK,
     SHINEBOOK,
     POCKETBOOK360,
@@ -572,6 +573,8 @@ plugins += [
     PDNOVEL,
     SPECTRA,
     GEMEI,
+    VELOCITYMICRO,
+    PDNOVEL_KOBO,
     ITUNES,
 ]
 plugins += [x for x in list(locals().values()) if isinstance(x, type) and \
@@ -797,6 +800,17 @@ class Sending(PreferencesPlugin):
     description = _('Control how calibre transfers files to your '
             'ebook reader')
 
+class Plugboard(PreferencesPlugin):
+    name = 'Plugboard'
+    icon = I('plugboard.png')
+    gui_name = _('Metadata plugboards')
+    category = 'Import/Export'
+    gui_category = _('Import/Export')
+    category_order = 3
+    name_order = 4
+    config_widget = 'calibre.gui2.preferences.plugboard'
+    description = _('Change metadata fields before saving/sending')
+
 class Email(PreferencesPlugin):
     name = 'Email'
     icon = I('mail.png')
@@ -857,8 +871,8 @@ class Misc(PreferencesPlugin):
     description = _('Miscellaneous advanced configuration')
 
 plugins += [LookAndFeel, Behavior, Columns, Toolbar, InputOptions,
-        CommonOptions, OutputOptions, Adding, Saving, Sending, Email, Server,
-        Plugins, Tweaks, Misc]
+        CommonOptions, OutputOptions, Adding, Saving, Sending, Plugboard,
+        Email, Server, Plugins, Tweaks, Misc]
 
 #}}}
 

src/calibre/customize/profiles.py

@@ -1,3 +1,4 @@
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
 from __future__ import with_statement
 __license__ = 'GPL 3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
@@ -251,6 +252,12 @@ class OutputProfile(Plugin):
     #: The character used to represent a star in ratings
     ratings_char = u'*'
 
+    #: Unsupported unicode characters to be replaced during preprocessing
+    unsupported_unicode_chars = []
+
+    #: Number of ems that the left margin of a blockquote is rendered as
+    mobi_ems_per_blockquote = 1.0
+
     @classmethod
     def tags_to_string(cls, tags):
         return escape(', '.join(tags))
@@ -422,6 +429,8 @@ class SonyReaderOutput(OutputProfile):
     dpi                       = 168.451
     fbase                     = 12
     fsizes                    = [7.5, 9, 10, 12, 15.5, 20, 22, 24]
+    unsupported_unicode_chars = [u'\u201f', u'\u201b']
+
 
 class KoboReaderOutput(OutputProfile):
 
@@ -558,6 +567,7 @@ class KindleOutput(OutputProfile):
     supports_mobi_indexing = True
     periodical_date_in_title = False
     ratings_char = u'\u2605'
+    mobi_ems_per_blockquote = 2.0
 
     @classmethod
     def tags_to_string(cls, tags):
@@ -576,6 +586,7 @@ class KindleDXOutput(OutputProfile):
     comic_screen_size         = (741, 1022)
     supports_mobi_indexing = True
     periodical_date_in_title = False
+    mobi_ems_per_blockquote = 2.0
 
     @classmethod
     def tags_to_string(cls, tags):

src/calibre/customize/ui.py

@@ -120,7 +120,7 @@ def enable_plugin(plugin_or_name):
     config['enabled_plugins'] = ep
 
 default_disabled_plugins = set([
-    'Douban Books',
+    'Douban Books', 'Douban.com covers',
 ])
 
 def is_disabled(plugin):

src/calibre/debug.py

@@ -36,11 +36,17 @@ Run an embedded python interpreter.
             'plugin code.')
     parser.add_option('--reinitialize-db', default=None,
             help='Re-initialize the sqlite calibre database at the '
-            'specified path. Useful to recover from db corruption.')
+            'specified path. Useful to recover from db corruption.'
+            ' You can also specify the path to an SQL dump which '
+            'will be used instead of trying to dump the database.'
+            ' This can be useful when dumping fails, but dumping '
+            'with sqlite3 works.')
+    parser.add_option('-p', '--py-console', help='Run python console',
+            default=False, action='store_true')
 
     return parser
 
-def reinit_db(dbpath, callback=None):
+def reinit_db(dbpath, callback=None, sql_dump=None):
     if not os.path.exists(dbpath):
         raise ValueError(dbpath + ' does not exist')
     from calibre.library.sqlite import connect
@@ -50,26 +56,32 @@ def reinit_db(dbpath, callback=None):
     uv = conn.get('PRAGMA user_version;', all=False)
     conn.execute('PRAGMA writable_schema=ON')
     conn.commit()
-    sql_lines = conn.dump()
+    if sql_dump is None:
+        sql_lines = conn.dump()
+    else:
+        sql_lines = open(sql_dump, 'rb').read()
     conn.close()
     dest = dbpath + '.tmp'
     try:
         with closing(connect(dest, False)) as nconn:
             nconn.execute('create temporary table temp_sequence(id INTEGER PRIMARY KEY AUTOINCREMENT)')
             nconn.commit()
-            if callable(callback):
-                callback(len(sql_lines), True)
-            for i, line in enumerate(sql_lines):
-                try:
-                    nconn.execute(line)
-                except:
-                    import traceback
-                    prints('SQL line %r failed with error:'%line)
-                    prints(traceback.format_exc())
-                    continue
-                finally:
-                    if callable(callback):
-                        callback(i, False)
+            if sql_dump is None:
+                if callable(callback):
+                    callback(len(sql_lines), True)
+                for i, line in enumerate(sql_lines):
+                    try:
+                        nconn.execute(line)
+                    except:
+                        import traceback
+                        prints('SQL line %r failed with error:'%line)
+                        prints(traceback.format_exc())
+                        continue
+                    finally:
+                        if callable(callback):
+                            callback(i, False)
+            else:
+                nconn.executescript(sql_lines)
             nconn.execute('pragma user_version=%d'%int(uv))
             nconn.commit()
         os.remove(dbpath)
@@ -148,6 +160,9 @@ def main(args=sys.argv):
         if len(args) > 1:
             vargs.append(args[-1])
         main(vargs)
+    elif opts.py_console:
+        from calibre.utils.pyconsole.main import main
+        main()
     elif opts.command:
         sys.argv = args[:1]
         exec opts.command
@@ -165,7 +180,10 @@ def main(args=sys.argv):
         prints('CALIBRE_EXTENSIONS_PATH='+sys.extensions_location)
         prints('CALIBRE_PYTHON_PATH='+os.pathsep.join(sys.path))
     elif opts.reinitialize_db is not None:
-        reinit_db(opts.reinitialize_db)
+        sql_dump = None
+        if len(args) > 1 and os.access(args[-1], os.R_OK):
+            sql_dump = args[-1]
+        reinit_db(opts.reinitialize_db, sql_dump=sql_dump)
     else:
         from calibre import ipython
         ipython()

src/calibre/devices/__init__.py

@@ -56,6 +56,7 @@ def get_connected_device():
     return dev
 
 def debug(ioreg_to_tmp=False, buf=None):
+    import textwrap
     from calibre.customize.ui import device_plugins
     from calibre.devices.scanner import DeviceScanner, win_pnp_drives
     from calibre.constants import iswindows, isosx, __version__
@@ -95,13 +96,19 @@ def debug(ioreg_to_tmp=False, buf=None):
             ioreg += 'Output from osx_get_usb_drives:\n'+drives+'\n\n'
             ioreg += Device.run_ioreg()
         connected_devices = []
-        for dev in sorted(device_plugins(), cmp=lambda
-                x,y:cmp(x.__class__.__name__, y.__class__.__name__)):
-            out('Looking for', dev.__class__.__name__)
+        devplugins = list(sorted(device_plugins(), cmp=lambda
+                x,y:cmp(x.__class__.__name__, y.__class__.__name__)))
+        out('Available plugins:', textwrap.fill(' '.join([x.__class__.__name__ for x in
+            devplugins])))
+        out(' ')
+        out('Looking for devices...')
+        for dev in devplugins:
             connected, det = s.is_device_connected(dev, debug=True)
             if connected:
+                out('\t\tDetected possible device', dev.__class__.__name__)
                 connected_devices.append((dev, det))
 
+        out(' ')
         errors = {}
         success = False
         out('Devices possibly connected:', end=' ')

src/calibre/devices/apple/driver.py

@@ -13,12 +13,12 @@ from calibre.devices.errors import UserFeedback
 from calibre.devices.usbms.deviceconfig import DeviceConfig
 from calibre.devices.interface import DevicePlugin
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
-from calibre.ebooks.metadata import MetaInformation, authors_to_string
+from calibre.ebooks.metadata import authors_to_string, MetaInformation
+from calibre.ebooks.metadata.book.base import Metadata
 from calibre.ebooks.metadata.epub import set_metadata
 from calibre.library.server.utils import strftime
-from calibre.utils.config import config_dir
+from calibre.utils.config import config_dir, prefs
 from calibre.utils.date import isoformat, now, parse_date
-from calibre.utils.localization import get_lang
 from calibre.utils.logging import Log
 from calibre.utils.zipfile import ZipFile
 
@@ -67,6 +67,8 @@ class ITUNES(DriverBase):
     Delete:
         delete_books()
         remove_books_from_metadata()
+        use_plugboard_ext()
+        set_plugboard()
         sync_booklists()
         card_prefix()
         free_space()
@@ -75,6 +77,8 @@ class ITUNES(DriverBase):
         set_progress_reporter()
         upload_books()
         add_books_to_metadata()
+        use_plugboard_ext()
+        set_plugboard()
         set_progress_reporter()
         sync_booklists()
         card_prefix()
@@ -105,6 +109,9 @@ class ITUNES(DriverBase):
     PRODUCT_ID = [0x1292,0x1293,0x1294,0x1297,0x1299,0x129a]
     BCD = [0x01]
 
+    # Plugboard ID
+    DEVICE_PLUGBOARD_NAME = 'APPLE'
+
     # iTunes enumerations
     Audiobooks = [
         'Audible file',
@@ -163,6 +170,7 @@ class ITUNES(DriverBase):
     # Properties
     cached_books = {}
     cache_dir = os.path.join(config_dir, 'caches', 'itunes')
+    calibre_library_path = prefs['library_path']
     archive_path = os.path.join(cache_dir, "thumbs.zip")
     description_prefix = "added by calibre"
     ejected = False
@@ -172,6 +180,8 @@ class ITUNES(DriverBase):
     log = Log()
     manual_sync_mode = False
     path_template = 'iTunes/%s - %s.%s'
+    plugboards = None
+    plugboard_func = None
     problem_titles = []
     problem_msg = None
     report_progress = None
@@ -249,6 +259,8 @@ class ITUNES(DriverBase):
                 self.report_progress(1.0, _('Updating device metadata listing...'))
 
         # Add new books to booklists[0]
+        # Charles thinks this should be
+        # for new_book in metadata[0]:
         for new_book in locations[0]:
             if DEBUG:
                 self.log.info("  adding '%s' by '%s' to booklists[0]" %
@@ -813,6 +825,15 @@ class ITUNES(DriverBase):
         '''
         self.report_progress = report_progress
 
+    def set_plugboards(self, plugboards, pb_func):
+        # This method is called with the plugboard that matches the format
+        # declared in use_plugboard_ext and a device name of ITUNES
+        if DEBUG:
+            self.log.info("ITUNES.set_plugboard()")
+            #self.log.info('  using plugboard %s' % plugboards)
+        self.plugboards = plugboards
+        self.plugboard_func = pb_func
+
     def sync_booklists(self, booklists, end_session=True):
         '''
         Update metadata on device.
@@ -871,7 +892,7 @@ class ITUNES(DriverBase):
         once uploaded to the device. len(names) == len(files)
         :return: A list of 3-element tuples. The list is meant to be passed
         to L{add_books_to_metadata}.
-        :metadata: If not None, it is a list of :class:`MetaInformation` objects.
+        :metadata: If not None, it is a list of :class:`Metadata` objects.
         The idea is to use the metadata to determine where on the device to
         put the book. len(metadata) == len(files). Apart from the regular
         cover (path to cover), there may also be a thumbnail attribute, which should
@@ -976,7 +997,6 @@ class ITUNES(DriverBase):
             self._dump_cached_books(header="after upload_books()",indent=2)
         return (new_booklist, [], [])
 
-
     # Private methods
     def _add_device_book(self,fpath, metadata):
         '''
@@ -1190,6 +1210,10 @@ class ITUNES(DriverBase):
                 except:
                     self.problem_titles.append("'%s' by %s" % (metadata.title, metadata.author[0]))
                     self.log.error("  error scaling '%s' for '%s'" % (metadata.cover,metadata.title))
+
+                    import traceback
+                    traceback.print_exc()
+
                     return thumb
 
                 if isosx:
@@ -1255,7 +1279,10 @@ class ITUNES(DriverBase):
                 self.problem_titles.append("'%s' by %s" % (metadata.title, metadata.author[0]))
                 self.log.error("   error converting '%s' to thumb for '%s'" % (metadata.cover,metadata.title))
             finally:
-                zfw.close()
+                try:
+                    zfw.close()
+                except:
+                    pass
         else:
             if DEBUG:
                 self.log.info("   no cover defined in metadata for '%s'" % metadata.title)
@@ -1272,10 +1299,10 @@ class ITUNES(DriverBase):
         this_book.db_id = None
         this_book.device_collections = []
         this_book.format = format
-        this_book.library_id = lb_added
+        this_book.library_id = lb_added     # ??? GR
         this_book.path = path
         this_book.thumbnail = thumb
-        this_book.iTunes_id = lb_added
+        this_book.iTunes_id = lb_added      # ??? GR
         this_book.uuid = metadata.uuid
 
         if isosx:
@@ -1321,8 +1348,8 @@ class ITUNES(DriverBase):
             plist = None
         if plist:
             if DEBUG:
-                self.log.info(" _delete_iTunesMetadata_plist():")
-                self.log.info("   deleting '%s'\n   from '%s'" % (pl_name,fpath))
+                self.log.info("  _delete_iTunesMetadata_plist():")
+                self.log.info("    deleting '%s'\n   from '%s'" % (pl_name,fpath))
                 zf.delete(pl_name)
         zf.close()
 
@@ -2212,6 +2239,7 @@ class ITUNES(DriverBase):
                  (self.iTunes.name(), self.iTunes.version(), self.initial_status,
                   self.version[0],self.version[1],self.version[2]))
                 self.log.info("  iTunes_media: %s" % self.iTunes_media)
+                self.log.info("  calibre_library_path: %s" % self.calibre_library_path)
 
         if iswindows:
             '''
@@ -2265,6 +2293,7 @@ class ITUNES(DriverBase):
                  (self.iTunes.Windows[0].name, self.iTunes.Version, self.initial_status,
                   self.version[0],self.version[1],self.version[2]))
                 self.log.info("  iTunes_media: %s" % self.iTunes_media)
+                self.log.info("  calibre_library_path: %s" % self.calibre_library_path)
 
     def _purge_orphans(self,library_books, cached_books):
         '''
@@ -2367,7 +2396,8 @@ class ITUNES(DriverBase):
         '''
         iTunes does not delete books from storage when removing from database
         We only want to delete stored copies if the file is stored in iTunes
-        We don't want to delete files stored outside of iTunes
+        We don't want to delete files stored outside of iTunes.
+        Also confirm that storage_path does not point into calibre's storage.
         '''
         if DEBUG:
             self.log.info(" ITUNES._remove_from_iTunes():")
@@ -2375,7 +2405,8 @@ class ITUNES(DriverBase):
         if isosx:
             try:
                 storage_path = os.path.split(cached_book['lib_book'].location().path)
-                if cached_book['lib_book'].location().path.startswith(self.iTunes_media):
+                if cached_book['lib_book'].location().path.startswith(self.iTunes_media) and \
+                   not storage_path[0].startswith(prefs['library_path']):
                     title_storage_path = storage_path[0]
                     if DEBUG:
                         self.log.info("  removing title_storage_path: %s" % title_storage_path)
@@ -2426,7 +2457,8 @@ class ITUNES(DriverBase):
                     path = book.Location
 
             if book:
-                if self.iTunes_media and path.startswith(self.iTunes_media):
+                if self.iTunes_media and path.startswith(self.iTunes_media) and \
+                   not path.startswith(prefs['library_path']):
                     storage_path = os.path.split(path)
                     if DEBUG:
                         self.log.info("   removing '%s' at %s" %
@@ -2453,11 +2485,17 @@ class ITUNES(DriverBase):
                 if DEBUG:
                     self.log.info("   unable to remove '%s' from iTunes" % cached_book['title'])
 
+    def title_sorter(self, title):
+        return re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', title).rstrip()
+
     def _update_epub_metadata(self, fpath, metadata):
         '''
         '''
         self.log.info(" ITUNES._update_epub_metadata()")
 
+        # Fetch plugboard updates
+        metadata_x = self._xform_metadata_via_plugboard(metadata, 'epub')
+
         # Refresh epub metadata
         with open(fpath,'r+b') as zfo:
             # Touch the OPF timestamp
@@ -2489,9 +2527,14 @@ class ITUNES(DriverBase):
                         self.log.info("   add timestamp: %s" % metadata.timestamp)
 
                 # Force the language declaration for iBooks 1.1
-                metadata.language = get_lang().replace('_', '-')
+                #metadata.language = get_lang().replace('_', '-')
+
+                # Updates from metadata plugboard (ignoring publisher)
+                metadata.language = metadata_x.language
+
                 if DEBUG:
-                    self.log.info("   rewriting language: <dc:language>%s</dc:language>" % metadata.language)
+                    if metadata.language != metadata_x.language:
+                        self.log.info("   rewriting language: <dc:language>%s</dc:language>" % metadata.language)
 
             zf_opf.close()
 
@@ -2569,75 +2612,97 @@ class ITUNES(DriverBase):
         if DEBUG:
             self.log.info(" ITUNES._update_iTunes_metadata()")
 
-        strip_tags = re.compile(r'<[^<]*?/?>')
+        STRIP_TAGS = re.compile(r'<[^<]*?/?>')
+
+        # Update metadata from plugboard
+        # If self.plugboard is None (no transforms), original metadata is returned intact
+        metadata_x = self._xform_metadata_via_plugboard(metadata, this_book.format)
 
         if isosx:
             if lb_added:
-                lb_added.album.set(metadata.title)
-                lb_added.artist.set(authors_to_string(metadata.authors))
-                lb_added.composer.set(metadata.uuid)
+                lb_added.name.set(metadata_x.title)
+                lb_added.album.set(metadata_x.title)
+                lb_added.artist.set(authors_to_string(metadata_x.authors))
+                lb_added.composer.set(metadata_x.uuid)
                 lb_added.description.set("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S')))
                 lb_added.enabled.set(True)
-                lb_added.sort_artist.set(metadata.author_sort.title())
-                lb_added.sort_name.set(this_book.title_sorter)
-                if this_book.format == 'pdf':
-                    lb_added.name.set(metadata.title)
+                lb_added.sort_artist.set(metadata_x.author_sort.title())
+                lb_added.sort_name.set(metadata.title_sort)
+
 
             if db_added:
-                db_added.album.set(metadata.title)
-                db_added.artist.set(authors_to_string(metadata.authors))
-                db_added.composer.set(metadata.uuid)
+                db_added.name.set(metadata_x.title)
+                db_added.album.set(metadata_x.title)
+                db_added.artist.set(authors_to_string(metadata_x.authors))
+                db_added.composer.set(metadata_x.uuid)
                 db_added.description.set("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S')))
                 db_added.enabled.set(True)
-                db_added.sort_artist.set(metadata.author_sort.title())
-                db_added.sort_name.set(this_book.title_sorter)
-                if this_book.format == 'pdf':
-                    db_added.name.set(metadata.title)
+                db_added.sort_artist.set(metadata_x.author_sort.title())
+                db_added.sort_name.set(metadata.title_sort)
 
-            if metadata.comments:
+            if metadata_x.comments:
                 if lb_added:
-                    lb_added.comment.set(strip_tags.sub('',metadata.comments))
+                    lb_added.comment.set(STRIP_TAGS.sub('',metadata_x.comments))
                 if db_added:
-                    db_added.comment.set(strip_tags.sub('',metadata.comments))
+                    db_added.comment.set(STRIP_TAGS.sub('',metadata_x.comments))
 
-            if metadata.rating:
+            if metadata_x.rating:
                 if lb_added:
-                    lb_added.rating.set(metadata.rating*10)
+                    lb_added.rating.set(metadata_x.rating*10)
                 # iBooks currently doesn't allow setting rating ... ?
                 try:
                     if db_added:
-                        db_added.rating.set(metadata.rating*10)
+                        db_added.rating.set(metadata_x.rating*10)
                 except:
                     pass
 
             # Set genre from series if available, else first alpha tag
             # Otherwise iTunes grabs the first dc:subject from the opf metadata
-            if metadata.series and self.settings().read_metadata:
+            # self.settings().read_metadata is used as a surrogate for "Use Series name as Genre"
+            if metadata_x.series and self.settings().read_metadata:
                 if DEBUG:
+                    self.log.info(" ITUNES._update_iTunes_metadata()")
                     self.log.info("   using Series name as Genre")
 
                 # Format the index as a sort key
-                index = metadata.series_index
+                index = metadata_x.series_index
                 integer = int(index)
                 fraction = index-integer
                 series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0'))
                 if lb_added:
-                    lb_added.sort_name.set("%s %s" % (metadata.series, series_index))
-                    lb_added.genre.set(metadata.series)
-                    lb_added.episode_ID.set(metadata.series)
-                    lb_added.episode_number.set(metadata.series_index)
+                    lb_added.sort_name.set("%s %s" % (self.title_sorter(metadata_x.series), series_index))
+                    lb_added.episode_ID.set(metadata_x.series)
+                    lb_added.episode_number.set(metadata_x.series_index)
+
+                    # If no plugboard transform applied to tags, change the Genre/Category to Series
+                    if metadata.tags == metadata_x.tags:
+                        lb_added.genre.set(self.title_sorter(metadata_x.series))
+                    else:
+                        for tag in metadata_x.tags:
+                            if self._is_alpha(tag[0]):
+                                lb_added.genre.set(tag)
+                                break
 
                 if db_added:
-                    db_added.sort_name.set("%s %s" % (metadata.series, series_index))
-                    db_added.genre.set(metadata.series)
-                    db_added.episode_ID.set(metadata.series)
-                    db_added.episode_number.set(metadata.series_index)
+                    db_added.sort_name.set("%s %s" % (self.title_sorter(metadata_x.series), series_index))
+                    db_added.episode_ID.set(metadata_x.series)
+                    db_added.episode_number.set(metadata_x.series_index)
 
-            elif metadata.tags:
+                    # If no plugboard transform applied to tags, change the Genre/Category to Series
+                    if metadata.tags == metadata_x.tags:
+                        db_added.genre.set(self.title_sorter(metadata_x.series))
+                    else:
+                        for tag in metadata_x.tags:
+                            if self._is_alpha(tag[0]):
+                                db_added.genre.set(tag)
+                                break
+
+
+            elif metadata_x.tags is not None:
                 if DEBUG:
                     self.log.info("   %susing Tag as Genre" %
                                   "no Series name available, " if self.settings().read_metadata else '')
-                for tag in metadata.tags:
+                for tag in metadata_x.tags:
                     if self._is_alpha(tag[0]):
                         if lb_added:
                             lb_added.genre.set(tag)
@@ -2647,40 +2712,38 @@ class ITUNES(DriverBase):
 
         elif iswindows:
             if lb_added:
-                lb_added.Album = metadata.title
-                lb_added.Artist = authors_to_string(metadata.authors)
-                lb_added.Composer = metadata.uuid
+                lb_added.Name = metadata_x.title
+                lb_added.Album = metadata_x.title
+                lb_added.Artist = authors_to_string(metadata_x.authors)
+                lb_added.Composer = metadata_x.uuid
                 lb_added.Description = ("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S')))
                 lb_added.Enabled = True
-                lb_added.SortArtist = (metadata.author_sort.title())
-                lb_added.SortName = (this_book.title_sorter)
-                if this_book.format == 'pdf':
-                    lb_added.Name = metadata.title
+                lb_added.SortArtist = metadata_x.author_sort.title()
+                lb_added.SortName = metadata.title_sort
 
             if db_added:
-                db_added.Album = metadata.title
-                db_added.Artist = authors_to_string(metadata.authors)
-                db_added.Composer = metadata.uuid
+                db_added.Name = metadata_x.title
+                db_added.Album = metadata_x.title
+                db_added.Artist = authors_to_string(metadata_x.authors)
+                db_added.Composer = metadata_x.uuid
                 db_added.Description = ("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S')))
                 db_added.Enabled = True
-                db_added.SortArtist = (metadata.author_sort.title())
-                db_added.SortName = (this_book.title_sorter)
-                if this_book.format == 'pdf':
-                    db_added.Name = metadata.title
+                db_added.SortArtist = metadata_x.author_sort.title()
+                db_added.SortName = metadata.title_sort
 
-            if metadata.comments:
+            if metadata_x.comments:
                 if lb_added:
-                    lb_added.Comment = (strip_tags.sub('',metadata.comments))
+                    lb_added.Comment = (STRIP_TAGS.sub('',metadata_x.comments))
                 if db_added:
-                    db_added.Comment = (strip_tags.sub('',metadata.comments))
+                    db_added.Comment = (STRIP_TAGS.sub('',metadata_x.comments))
 
-            if metadata.rating:
+            if metadata_x.rating:
                 if lb_added:
-                    lb_added.AlbumRating = (metadata.rating*10)
+                    lb_added.AlbumRating = (metadata_x.rating*10)
                 # iBooks currently doesn't allow setting rating ... ?
                 try:
                     if db_added:
-                        db_added.AlbumRating = (metadata.rating*10)
+                        db_added.AlbumRating = (metadata_x.rating*10)
                 except:
                     if DEBUG:
                         self.log.warning("  iTunes automation interface reported an error"
@@ -2690,36 +2753,54 @@ class ITUNES(DriverBase):
             # Otherwise iBooks uses first <dc:subject> from opf
             # iTunes balks on setting EpisodeNumber, but it sticks (9.1.1.12)
 
-            if metadata.series and self.settings().read_metadata:
+            if metadata_x.series and self.settings().read_metadata:
                 if DEBUG:
                     self.log.info("   using Series name as Genre")
                 # Format the index as a sort key
-                index = metadata.series_index
+                index = metadata_x.series_index
                 integer = int(index)
                 fraction = index-integer
                 series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0'))
                 if lb_added:
-                    lb_added.SortName = "%s %s" % (metadata.series, series_index)
-                    lb_added.Genre = metadata.series
-                    lb_added.EpisodeID = metadata.series
+                    lb_added.SortName = "%s %s" % (self.title_sorter(metadata_x.series), series_index)
+                    lb_added.EpisodeID = metadata_x.series
                     try:
-                        lb_added.EpisodeNumber = metadata.series_index
+                        lb_added.EpisodeNumber = metadata_x.series_index
                     except:
                         pass
+
+                    # If no plugboard transform applied to tags, change the Genre/Category to Series
+                    if metadata.tags == metadata_x.tags:
+                        lb_added.Genre = self.title_sorter(metadata_x.series)
+                    else:
+                        for tag in metadata_x.tags:
+                            if self._is_alpha(tag[0]):
+                                lb_added.Genre = tag
+                                break
+
                 if db_added:
-                    db_added.SortName = "%s %s" % (metadata.series, series_index)
-                    db_added.Genre = metadata.series
-                    db_added.EpisodeID = metadata.series
+                    db_added.SortName = "%s %s" % (self.title_sorter(metadata_x.series), series_index)
+                    db_added.EpisodeID = metadata_x.series
                     try:
-                        db_added.EpisodeNumber = metadata.series_index
+                        db_added.EpisodeNumber = metadata_x.series_index
                     except:
                         if DEBUG:
                             self.log.warning("  iTunes automation interface reported an error"
                                              " setting EpisodeNumber on iDevice")
-            elif metadata.tags:
+
+                    # If no plugboard transform applied to tags, change the Genre/Category to Series
+                    if metadata.tags == metadata_x.tags:
+                        db_added.Genre = self.title_sorter(metadata_x.series)
+                    else:
+                        for tag in metadata_x.tags:
+                            if self._is_alpha(tag[0]):
+                                db_added.Genre = tag
+                                break
+
+            elif metadata_x.tags is not None:
                 if DEBUG:
                     self.log.info("   using Tag as Genre")
-                for tag in metadata.tags:
+                for tag in metadata_x.tags:
                     if self._is_alpha(tag[0]):
                         if lb_added:
                             lb_added.Genre = tag
@@ -2727,6 +2808,36 @@ class ITUNES(DriverBase):
                             db_added.Genre = tag
                         break
 
+    def _xform_metadata_via_plugboard(self, book, format):
+        ''' Transform book metadata from plugboard templates '''
+        if DEBUG:
+            self.log.info("  ITUNES._update_metadata_from_plugboard()")
+
+        if self.plugboard_func:
+            pb = self.plugboard_func(self.DEVICE_PLUGBOARD_NAME, format, self.plugboards)
+            newmi = book.deepcopy_metadata()
+            newmi.template_to_attribute(book, pb)
+            if DEBUG:
+                self.log.info(" transforming %s using %s:" % (format, pb))
+                self.log.info("       title: %s %s" % (book.title, ">>> %s" %
+                                           newmi.title if book.title != newmi.title else ''))
+                self.log.info("  title_sort: %s %s" % (book.title_sort, ">>> %s" %
+                                           newmi.title_sort if book.title_sort != newmi.title_sort else ''))
+                self.log.info("     authors: %s %s" % (book.authors, ">>> %s" %
+                                           newmi.authors if book.authors != newmi.authors else ''))
+                self.log.info(" author_sort: %s %s" % (book.author_sort, ">>> %s" %
+                                           newmi.author_sort if book.author_sort != newmi.author_sort else ''))
+                self.log.info("    language: %s %s" % (book.language, ">>> %s" %
+                                           newmi.language if book.language != newmi.language else ''))
+                self.log.info("   publisher: %s %s" % (book.publisher, ">>> %s" %
+                                           newmi.publisher if book.publisher != newmi.publisher else ''))
+                self.log.info("        tags: %s %s" % (book.tags, ">>> %s" %
+                                           newmi.tags if book.tags != newmi.tags else ''))
+        else:
+            newmi = book
+        return newmi
+
+
 class ITUNES_ASYNC(ITUNES):
     '''
     This subclass allows the user to interact directly with iTunes via a menu option
@@ -2737,6 +2848,9 @@ class ITUNES_ASYNC(ITUNES):
     icon = I('devices/itunes.png')
     description    = _('Communicate with iTunes.')
 
+    # Plugboard ID
+    DEVICE_PLUGBOARD_NAME = 'APPLE'
+
     connected = False
 
     def __init__(self,path):
@@ -3008,18 +3122,12 @@ class BookList(list):
         '''
         return {}
 
-class Book(MetaInformation):
+class Book(Metadata):
     '''
     A simple class describing a book in the iTunes Books Library.
-    - See ebooks.metadata.__init__ for all fields
+    See ebooks.metadata.book.base
     '''
     def __init__(self,title,author):
 
-        MetaInformation.__init__(self, title, authors=[author])
+        Metadata.__init__(self, title, authors=[author])
 
-    @dynamic_property
-    def title_sorter(self):
-        doc = '''String to sort the title. If absent, title is returned'''
-        def fget(self):
-            return re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', self.title).rstrip()
-        return property(doc=doc, fget=fget)

src/calibre/devices/folder_device/driver.py

@@ -16,10 +16,12 @@ class FOLDER_DEVICE_FOR_CONFIG(USBMS):
     description    = _('Use an arbitrary folder as a device.')
     author         = 'John Schember/Charles Haley'
     supported_platforms = ['windows', 'osx', 'linux']
-    FORMATS     = ['epub', 'fb2', 'mobi', 'azw', 'lrf', 'tcr', 'pmlz', 'lit', 'rtf', 'rb', 'pdf', 'oeb', 'txt', 'pdb']
+    FORMATS     = ['epub', 'fb2', 'mobi', 'azw', 'lrf', 'tcr', 'pmlz', 'lit',
+                   'rtf', 'rb', 'pdf', 'oeb', 'txt', 'pdb', 'prc']
     VENDOR_ID   = 0xffff
     PRODUCT_ID  = 0xffff
     BCD         = 0xffff
+    DEVICE_PLUGBOARD_NAME = 'FOLDER_DEVICE'
 
 
 class FOLDER_DEVICE(USBMS):
@@ -30,15 +32,16 @@ class FOLDER_DEVICE(USBMS):
     description    = _('Use an arbitrary folder as a device.')
     author         = 'John Schember/Charles Haley'
     supported_platforms = ['windows', 'osx', 'linux']
-    FORMATS     = ['epub', 'fb2', 'mobi', 'azw', 'lrf', 'tcr', 'pmlz', 'lit', 'rtf', 'rb', 'pdf', 'oeb', 'txt', 'pdb']
+    FORMATS     = FOLDER_DEVICE_FOR_CONFIG.FORMATS
 
     VENDOR_ID   = 0xffff
     PRODUCT_ID  = 0xffff
     BCD         = 0xffff
+    DEVICE_PLUGBOARD_NAME = 'FOLDER_DEVICE'
 
     THUMBNAIL_HEIGHT = 68 # Height for thumbnails on device
 
-    CAN_SET_METADATA = True
+    CAN_SET_METADATA = ['title', 'authors']
     SUPPORTS_SUB_DIRS = True
 
     #: Icon for this device

src/calibre/devices/hanvon/driver.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 '''
 Device driver for Hanvon devices
 '''
-import re
+import re, os
 
 from calibre.devices.usbms.driver import USBMS
 
@@ -59,18 +59,59 @@ class ALEX(N516):
     description    = _('Communicate with the SpringDesign Alex eBook reader.')
     author         = 'Kovid Goyal'
 
-    FORMATS     = ['epub', 'pdf']
+    FORMATS     = ['epub', 'fb2', 'pdf']
     VENDOR_NAME      = 'ALEX'
     WINDOWS_MAIN_MEM = 'READER'
 
     MAIN_MEMORY_VOLUME_LABEL  = 'Alex Internal Memory'
 
     EBOOK_DIR_MAIN = 'eBooks'
-    SUPPORTS_SUB_DIRS = True
+    SUPPORTS_SUB_DIRS = False
+    THUMBNAIL_HEIGHT = 120
 
     def can_handle(self, device_info, debug=False):
         return is_alex(device_info)
 
+    def alex_cpath(self, file_abspath):
+        base = os.path.dirname(file_abspath)
+        name = os.path.splitext(os.path.basename(file_abspath))[0] + '.png'
+        return os.path.join(base, 'covers', name)
+
+    def upload_cover(self, path, filename, metadata):
+        from calibre.ebooks import calibre_cover
+        from calibre.utils.magick.draw import thumbnail
+        coverdata = getattr(metadata, 'thumbnail', None)
+        if coverdata and coverdata[2]:
+            cover = coverdata[2]
+        else:
+            cover = calibre_cover(metadata.get('title', _('Unknown')),
+                    metadata.get('authors', _('Unknown')))
+
+        cover = thumbnail(cover, width=self.THUMBNAIL_HEIGHT,
+                height=self.THUMBNAIL_HEIGHT, fmt='png')[-1]
+
+        cpath = self.alex_cpath(os.path.join(path, filename))
+        cdir = os.path.dirname(cpath)
+        if not os.path.exists(cdir):
+            os.makedirs(cdir)
+        with open(cpath, 'wb') as coverfile:
+            coverfile.write(cover)
+
+    def delete_books(self, paths, end_session=True):
+        for i, path in enumerate(paths):
+            self.report_progress((i+1) / float(len(paths)), _('Removing books from device...'))
+            path = self.normalize_path(path)
+            if os.path.exists(path):
+                # Delete the ebook
+                os.unlink(path)
+            try:
+                cpath = self.alex_cpath(path)
+                if os.path.exists(cpath):
+                    os.remove(cpath)
+            except:
+                pass
+        self.report_progress(1.0, _('Removing books from device...'))
+
 class AZBOOKA(ALEX):
 
     name = 'Azbooka driver'
@@ -83,10 +124,13 @@ class AZBOOKA(ALEX):
     MAIN_MEMORY_VOLUME_LABEL  = 'Azbooka Internal Memory'
 
     EBOOK_DIR_MAIN = ''
+    SUPPORTS_SUB_DIRS = True
 
     def can_handle(self, device_info, debug=False):
         return not is_alex(device_info)
 
+    def upload_cover(self, path, filename, metadata):
+        pass
 
 class EB511(USBMS):
     name           = 'Elonex EB 511 driver'

src/calibre/devices/interface.py

@@ -37,7 +37,7 @@ class DevicePlugin(Plugin):
     THUMBNAIL_HEIGHT = 68
 
     #: Whether the metadata on books can be set via the GUI.
-    CAN_SET_METADATA = True
+    CAN_SET_METADATA = ['title', 'authors', 'collections']
 
     #: Path separator for paths to books on device
     path_sep = os.sep
@@ -316,7 +316,7 @@ class DevicePlugin(Plugin):
                       being uploaded to the device.
         :param names: A list of file names that the books should have
                       once uploaded to the device. len(names) == len(files)
-        :param metadata: If not None, it is a list of :class:`MetaInformation` objects.
+        :param metadata: If not None, it is a list of :class:`Metadata` objects.
                          The idea is to use the metadata to determine where on the device to
                          put the book. len(metadata) == len(files). Apart from the regular
                          cover (path to cover), there may also be a thumbnail attribute, which should
@@ -335,7 +335,7 @@ class DevicePlugin(Plugin):
         the device.
 
         :param locations: Result of a call to L{upload_books}
-        :param metadata: List of :class:`MetaInformation` objects, same as for
+        :param metadata: List of :class:`Metadata` objects, same as for
                          :meth:`upload_books`.
         :param booklists: A tuple containing the result of calls to
                           (:meth:`books(oncard=None)`,
@@ -411,6 +411,24 @@ class DevicePlugin(Plugin):
         '''
         raise NotImplementedError()
 
+    def set_plugboards(self, plugboards, pb_func):
+        '''
+        provide the driver the current set of plugboards and a function to
+        select a specific plugboard. This method is called immediately before
+        add_books and sync_booklists.
+
+        pb_func is a callable with the following signature::
+            def pb_func(device_name, format, plugboards)
+
+        You give it the current device name (either the class name or
+        DEVICE_PLUGBOARD_NAME), the format you are interested in (a 'real'
+        format or 'device_db'), and the plugboards (you were given those by
+        set_plugboards, the same place you got this method).
+
+        :return: None or a single plugboard instance.
+
+        '''
+        pass
 
 class BookList(list):
     '''

src/calibre/devices/irexdr/driver.py

@@ -20,7 +20,7 @@ class IREXDR1000(USBMS):
 
     # Ordered list of supported formats
     # Be sure these have an entry in calibre.devices.mime
-    FORMATS     = ['epub', 'mobi', 'prc', 'html', 'pdf', 'txt']
+    FORMATS     = ['epub', 'mobi', 'prc', 'html', 'pdf', 'djvu', 'txt']
 
     VENDOR_ID   = [0x1e6b]
     PRODUCT_ID  = [0x001]

src/calibre/devices/jetbook/driver.py

@@ -99,4 +99,30 @@ class MIBUK(USBMS):
     VENDOR_NAME      = 'LINUX'
     WINDOWS_MAIN_MEM = 'WOLDERMIBUK'
 
+class JETBOOK_MINI(USBMS):
+
+    '''
+    ['0x4b8',
+  '0x507',
+  '0x100',
+  'ECTACO',
+  'ECTACO ATA/ATAPI Bridge (Bulk-Only)',
+  'Rev.0.20']
+    '''
+    FORMATS     = ['fb2', 'txt']
+
+    gui_name = 'JetBook Mini'
+    name = 'JetBook Mini Device Interface'
+    description    = _('Communicate with the JetBook Mini reader.')
+    author         = 'Kovid Goyal'
+
+    VENDOR_ID = [0x4b8]
+    PRODUCT_ID = [0x507]
+    BCD = [0x100]
+    VENDOR_NAME      = 'ECTACO'
+    WINDOWS_MAIN_MEM = '' # Matches PROD_
+    MAIN_MEMORY_VOLUME_LABEL  = 'Jetbook Mini'
+
+    SUPPORTS_SUB_DIRS = True
+
 

src/calibre/devices/kobo/books.py

@@ -4,37 +4,15 @@ __copyright__ = '2010, Timothy Legge <timlegge at gmail.com>'
 '''
 
 import os
-import re
 import time
 
-from calibre.ebooks.metadata import MetaInformation
-from calibre.constants import filesystem_encoding, preferred_encoding
-from calibre import isbytestring
+from calibre.devices.usbms.books import Book as Book_
 
-class Book(MetaInformation):
+class Book(Book_):
 
-    BOOK_ATTRS = ['lpath', 'size', 'mime', 'device_collections', '_new_book']
-
-    JSON_ATTRS = [
-        'lpath', 'title', 'authors', 'mime', 'size', 'tags', 'author_sort',
-        'title_sort', 'comments', 'category', 'publisher', 'series',
-        'series_index', 'rating', 'isbn', 'language', 'application_id',
-        'book_producer', 'lccn', 'lcc', 'ddc', 'rights', 'publication_type',
-        'uuid', 'device_collections',
-    ]
-
-    def __init__(self, prefix, lpath, title, authors, mime, date, ContentType, thumbnail_name, size=None, other=None):
-
-        MetaInformation.__init__(self, '')
-        self.device_collections = []
-        self._new_book = False
-
-        self.path = os.path.join(prefix, lpath)
-        if os.sep == '\\':
-            self.path = self.path.replace('/', '\\')
-            self.lpath = lpath.replace('\\', '/')
-        else:
-             self.lpath = lpath
+    def __init__(self, prefix, lpath, title, authors, mime, date, ContentType,
+                 thumbnail_name, size=None, other=None):
+        Book_.__init__(self, prefix, lpath)
 
         self.title = title
         if not authors:
@@ -63,57 +41,7 @@ class Book(MetaInformation):
         if other:
             self.smart_update(other)
 
-    def __eq__(self, other):
-        return self.path == getattr(other, 'path', None)
-
-    @dynamic_property
-    def db_id(self):
-        doc = '''The database id in the application database that this file corresponds to'''
-        def fget(self):
-            match = re.search(r'_(\d+)$', self.lpath.rpartition('.')[0])
-            if match:
-                return int(match.group(1))
-            return None
-        return property(fget=fget, doc=doc)
-
-    @dynamic_property
-    def title_sorter(self):
-        doc = '''String to sort the title. If absent, title is returned'''
-        def fget(self):
-            return re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', self.title).rstrip()
-        return property(doc=doc, fget=fget)
-
-    @dynamic_property
-    def thumbnail(self):
-        return None
-
-    def smart_update(self, other, replace_metadata=False):
-        '''
-        Merge the information in C{other} into self. In case of conflicts, the information
-        in C{other} takes precedence, unless the information in C{other} is NULL.
-        '''
-
-        MetaInformation.smart_update(self, other)
-
-        for attr in self.BOOK_ATTRS:
-            if hasattr(other, attr):
-                val = getattr(other, attr, None)
-                setattr(self, attr, val)
-
-    def to_json(self):
-        json = {}
-        for attr in self.JSON_ATTRS:
-            val = getattr(self, attr)
-            if isbytestring(val):
-                 enc = filesystem_encoding if attr == 'lpath' else preferred_encoding
-                 val = val.decode(enc, 'replace')
-            elif isinstance(val, (list, tuple)):
-                val = [x.decode(preferred_encoding, 'replace') if
-                        isbytestring(x) else x for x in val]
-            json[attr] = val
-        return json
-
 class ImageWrapper(object):
     def __init__(self, image_path):
-       self.image_path = image_path
+        self.image_path = image_path
 

src/calibre/devices/kobo/driver.py

@@ -30,7 +30,7 @@ class KOBO(USBMS):
 
     # Ordered list of supported formats
     FORMATS     = ['epub', 'pdf']
-    CAN_SET_METADATA = True
+    CAN_SET_METADATA = ['collections']
 
     VENDOR_ID   = [0x2237]
     PRODUCT_ID  = [0x4161]
@@ -126,7 +126,7 @@ class KOBO(USBMS):
                         book = self.book_from_path(prefix, lpath, title, authors, mime, date, ContentType, ImageID)
                     # print 'Update booklist'
                     book.device_collections = [playlist_map[lpath]] if lpath in playlist_map else []
-                                       
+
                     if bl.add_book(book, replace_metadata=False):
                         changed = True
             except: # Probably a path encoding error
@@ -150,7 +150,7 @@ class KOBO(USBMS):
 
         changed = False
         for i, row in enumerate(cursor):
-         #  self.report_progress((i+1) / float(numrows), _('Getting list of books on device...'))
+        #  self.report_progress((i+1) / float(numrows), _('Getting list of books on device...'))
 
             path = self.path_from_contentid(row[3], row[5], oncard)
             mime = mime_type_ext(path_to_ext(row[3]))
@@ -250,7 +250,7 @@ class KOBO(USBMS):
             # print "Delete file normalized path: " + path
             extension =  os.path.splitext(path)[1]
             ContentType = self.get_content_type_from_extension(extension)
-            
+
             ContentID = self.contentid_from_path(path, ContentType)
 
             ImageID = self.delete_via_sql(ContentID, ContentType)
@@ -325,8 +325,9 @@ class KOBO(USBMS):
             book = Book(prefix, lpath, '', '', '', '', '', '', other=info)
             if book.size is None:
                 book.size = os.stat(self.normalize_path(path)).st_size
-            book._new_book = True # Must be before add_book
-            booklists[blist].add_book(book, replace_metadata=True)
+            b = booklists[blist].add_book(book, replace_metadata=True)
+            if b:
+                b._new_book = True
         self.report_progress(1.0, _('Adding books to device metadata listing...'))
 
     def contentid_from_path(self, path, ContentType):
@@ -453,7 +454,7 @@ class KOBO(USBMS):
                         query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ReadStatus = 1 and ContentID like \'file:///mnt/sd/%\''
                     elif oncard != 'carda' and oncard != 'cardb':
                         query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ReadStatus = 1 and ContentID not like \'file:///mnt/sd/%\''
-                    
+
                     try:
                         cursor.execute (query)
                     except:
@@ -489,7 +490,7 @@ class KOBO(USBMS):
                         query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ReadStatus = 2 and ContentID like \'file:///mnt/sd/%\''
                     elif oncard != 'carda' and oncard != 'cardb':
                         query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ReadStatus = 2 and ContentID not like \'file:///mnt/sd/%\''
-                    
+
                     try:
                         cursor.execute (query)
                     except:
@@ -519,7 +520,7 @@ class KOBO(USBMS):
                         else:
                             connection.commit()
 #                            debug_print('Database: Commit set ReadStatus as Finished')
-        else: # No collections 
+        else: # No collections
             # Since no collections exist the ReadStatus needs to be reset to 0 (Unread)
             print "Reseting ReadStatus to 0"
             # Reset Im_Reading list in the database
@@ -527,7 +528,7 @@ class KOBO(USBMS):
                 query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ContentID like \'file:///mnt/sd/%\''
             elif oncard != 'carda' and oncard != 'cardb':
                 query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ContentID not like \'file:///mnt/sd/%\''
-                    
+
             try:
                 cursor.execute (query)
             except:
@@ -541,7 +542,7 @@ class KOBO(USBMS):
         connection.close()
 
 #        debug_print('Finished update_device_database_collections', collections_attributes)
-        
+
     def sync_booklists(self, booklists, end_session=True):
 #        debug_print('KOBO: started sync_booklists')
         paths = self.get_device_paths()

src/calibre/devices/misc.py

@@ -108,6 +108,34 @@ class PDNOVEL(USBMS):
             with open('%s.jpg' % os.path.join(path, filename), 'wb') as coverfile:
                 coverfile.write(coverdata[2])
 
+class PDNOVEL_KOBO(PDNOVEL):
+    name = 'Pandigital Kobo device interface'
+    gui_name = 'PD Novel (Kobo)'
+    description = _('Communicate with the Pandigital Novel')
+
+    BCD         = [0x222]
+
+    EBOOK_DIR_MAIN = 'eBooks/Kobo'
+
+
+class VELOCITYMICRO(USBMS):
+    name = 'VelocityMicro device interface'
+    gui_name = 'VelocityMicro'
+    description = _('Communicate with the VelocityMicro')
+    author = 'Kovid Goyal'
+    supported_platforms = ['windows', 'linux', 'osx']
+    FORMATS = ['epub', 'pdb', 'txt', 'html', 'pdf']
+
+    VENDOR_ID   = [0x18d1]
+    PRODUCT_ID  = [0xb015]
+    BCD         = [0x224]
+
+    VENDOR_NAME = 'ANDROID'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = '__UMS_COMPOSITE'
+
+    EBOOK_DIR_MAIN = 'eBooks'
+    SUPPORTS_SUB_DIRS = False
+
 class GEMEI(USBMS):
     name           = 'Gemei Device Interface'
     gui_name       = 'GM2000'

src/calibre/devices/prs505/driver.py

@@ -27,7 +27,7 @@ class PRS505(USBMS):
 
 
     FORMATS      = ['epub', 'lrf', 'lrx', 'rtf', 'pdf', 'txt']
-    CAN_SET_METADATA = True
+    CAN_SET_METADATA = ['title', 'authors', 'collections']
 
     VENDOR_ID    = [0x054c]   #: SONY Vendor Id
     PRODUCT_ID   = [0x031e]
@@ -63,6 +63,9 @@ class PRS505(USBMS):
                     'series, tags, authors'
     EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(['series', 'tags'])
 
+    plugboard = None
+    plugboard_func = None
+
     def windows_filter_pnp_id(self, pnp_id):
         return '_LAUNCHER' in pnp_id
 
@@ -150,7 +153,12 @@ class PRS505(USBMS):
         else:
             collections = []
         debug_print('PRS505: collection fields:', collections)
-        c.update(blists, collections)
+        pb = None
+        if self.plugboard_func:
+            pb = self.plugboard_func(self.__class__.__name__,
+                                     'device_db', self.plugboards)
+        debug_print('PRS505: use plugboards', pb)
+        c.update(blists, collections, pb)
         c.write()
 
         USBMS.sync_booklists(self, booklists, end_session=end_session)
@@ -163,3 +171,6 @@ class PRS505(USBMS):
         c.write()
         debug_print('PRS505: finished rebuild_collections')
 
+    def set_plugboards(self, plugboards, pb_func):
+        self.plugboards = plugboards
+        self.plugboard_func = pb_func

src/calibre/devices/prs505/sony_cache.py

@@ -325,12 +325,6 @@ class XMLCache(object):
         for book in bl:
             record = lpath_map.get(book.lpath, None)
             if record is not None:
-                title = record.get('title', None)
-                if title is not None and title != book.title:
-                    debug_print('Renaming title', book.title, 'to', title)
-                    book.title = title
-                    # Don't set the author, because the reader strips all but
-                    # the first author.
                 for thumbnail in record.xpath(
                         'descendant::*[local-name()="thumbnail"]'):
                     for img in thumbnail.xpath(
@@ -350,7 +344,7 @@ class XMLCache(object):
     # }}}
 
     # Update XML from JSON {{{
-    def update(self, booklists, collections_attributes):
+    def update(self, booklists, collections_attributes, plugboard):
         debug_print('Starting update', collections_attributes)
         use_tz_var = False
         for i, booklist in booklists.items():
@@ -365,8 +359,14 @@ class XMLCache(object):
                 record = lpath_map.get(book.lpath, None)
                 if record is None:
                     record = self.create_text_record(root, i, book.lpath)
+                if plugboard is not None:
+                    newmi = book.deepcopy_metadata()
+                    newmi.template_to_attribute(book, plugboard)
+                    newmi.set('_new_book', getattr(book, '_new_book', False))
+                else:
+                    newmi = book
                 (gtz_count, ltz_count, use_tz_var) = \
-                    self.update_text_record(record, book, path, i,
+                    self.update_text_record(record, newmi, path, i,
                                             gtz_count, ltz_count, use_tz_var)
                 # Ensure the collections in the XML database are recorded for
                 # this book

src/calibre/devices/usbms/books.py

@@ -6,29 +6,18 @@ __docformat__ = 'restructuredtext en'
 
 import os, re, time, sys
 
-from calibre.ebooks.metadata import MetaInformation
+from calibre.ebooks.metadata.book.base import Metadata
 from calibre.devices.mime import mime_type_ext
 from calibre.devices.interface import BookList as _BookList
-from calibre.constants import filesystem_encoding, preferred_encoding
+from calibre.constants import preferred_encoding
 from calibre import isbytestring
-from calibre.utils.config import prefs
-
-class Book(MetaInformation):
-
-    BOOK_ATTRS = ['lpath', 'size', 'mime', 'device_collections', '_new_book']
-
-    JSON_ATTRS = [
-        'lpath', 'title', 'authors', 'mime', 'size', 'tags', 'author_sort',
-        'title_sort', 'comments', 'category', 'publisher', 'series',
-        'series_index', 'rating', 'isbn', 'language', 'application_id',
-        'book_producer', 'lccn', 'lcc', 'ddc', 'rights', 'publication_type',
-        'uuid',
-    ]
+from calibre.utils.config import prefs, tweaks
 
+class Book(Metadata):
     def __init__(self, prefix, lpath, size=None, other=None):
         from calibre.ebooks.metadata.meta import path_to_ext
 
-        MetaInformation.__init__(self, '')
+        Metadata.__init__(self, '')
 
         self._new_book = False
         self.device_collections = []
@@ -72,32 +61,6 @@ class Book(MetaInformation):
     def thumbnail(self):
         return None
 
-    def smart_update(self, other, replace_metadata=False):
-        '''
-        Merge the information in C{other} into self. In case of conflicts, the information
-        in C{other} takes precedence, unless the information in C{other} is NULL.
-        '''
-
-        MetaInformation.smart_update(self, other, replace_metadata)
-
-        for attr in self.BOOK_ATTRS:
-            if hasattr(other, attr):
-                val = getattr(other, attr, None)
-                setattr(self, attr, val)
-
-    def to_json(self):
-        json = {}
-        for attr in self.JSON_ATTRS:
-            val = getattr(self, attr)
-            if isbytestring(val):
-                enc = filesystem_encoding if attr == 'lpath' else preferred_encoding
-                val = val.decode(enc, 'replace')
-            elif isinstance(val, (list, tuple)):
-                val = [x.decode(preferred_encoding, 'replace') if
-                        isbytestring(x) else x for x in val]
-            json[attr] = val
-        return json
-
 class BookList(_BookList):
 
     def __init__(self, oncard, prefix, settings):
@@ -108,17 +71,21 @@ class BookList(_BookList):
         return False
 
     def add_book(self, book, replace_metadata):
+        '''
+        Add the book to the booklist, if needed. Return None if the book is
+        already there and not updated, otherwise return the book.
+        '''
         try:
             b = self.index(book)
         except (ValueError, IndexError):
             b = None
         if b is None:
             self.append(book)
-            return True
+            return book
         if replace_metadata:
             self[b].smart_update(book, replace_metadata=True)
-            return True
-        return False
+            return self[b]
+        return None
 
     def remove_book(self, book):
         self.remove(book)
@@ -131,11 +98,30 @@ class CollectionsBookList(BookList):
     def supports_collections(self):
         return True
 
+    def compute_category_name(self, attr, category, field_meta):
+        renames = tweaks['sony_collection_renaming_rules']
+        attr_name = renames.get(attr, None)
+        if attr_name is None:
+            if field_meta['is_custom']:
+                attr_name = '(%s)'%field_meta['name']
+            else:
+                attr_name = ''
+        elif attr_name != '':
+            attr_name = '(%s)'%attr_name
+        cat_name = '%s %s'%(category, attr_name)
+        return cat_name.strip()
+
     def get_collections(self, collection_attributes):
         from calibre.devices.usbms.driver import debug_print
         debug_print('Starting get_collections:', prefs['manage_device_metadata'])
+        debug_print('Renaming rules:', tweaks['sony_collection_renaming_rules'])
+
+        # Complexity: we can use renaming rules only when using automatic
+        # management. Otherwise we don't always have the metadata to make the
+        # right decisions
+        use_renaming_rules = prefs['manage_device_metadata'] == 'on_connect'
+
         collections = {}
-        series_categories = set([])
         # This map of sets is used to avoid linear searches when testing for
         # book equality
         collections_lpaths = {}
@@ -163,39 +149,72 @@ class CollectionsBookList(BookList):
                 attrs = collection_attributes
             for attr in attrs:
                 attr = attr.strip()
-                val = getattr(book, attr, None)
+                # If attr is device_collections, then we cannot use
+                # format_field, because we don't know the fields where the
+                # values came from.
+                if attr == 'device_collections':
+                    doing_dc = True
+                    val = book.device_collections # is a list
+                else:
+                    doing_dc = False
+                    ign, val, orig_val, fm = book.format_field_extended(attr)
+
                 if not val: continue
                 if isbytestring(val):
                     val = val.decode(preferred_encoding, 'replace')
                 if isinstance(val, (list, tuple)):
                     val = list(val)
-                elif isinstance(val, unicode):
+                elif fm['datatype'] == 'series':
+                    val = [orig_val]
+                elif fm['datatype'] == 'text' and fm['is_multiple']:
+                    val = orig_val
+                else:
                     val = [val]
+
                 for category in val:
-                    if attr == 'tags' and len(category) > 1 and \
-                            category[0] == '[' and category[-1] == ']':
+                    is_series = False
+                    if doing_dc:
+                        # Attempt to determine if this value is a series by
+                        # comparing it to the series name.
+                        if category == book.series:
+                            is_series = True
+                    elif fm['is_custom']: # is a custom field
+                        if fm['datatype'] == 'text' and len(category) > 1 and \
+                                category[0] == '[' and category[-1] == ']':
+                            continue
+                        if fm['datatype'] == 'series':
+                            is_series = True
+                    else:                       # is a standard field
+                        if attr == 'tags' and len(category) > 1 and \
+                                category[0] == '[' and category[-1] == ']':
+                            continue
+                        if attr == 'series' or \
+                                ('series' in collection_attributes and
+                                 book.get('series', None) == category):
+                            is_series = True
+                    if use_renaming_rules:
+                        cat_name = self.compute_category_name(attr, category, fm)
+                    else:
+                        cat_name = category
+
+                    if cat_name not in collections:
+                        collections[cat_name] = []
+                        collections_lpaths[cat_name] = set()
+                    if lpath in collections_lpaths[cat_name]:
                         continue
-                    if category not in collections:
-                        collections[category] = []
-                        collections_lpaths[category] = set()
-                    if lpath not in collections_lpaths[category]:
-                        collections_lpaths[category].add(lpath)
-                        collections[category].append(book)
-                    if attr == 'series' or \
-                            ('series' in collection_attributes and
-                             getattr(book, 'series', None) == category):
-                        series_categories.add(category)
+                    collections_lpaths[cat_name].add(lpath)
+                    if is_series:
+                        collections[cat_name].append(
+                            (book, book.get(attr+'_index', sys.maxint)))
+                    else:
+                        collections[cat_name].append(
+                            (book, book.get('title_sort', 'zzzz')))
         # Sort collections
+        result = {}
         for category, books in collections.items():
-            def tgetter(x):
-                return getattr(x, 'title_sort', 'zzzz')
-            books.sort(cmp=lambda x,y:cmp(tgetter(x), tgetter(y)))
-            if category in series_categories:
-                # Ensures books are sub sorted by title
-                def getter(x):
-                    return getattr(x, 'series_index', sys.maxint)
-                books.sort(cmp=lambda x,y:cmp(getter(x), getter(y)))
-        return collections
+            books.sort(cmp=lambda x,y:cmp(x[1], y[1]))
+            result[category] = [x[0] for x in books]
+        return result
 
     def rebuild_collections(self, booklist, oncard):
         '''

src/calibre/devices/usbms/device.py

@@ -829,12 +829,14 @@ class Device(DeviceConfig, DevicePlugin):
         ext = os.path.splitext(fname)[1]
 
         from calibre.library.save_to_disk import get_components
+        from calibre.library.save_to_disk import config
+        opts = config().parse()
         if not isinstance(template, unicode):
             template = template.decode('utf-8')
         app_id = str(getattr(mdata, 'application_id', ''))
         # The db id will be in the created filename
         extra_components = get_components(template, mdata, fname,
-                length=250-len(app_id)-1)
+                timefmt=opts.send_timefmt, length=250-len(app_id)-1)
         if not extra_components:
             extra_components.append(sanitize(self.filename_callback(fname,
                 mdata)))

src/calibre/devices/usbms/driver.py

@@ -13,7 +13,6 @@ for a particular device.
 import os
 import re
 import time
-import json
 from itertools import cycle
 
 from calibre import prints, isbytestring
@@ -21,6 +20,7 @@ from calibre.constants import filesystem_encoding, DEBUG
 from calibre.devices.usbms.cli import CLI
 from calibre.devices.usbms.device import Device
 from calibre.devices.usbms.books import BookList, Book
+from calibre.ebooks.metadata.book.json_codec import JsonCodec
 
 BASE_TIME = None
 def debug_print(*args):
@@ -50,7 +50,7 @@ class USBMS(CLI, Device):
     book_class = Book
 
     FORMATS = []
-    CAN_SET_METADATA = False
+    CAN_SET_METADATA = []
     METADATA_CACHE = 'metadata.calibre'
 
     def get_device_information(self, end_session=True):
@@ -242,8 +242,9 @@ class USBMS(CLI, Device):
             book = self.book_class(prefix, lpath, other=info)
             if book.size is None:
                 book.size = os.stat(self.normalize_path(path)).st_size
-            book._new_book = True # Must be before add_book
-            booklists[blist].add_book(book, replace_metadata=True)
+            b = booklists[blist].add_book(book, replace_metadata=True)
+            if b:
+                b._new_book = True
         self.report_progress(1.0, _('Adding books to device metadata listing...'))
         debug_print('USBMS: finished adding metadata')
 
@@ -288,6 +289,7 @@ class USBMS(CLI, Device):
     # at the end just before the return
     def sync_booklists(self, booklists, end_session=True):
         debug_print('USBMS: starting sync_booklists')
+        json_codec = JsonCodec()
 
         if not os.path.exists(self.normalize_path(self._main_prefix)):
             os.makedirs(self.normalize_path(self._main_prefix))
@@ -296,10 +298,8 @@ class USBMS(CLI, Device):
             if prefix is not None and isinstance(booklists[listid], self.booklist_class):
                 if not os.path.exists(prefix):
                     os.makedirs(self.normalize_path(prefix))
-                js = [item.to_json() for item in booklists[listid] if
-                        hasattr(item, 'to_json')]
                 with open(self.normalize_path(os.path.join(prefix, self.METADATA_CACHE)), 'wb') as f:
-                    f.write(json.dumps(js, indent=2, encoding='utf-8'))
+                    json_codec.encode_to_file(f, booklists[listid])
         write_prefix(self._main_prefix, 0)
         write_prefix(self._card_a_prefix, 1)
         write_prefix(self._card_b_prefix, 2)
@@ -345,19 +345,13 @@ class USBMS(CLI, Device):
 
     @classmethod
     def parse_metadata_cache(cls, bl, prefix, name):
-        # bl = cls.booklist_class()
-        js = []
+        json_codec = JsonCodec()
         need_sync = False
         cache_file = cls.normalize_path(os.path.join(prefix, name))
         if os.access(cache_file, os.R_OK):
             try:
                 with open(cache_file, 'rb') as f:
-                    js = json.load(f, encoding='utf-8')
-                for item in js:
-                    book = cls.book_class(prefix, item.get('lpath', None))
-                    for key in item.keys():
-                        setattr(book, key, item[key])
-                    bl.append(book)
+                    json_codec.decode_from_file(f, bl, cls.book_class, prefix)
             except:
                 import traceback
                 traceback.print_exc()
@@ -392,7 +386,7 @@ class USBMS(CLI, Device):
 
     @classmethod
     def book_from_path(cls, prefix, lpath):
-        from calibre.ebooks.metadata import MetaInformation
+        from calibre.ebooks.metadata.book.base import Metadata
 
         if cls.settings().read_metadata or cls.MUST_READ_METADATA:
             mi = cls.metadata_from_path(cls.normalize_path(os.path.join(prefix, lpath)))
@@ -401,7 +395,7 @@ class USBMS(CLI, Device):
             mi = metadata_from_filename(cls.normalize_path(os.path.basename(lpath)),
                                         cls.build_template_regexp())
         if mi is None:
-            mi = MetaInformation(os.path.splitext(os.path.basename(lpath))[0],
+            mi = Metadata(os.path.splitext(os.path.basename(lpath))[0],
                     [_('Unknown')])
         size = os.stat(cls.normalize_path(os.path.join(prefix, lpath))).st_size
         book = cls.book_class(prefix, lpath, other=mi, size=size)

src/calibre/ebooks/chm/reader.py

@@ -15,7 +15,6 @@ from calibre.utils.chm.chmlib import (
   chm_enumerate,
 )
 
-from calibre.utils.config import OptionParser
 from calibre.ebooks.metadata.toc import TOC
 from calibre.ebooks.chardet import xml_to_unicode
 
@@ -37,41 +36,6 @@ def check_empty(s, rex = re.compile(r'\S')):
     return rex.search(s) is None
 
 
-def option_parser():
-    parser = OptionParser(usage=_('%prog [options] mybook.chm'))
-    parser.add_option('--output-dir', '-d', default='.', help=_('Output directory. Defaults to current directory'), dest='output')
-    parser.add_option('--verbose', default=False, action='store_true', dest='verbose')
-    parser.add_option("-t", "--title", action="store", type="string", \
-                    dest="title", help=_("Set the book title"))
-    parser.add_option('--title-sort', action='store', type='string', default=None,
-                      dest='title_sort', help=_('Set sort key for the title'))
-    parser.add_option("-a", "--author", action="store", type="string", \
-                    dest="author", help=_("Set the author"))
-    parser.add_option('--author-sort', action='store', type='string', default=None,
-                      dest='author_sort', help=_('Set sort key for the author'))
-    parser.add_option("-c", "--category", action="store", type="string", \
-                    dest="category", help=_("The category this book belongs"
-                    " to. E.g.: History"))
-    parser.add_option("--thumbnail", action="store", type="string", \
-                    dest="thumbnail", help=_("Path to a graphic that will be"
-                    " set as this files' thumbnail"))
-    parser.add_option("--comment", action="store", type="string", \
-                    dest="freetext", help=_("Path to a txt file containing a comment."))
-    parser.add_option("--get-thumbnail", action="store_true", \
-                    dest="get_thumbnail", default=False, \
-                    help=_("Extract thumbnail from LRF file"))
-    parser.add_option('--publisher', default=None, help=_('Set the publisher'))
-    parser.add_option('--classification', default=None, help=_('Set the book classification'))
-    parser.add_option('--creator', default=None, help=_('Set the book creator'))
-    parser.add_option('--producer', default=None, help=_('Set the book producer'))
-    parser.add_option('--get-cover', action='store_true', default=False,
-                      help=_('Extract cover from LRF file. Note that the LRF format has no defined cover, so we use some heuristics to guess the cover.'))
-    parser.add_option('--bookid', action='store', type='string', default=None,
-                      dest='book_id', help=_('Set book ID'))
-    parser.add_option('--font-delta', action='store', type='int', default=0,
-                      dest='font_delta', help=_('Set font delta'))
-    return parser
-
 class CHMError(Exception):
     pass
 
@@ -151,7 +115,8 @@ class CHMReader(CHMFile):
                     continue
                 raise
         self._extracted = True
-        files = os.listdir(output_dir)
+        files = [x for x in os.listdir(output_dir) if
+                os.path.isfile(os.path.join(output_dir, x))]
         if self.hhc_path not in files:
             for f in files:
                 if f.lower() == self.hhc_path.lower():

src/calibre/ebooks/conversion/plumber.py

@@ -701,13 +701,13 @@ OptionRecommendation(name='timestamp',
                                             self.opts.read_metadata_from_opf)
             opf = OPF(open(self.opts.read_metadata_from_opf, 'rb'),
                       os.path.dirname(self.opts.read_metadata_from_opf))
-            mi = MetaInformation(opf)
+            mi = opf.to_book_metadata()
         self.opts_to_mi(mi)
         if mi.cover:
             if mi.cover.startswith('http:') or mi.cover.startswith('https:'):
                 mi.cover = self.download_cover(mi.cover)
             ext = mi.cover.rpartition('.')[-1].lower().strip()
-            if ext not in ('png', 'jpg', 'jpeg'):
+            if ext not in ('png', 'jpg', 'jpeg', 'gif'):
                 ext = 'jpg'
             mi.cover_data = (ext, open(mi.cover, 'rb').read())
             mi.cover = None

src/calibre/ebooks/conversion/preprocess.py

@@ -62,49 +62,104 @@ def wrap_lines(match):
     else:
                return ital+' '
 
-def line_length(format, raw, percent):
+class DocAnalysis(object):
     '''
-    raw is the raw text to find the line length to use for wrapping.
-    percentage is a decimal number, 0 - 1 which is used to determine
-    how far in the list of line lengths to use. The list of line lengths is
-    ordered smallest to larged and does not include duplicates. 0.5 is the
-    median value.
+    Provides various text analysis functions to determine how the document is structured.
+    format is the type of document analysis will be done against.
+    raw is the raw text to determine the line length to use for wrapping.
+    Blank lines are excluded from analysis
     '''
-    raw = raw.replace(' ', ' ')
-    if format == 'html':
-        linere = re.compile('(?<=<p).*?(?=</p>)', re.DOTALL)
-    elif format == 'pdf':
-        linere = re.compile('(?<=<br>).*?(?=<br>)', re.DOTALL)
-    elif format == 'spanned_html':
-        linere = re.compile('(?<=<span).*?(?=</span>)', re.DOTALL)
-    lines = linere.findall(raw)
 
-    lengths = []
-    for line in lines:
-        if len(line) > 0:
-            lengths.append(len(line))
+    def __init__(self, format='html', raw=''):
+        raw = raw.replace('&nbsp;', ' ')
+        if format == 'html':
+            linere = re.compile('(?<=<p)(?![^>]*>\s*</p>).*?(?=</p>)', re.DOTALL)
+        elif format == 'pdf':
+            linere = re.compile('(?<=<br>)(?!\s*<br>).*?(?=<br>)', re.DOTALL)
+        elif format == 'spanned_html':
+            linere = re.compile('(?<=<span).*?(?=</span>)', re.DOTALL)
+        self.lines = linere.findall(raw)
 
-    if not lengths:
-        return 0
+    def line_length(self, percent):
+        '''
+        Analyses the document to find the median line length.
+        percentage is a decimal number, 0 - 1 which is used to determine
+        how far in the list of line lengths to use. The list of line lengths is
+        ordered smallest to larged and does not include duplicates. 0.5 is the
+        median value.
+        '''
+        lengths = []
+        for line in self.lines:
+            if len(line) > 0:
+                lengths.append(len(line))
 
-    lengths = list(set(lengths))
-    total = sum(lengths)
-    avg = total / len(lengths)
-    max_line = avg * 2
+        if not lengths:
+            return 0
 
-    lengths = sorted(lengths)
-    for i in range(len(lengths) - 1, -1, -1):
-        if lengths[i] > max_line:
-            del lengths[i]
+        lengths = list(set(lengths))
+        total = sum(lengths)
+        avg = total / len(lengths)
+        max_line = avg * 2
 
-    if percent > 1:
-        percent = 1
-    if percent < 0:
-        percent = 0
+        lengths = sorted(lengths)
+        for i in range(len(lengths) - 1, -1, -1):
+            if lengths[i] > max_line:
+                del lengths[i]
 
-    index = int(len(lengths) * percent) - 1
+        if percent > 1:
+            percent = 1
+        if percent < 0:
+            percent = 0
 
-    return lengths[index]
+        index = int(len(lengths) * percent) - 1
+
+        return lengths[index]
+
+    def line_histogram(self, percent):
+        '''
+        Creates a broad histogram of the document to determine whether it incorporates hard
+        line breaks.  Lines are sorted into 20 'buckets' based on length.
+        percent is the percentage of lines that should be in a single bucket to return true
+        The majority of the lines will exist in 1-2 buckets in typical docs with hard line breaks
+        '''
+        minLineLength=20 # Ignore lines under 20 chars (typical of spaces)
+        maxLineLength=1900 # Discard larger than this to stay in range
+        buckets=20 # Each line is divided into a bucket based on length
+
+        #print "there are "+str(len(lines))+" lines"
+        #max = 0
+        #for line in self.lines:
+        #    l = len(line)
+        #    if l > max:
+        #        max = l
+        #print "max line found is "+str(max)
+        # Build the line length histogram
+        hRaw = [ 0 for i in range(0,buckets) ]
+        for line in self.lines:
+            l = len(line)
+            if l > minLineLength and l < maxLineLength:
+                    l = int(l/100)
+                    #print "adding "+str(l)
+                    hRaw[l]+=1
+
+        # Normalize the histogram into percents
+        totalLines = len(self.lines)
+        h = [ float(count)/totalLines for count in hRaw ]
+        #print "\nhRaw histogram lengths are: "+str(hRaw)
+        #print "              percents are: "+str(h)+"\n"
+
+        # Find the biggest bucket
+        maxValue = 0
+        for i in range(0,len(h)):
+            if h[i] > maxValue:
+                maxValue = h[i]
+
+        if maxValue < percent:
+            #print "Line lengths are too variable. Not unwrapping."
+            return False
+        else:
+            #print str(maxValue)+" of the lines were in one bucket"
+            return True
 
 class Dehyphenator(object):
     '''
@@ -117,42 +172,62 @@ class Dehyphenator(object):
     def __init__(self):
         # Add common suffixes to the regex below to increase the likelihood of a match -
         # don't add suffixes which are also complete words, such as 'able' or 'sex'
-        self.removesuffixes = re.compile(r"((ed)?ly|('e)?s|a?(t|s)ion(s|al(ly)?)?|ings?|(i)?ous|(i|a)ty|(it)?ies|ive|gence|istic|(e|a)nce|ment(s)?|ism|ated|(e|u)ct(ed)?|ed|(i|ed)?ness|(e|a)ncy|ble|ier|al|ex)$", re.IGNORECASE)
+        self.removesuffixes = re.compile(r"((ed)?ly|('e)?s|a?(t|s)?ion(s|al(ly)?)?|ings?|er|(i)?ous|(i|a)ty|(it)?ies|ive|gence|istic(ally)?|(e|a)nce|ment(s)?|ism|ated|(e|u)ct(ed)?|ed|(i|ed)?ness|(e|a)ncy|ble|ier|al|ex)$", re.IGNORECASE)
         # remove prefixes if the prefix was not already the point of hyphenation
-        self.prefixes = re.compile(r'^(un|in|ex)$', re.IGNORECASE)
-        self.removeprefix = re.compile(r'^(un|in|ex)', re.IGNORECASE)
+        self.prefixes = re.compile(r'^(dis|re|un|in|ex)$', re.IGNORECASE)
+        self.removeprefix = re.compile(r'^(dis|re|un|in|ex)', re.IGNORECASE)
 
     def dehyphenate(self, match):
         firsthalf = match.group('firstpart')
         secondhalf = match.group('secondpart')
-        hyphenated = str(firsthalf) + "-" + str(secondhalf)
-        dehyphenated = str(firsthalf) + str(secondhalf)
+        try:
+            wraptags = match.group('wraptags')
+        except:
+            wraptags = ''
+        hyphenated = unicode(firsthalf) + "-" + unicode(secondhalf)
+        dehyphenated = unicode(firsthalf) + unicode(secondhalf)
         lookupword = self.removesuffixes.sub('', dehyphenated)
         if self.prefixes.match(firsthalf) is None:
            lookupword = self.removeprefix.sub('', lookupword)
-        booklookup = re.compile(u'%s' % lookupword, re.IGNORECASE)
         #print "lookup word is: "+str(lookupword)+", orig is: " + str(hyphenated)
-        match = booklookup.search(self.html)
-        if match:
-            #print "returned dehyphenated word: " + str(dehyphenated)
-            return dehyphenated
-        else:
-            #print "returned hyphenated word: " + str(hyphenated)
+        try:
+            searchresult = self.html.find(lookupword.lower())
+        except:
             return hyphenated
+        if self.format == 'html_cleanup':
+            if self.html.find(lookupword) != -1 or searchresult != -1:
+                #print "Cleanup:returned dehyphenated word: " + str(dehyphenated)
+                return dehyphenated
+            elif self.html.find(hyphenated) != -1:
+                #print "Cleanup:returned hyphenated word: " + str(hyphenated)
+                return hyphenated
+            else:
+                #print "Cleanup:returning original text "+str(firsthalf)+" + linefeed "+str(secondhalf)
+                return firsthalf+u'\u2014'+wraptags+secondhalf
+
+        else:
+            if self.html.find(lookupword) != -1 or searchresult != -1:
+                #print "returned dehyphenated word: " + str(dehyphenated)
+                return dehyphenated
+            else:
+                #print "           returned hyphenated word: " + str(hyphenated)
+                return hyphenated
 
     def __call__(self, html, format, length=1):
         self.html = html
+        self.format = format
         if format == 'html':
-            intextmatch = re.compile(u'(?<=.{%i})(?P<firstpart>[^“"\s>]+)-\s*(?=<)(</span>\s*(</[iubp]>\s*<[iubp][^>]*>\s*)?<span[^>]*>|</[iubp]>\s*<[iubp][^>]*>)?\s*(?P<secondpart>[\w\d]+)' % length)
+            intextmatch = re.compile(u'(?<=.{%i})(?P<firstpart>[^\[\]\\\^\$\.\|\?\*\+\(\)“"\s>]+)-\s*(?=<)(?P<wraptags></span>\s*(</[iubp]>\s*<[iubp][^>]*>\s*)?<span[^>]*>|</[iubp]>\s*<[iubp][^>]*>)?\s*(?P<secondpart>[\w\d]+)' % length)
         elif format == 'pdf':
-            intextmatch = re.compile(u'(?<=.{%i})(?P<firstpart>[^“"\s>]+)-\s*(<p>|</[iub]>\s*<p>\s*<[iub]>)\s*(?P<secondpart>[\w\d]+)'% length)
+            intextmatch = re.compile(u'(?<=.{%i})(?P<firstpart>[^\[\]\\\^\$\.\|\?\*\+\(\)“"\s>]+)-\s*(?P<wraptags><p>|</[iub]>\s*<p>\s*<[iub]>)\s*(?P<secondpart>[\w\d]+)'% length)
         elif format == 'individual_words':
-            intextmatch = re.compile('>[^<]*\b(?P<firstpart>[^"\s>]+)-(?P<secondpart)\w+)\b[^<]*<') # for later, not called anywhere yet
+            intextmatch = re.compile(u'>[^<]*\b(?P<firstpart>[^\[\]\\\^\$\.\|\?\*\+\(\)"\s>]+)-(?P<secondpart)\w+)\b[^<]*<') # for later, not called anywhere yet
+        elif format == 'html_cleanup':
+            intextmatch = re.compile(u'(?P<firstpart>[^\[\]\\\^\$\.\|\?\*\+\(\)“"\s>]+)-\s*(?=<)(?P<wraptags></span>\s*(</[iubp]>\s*<[iubp][^>]*>\s*)?<span[^>]*>|</[iubp]>\s*<[iubp][^>]*>)?\s*(?P<secondpart>[\w\d]+)')
 
         html = intextmatch.sub(self.dehyphenate, html)
         return html
 
-
 class CSSPreProcessor(object):
 
     PAGE_PAT   = re.compile(r'@page[^{]*?{[^}]*?}')
@@ -286,7 +361,7 @@ class HTMLPreProcessor(object):
                   (re.compile(r'<BODY[^<>]+>'), lambda match : '<BODY>'),
 
                   # Detect Chapters to match default XPATH in GUI
-                  (re.compile(r'<br>\s*(?P<chap>(<[ibu]>){0,2}\s*.?(Introduction|Chapter|Epilogue|Prologue|Book|Part|Dedication|Volume|Preface|Acknowledgments)\s*([\d\w-]+\s*){0,3}\s*(</[ibu]>){0,2})\s*(<br>\s*){1,3}\s*(?P<title>(<[ibu]>){0,2}(\s*\w+){1,4}\s*(</[ibu]>){0,2}\s*<br>)?', re.IGNORECASE), chap_head),
+                  (re.compile(r'<br>\s*(?P<chap>(<[ibu]>){0,2}\s*.?(Introduction|Chapter|Kapitel|Epilogue|Prologue|Book|Part|Dedication|Volume|Preface|Acknowledgments)\s*([\d\w-]+\s*){0,3}\s*(</[ibu]>){0,2})\s*(<br>\s*){1,3}\s*(?P<title>(<[ibu]>){0,2}(\s*\w+){1,4}\s*(</[ibu]>){0,2}\s*<br>)?', re.IGNORECASE), chap_head),
                   # Cover the case where every letter in a chapter title is separated by a space
                   (re.compile(r'<br>\s*(?P<chap>([A-Z]\s+){4,}\s*([\d\w-]+\s*){0,3}\s*)\s*(<br>\s*){1,3}\s*(?P<title>(<[ibu]>){0,2}(\s*\w+){1,4}\s*(</[ibu]>){0,2}\s*(<br>))?'), chap_head),
 
@@ -374,10 +449,8 @@ class HTMLPreProcessor(object):
                 print 'Failed to parse remove_footer regexp'
                 traceback.print_exc()
 
-        # unwrap em/en dashes, delete soft hyphens - moved here so it's executed after header/footer removal
+        # delete soft hyphens - moved here so it's executed after header/footer removal
         if is_pdftohtml:
-            # unwrap em/en dashes
-            end_rules.append((re.compile(u'(?<=[–—])\s*<p>\s*(?=[[a-z\d])'), lambda match: ''))
             # unwrap/delete soft hyphens
             end_rules.append((re.compile(u'[­](\s*<p>)+\s*(?=[[a-z\d])'), lambda match: ''))
             # unwrap/delete soft hyphens with formatting
@@ -391,12 +464,15 @@ class HTMLPreProcessor(object):
 
         length = -1
         if getattr(self.extra_opts, 'unwrap_factor', 0.0) > 0.01:
-            length = line_length('pdf', html, getattr(self.extra_opts, 'unwrap_factor'))
+            docanalysis = DocAnalysis('pdf', html)
+            length = docanalysis.line_length(getattr(self.extra_opts, 'unwrap_factor'))
             if length:
-                # print "The pdf line length returned is " + str(length)
+                #print "The pdf line length returned is " + str(length)
+                # unwrap em/en dashes
+                end_rules.append((re.compile(u'(?<=.{%i}[–—])\s*<p>\s*(?=[[a-z\d])' % length), lambda match: ''))
                 end_rules.append(
                     # Un wrap using punctuation
-                    (re.compile(r'(?<=.{%i}([a-z,:)\IA]|(?<!\&\w{4});))\s*(?P<ital></(i|b|u)>)?\s*(<p.*?>\s*)+\s*(?=(<(i|b|u)>)?\s*[\w\d$(])' % length, re.UNICODE), wrap_lines),
+                    (re.compile(u'(?<=.{%i}([a-z,:)\IA\u00DF]|(?<!\&\w{4});))\s*(?P<ital></(i|b|u)>)?\s*(<p.*?>\s*)+\s*(?=(<(i|b|u)>)?\s*[\w\d$(])' % length, re.UNICODE), wrap_lines),
                 )
 
         for rule in self.PREPROCESS + start_rules:
@@ -454,6 +530,14 @@ class HTMLPreProcessor(object):
         if getattr(self.extra_opts, 'smarten_punctuation', False):
             html = self.smarten_punctuation(html)
 
+        unsupported_unicode_chars = self.extra_opts.output_profile.unsupported_unicode_chars
+        if unsupported_unicode_chars:
+            from calibre.ebooks.unidecode.unidecoder import Unidecoder
+            unidecoder = Unidecoder()
+            for char in unsupported_unicode_chars:
+                asciichar = unidecoder.decode(char)
+                html = html.replace(char, asciichar)
+
         return html
 
     def smarten_punctuation(self, html):

src/calibre/ebooks/conversion/utils.py

@@ -6,7 +6,7 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import re
-from calibre.ebooks.conversion.preprocess import line_length, Dehyphenator
+from calibre.ebooks.conversion.preprocess import DocAnalysis, Dehyphenator
 from calibre.utils.logging import default_log
 
 class PreProcessor(object):
@@ -22,18 +22,21 @@ class PreProcessor(object):
         title = match.group('title')
         if not title:
             self.html_preprocess_sections = self.html_preprocess_sections + 1
-            self.log("found " + str(self.html_preprocess_sections) + " chapters. - " + str(chap))
+            self.log("found " + unicode(self.html_preprocess_sections) +
+                    " chapters. - " + unicode(chap))
             return '<h2>'+chap+'</h2>\n'
         else:
             self.html_preprocess_sections = self.html_preprocess_sections + 1
-            self.log("found " + str(self.html_preprocess_sections) + " chapters & titles. - " + str(chap) + ", " + str(title))
+            self.log("found " + unicode(self.html_preprocess_sections) +
+                    " chapters & titles. - " + unicode(chap) + ", " + unicode(title))
             return '<h2>'+chap+'</h2>\n<h3>'+title+'</h3>\n'
 
     def chapter_break(self, match):
         chap = match.group('section')
         styles = match.group('styles')
         self.html_preprocess_sections = self.html_preprocess_sections + 1
-        self.log("marked " + str(self.html_preprocess_sections) + " section markers based on punctuation. - " + str(chap))
+        self.log("marked " + unicode(self.html_preprocess_sections) +
+                " section markers based on punctuation. - " + unicode(chap))
         return '<'+styles+' style="page-break-before:always">'+chap
 
     def insert_indent(self, match):
@@ -63,7 +66,8 @@ class PreProcessor(object):
         line_end = line_end_ere.findall(raw)
         tot_htm_ends = len(htm_end)
         tot_ln_fds = len(line_end)
-        self.log("There are " + str(tot_ln_fds) + " total Line feeds, and " + str(tot_htm_ends) + " marked up endings")
+        self.log("There are " + unicode(tot_ln_fds) + " total Line feeds, and " +
+                unicode(tot_htm_ends) + " marked up endings")
 
         if percent > 1:
             percent = 1
@@ -71,19 +75,24 @@ class PreProcessor(object):
             percent = 0
 
         min_lns = tot_ln_fds * percent
-        self.log("There must be fewer than " + str(min_lns) + " unmarked lines to add markup")
+        self.log("There must be fewer than " + unicode(min_lns) + " unmarked lines to add markup")
         if min_lns > tot_htm_ends:
             return True
 
     def __call__(self, html):
         self.log("*********  Preprocessing HTML  *********")
+
+        # Arrange line feeds and </p> tags so the line_length and no_markup functions work correctly
+        html = re.sub(r"\s*</p>", "</p>\n", html)
+        html = re.sub(r"\s*<p>\s*", "\n<p>", html)
+
         ###### Check Markup ######
         #
         # some lit files don't have any <p> tags or equivalent (generally just plain text between
         # <pre> tags), check and  mark up line endings if required before proceeding
         if self.no_markup(html, 0.1):
              self.log("not enough paragraph markers, adding now")
-             # check if content is in pre tags, use txt procesor to mark up if so
+             # check if content is in pre tags, use txt processor to mark up if so
              pre = re.compile(r'<pre>', re.IGNORECASE)
              if len(pre.findall(html)) == 1:
                  self.log("Running Text Processing")
@@ -107,53 +116,84 @@ class PreProcessor(object):
         txtindent = re.compile(ur'<p(?P<formatting>[^>]*)>\s*(?P<span>(<span[^>]*>\s*)+)?\s*(\u00a0){2,}', re.IGNORECASE)
         html = txtindent.sub(self.insert_indent, html)
         if self.found_indents > 1:
-            self.log("replaced "+str(self.found_indents)+ " nbsp indents with inline styles")
+            self.log("replaced "+unicode(self.found_indents)+ " nbsp indents with inline styles")
         # remove remaining non-breaking spaces
         html = re.sub(ur'\u00a0', ' ', html)
         # Get rid of empty <o:p> tags to simplify other processing
         html = re.sub(ur'\s*<o:p>\s*</o:p>', ' ', html)
         # Get rid of empty span, bold, & italics tags
-        html = re.sub(r"\s*<span[^>]*>\s*(<span[^>]>\s*</span>){0,2}\s*</span>\s*", " ", html)
+        html = re.sub(r"\s*<span[^>]*>\s*(<span[^>]*>\s*</span>){0,2}\s*</span>\s*", " ", html)
         html = re.sub(r"\s*<[ibu][^>]*>\s*(<[ibu][^>]*>\s*</[ibu]>\s*){0,2}\s*</[ibu]>", " ", html)
         html = re.sub(r"\s*<span[^>]*>\s*(<span[^>]>\s*</span>){0,2}\s*</span>\s*", " ", html)
 
-        # If more than 40% of the lines are empty paragraphs then delete them to clean up spacing
+        # If more than 40% of the lines are empty paragraphs and the user has enabled remove
+        # paragraph spacing then delete blank lines to clean up spacing
         linereg = re.compile('(?<=<p).*?(?=</p>)', re.IGNORECASE|re.DOTALL)
         blankreg = re.compile(r'\s*(?P<openline><p[^>]*>)\s*(?P<closeline></p>)', re.IGNORECASE)
         #multi_blank = re.compile(r'(\s*<p[^>]*>\s*(<(b|i|u)>)?\s*(</(b|i|u)>)?\s*</p>){2,}', re.IGNORECASE)
         blanklines = blankreg.findall(html)
         lines = linereg.findall(html)
+        blanks_between_paragraphs = False
         if len(lines) > 1:
-            self.log("There are " + str(len(blanklines)) + " blank lines. " + str(float(len(blanklines)) / float(len(lines))) + " percent blank")
+            self.log("There are " + unicode(len(blanklines)) + " blank lines. " +
+                    unicode(float(len(blanklines)) / float(len(lines))) + " percent blank")
             if float(len(blanklines)) / float(len(lines)) > 0.40 and getattr(self.extra_opts,
             'remove_paragraph_spacing', False):
                 self.log("deleting blank lines")
                 html = blankreg.sub('', html)
-        # Arrange line feeds and </p> tags so the line_length and no_markup functions work correctly
-        html = re.sub(r"\s*</p>", "</p>\n", html)
-        html = re.sub(r"\s*<p>\s*", "\n<p>", html)
+            elif float(len(blanklines)) / float(len(lines)) > 0.40:
+               blanks_between_paragraphs = True
+               #print "blanks between paragraphs is marked True"
+            else:
+                blanks_between_paragraphs = False
+        #self.log("\n\n\n\n\n\n\n\n\n\n\n"+html+"\n\n\n\n\n\n\n\n\n\n\n\n\n")
         # detect chapters/sections to match xpath or splitting logic
+        #
+        # Build the Regular Expressions in pieces
+        lookahead = "(?=<(p|div))"
+        chapter_line_open = "<(?P<outer>p|div)[^>]*>\s*(<(?P<inner1>font|span|[ibu])[^>]*>)?\s*(<(?P<inner2>font|span|[ibu])[^>]*>)?\s*(<(?P<inner3>font|span|[ibu])[^>]*>)?\s*"
+        chapter_header_open = r"(?P<chap>"
+        chapter_header_close = ")\s*"
+        chapter_line_close = "(</(?P=inner3)>)?\s*(</(?P=inner2)>)?\s*(</(?P=inner1)\s[^>]*>)?\s*</(?P=outer)>\s*"
+        if blanks_between_paragraphs:
+            blank_lines = "(\s*<p[^>]*>\s*</p>){0,2}\s*"
+        else:
+            blank_lines = ""
+        opt_title_open = "("
+        title_line_open = "<(?P<outer2>p|div)[^>]*>\s*(<(?P<inner4>font|span|[ibu])[^>]*>)?\s*(<(?P<inner5>font|span|[ibu])[^>]*>)?\s*(<(?P<inner6>font|span|[ibu])[^>]*>)?\s*"
+        title_header_open = "(?P<title>"
+        title_header_close = ")\s*"
+        title_line_close = "(</(?P=inner6)>)?\s*(</(?P=inner5)>)?\s*(</(?P=inner4)\s[^>]*>)?\s*</(?P=outer2)>"
+        opt_title_close = ")?"
+
+        default_title = r"(\s*[\w\'\"-]+){1,5}(?!<)"
+        typical_chapters = r".?(Introduction|Synopsis|Acknowledgements|Chapter|Kapitel|Epilogue|Volume\s|Prologue|Book\s|Part\s|Dedication)\s*([\d\w-]+\:?\s*){0,4}"
+        numeric_chapters = r".?(\d+\.?|(CHAPTER\s*([\dA-Z\-\'\"\?\.!#,]+\s*){1,10}))\s*"
+        uppercase_chapters = r"\s*.?([A-Z#]+(\s|-){0,3}){1,5}\s*"
+
+        chapter_marker = lookahead+chapter_line_open+chapter_header_open+typical_chapters+chapter_header_close+chapter_line_close+blank_lines+opt_title_open+title_line_open+title_header_open+default_title+title_header_close+title_line_close+opt_title_close
+        #print chapter_marker
         heading = re.compile('<h[1-3][^>]*>', re.IGNORECASE)
         self.html_preprocess_sections = len(heading.findall(html))
-        self.log("found " + str(self.html_preprocess_sections) + " pre-existing headings")
+        self.log("found " + unicode(self.html_preprocess_sections) + " pre-existing headings")
         #
         # Start with most typical chapter headings, get more aggressive until one works
         if self.html_preprocess_sections < 10:
-            chapdetect = re.compile(r'(?=</?(br|p))(<(/?br|p)[^>]*>)\s*(<[ibu][^>]*>){0,2}\s*(<span[^>]*>)?\s*(<[ibu][^>]*>){0,2}\s*(<span[^>]*>)?\s*(?P<chap>(<[ibu][^>]*>){0,2}\s*.?(Introduction|Synopsis|Acknowledgements|Chapter|Epilogue|Volume|Prologue|Book\s|Part\s|Dedication)\s*([\d\w-]+\:?\s*){0,8}\s*(</[ibu]>){0,2})\s*(</span>)?s*(</[ibu]>){0,2}\s*(</span>)?\s*(</(p|/?br)>)\s*\s*(\s*<p[^>]*>\s*</p>){0,2}\s*(<(/?br|p)[^>]*>\s*(<[ibu][^>]*>){0,2}\s*(<span[^>]*>)?\s*(?P<title>(<[ibu][^>]*>){0,2}(\s*[\w\'\"-]+){1,5}\s*(</[ibu]>){0,2})\s*(</span>)?\s*(</[ibu]>){0,2}\s*(</(br|p)>))?', re.IGNORECASE|re.VERBOSE)
+            chapdetect = re.compile(r'%s' % chapter_marker, re.IGNORECASE)
             html = chapdetect.sub(self.chapter_head, html)
         if self.html_preprocess_sections < 10:
-            self.log("not enough chapters, only " + str(self.html_preprocess_sections) + ", trying numeric chapters")
-            chapdetect2 = re.compile(r'(?=</?(br|p))(<(/?br|p)[^>]*>)\s*(<[ibu][^>]*>){0,2}\s*(<span[^>]*>)?\s*(?P<chap>(<[ibu][^>]*>){0,2}\s*.?(\d+\.?|(CHAPTER\s*([\dA-Z\-\'\"\?\.!#,]+\s*){1,10}))\s*(</[ibu]>){0,2})\s*(</span>)?\s*(</[ibu]>){0,2}\s*(</(p|/?br)>)\s*(<(/?br|p)[^>]*>\s*(<[ibu][^>]*>){0,2}\s*(<span[^>]*>)?\s*(?P<title>(<[ibu][^>]*>){0,2}(\s*[\w\'\"-]+){1,5}\s*(</[ibu]>){0,2})\s*(</span>)?\s*(</[ibu]>){0,2}\s*(</(br|p)>))?', re.UNICODE)
+            self.log("not enough chapters, only " + unicode(self.html_preprocess_sections) + ", trying numeric chapters")
+            chapter_marker = lookahead+chapter_line_open+chapter_header_open+numeric_chapters+chapter_header_close+chapter_line_close+blank_lines+opt_title_open+title_line_open+title_header_open+default_title+title_header_close+title_line_close+opt_title_close
+            chapdetect2 = re.compile(r'%s' % chapter_marker, re.IGNORECASE)
             html = chapdetect2.sub(self.chapter_head, html)
 
         if self.html_preprocess_sections < 10:
-            self.log("not enough chapters, only " + str(self.html_preprocess_sections) + ", trying with uppercase words")
-            chapdetect2 = re.compile(r'(?=</?(br|p))(<(/?br|p)[^>]*>)\s*(<[ibu][^>]*>){0,2}\s*(<span[^>]*>)?\s*(?P<chap>(<[ibu][^>]*>){0,2}\s*.?([A-Z#\-\s]+)\s*(</[ibu]>){0,2})\s*(</span>)?\s*(</[ibu]>){0,2}\s*(</(p|/?br)>)\s*(<(/?br|p)[^>]*>\s*(<[ibu][^>]*>){0,2}\s*(<span[^>]*>)?\s*(?P<title>(<[ibu][^>]*>){0,2}(\s*[\w\'\"-]+){1,5}\s*(</[ibu]>){0,2})\s*(</span>)?\s*(</[ibu]>){0,2}\s*(</(br|p)>))?', re.UNICODE)
+            self.log("not enough chapters, only " + unicode(self.html_preprocess_sections) + ", trying with uppercase words")
+            chapter_marker = lookahead+chapter_line_open+chapter_header_open+uppercase_chapters+chapter_header_close+chapter_line_close+blank_lines+opt_title_open+title_line_open+title_header_open+default_title+title_header_close+title_line_close+opt_title_close
+            chapdetect2 = re.compile(r'%s' % chapter_marker,  re.UNICODE)
             html = chapdetect2.sub(self.chapter_head, html)
-
         ###### Unwrap lines ######
         #
-        self.log("Unwrapping Lines")
         # Some OCR sourced files have line breaks in the html using a combination of span & p tags
         # span are used for hard line breaks, p for new paragraphs.  Determine which is used so
         # that lines can be un-wrapped across page boundaries
@@ -168,29 +208,45 @@ class PreProcessor(object):
                 format = 'html'
         else:
             format = 'html'
-
+        # Check Line histogram to determine if the document uses hard line breaks, If 50% or
+        # more of the lines break in the same region of the document then unwrapping is required
+        docanalysis = DocAnalysis(format, html)
+        hardbreaks = docanalysis.line_histogram(.50)
+        self.log("Hard line breaks check returned "+unicode(hardbreaks))
         # Calculate Length
-        length = line_length(format, html, getattr(self.extra_opts,
-            'html_unwrap_factor', 0.4))
-        self.log("*** Median line length is " + str(length) + ", calculated with " + format + " format ***")
-        max_length = length * 1.4
-        min_max = str("(?<=.{"+str(length)+"})(?<!.{"+str(max_length)+"})")
-        #
-        # Unwrap em/en dashes, delete soft-hyphens
-        #self.log("\n\n\n\n\n\n\n\n\n\n\n"+html+"\n\n\n\n\n\n\n\n\n\n\n\n\n")
-        html = re.sub(u'\xad\s*(</span>\s*(</[iubp]>\s*<[iubp][^>]*>\s*)?<span[^>]*>|</[iubp]>\s*<[iubp][^>]*>)?\s*', '', html)
-        html = re.sub(u'%s(?<=[\u2013\u2014])\s*(?=<)(</span>\s*(</[iubp]>\s*<[iubp][^>]*>\s*)?<span[^>]*>|</[iubp]>\s*<[iubp][^>]*>)?\s*(?=[[a-z\d])' % min_max, '', html)
-        # Dehyphenate
-        dehyphenator = Dehyphenator()
-        html = dehyphenator(html,'html', length)
+        unwrap_factor = getattr(self.extra_opts, 'html_unwrap_factor', 0.4)
+        length = docanalysis.line_length(unwrap_factor)
+        self.log("*** Median line length is " + unicode(length) + ", calculated with " + format + " format ***")
+        # only go through unwrapping code if the histogram shows unwrapping is required or if the user decreased the default unwrap_factor
+        if hardbreaks or unwrap_factor < 0.4:
+            self.log("Unwrapping required, unwrapping Lines")
+            # Unwrap em/en dashes
+            html = re.sub(u'(?<=.{%i}[\u2013\u2014])\s*(?=<)(</span>\s*(</[iubp]>\s*<[iubp][^>]*>\s*)?<span[^>]*>|</[iubp]>\s*<[iubp][^>]*>)?\s*(?=[[a-z\d])' % length, '', html)
+            # Dehyphenate
+            self.log("Unwrapping/Removing hyphens")
+            dehyphenator = Dehyphenator()
+            html = dehyphenator(html,'html', length)
+            self.log("Done dehyphenating")
+            # Unwrap lines using punctation and line length
+            unwrap = re.compile(u"(?<=.{%i}([a-z,:)\IA\u00DF]|(?<!\&\w{4});))\s*</(span|p|div)>\s*(</(p|span|div)>)?\s*(?P<up2threeblanks><(p|span|div)[^>]*>\s*(<(p|span|div)[^>]*>\s*</(span|p|div)>\s*)</(span|p|div)>\s*){0,3}\s*<(span|div|p)[^>]*>\s*(<(span|div|p)[^>]*>)?\s*" % length, re.UNICODE)
+            html = unwrap.sub(' ', html)
+            #check any remaining hyphens, but only unwrap if there is a match
+            dehyphenator = Dehyphenator()
+            html = dehyphenator(html,'html_cleanup', length)
+        else:
+            # dehyphenate in cleanup mode to fix anything previous conversions/editing missed
+            self.log("Cleaning up hyphenation")
+            dehyphenator = Dehyphenator()
+            html = dehyphenator(html,'html_cleanup', length)
+            self.log("Done dehyphenating")
 
-        # Unwrap lines using punctation and line length
-        unwrap = re.compile(r"(?<=.{%i}([a-z,;):\IA]|(?<!\&\w{4});))\s*</(span|p|div)>\s*(</(p|span|div)>)?\s*(?P<up2threeblanks><(p|span|div)[^>]*>\s*(<(p|span|div)[^>]*>\s*</(span|p|div)>\s*)</(span|p|div)>\s*){0,3}\s*<(span|div|p)[^>]*>\s*(<(span|div|p)[^>]*>)?\s*" % length, re.UNICODE)
-        html = unwrap.sub(' ', html)
+        # delete soft hyphens
+        html = re.sub(u'\xad\s*(</span>\s*(</[iubp]>\s*<[iubp][^>]*>\s*)?<span[^>]*>|</[iubp]>\s*<[iubp][^>]*>)?\s*', '', html)
 
         # If still no sections after unwrapping mark split points on lines with no punctuation
         if self.html_preprocess_sections < 10:
-            self.log("Looking for more split points based on punctuation, currently have " + str(self.html_preprocess_sections))
+            self.log("Looking for more split points based on punctuation,"
+                    " currently have " + unicode(self.html_preprocess_sections))
             chapdetect3 = re.compile(r'<(?P<styles>(p|div)[^>]*)>\s*(?P<section>(<span[^>]*>)?\s*(<[ibu][^>]*>){0,2}\s*(<span[^>]*>)?\s*(<[ibu][^>]*>){0,2}\s*(<span[^>]*>)?\s*.?(?=[a-z#\-*\s]+<)([a-z#-*]+\s*){1,5}\s*\s*(</span>)?(</[ibu]>){0,2}\s*(</span>)?\s*(</[ibu]>){0,2}\s*(</span>)?\s*</(p|div)>)', re.IGNORECASE)
             html = chapdetect3.sub(self.chapter_break, html)
         # search for places where a first or second level heading is immediately followed by another

src/calibre/ebooks/epub/fix/epubcheck.py

@@ -43,7 +43,11 @@ class Epubcheck(ePubFixer):
                         default=default)
             except:
                 raise InvalidEpub('Invalid date set in OPF', raw)
-            sval = ts.strftime('%Y-%m-%d')
+            try:
+                sval = ts.strftime('%Y-%m-%d')
+            except:
+                from calibre import strftime
+                sval = strftime('%Y-%m-%d', ts.timetuple())
             if sval != raw:
                 self.log.error(
                     'OPF contains date', raw, 'that epubcheck does not like')

src/calibre/ebooks/epub/input.py

@@ -117,7 +117,8 @@ class EPUBInput(InputFormatPlugin):
         encfile = os.path.abspath(os.path.join('META-INF', 'encryption.xml'))
         opf = None
         for f in walk(u'.'):
-            if f.lower().endswith('.opf') and '__MACOSX' not in f:
+            if f.lower().endswith('.opf') and '__MACOSX' not in f and \
+                    not os.path.basename(f).startswith('.'):
                 opf = os.path.abspath(f)
                 break
         path = getattr(stream, 'name', 'stream')

src/calibre/ebooks/metadata/__init__.py

@@ -10,10 +10,9 @@ import os, mimetypes, sys, re
 from urllib import unquote, quote
 from urlparse import urlparse
 
-from calibre import relpath, prints
+from calibre import relpath
 
 from calibre.utils.config import tweaks
-from calibre.utils.date import isoformat
 
 _author_pat = re.compile(',?\s+(and|with)\s+', re.IGNORECASE)
 def string_to_authors(raw):
@@ -45,7 +44,15 @@ def author_to_author_sort(author):
 def authors_to_sort_string(authors):
     return ' & '.join(map(author_to_author_sort, authors))
 
-_title_pat = re.compile('^(A|The|An)\s+', re.IGNORECASE)
+try:
+    _title_pat = re.compile(tweaks.get('title_sort_articles',
+                                       r'^(A|The|An)\s+'), re.IGNORECASE)
+except:
+    print 'Error in title sort pattern'
+    import traceback
+    traceback.print_exc()
+    _title_pat = re.compile('^(A|The|An)\s+', re.IGNORECASE)
+
 _ignore_starts = u'\'"'+u''.join(unichr(x) for x in range(0x2018, 0x201e)+[0x2032, 0x2033])
 
 def title_sort(title):
@@ -221,214 +228,18 @@ class ResourceCollection(object):
 
 
 
-class MetaInformation(object):
-    '''Convenient encapsulation of book metadata'''
-
-    @staticmethod
-    def copy(mi):
-        ans = MetaInformation(mi.title, mi.authors)
-        for attr in ('author_sort', 'title_sort', 'comments', 'category',
-                     'publisher', 'series', 'series_index', 'rating',
-                     'isbn', 'tags', 'cover_data', 'application_id', 'guide',
-                     'manifest', 'spine', 'toc', 'cover', 'language',
-                     'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc',
-                     'author_sort_map',
-                     'pubdate', 'rights', 'publication_type', 'uuid'):
-            if hasattr(mi, attr):
-                setattr(ans, attr, getattr(mi, attr))
-
-    def __init__(self, title, authors=(_('Unknown'),)):
-        '''
+def MetaInformation(title, authors=(_('Unknown'),)):
+    ''' Convenient encapsulation of book metadata, needed for compatibility
         @param title: title or ``_('Unknown')`` or a MetaInformation object
         @param authors: List of strings or []
-        '''
-        mi = None
-        if hasattr(title, 'title') and hasattr(title, 'authors'):
-            mi = title
-            title = mi.title
-            authors = mi.authors
-        self.title = title
-        self.author = list(authors) if authors else []# Needed for backward compatibility
-        #: List of strings or []
-        self.authors = list(authors) if authors else []
-        self.tags = getattr(mi, 'tags', [])
-        #: mi.cover_data = (ext, data)
-        self.cover_data   = getattr(mi, 'cover_data', (None, None))
-        self.author_sort_map = getattr(mi, 'author_sort_map', {})
-
-        for x in ('author_sort', 'title_sort', 'comments', 'category', 'publisher',
-                  'series', 'series_index', 'rating', 'isbn', 'language',
-                  'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover',
-                  'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate',
-                  'rights', 'publication_type', 'uuid',
-                  ):
-            setattr(self, x, getattr(mi, x, None))
-
-    def print_all_attributes(self):
-        for x in ('title','author', 'author_sort', 'title_sort', 'comments', 'category', 'publisher',
-                  'series', 'series_index', 'tags', 'rating', 'isbn', 'language',
-                  'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover',
-                  'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate',
-                  'rights', 'publication_type', 'uuid', 'author_sort_map'
-                  ):
-            prints(x, getattr(self, x, 'None'))
-
-    def smart_update(self, mi, replace_metadata=False):
-        '''
-        Merge the information in C{mi} into self. In case of conflicts, the
-        information in C{mi} takes precedence, unless the information in mi is
-        NULL. If replace_metadata is True, then the information in mi always
-        takes precedence.
-        '''
-        if mi.title and mi.title != _('Unknown'):
-            self.title = mi.title
-
-        if mi.authors and mi.authors[0] != _('Unknown'):
-            self.authors = mi.authors
-
-        for attr in ('author_sort', 'title_sort', 'category',
-                     'publisher', 'series', 'series_index', 'rating',
-                     'isbn', 'application_id', 'manifest', 'spine', 'toc',
-                     'cover', 'guide', 'book_producer',
-                     'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate', 'rights',
-                     'publication_type', 'uuid'):
-            if replace_metadata:
-                setattr(self, attr, getattr(mi, attr, 1.0 if \
-                        attr == 'series_index' else None))
-            elif hasattr(mi, attr):
-                val = getattr(mi, attr)
-                if val is not None:
-                    setattr(self, attr, val)
-
-        if replace_metadata:
-            self.tags = mi.tags
-        elif mi.tags:
-            self.tags += mi.tags
-        self.tags = list(set(self.tags))
-
-        if mi.author_sort_map:
-            self.author_sort_map.update(mi.author_sort_map)
-
-        if getattr(mi, 'cover_data', False):
-            other_cover = mi.cover_data[-1]
-            self_cover = self.cover_data[-1] if self.cover_data else ''
-            if not self_cover: self_cover = ''
-            if not other_cover: other_cover = ''
-            if len(other_cover) > len(self_cover):
-                self.cover_data = mi.cover_data
-
-        if replace_metadata:
-            self.comments = getattr(mi, 'comments', '')
-        else:
-            my_comments = getattr(self, 'comments', '')
-            other_comments = getattr(mi, 'comments', '')
-            if not my_comments:
-                my_comments = ''
-            if not other_comments:
-                other_comments = ''
-            if len(other_comments.strip()) > len(my_comments.strip()):
-                self.comments = other_comments
-
-        other_lang = getattr(mi, 'language', None)
-        if other_lang and other_lang.lower() != 'und':
-            self.language = other_lang
-
-
-    def format_series_index(self):
-        try:
-            x = float(self.series_index)
-        except ValueError:
-            x = 1
-        return fmt_sidx(x)
-
-    def authors_from_string(self, raw):
-        self.authors = string_to_authors(raw)
-
-    def format_authors(self):
-        return authors_to_string(self.authors)
-
-    def format_tags(self):
-        return u', '.join([unicode(t) for t in self.tags])
-
-    def format_rating(self):
-        return unicode(self.rating)
-
-    def __unicode__(self):
-        ans = []
-        def fmt(x, y):
-            ans.append(u'%-20s: %s'%(unicode(x), unicode(y)))
-
-        fmt('Title', self.title)
-        if self.title_sort:
-            fmt('Title sort', self.title_sort)
-        if self.authors:
-            fmt('Author(s)',  authors_to_string(self.authors) + \
-               ((' [' + self.author_sort + ']') if self.author_sort else ''))
-        if self.publisher:
-            fmt('Publisher', self.publisher)
-        if getattr(self, 'book_producer', False):
-            fmt('Book Producer', self.book_producer)
-        if self.category:
-            fmt('Category', self.category)
-        if self.comments:
-            fmt('Comments', self.comments)
-        if self.isbn:
-            fmt('ISBN', self.isbn)
-        if self.tags:
-            fmt('Tags', u', '.join([unicode(t) for t in self.tags]))
-        if self.series:
-            fmt('Series', self.series + ' #%s'%self.format_series_index())
-        if self.language:
-            fmt('Language', self.language)
-        if self.rating is not None:
-            fmt('Rating', self.rating)
-        if self.timestamp is not None:
-            fmt('Timestamp', isoformat(self.timestamp))
-        if self.pubdate is not None:
-            fmt('Published', isoformat(self.pubdate))
-        if self.rights is not None:
-            fmt('Rights', unicode(self.rights))
-        if self.lccn:
-            fmt('LCCN', unicode(self.lccn))
-        if self.lcc:
-            fmt('LCC', unicode(self.lcc))
-        if self.ddc:
-            fmt('DDC', unicode(self.ddc))
-
-        return u'\n'.join(ans)
-
-    def to_html(self):
-        ans = [(_('Title'), unicode(self.title))]
-        ans += [(_('Author(s)'), (authors_to_string(self.authors) if self.authors else _('Unknown')))]
-        ans += [(_('Publisher'), unicode(self.publisher))]
-        ans += [(_('Producer'), unicode(self.book_producer))]
-        ans += [(_('Comments'), unicode(self.comments))]
-        ans += [('ISBN', unicode(self.isbn))]
-        if self.lccn:
-            ans += [('LCCN', unicode(self.lccn))]
-        if self.lcc:
-            ans += [('LCC', unicode(self.lcc))]
-        if self.ddc:
-            ans += [('DDC', unicode(self.ddc))]
-        ans += [(_('Tags'), u', '.join([unicode(t) for t in self.tags]))]
-        if self.series:
-            ans += [(_('Series'), unicode(self.series)+ ' #%s'%self.format_series_index())]
-        ans += [(_('Language'), unicode(self.language))]
-        if self.timestamp is not None:
-            ans += [(_('Timestamp'), unicode(self.timestamp.isoformat(' ')))]
-        if self.pubdate is not None:
-            ans += [(_('Published'), unicode(self.pubdate.isoformat(' ')))]
-        if self.rights is not None:
-            ans += [(_('Rights'), unicode(self.rights))]
-        for i, x in enumerate(ans):
-            ans[i] = u'<tr><td><b>%s</b></td><td>%s</td></tr>'%x
-        return u'<table>%s</table>'%u'\n'.join(ans)
-
-    def __str__(self):
-        return self.__unicode__().encode('utf-8')
-
-    def __nonzero__(self):
-        return bool(self.title or self.author or self.comments or self.tags)
+    '''
+    from calibre.ebooks.metadata.book.base import Metadata
+    mi = None
+    if hasattr(title, 'title') and hasattr(title, 'authors'):
+        mi = title
+        title = mi.title
+        authors = mi.authors
+    return Metadata(title, authors, other=mi)
 
 def check_isbn10(isbn):
     try:

src/calibre/ebooks/metadata/book/__init__.py

@@ -11,48 +11,45 @@ an empty list/dictionary for complex types and (None, None) for cover_data
 '''
 
 SOCIAL_METADATA_FIELDS = frozenset([
-    'tags', # Ordered list
-    # A floating point number between 0 and 10
-    'rating',
-    # A simple HTML enabled string
-    'comments',
-    # A simple string
-    'series',
-    # A floating point number
-    'series_index',
+    'tags',             # Ordered list
+    'rating',           # A floating point number between 0 and 10
+    'comments',         # A simple HTML enabled string
+    'series',           # A simple string
+    'series_index',     # A floating point number
     # Of the form { scheme1:value1, scheme2:value2}
     # For example: {'isbn':'123456789', 'doi':'xxxx', ... }
     'classifiers',
-    'isbn', # Pseudo field for convenience, should get/set isbn classifier
+])
 
+'''
+The list of names that convert to classifiers when in get and set.
+'''
+
+TOP_LEVEL_CLASSIFIERS = frozenset([
+    'isbn',
 ])
 
 PUBLICATION_METADATA_FIELDS = frozenset([
-    # title must never be None. Should be _('Unknown')
-    'title',
+    'title',            # title must never be None. Should be _('Unknown')
     # Pseudo field that can be set, but if not set is auto generated
     # from title and languages
     'title_sort',
-    # Ordered list of authors. Must never be None, can be [_('Unknown')]
-    'authors',
-    # Map of sort strings for each author
-    'author_sort_map',
+    'authors',          # Ordered list. Must never be None, can be [_('Unknown')]
+    'author_sort_map',  # Map of sort strings for each author
     # Pseudo field that can be set, but if not set is auto generated
     # from authors and languages
     'author_sort',
     'book_producer',
-    # Dates and times must be timezone aware
-    'timestamp',
+    'timestamp',        # Dates and times must be timezone aware
     'pubdate',
     'rights',
     # So far only known publication type is periodical:calibre
     # If None, means book
     'publication_type',
-    # A UUID usually of type 4
-    'uuid',
-    'languages', # ordered list
-    # Simple string, no special semantics
-    'publisher',
+    'uuid',             # A UUID usually of type 4
+    'language',         # the primary language of this book
+    'languages',        # ordered list
+    'publisher',        # Simple string, no special semantics
     # Absolute path to image file encoded in filesystem_encoding
     'cover',
     # Of the form (format, data) where format is, for e.g. 'jpeg', 'png', 'gif'...
@@ -69,33 +66,63 @@ BOOK_STRUCTURE_FIELDS = frozenset([
     ])
 
 USER_METADATA_FIELDS = frozenset([
-    # A dict of a form to be specified
+    # A dict of dicts similar to field_metadata. Each field description dict
+    # also contains a value field with the key #value#.
     'user_metadata',
 ])
 
 DEVICE_METADATA_FIELDS = frozenset([
-    # Ordered list of strings
-    'device_collections',
-    'lpath', # Unicode, / separated
-    # In bytes
-    'size',
-    # Mimetype of the book file being represented
-    'mime',
+    'device_collections',   # Ordered list of strings
+    'lpath',                # Unicode, / separated
+    'size',                 # In bytes
+    'mime',                 # Mimetype of the book file being represented
+
 ])
 
 CALIBRE_METADATA_FIELDS = frozenset([
-    # An application id
-    # Semantics to be defined. Is it a db key? a db name + key? A uuid?
-    'application_id',
+    'application_id',   # An application id, currently set to the db_id.
+    'db_id',            # the calibre primary key of the item.
+    'formats',          # list of formats (extensions) for this book
     ]
 )
 
+ALL_METADATA_FIELDS =      SOCIAL_METADATA_FIELDS.union(
+                           PUBLICATION_METADATA_FIELDS).union(
+                           BOOK_STRUCTURE_FIELDS).union(
+                           USER_METADATA_FIELDS).union(
+                           DEVICE_METADATA_FIELDS).union(
+                           CALIBRE_METADATA_FIELDS)
 
-SERIALIZABLE_FIELDS = SOCIAL_METADATA_FIELDS.union(
-                      USER_METADATA_FIELDS).union(
-                      PUBLICATION_METADATA_FIELDS).union(
-                      CALIBRE_METADATA_FIELDS).union(
-        frozenset(['lpath'])) # I don't think we need device_collections
+# All fields except custom fields
+STANDARD_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union(
+                           PUBLICATION_METADATA_FIELDS).union(
+                           BOOK_STRUCTURE_FIELDS).union(
+                           DEVICE_METADATA_FIELDS).union(
+                           CALIBRE_METADATA_FIELDS)
 
-# Serialization of covers/thumbnails will have to be handled carefully, maybe
-# as an option to the serializer class
+# Metadata fields that smart update must do special processing to copy.
+SC_FIELDS_NOT_COPIED =     frozenset(['title', 'title_sort', 'authors',
+                                      'author_sort', 'author_sort_map',
+                                      'cover_data', 'tags', 'language',
+                                      'classifiers'])
+
+# Metadata fields that smart update should copy only if the source is not None
+SC_FIELDS_COPY_NOT_NULL =  frozenset(['lpath', 'size', 'comments', 'thumbnail'])
+
+# Metadata fields that smart update should copy without special handling
+SC_COPYABLE_FIELDS =       SOCIAL_METADATA_FIELDS.union(
+                           PUBLICATION_METADATA_FIELDS).union(
+                           BOOK_STRUCTURE_FIELDS).union(
+                           DEVICE_METADATA_FIELDS).union(
+                           CALIBRE_METADATA_FIELDS) - \
+                           SC_FIELDS_NOT_COPIED.union(
+                           SC_FIELDS_COPY_NOT_NULL)
+
+SERIALIZABLE_FIELDS =      SOCIAL_METADATA_FIELDS.union(
+                           USER_METADATA_FIELDS).union(
+                           PUBLICATION_METADATA_FIELDS).union(
+                           CALIBRE_METADATA_FIELDS).union(
+                           DEVICE_METADATA_FIELDS) - \
+                           frozenset(['device_collections', 'formats',
+                               'cover_data'])
+                           # these are rebuilt when needed

src/calibre/ebooks/metadata/book/base.py

@@ -5,9 +5,19 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import copy
+import copy, traceback
+
+from calibre import prints
+from calibre.constants import DEBUG
+from calibre.ebooks.metadata.book import SC_COPYABLE_FIELDS
+from calibre.ebooks.metadata.book import SC_FIELDS_COPY_NOT_NULL
+from calibre.ebooks.metadata.book import STANDARD_METADATA_FIELDS
+from calibre.ebooks.metadata.book import TOP_LEVEL_CLASSIFIERS
+from calibre.ebooks.metadata.book import ALL_METADATA_FIELDS
+from calibre.library.field_metadata import FieldMetadata
+from calibre.utils.date import isoformat, format_date
+from calibre.utils.formatter import TemplateFormatter
 
-from calibre.ebooks.metadata.book import RESERVED_METADATA_FIELDS
 
 NULL_VALUES = {
                 'user_metadata': {},
@@ -19,103 +29,609 @@ NULL_VALUES = {
                 'author_sort_map': {},
                 'authors'      : [_('Unknown')],
                 'title'        : _('Unknown'),
+                'language'     : 'und'
 }
 
+field_metadata = FieldMetadata()
+
+class SafeFormat(TemplateFormatter):
+
+    def get_value(self, key, args, kwargs):
+        try:
+            if key != 'title_sort':
+                key = field_metadata.search_term_to_field_key(key.lower())
+            b = self.book.get_user_metadata(key, False)
+            if b and b['datatype'] == 'int' and self.book.get(key, 0) == 0:
+                v = ''
+            elif b and b['datatype'] == 'float' and b.get(key, 0.0) == 0.0:
+                v = ''
+            else:
+                ign, v = self.book.format_field(key.lower(), series_with_index=False)
+            if v is None:
+                return ''
+            if v == '':
+                return ''
+            return v
+        except:
+            if DEBUG:
+                traceback.print_exc()
+            return key
+
+composite_formatter = SafeFormat()
+
 class Metadata(object):
 
     '''
-    This class must expose a superset of the API of MetaInformation in terms
-    of attribute access and methods. Only the __init__ method is different.
-    MetaInformation will simply become a function that creates and fills in
-    the attributes of this class.
+    A class representing all the metadata for a book.
 
     Please keep the method based API of this class to a minimum. Every method
     becomes a reserved field name.
     '''
 
-    def __init__(self):
-        object.__setattr__(self, '_data', copy.deepcopy(NULL_VALUES))
+    def __init__(self, title, authors=(_('Unknown'),), other=None):
+        '''
+        @param title: title or ``_('Unknown')``
+        @param authors: List of strings or []
+        @param other: None or a metadata object
+        '''
+        _data = copy.deepcopy(NULL_VALUES)
+        object.__setattr__(self, '_data', _data)
+        if other is not None:
+            self.smart_update(other)
+        else:
+            if title:
+                self.title = title
+            if authors:
+                #: List of strings or []
+                self.author = list(authors) if authors else []# Needed for backward compatibility
+                self.authors = list(authors) if authors else []
+
+    def is_null(self, field):
+        null_val = NULL_VALUES.get(field, None)
+        val = getattr(self, field, None)
+        return not val or val == null_val
 
     def __getattribute__(self, field):
         _data = object.__getattribute__(self, '_data')
-        if field in RESERVED_METADATA_FIELDS:
+        if field in TOP_LEVEL_CLASSIFIERS:
+            return _data.get('classifiers').get(field, None)
+        if field in STANDARD_METADATA_FIELDS:
             return _data.get(field, None)
         try:
             return object.__getattribute__(self, field)
         except AttributeError:
             pass
         if field in _data['user_metadata'].iterkeys():
-            # TODO: getting user metadata values
-            pass
+            d = _data['user_metadata'][field]
+            val = d['#value#']
+            if d['datatype'] != 'composite':
+                return val
+            if val is None:
+                d['#value#'] = 'RECURSIVE_COMPOSITE FIELD (Metadata) ' + field
+                val = d['#value#'] = composite_formatter.safe_format(
+                                            d['display']['composite_template'],
+                                            self,
+                                            _('TEMPLATE ERROR'),
+                                            self).strip()
+            return val
+
         raise AttributeError(
                 'Metadata object has no attribute named: '+ repr(field))
 
-
-    def __setattr__(self, field, val):
+    def __setattr__(self, field, val, extra=None):
         _data = object.__getattribute__(self, '_data')
-        if field in RESERVED_METADATA_FIELDS:
-            if field != 'user_metadata':
-                if not val:
-                    val = NULL_VALUES[field]
-                _data[field] = val
-            else:
-                raise AttributeError('You cannot set user_metadata directly.')
+        if field in TOP_LEVEL_CLASSIFIERS:
+            _data['classifiers'].update({field: val})
+        elif field in STANDARD_METADATA_FIELDS:
+            if val is None:
+                val = NULL_VALUES.get(field, None)
+            _data[field] = val
         elif field in _data['user_metadata'].iterkeys():
-            # TODO: Setting custom column values
-            pass
+            if _data['user_metadata'][field]['datatype'] == 'composite':
+                _data['user_metadata'][field]['#value#'] = None
+            else:
+                _data['user_metadata'][field]['#value#'] = val
+            _data['user_metadata'][field]['#extra#'] = extra
         else:
             # You are allowed to stick arbitrary attributes onto this object as
-            # long as they dont conflict with global or user metadata names
+            # long as they don't conflict with global or user metadata names
             # Don't abuse this privilege
             self.__dict__[field] = val
 
-    @property
-    def user_metadata_names(self):
-        'The set of user metadata names this object knows about'
+    def __iter__(self):
+        return object.__getattribute__(self, '_data').iterkeys()
+
+    def has_key(self, key):
+        return key in object.__getattribute__(self, '_data')
+
+    def deepcopy(self):
+        m = Metadata(None)
+        m.__dict__ = copy.deepcopy(self.__dict__)
+        object.__setattr__(m, '_data', copy.deepcopy(object.__getattribute__(self, '_data')))
+        return m
+
+    def deepcopy_metadata(self):
+        m = Metadata(None)
+        object.__setattr__(m, '_data', copy.deepcopy(object.__getattribute__(self, '_data')))
+        return m
+
+    def get(self, field, default=None):
+        try:
+            return self.__getattribute__(field)
+        except AttributeError:
+            return default
+
+    def get_extra(self, field):
         _data = object.__getattribute__(self, '_data')
-        return frozenset(_data['user_metadata'].iterkeys())
+        if field in _data['user_metadata'].iterkeys():
+            return _data['user_metadata'][field]['#extra#']
+        raise AttributeError(
+                'Metadata object has no attribute named: '+ repr(field))
 
-    # Old MetaInformation API {{{
-    def copy(self):
-        pass
+    def set(self, field, val, extra=None):
+        self.__setattr__(field, val, extra)
 
+    def get_classifiers(self):
+        '''
+        Return a copy of the classifiers dictionary.
+        The dict is small, and the penalty for using a reference where a copy is
+        needed is large. Also, we don't want any manipulations of the returned
+        dict to show up in the book.
+        '''
+        return copy.deepcopy(object.__getattribute__(self, '_data')['classifiers'])
+
+    def set_classifiers(self, classifiers):
+        object.__getattribute__(self, '_data')['classifiers'] = classifiers
+
+    # field-oriented interface. Intended to be the same as in LibraryDatabase
+
+    def standard_field_keys(self):
+        '''
+        return a list of all possible keys, even if this book doesn't have them
+        '''
+        return STANDARD_METADATA_FIELDS
+
+    def custom_field_keys(self):
+        '''
+        return a list of the custom fields in this book
+        '''
+        return object.__getattribute__(self, '_data')['user_metadata'].iterkeys()
+
+    def all_field_keys(self):
+        '''
+        All field keys known by this instance, even if their value is None
+        '''
+        _data = object.__getattribute__(self, '_data')
+        return frozenset(ALL_METADATA_FIELDS.union(_data['user_metadata'].iterkeys()))
+
+    def metadata_for_field(self, key):
+        '''
+        return metadata describing a standard or custom field.
+        '''
+        if key not in self.custom_field_keys():
+            return self.get_standard_metadata(key, make_copy=False)
+        return self.get_user_metadata(key, make_copy=False)
+
+    def all_non_none_fields(self):
+        '''
+        Return a dictionary containing all non-None metadata fields, including
+        the custom ones.
+        '''
+        result = {}
+        _data = object.__getattribute__(self, '_data')
+        for attr in STANDARD_METADATA_FIELDS:
+            v = _data.get(attr, None)
+            if v is not None:
+                result[attr] = v
+        # separate these because it uses the self.get(), not _data.get()
+        for attr in TOP_LEVEL_CLASSIFIERS:
+            v = self.get(attr, None)
+            if v is not None:
+                result[attr] = v
+        for attr in _data['user_metadata'].iterkeys():
+            v = self.get(attr, None)
+            if v is not None:
+                result[attr] = v
+                if _data['user_metadata'][attr]['datatype'] == 'series':
+                    result[attr+'_index'] = _data['user_metadata'][attr]['#extra#']
+        return result
+
+    # End of field-oriented interface
+
+    # Extended interfaces. These permit one to get copies of metadata dictionaries, and to
+    # get and set custom field metadata
+
+    def get_standard_metadata(self, field, make_copy):
+        '''
+        return field metadata from the field if it is there. Otherwise return
+        None. field is the key name, not the label. Return a copy if requested,
+        just in case the user wants to change values in the dict.
+        '''
+        if field in field_metadata and field_metadata[field]['kind'] == 'field':
+            if make_copy:
+                return copy.deepcopy(field_metadata[field])
+            return field_metadata[field]
+        return None
+
+    def get_all_standard_metadata(self, make_copy):
+        '''
+        return a dict containing all the standard field metadata associated with
+        the book.
+        '''
+        if not make_copy:
+            return field_metadata
+        res = {}
+        for k in field_metadata:
+            if field_metadata[k]['kind'] == 'field':
+                res[k] = copy.deepcopy(field_metadata[k])
+        return res
+
+    def get_all_user_metadata(self, make_copy):
+        '''
+        return a dict containing all the custom field metadata associated with
+        the book.
+        '''
+        _data = object.__getattribute__(self, '_data')
+        user_metadata = _data['user_metadata']
+        if not make_copy:
+            return user_metadata
+        res = {}
+        for k in user_metadata:
+            res[k] = copy.deepcopy(user_metadata[k])
+        return res
+
+    def get_user_metadata(self, field, make_copy):
+        '''
+        return field metadata from the object if it is there. Otherwise return
+        None. field is the key name, not the label. Return a copy if requested,
+        just in case the user wants to change values in the dict.
+        '''
+        _data = object.__getattribute__(self, '_data')
+        _data = _data['user_metadata']
+        if field in _data:
+            if make_copy:
+                return copy.deepcopy(_data[field])
+            return _data[field]
+        return None
+
+    def set_all_user_metadata(self, metadata):
+        '''
+        store custom field metadata into the object. Field is the key name
+        not the label
+        '''
+        if metadata is None:
+            traceback.print_stack()
+        else:
+            for key in metadata:
+                self.set_user_metadata(key, metadata[key])
+
+    def set_user_metadata(self, field, metadata):
+        '''
+        store custom field metadata for one column into the object. Field is
+        the key name not the label
+        '''
+        if field is not None:
+            if not field.startswith('#'):
+                raise AttributeError(
+                        'Custom field name %s must begin with \'#\''%repr(field))
+            if metadata is None:
+                traceback.print_stack()
+                return
+            metadata = copy.deepcopy(metadata)
+            if '#value#' not in metadata:
+                if metadata['datatype'] == 'text' and metadata['is_multiple']:
+                    metadata['#value#'] = []
+                else:
+                    metadata['#value#'] = None
+            _data = object.__getattribute__(self, '_data')
+            _data['user_metadata'][field] = metadata
+
+    def template_to_attribute(self, other, ops):
+        '''
+        Takes a list [(src,dest), (src,dest)], evaluates the template in the
+        context of other, then copies the result to self[dest]. This is on a
+        best-efforts basis. Some assignments can make no sense.
+        '''
+        if not ops:
+            return
+        for op in ops:
+            try:
+                src = op[0]
+                dest = op[1]
+                val = composite_formatter.safe_format\
+                    (src, other, 'PLUGBOARD TEMPLATE ERROR', other)
+                if dest == 'tags':
+                    self.set(dest, [f.strip() for f in val.split(',') if f.strip()])
+                elif dest == 'authors':
+                    self.set(dest, [f.strip() for f in val.split('&') if f.strip()])
+                else:
+                    self.set(dest, val)
+            except:
+                if DEBUG:
+                    traceback.print_exc()
+
+    # Old Metadata API {{{
     def print_all_attributes(self):
-        pass
+        for x in STANDARD_METADATA_FIELDS:
+            prints('%s:'%x, getattr(self, x, 'None'))
+        for x in self.custom_field_keys():
+            meta = self.get_user_metadata(x, make_copy=False)
+            if meta is not None:
+                prints(x, meta)
+        prints('--------------')
 
     def smart_update(self, other, replace_metadata=False):
-        pass
+        '''
+        Merge the information in `other` into self. In case of conflicts, the information
+        in `other` takes precedence, unless the information in `other` is NULL.
+        '''
+        def copy_not_none(dest, src, attr):
+            v = getattr(src, attr, None)
+            if v not in (None, NULL_VALUES.get(attr, None)):
+                setattr(dest, attr, copy.deepcopy(v))
 
-    def format_series_index(self):
-        pass
+        if other.title and other.title != _('Unknown'):
+            self.title = other.title
+            if hasattr(other, 'title_sort'):
+                self.title_sort = other.title_sort
+
+        if other.authors and other.authors[0] != _('Unknown'):
+            self.authors = list(other.authors)
+            if hasattr(other, 'author_sort_map'):
+                self.author_sort_map = dict(other.author_sort_map)
+            if hasattr(other, 'author_sort'):
+                self.author_sort = other.author_sort
+
+        if replace_metadata:
+            # SPECIAL_FIELDS = frozenset(['lpath', 'size', 'comments', 'thumbnail'])
+            for attr in SC_COPYABLE_FIELDS:
+                setattr(self, attr, getattr(other, attr, 1.0 if \
+                        attr == 'series_index' else None))
+            self.tags = other.tags
+            self.cover_data = getattr(other, 'cover_data',
+                                      NULL_VALUES['cover_data'])
+            self.set_all_user_metadata(other.get_all_user_metadata(make_copy=True))
+            for x in SC_FIELDS_COPY_NOT_NULL:
+                copy_not_none(self, other, x)
+            if callable(getattr(other, 'get_classifiers', None)):
+                self.set_classifiers(other.get_classifiers())
+            # language is handled below
+        else:
+            for attr in SC_COPYABLE_FIELDS:
+                copy_not_none(self, other, attr)
+            for x in SC_FIELDS_COPY_NOT_NULL:
+                copy_not_none(self, other, x)
+
+            if other.tags:
+                # Case-insensitive but case preserving merging
+                lotags = [t.lower() for t in other.tags]
+                lstags = [t.lower() for t in self.tags]
+                ot, st = map(frozenset, (lotags, lstags))
+                for t in st.intersection(ot):
+                    sidx = lstags.index(t)
+                    oidx = lotags.index(t)
+                    self.tags[sidx] = other.tags[oidx]
+                self.tags += [t for t in other.tags if t.lower() in ot-st]
+
+            if getattr(other, 'cover_data', False):
+                other_cover = other.cover_data[-1]
+                self_cover = self.cover_data[-1] if self.cover_data else ''
+                if not self_cover: self_cover = ''
+                if not other_cover: other_cover = ''
+                if len(other_cover) > len(self_cover):
+                    self.cover_data = other.cover_data
+
+            if callable(getattr(other, 'custom_field_keys', None)):
+                for x in other.custom_field_keys():
+                    meta = other.get_user_metadata(x, make_copy=True)
+                    if meta is not None:
+                        self_tags = self.get(x, [])
+                        self.set_user_metadata(x, meta) # get... did the deepcopy
+                        other_tags = other.get(x, [])
+                        if meta['is_multiple']:
+                            # Case-insensitive but case preserving merging
+                            lotags = [t.lower() for t in other_tags]
+                            lstags = [t.lower() for t in self_tags]
+                            ot, st = map(frozenset, (lotags, lstags))
+                            for t in st.intersection(ot):
+                                sidx = lstags.index(t)
+                                oidx = lotags.index(t)
+                                self_tags[sidx] = other.tags[oidx]
+                            self_tags += [t for t in other.tags if t.lower() in ot-st]
+                            setattr(self, x, self_tags)
+
+            my_comments = getattr(self, 'comments', '')
+            other_comments = getattr(other, 'comments', '')
+            if not my_comments:
+                my_comments = ''
+            if not other_comments:
+                other_comments = ''
+            if len(other_comments.strip()) > len(my_comments.strip()):
+                self.comments = other_comments
+
+            # Copy all the non-none classifiers
+            if callable(getattr(other, 'get_classifiers', None)):
+                d = self.get_classifiers()
+                s = other.get_classifiers()
+                d.update([v for v in s.iteritems() if v[1] is not None])
+                self.set_classifiers(d)
+            else:
+                # other structure not Metadata. Copy the top-level classifiers
+                for attr in TOP_LEVEL_CLASSIFIERS:
+                    copy_not_none(self, other, attr)
+
+        other_lang = getattr(other, 'language', None)
+        if other_lang and other_lang.lower() != 'und':
+            self.language = other_lang
+
+    def format_series_index(self, val=None):
+        from calibre.ebooks.metadata import fmt_sidx
+        v = self.series_index if val is None else val
+        try:
+            x = float(v)
+        except (ValueError, TypeError):
+            x = 1
+        return fmt_sidx(x)
 
     def authors_from_string(self, raw):
-        pass
+        from calibre.ebooks.metadata import string_to_authors
+        self.authors = string_to_authors(raw)
 
     def format_authors(self):
-        pass
+        from calibre.ebooks.metadata import authors_to_string
+        return authors_to_string(self.authors)
 
     def format_tags(self):
-        pass
+        return u', '.join([unicode(t) for t in self.tags])
 
     def format_rating(self):
         return unicode(self.rating)
 
+    def format_field(self, key, series_with_index=True):
+        name, val, ign, ign = self.format_field_extended(key, series_with_index)
+        return (name, val)
+
+    def format_field_extended(self, key, series_with_index=True):
+        from calibre.ebooks.metadata import authors_to_string
+        '''
+        returns the tuple (field_name, formatted_value)
+        '''
+
+        # Handle custom series index
+        if key.startswith('#') and key.endswith('_index'):
+            tkey = key[:-6] # strip the _index
+            cmeta = self.get_user_metadata(tkey, make_copy=False)
+            if cmeta['datatype'] == 'series':
+                if self.get(tkey):
+                    res = self.get_extra(tkey)
+                    return (unicode(cmeta['name']+'_index'),
+                            self.format_series_index(res), res, cmeta)
+                else:
+                    return (unicode(cmeta['name']+'_index'), '', '', cmeta)
+
+        if key in self.custom_field_keys():
+            res = self.get(key, None)
+            cmeta = self.get_user_metadata(key, make_copy=False)
+            name = unicode(cmeta['name'])
+            if cmeta['datatype'] != 'composite' and (res is None or res == ''):
+                return (name, res, None, None)
+            orig_res = res
+            cmeta = self.get_user_metadata(key, make_copy=False)
+            if res is None or res == '':
+                return (name, res, None, None)
+            orig_res = res
+            datatype = cmeta['datatype']
+            if datatype == 'text' and cmeta['is_multiple']:
+                res = u', '.join(res)
+            elif datatype == 'series' and series_with_index:
+                if self.get_extra(key) is not None:
+                    res = res + \
+                        ' [%s]'%self.format_series_index(val=self.get_extra(key))
+            elif datatype == 'datetime':
+                res = format_date(res, cmeta['display'].get('date_format','dd MMM yyyy'))
+            elif datatype == 'bool':
+                res = _('Yes') if res else _('No')
+            return (name, unicode(res), orig_res, cmeta)
+
+        # Translate aliases into the standard field name
+        fmkey = field_metadata.search_term_to_field_key(key)
+
+        if fmkey in field_metadata and field_metadata[fmkey]['kind'] == 'field':
+            res = self.get(key, None)
+            fmeta = field_metadata[fmkey]
+            name = unicode(fmeta['name'])
+            if res is None or res == '':
+                return (name, res, None, None)
+            orig_res = res
+            name = unicode(fmeta['name'])
+            datatype = fmeta['datatype']
+            if key == 'authors':
+                res = authors_to_string(res)
+            elif key == 'series_index':
+                res = self.format_series_index(res)
+            elif datatype == 'text' and fmeta['is_multiple']:
+                res = u', '.join(res)
+            elif datatype == 'series' and series_with_index:
+                res = res + ' [%s]'%self.format_series_index()
+            elif datatype == 'datetime':
+                res = format_date(res, fmeta['display'].get('date_format','dd MMM yyyy'))
+            return (name, unicode(res), orig_res, fmeta)
+
+        return (None, None, None, None)
+
     def __unicode__(self):
-        pass
+        from calibre.ebooks.metadata import authors_to_string
+        ans = []
+        def fmt(x, y):
+            ans.append(u'%-20s: %s'%(unicode(x), unicode(y)))
+
+        fmt('Title', self.title)
+        if self.title_sort:
+            fmt('Title sort', self.title_sort)
+        if self.authors:
+            fmt('Author(s)',  authors_to_string(self.authors) + \
+               ((' [' + self.author_sort + ']') if self.author_sort else ''))
+        if self.publisher:
+            fmt('Publisher', self.publisher)
+        if getattr(self, 'book_producer', False):
+            fmt('Book Producer', self.book_producer)
+        if self.comments:
+            fmt('Comments', self.comments)
+        if self.isbn:
+            fmt('ISBN', self.isbn)
+        if self.tags:
+            fmt('Tags', u', '.join([unicode(t) for t in self.tags]))
+        if self.series:
+            fmt('Series', self.series + ' #%s'%self.format_series_index())
+        if self.language:
+            fmt('Language', self.language)
+        if self.rating is not None:
+            fmt('Rating', self.rating)
+        if self.timestamp is not None:
+            fmt('Timestamp', isoformat(self.timestamp))
+        if self.pubdate is not None:
+            fmt('Published', isoformat(self.pubdate))
+        if self.rights is not None:
+            fmt('Rights', unicode(self.rights))
+        for key in self.custom_field_keys():
+            val = self.get(key, None)
+            if val:
+                (name, val) = self.format_field(key)
+                fmt(name, unicode(val))
+        return u'\n'.join(ans)
 
     def to_html(self):
-        pass
+        from calibre.ebooks.metadata import authors_to_string
+        ans = [(_('Title'), unicode(self.title))]
+        ans += [(_('Author(s)'), (authors_to_string(self.authors) if self.authors else _('Unknown')))]
+        ans += [(_('Publisher'), unicode(self.publisher))]
+        ans += [(_('Producer'), unicode(self.book_producer))]
+        ans += [(_('Comments'), unicode(self.comments))]
+        ans += [('ISBN', unicode(self.isbn))]
+        ans += [(_('Tags'), u', '.join([unicode(t) for t in self.tags]))]
+        if self.series:
+            ans += [(_('Series'), unicode(self.series)+ ' #%s'%self.format_series_index())]
+        ans += [(_('Language'), unicode(self.language))]
+        if self.timestamp is not None:
+            ans += [(_('Timestamp'), unicode(self.timestamp.isoformat(' ')))]
+        if self.pubdate is not None:
+            ans += [(_('Published'), unicode(self.pubdate.isoformat(' ')))]
+        if self.rights is not None:
+            ans += [(_('Rights'), unicode(self.rights))]
+        for key in self.custom_field_keys():
+            val = self.get(key, None)
+            if val:
+                (name, val) = self.format_field(key)
+                ans += [(name, val)]
+        for i, x in enumerate(ans):
+            ans[i] = u'<tr><td><b>%s</b></td><td>%s</td></tr>'%x
+        return u'<table>%s</table>'%u'\n'.join(ans)
 
     def __str__(self):
         return self.__unicode__().encode('utf-8')
 
     def __nonzero__(self):
-        return True
+        return bool(self.title or self.author or self.comments or self.tags)
 
     # }}}
 
-# We don't need reserved field names for this object any more. Lets just use a
-# protocol like the last char of a user field label should be _ when using this
-# object
-# So mi.tags returns the builtin tags and mi.tags_ returns the user tags
-

src/calibre/ebooks/metadata/book/json_codec.py

@@ -0,0 +1,143 @@
+'''
+Created on 4 Jun 2010
+
+@author: charles
+'''
+
+from base64 import b64encode, b64decode
+import json
+import traceback
+
+from calibre.ebooks.metadata.book import SERIALIZABLE_FIELDS
+from calibre.constants import filesystem_encoding, preferred_encoding
+from calibre.library.field_metadata import FieldMetadata
+from calibre.utils.date import parse_date, isoformat, UNDEFINED_DATE
+from calibre.utils.magick import Image
+from calibre import isbytestring
+
+# Translate datetimes to and from strings. The string form is the datetime in
+# UTC. The returned date is also UTC
+def string_to_datetime(src):
+    if src == "None":
+        return None
+    return parse_date(src)
+
+def datetime_to_string(dateval):
+    if dateval is None or dateval == UNDEFINED_DATE:
+        return "None"
+    return isoformat(dateval)
+
+def encode_thumbnail(thumbnail):
+    '''
+    Encode the image part of a thumbnail, then return the 3 part tuple
+    '''
+    if thumbnail is None:
+        return None
+    if not isinstance(thumbnail, (tuple, list)):
+        try:
+            img = Image()
+            img.load(thumbnail)
+            width, height = img.size
+            thumbnail = (width, height, thumbnail)
+        except:
+            return None
+    return (thumbnail[0], thumbnail[1], b64encode(str(thumbnail[2])))
+
+def decode_thumbnail(tup):
+    '''
+    Decode an encoded thumbnail into its 3 component parts
+    '''
+    if tup is None:
+        return None
+    return (tup[0], tup[1], b64decode(tup[2]))
+
+def object_to_unicode(obj, enc=preferred_encoding):
+
+    def dec(x):
+        return x.decode(enc, 'replace')
+
+    if isbytestring(obj):
+        return dec(obj)
+    if isinstance(obj, (list, tuple)):
+        return [dec(x) if isbytestring(x) else x for x in obj]
+    if isinstance(obj, dict):
+        ans = {}
+        for k, v in obj.items():
+            k = object_to_unicode(k)
+            v = object_to_unicode(v)
+            ans[k] = v
+        return ans
+    return obj
+
+class JsonCodec(object):
+
+    def __init__(self):
+        self.field_metadata = FieldMetadata()
+
+    def encode_to_file(self, file, booklist):
+        file.write(json.dumps(self.encode_booklist_metadata(booklist),
+                              indent=2, encoding='utf-8'))
+
+    def encode_booklist_metadata(self, booklist):
+        result = []
+        for book in booklist:
+            result.append(self.encode_book_metadata(book))
+        return result
+
+    def encode_book_metadata(self, book):
+        result = {}
+        for key in SERIALIZABLE_FIELDS:
+            result[key] = self.encode_metadata_attr(book, key)
+        return result
+
+    def encode_metadata_attr(self, book, key):
+        if key == 'user_metadata':
+            meta = book.get_all_user_metadata(make_copy=True)
+            for k in meta:
+                if meta[k]['datatype'] == 'datetime':
+                    meta[k]['#value#'] = datetime_to_string(meta[k]['#value#'])
+            return meta
+        if key in self.field_metadata:
+            datatype = self.field_metadata[key]['datatype']
+        else:
+            datatype = None
+        value = book.get(key)
+        if key == 'thumbnail':
+            return encode_thumbnail(value)
+        elif isbytestring(value): # str includes bytes
+            enc = filesystem_encoding if key == 'lpath' else preferred_encoding
+            return object_to_unicode(value, enc=enc)
+        elif datatype == 'datetime':
+            return datetime_to_string(value)
+        else:
+            return object_to_unicode(value)
+
+    def decode_from_file(self, file, booklist, book_class, prefix):
+        js = []
+        try:
+            js = json.load(file, encoding='utf-8')
+            for item in js:
+                book = book_class(prefix, item.get('lpath', None))
+                for key in item.keys():
+                    meta = self.decode_metadata(key, item[key])
+                    if key == 'user_metadata':
+                        book.set_all_user_metadata(meta)
+                    else:
+                        setattr(book, key, meta)
+                booklist.append(book)
+        except:
+            print 'exception during JSON decoding'
+            traceback.print_exc()
+
+    def decode_metadata(self, key, value):
+        if key == 'user_metadata':
+            for k in value:
+                if value[k]['datatype'] == 'datetime':
+                    value[k]['#value#'] = string_to_datetime(value[k]['#value#'])
+            return value
+        elif key in self.field_metadata:
+            if self.field_metadata[key]['datatype'] == 'datetime':
+                return string_to_datetime(value)
+        if key == 'thumbnail':
+            return decode_thumbnail(value)
+        return value

src/calibre/ebooks/metadata/cli.py

@@ -109,7 +109,7 @@ def do_set_metadata(opts, mi, stream, stream_type):
     from_opf = getattr(opts, 'from_opf', None)
     if from_opf is not None:
         from calibre.ebooks.metadata.opf2 import OPF
-        opf_mi = MetaInformation(OPF(open(from_opf, 'rb')))
+        opf_mi = OPF(open(from_opf, 'rb')).to_book_metadata()
         mi.smart_update(opf_mi)
 
     for pref in config().option_set.preferences:

src/calibre/ebooks/metadata/covers.py

@@ -9,6 +9,7 @@ import traceback, socket, re, sys
 from functools import partial
 from threading import Thread, Event
 from Queue import Queue, Empty
+from lxml import etree
 
 import mechanize
 
@@ -216,6 +217,68 @@ def download_covers(mi, result_queue, max_covers=50, timeout=5.): # {{{
 
 # }}}
 
+class DoubanCovers(CoverDownload): # {{{
+    'Download covers from Douban.com'
+
+    DOUBAN_ISBN_URL = 'http://api.douban.com/book/subject/isbn/'
+    CALIBRE_DOUBAN_API_KEY = '0bd1672394eb1ebf2374356abec15c3d'
+    name = 'Douban.com covers'
+    description = _('Download covers from Douban.com')
+    author = 'Li Fanxi'
+
+    def get_cover_url(self, isbn, br, timeout=5.):
+        try:
+            url = self.DOUBAN_ISBN_URL + isbn + "?apikey=" + self.CALIBRE_DOUBAN_API_KEY
+            src = br.open(url, timeout=timeout).read()
+        except Exception, err:
+            if isinstance(getattr(err, 'args', [None])[0], socket.timeout):
+                err = Exception(_('Douban.com API timed out. Try again later.'))
+            raise err
+        else:
+            feed = etree.fromstring(src)
+            NAMESPACES = {
+              'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/',
+              'atom' : 'http://www.w3.org/2005/Atom',
+              'db': 'http://www.douban.com/xmlns/'
+            }
+            XPath = partial(etree.XPath, namespaces=NAMESPACES)
+            entries = XPath('//atom:entry')(feed)
+            if len(entries) < 1:
+                return None
+            try:
+                cover_url = XPath("descendant::atom:link[@rel='image']/attribute::href")
+                u = cover_url(entries[0])[0].replace('/spic/', '/lpic/');
+                # If URL contains "book-default", the book doesn't have a cover
+                if u.find('book-default') != -1:
+                    return None
+            except:
+                return None
+            return u
+
+    def has_cover(self, mi, ans, timeout=5.):
+        if not mi.isbn:
+            return False
+        br = browser()
+        try:
+            if self.get_cover_url(mi.isbn, br, timeout=timeout) != None:
+                self.debug('cover for', mi.isbn, 'found')
+                ans.set()
+        except Exception, e:
+            self.debug(e)
+
+    def get_covers(self, mi, result_queue, abort, timeout=5.):
+        if not mi.isbn:
+            return
+        br = browser()
+        try:
+            url = self.get_cover_url(mi.isbn, br, timeout=timeout)
+            cover_data = br.open_novisit(url).read()
+            result_queue.put((True, cover_data, 'jpg', self.name))
+        except Exception, e:
+            result_queue.put((False, self.exception_to_string(e),
+                traceback.format_exc(), self.name))
+# }}}
+
 def download_cover(mi, timeout=5.): # {{{
     results = Queue()
     download_covers(mi, results, max_covers=1, timeout=timeout)

src/calibre/ebooks/metadata/epub.py

@@ -164,10 +164,10 @@ def get_cover(opf, opf_path, stream, reader=None):
             return render_html_svg_workaround(cpage, default_log)
 
 def get_metadata(stream, extract_cover=True):
-    """ Return metadata as a :class:`MetaInformation` object """
+    """ Return metadata as a :class:`Metadata` object """
     stream.seek(0)
     reader = OCFZipReader(stream)
-    mi = MetaInformation(reader.opf)
+    mi = reader.opf.to_book_metadata()
     if extract_cover:
         try:
             cdata = get_cover(reader.opf, reader.opf_path, stream, reader=reader)

src/calibre/ebooks/metadata/fb2.py

@@ -33,7 +33,10 @@ def get_metadata(stream):
         le = XPath('descendant::fb2:last-name')(au)
         if le:
             lname = tostring(le[0])
-            author += ' '+lname
+            if author:
+                author += ' '+lname
+            else:
+                author = lname
         if author:
             authors.append(author)
         if len(authors) == 1 and author is not None:

src/calibre/ebooks/metadata/fetch.py

@@ -29,7 +29,7 @@ class MetadataSource(Plugin): # {{{
     future use.
 
     The fetch method must store the results in `self.results` as a list of
-    :class:`MetaInformation` objects. If there is an error, it should be stored
+    :class:`Metadata` objects. If there is an error, it should be stored
     in `self.exception` and `self.tb` (for the traceback).
     '''
 

src/calibre/ebooks/metadata/isbndb.py

@@ -8,7 +8,7 @@ import sys, re
 from urllib import quote
 
 from calibre.utils.config import OptionParser
-from calibre.ebooks.metadata import MetaInformation
+from calibre.ebooks.metadata.book.base import Metadata
 from calibre.ebooks.BeautifulSoup import BeautifulStoneSoup
 from calibre import browser
 
@@ -42,34 +42,48 @@ def fetch_metadata(url, max=100, timeout=5.):
     return books
 
 
-class ISBNDBMetadata(MetaInformation):
+class ISBNDBMetadata(Metadata):
 
     def __init__(self, book):
-        MetaInformation.__init__(self, None, [])
+        Metadata.__init__(self, None, [])
 
-        self.isbn = book.get('isbn13', book.get('isbn'))
-        self.title = book.find('titlelong').string
+        def tostring(e):
+            if not hasattr(e, 'string'):
+                return None
+            ans = e.string
+            if ans is not None:
+                ans = unicode(ans).strip()
+            if not ans:
+                ans = None
+            return ans
+
+        self.isbn = unicode(book.get('isbn13', book.get('isbn')))
+        self.title = tostring(book.find('titlelong'))
         if not self.title:
-            self.title = book.find('title').string
+            self.title = tostring(book.find('title'))
+        if not self.title:
+            self.title = _('Unknown')
         self.title = unicode(self.title).strip()
-        au = unicode(book.find('authorstext').string).strip()
-        temp = au.split(',')
         self.authors = []
-        for au in temp:
-            if not au: continue
-            self.authors.extend([a.strip() for a in au.split('&')])
+        au = tostring(book.find('authorstext'))
+        if au:
+            au = au.strip()
+            temp = au.split(',')
+            for au in temp:
+                if not au: continue
+                self.authors.extend([a.strip() for a in au.split('&')])
 
         try:
-            self.author_sort = book.find('authors').find('person').string
+            self.author_sort = tostring(book.find('authors').find('person'))
             if self.authors and self.author_sort == self.authors[0]:
                 self.author_sort = None
         except:
             pass
-        self.publisher = book.find('publishertext').string
+        self.publisher = tostring(book.find('publishertext'))
 
-        summ = book.find('summary')
-        if summ and hasattr(summ, 'string') and summ.string:
-            self.comments = 'SUMMARY:\n'+summ.string
+        summ = tostring(book.find('summary'))
+        if summ:
+            self.comments = 'SUMMARY:\n'+summ
 
 
 def build_isbn(base_url, opts):

src/calibre/ebooks/metadata/library_thing.py

@@ -12,6 +12,7 @@ import mechanize
 from calibre import browser, prints
 from calibre.utils.config import OptionParser
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
+from calibre.ebooks.chardet import strip_encoding_declarations
 
 OPENLIBRARY = 'http://covers.openlibrary.org/b/isbn/%s-L.jpg?default=false'
 
@@ -110,6 +111,8 @@ def get_social_metadata(title, authors, publisher, isbn, username=None,
                     +isbn).read()
         if not raw:
             return mi
+        raw = raw.decode('utf-8', 'replace')
+        raw = strip_encoding_declarations(raw)
         root = html.fromstring(raw)
         h1 = root.xpath('//div[@class="headsummary"]/h1')
         if h1 and not mi.title:

src/calibre/ebooks/metadata/lit.py

@@ -6,7 +6,6 @@ Support for reading the metadata from a LIT file.
 
 import cStringIO, os
 
-from calibre.ebooks.metadata import MetaInformation
 from calibre.ebooks.metadata.opf2 import OPF
 
 def get_metadata(stream):
@@ -16,7 +15,7 @@ def get_metadata(stream):
     src = litfile.get_metadata().encode('utf-8')
     litfile = litfile._litfile
     opf = OPF(cStringIO.StringIO(src), os.getcwd())
-    mi = MetaInformation(opf)
+    mi = opf.to_book_metadata()
     covers = []
     for item in opf.iterguide():
         if 'cover' not in item.get('type', '').lower():

src/calibre/ebooks/metadata/meta.py

@@ -108,7 +108,8 @@ def _get_metadata(stream, stream_type, use_libprs_metadata,
     base = metadata_from_filename(name, pat=pattern)
     if force_read_metadata or is_recipe(name) or prefs['read_file_metadata']:
         mi = get_file_type_metadata(stream, stream_type)
-    if base.title == os.path.splitext(name)[0] and base.authors is None:
+    if base.title == os.path.splitext(name)[0] and \
+            base.is_null('authors') and base.is_null('isbn'):
         # Assume that there was no metadata in the file and the user set pattern
         # to match meta info from the file name did not match.
         # The regex is meant to match the standard format filenames are written
@@ -181,7 +182,7 @@ def metadata_from_filename(name, pat=None):
             mi.isbn = si
         except (IndexError, ValueError):
             pass
-    if not mi.title:
+    if mi.is_null('title'):
         mi.title = name
     return mi
 
@@ -194,7 +195,7 @@ def opf_metadata(opfpath):
     try:
         opf = OPF(f, os.path.dirname(opfpath))
         if opf.application_id is not None:
-            mi = MetaInformation(opf)
+            mi = opf.to_book_metadata()
             if hasattr(opf, 'cover') and opf.cover:
                 cpath = os.path.join(os.path.dirname(opfpath), opf.cover)
                 if os.access(cpath, os.R_OK):

src/calibre/ebooks/metadata/mobi.py

@@ -404,14 +404,16 @@ class MetadataUpdater(object):
                 if self.cover_record is not None:
                     size = len(self.cover_record)
                     cover = rescale_image(data, size)
-                    cover += '\0' * (size - len(cover))
-                    self.cover_record[:] = cover
+                    if len(cover) <= size:
+                        cover += '\0' * (size - len(cover))
+                        self.cover_record[:] = cover
                 if self.thumbnail_record is not None:
                     size = len(self.thumbnail_record)
                     thumbnail = rescale_image(data, size, dimen=MAX_THUMB_DIMEN)
-                    thumbnail += '\0' * (size - len(thumbnail))
-                    self.thumbnail_record[:] = thumbnail
-        return
+                    if len(thumbnail) <= size:
+                        thumbnail += '\0' * (size - len(thumbnail))
+                        self.thumbnail_record[:] = thumbnail
+                return
 
 def set_metadata(stream, mi):
     mu = MetadataUpdater(stream)

src/calibre/ebooks/metadata/opf2.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 lxml based OPF parser.
 '''
 
-import re, sys, unittest, functools, os, mimetypes, uuid, glob, cStringIO
+import re, sys, unittest, functools, os, mimetypes, uuid, glob, cStringIO, json
 from urllib import unquote
 from urlparse import urlparse
 
@@ -16,11 +16,13 @@ from lxml import etree
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.constants import __appname__, __version__, filesystem_encoding
 from calibre.ebooks.metadata.toc import TOC
-from calibre.ebooks.metadata import MetaInformation, string_to_authors
+from calibre.ebooks.metadata import string_to_authors, MetaInformation
+from calibre.ebooks.metadata.book.base import Metadata
 from calibre.utils.date import parse_date, isoformat
 from calibre.utils.localization import get_lang
+from calibre import prints
 
-class Resource(object):
+class Resource(object): # {{{
     '''
     Represents a resource (usually a file on the filesystem or a URL pointing
     to the web. Such resources are commonly referred to in OPF files.
@@ -101,8 +103,9 @@ class Resource(object):
     def __repr__(self):
         return 'Resource(%s, %s)'%(repr(self.path), repr(self.href()))
 
+# }}}
 
-class ResourceCollection(object):
+class ResourceCollection(object): # {{{
 
     def __init__(self):
         self._resources = []
@@ -153,10 +156,9 @@ class ResourceCollection(object):
         for res in self:
             res.set_basedir(path)
 
+# }}}
 
-
-
-class ManifestItem(Resource):
+class ManifestItem(Resource): # {{{
 
     @staticmethod
     def from_opf_manifest_item(item, basedir):
@@ -194,8 +196,9 @@ class ManifestItem(Resource):
             return self.media_type
         raise IndexError('%d out of bounds.'%index)
 
+# }}}
 
-class Manifest(ResourceCollection):
+class Manifest(ResourceCollection): # {{{
 
     @staticmethod
     def from_opf_manifest_element(items, dir):
@@ -262,7 +265,9 @@ class Manifest(ResourceCollection):
             if i.id == id:
                 return i.mime_type
 
-class Spine(ResourceCollection):
+# }}}
+
+class Spine(ResourceCollection): # {{{
 
     class Item(Resource):
 
@@ -334,7 +339,9 @@ class Spine(ResourceCollection):
         for i in self:
             yield i.path
 
-class Guide(ResourceCollection):
+# }}}
+
+class Guide(ResourceCollection): # {{{
 
     class Reference(Resource):
 
@@ -371,6 +378,7 @@ class Guide(ResourceCollection):
             self[-1].type = type
             self[-1].title = ''
 
+# }}}
 
 class MetadataField(object):
 
@@ -412,7 +420,29 @@ class MetadataField(object):
             elem = obj.create_metadata_element(self.name, is_dc=self.is_dc)
         obj.set_text(elem, unicode(val))
 
-class OPF(object):
+
+def serialize_user_metadata(metadata_elem, all_user_metadata, tail='\n'+(' '*8)):
+    from calibre.utils.config import to_json
+    from calibre.ebooks.metadata.book.json_codec import object_to_unicode
+
+    for name, fm in all_user_metadata.items():
+        try:
+            fm = object_to_unicode(fm)
+            fm = json.dumps(fm, default=to_json, ensure_ascii=False)
+        except:
+            prints('Failed to write user metadata:', name)
+            import traceback
+            traceback.print_exc()
+            continue
+        meta = metadata_elem.makeelement('meta')
+        meta.set('name', 'calibre:user_metadata:'+name)
+        meta.set('content', fm)
+        meta.tail = tail
+        metadata_elem.append(meta)
+
+
+class OPF(object): # {{{
+
     MIMETYPE         = 'application/oebps-package+xml'
     PARSER           = etree.XMLParser(recover=True)
     NAMESPACES       = {
@@ -497,6 +527,43 @@ class OPF(object):
         self.guide = Guide.from_opf_guide(guide, basedir) if guide else None
         self.cover_data = (None, None)
         self.find_toc()
+        self.read_user_metadata()
+
+    def read_user_metadata(self):
+        self._user_metadata_ = {}
+        temp = Metadata('x', ['x'])
+        from calibre.utils.config import from_json
+        elems = self.root.xpath('//*[name() = "meta" and starts-with(@name,'
+                '"calibre:user_metadata:") and @content]')
+        for elem in elems:
+            name = elem.get('name')
+            name = ':'.join(name.split(':')[2:])
+            if not name or not name.startswith('#'):
+                continue
+            fm = elem.get('content')
+            try:
+                fm = json.loads(fm, object_hook=from_json)
+                temp.set_user_metadata(name, fm)
+            except:
+                prints('Failed to read user metadata:', name)
+                import traceback
+                traceback.print_exc()
+                continue
+            self._user_metadata_ = temp.get_all_user_metadata(True)
+
+    def to_book_metadata(self):
+        ans = MetaInformation(self)
+        for n, v in self._user_metadata_.items():
+            ans.set_user_metadata(n, v)
+        return ans
+
+    def write_user_metadata(self):
+        elems = self.root.xpath('//*[name() = "meta" and starts-with(@name,'
+                '"calibre:user_metadata:") and @content]')
+        for elem in elems:
+            elem.getparent().remove(elem)
+        serialize_user_metadata(self.metadata,
+                self._user_metadata_)
 
     def find_toc(self):
         self.toc = None
@@ -911,6 +978,7 @@ class OPF(object):
         return elem
 
     def render(self, encoding='utf-8'):
+        self.write_user_metadata()
         raw = etree.tostring(self.root, encoding=encoding, pretty_print=True)
         if not raw.lstrip().startswith('<?xml '):
             raw = '<?xml version="1.0"  encoding="%s"?>\n'%encoding.upper()+raw
@@ -924,18 +992,22 @@ class OPF(object):
             val = getattr(mi, attr, None)
             if val is not None and val != [] and val != (None, None):
                 setattr(self, attr, val)
+        temp = self.to_book_metadata()
+        temp.smart_update(mi, replace_metadata=replace_metadata)
+        self._user_metadata_ = temp.get_all_user_metadata(True)
 
+# }}}
 
-class OPFCreator(MetaInformation):
+class OPFCreator(Metadata):
 
-    def __init__(self, base_path, *args, **kwargs):
+    def __init__(self, base_path, other):
         '''
         Initialize.
         @param base_path: An absolute path to the directory in which this OPF file
         will eventually be. This is used by the L{create_manifest} method
         to convert paths to files into relative paths.
         '''
-        MetaInformation.__init__(self, *args, **kwargs)
+        Metadata.__init__(self, title='', other=other)
         self.base_path = os.path.abspath(base_path)
         if self.application_id is None:
             self.application_id = str(uuid.uuid4())
@@ -1115,6 +1187,8 @@ class OPFCreator(MetaInformation):
                     item.set('title', ref.title)
                 guide.append(item)
 
+        serialize_user_metadata(metadata, self.get_all_user_metadata(False))
+
         root = E.package(
                 metadata,
                 manifest,
@@ -1156,7 +1230,7 @@ def metadata_to_opf(mi, as_string=True):
         <metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf">
             <dc:identifier opf:scheme="%(a)s" id="%(a)s_id">%(id)s</dc:identifier>
             <dc:identifier opf:scheme="uuid" id="uuid_id">%(uuid)s</dc:identifier>
-        </metadata>
+            </metadata>
         <guide/>
     </package>
     '''%dict(a=__appname__, id=mi.application_id, uuid=mi.uuid)))
@@ -1188,7 +1262,7 @@ def metadata_to_opf(mi, as_string=True):
     factory(DC('contributor'), mi.book_producer, __appname__, 'bkp')
     if hasattr(mi.pubdate, 'isoformat'):
         factory(DC('date'), isoformat(mi.pubdate))
-    if mi.category:
+    if hasattr(mi, 'category') and mi.category:
         factory(DC('type'), mi.category)
     if mi.comments:
         factory(DC('description'), mi.comments)
@@ -1217,6 +1291,8 @@ def metadata_to_opf(mi, as_string=True):
     if mi.title_sort:
         meta('title_sort', mi.title_sort)
 
+    serialize_user_metadata(metadata, mi.get_all_user_metadata(False))
+
     metadata[-1].tail = '\n' +(' '*4)
 
     if mi.cover:
@@ -1334,5 +1410,30 @@ def suite():
 def test():
     unittest.TextTestRunner(verbosity=2).run(suite())
 
+def test_user_metadata():
+    from cStringIO import StringIO
+    mi = Metadata('Test title', ['test author1', 'test author2'])
+    um = {
+        '#myseries': { '#value#': u'test series\xe4', 'datatype':'text',
+            'is_multiple': None, 'name': u'My Series'},
+        '#myseries_index': { '#value#': 2.45, 'datatype': 'float',
+            'is_multiple': None},
+        '#mytags': {'#value#':['t1','t2','t3'], 'datatype':'text',
+            'is_multiple': '|', 'name': u'My Tags'}
+        }
+    mi.set_all_user_metadata(um)
+    raw = metadata_to_opf(mi)
+    opfc = OPFCreator(os.getcwd(), other=mi)
+    out = StringIO()
+    opfc.render(out)
+    raw2 = out.getvalue()
+    f = StringIO(raw)
+    opf = OPF(f)
+    f2 = StringIO(raw2)
+    opf2 = OPF(f2)
+    assert um == opf._user_metadata_
+    assert um == opf2._user_metadata_
+    print opf.render()
+
 if __name__ == '__main__':
-    test()
+    test_user_metadata()

src/calibre/ebooks/metadata/rtf.py

@@ -125,7 +125,7 @@ def create_metadata(stream, options):
             au = u', '.join(au)
         author = au.encode('ascii', 'ignore')
         md += r'{\author %s}'%(author,)
-    if options.category:
+    if options.get('category', None):
         category = options.category.encode('ascii', 'ignore')
         md += r'{\category %s}'%(category,)
     comp = options.comment if hasattr(options, 'comment') else options.comments
@@ -180,7 +180,7 @@ def set_metadata(stream, options):
                 src = pat.sub(r'{\\author ' + author + r'}', src)
             else:
                 src = add_metadata_item(src, 'author', author)
-        category = options.category
+        category = options.get('category', None)
         if category != None:
             category = category.encode('ascii', 'replace')
             pat = re.compile(base_pat.replace('name', 'category'), re.DOTALL)

src/calibre/ebooks/mobi/mobiml.py

@@ -184,13 +184,14 @@ class MobiMLizer(object):
             elif tag in NESTABLE_TAGS and istate.rendered:
                 para = wrapper = bstate.nested[-1]
             elif left > 0 and indent >= 0:
+                ems = self.profile.mobi_ems_per_blockquote
                 para = wrapper = etree.SubElement(parent, XHTML('blockquote'))
                 para = wrapper
-                emleft = int(round(left / self.profile.fbase)) - 1
+                emleft = int(round(left / self.profile.fbase)) - ems
                 emleft = min((emleft, 10))
-                while emleft > 0:
+                while emleft > ems/2.0:
                     para = etree.SubElement(para, XHTML('blockquote'))
-                    emleft -= 1
+                    emleft -= ems
             else:
                 para = wrapper = etree.SubElement(parent, XHTML('p'))
             bstate.inline = bstate.para = para

src/calibre/ebooks/mobi/reader.py

@@ -234,7 +234,7 @@ class MobiReader(object):
         self.debug = debug
         self.embedded_mi = None
         self.base_css_rules = textwrap.dedent('''
-                blockquote { margin: 0em 0em 0em 1.25em; text-align: justify }
+                blockquote { margin: 0em 0em 0em 2em; text-align: justify }
 
                 p { margin: 0em; text-align: justify }
 
@@ -441,7 +441,7 @@ class MobiReader(object):
                 html.tostring(elem, encoding='utf-8') + '</package>'
         stream = cStringIO.StringIO(raw)
         opf = OPF(stream)
-        self.embedded_mi = MetaInformation(opf)
+        self.embedded_mi = opf.to_book_metadata()
         if guide is not None:
             for ref in guide.xpath('descendant::reference'):
                 if 'cover' in ref.get('type', '').lower():

src/calibre/ebooks/mobi/writer.py

@@ -15,7 +15,6 @@ from struct import pack
 import time
 from urlparse import urldefrag
 
-from PIL import Image
 from cStringIO import StringIO
 from calibre.ebooks.mobi.langcodes import iana2mobi
 from calibre.ebooks.mobi.mobiml import MBP_NS
@@ -28,6 +27,7 @@ from calibre.ebooks.oeb.base import namespace
 from calibre.ebooks.oeb.base import prefixname
 from calibre.ebooks.oeb.base import urlnormalize
 from calibre.ebooks.compression.palmdoc import compress_doc
+from calibre.utils.magick.draw import Image, save_cover_data_to, thumbnail
 
 INDEXING = True
 FCIS_FLIS = True
@@ -111,46 +111,35 @@ def align_block(raw, multiple=4, pad='\0'):
     return raw + pad*(multiple - extra)
 
 def rescale_image(data, maxsizeb, dimen=None):
-    image = Image.open(StringIO(data))
-    format = image.format
-    changed = False
-    if image.format not in ('JPEG', 'GIF'):
-        width, height = image.size
-        area = width * height
-        if area <= 40000:
-            format = 'GIF'
-        else:
-            image = image.convert('RGBA')
-            format = 'JPEG'
-        changed = True
     if dimen is not None:
-        image.thumbnail(dimen, Image.ANTIALIAS)
-        changed = True
-    if changed:
-        data = StringIO()
-        image.save(data, format)
-        data = data.getvalue()
+        data = thumbnail(data, width=dimen, height=dimen)[-1]
+    else:
+        # Replace transparent pixels with white pixels and convert to JPEG
+        data = save_cover_data_to(data, 'img.jpg', return_data=True)
     if len(data) <= maxsizeb:
         return data
-    image = image.convert('RGBA')
-    for quality in xrange(95, -1, -1):
-        data = StringIO()
-        image.save(data, 'JPEG', quality=quality)
-        data = data.getvalue()
-        if len(data) <= maxsizeb:
-            return data
-    width, height = image.size
-    for scale in xrange(99, 0, -1):
-        scale = scale / 100.
-        data = StringIO()
-        scaled = image.copy()
-        size = (int(width * scale), (height * scale))
-        scaled.thumbnail(size, Image.ANTIALIAS)
-        scaled.save(data, 'JPEG', quality=0)
-        data = data.getvalue()
-        if len(data) <= maxsizeb:
-            return data
-    # Well, we tried?
+    orig_data = data
+    img = Image()
+    quality = 95
+
+    img.load(data)
+    while len(data) >= maxsizeb and quality >= 10:
+        quality -= 5
+        img.set_compression_quality(quality)
+        data = img.export('jpg')
+    if len(data) <= maxsizeb:
+        return data
+    orig_data = data
+
+    scale = 0.9
+    while len(data) >= maxsizeb and scale >= 0.05:
+        img = Image()
+        img.load(orig_data)
+        w, h = img.size
+        img.size = (int(scale*w), int(scale*h))
+        img.set_compression_quality(quality)
+        data = img.export('jpg')
+        scale -= 0.05
     return data
 
 class Serializer(object):
@@ -1796,12 +1785,13 @@ class MobiWriter(object):
                 self._oeb.log.debug('Index records dumped to', t)
 
     def _clean_text_value(self, text):
-        if not text:
-            text = u'(none)'
-        text = text.strip()
-        if not isinstance(text, unicode):
-            text = text.decode('utf-8', 'replace')
-        text = text.encode('ascii','replace')
+        if text is not None and text.strip() :
+            text = text.strip()
+            if not isinstance(text, unicode):
+                text = text.decode('utf-8', 'replace')
+            text = text.encode('utf-8')
+        else :
+            text = "(none)".encode('utf-8')
         return text
 
     def _add_to_ctoc(self, ctoc_str, record_offset):

src/calibre/ebooks/oeb/base.py

@@ -654,8 +654,6 @@ class Metadata(object):
             if predicate(x):
                 l.remove(x)
 
-
-
     def __getitem__(self, key):
         return self.items[key]
 

src/calibre/ebooks/oeb/reader.py

@@ -126,24 +126,29 @@ class OEBReader(object):
 
     def _metadata_from_opf(self, opf):
         from calibre.ebooks.metadata.opf2 import OPF
-        from calibre.ebooks.metadata import MetaInformation
         from calibre.ebooks.oeb.transforms.metadata import meta_info_to_oeb_metadata
         stream = cStringIO.StringIO(etree.tostring(opf))
-        mi = MetaInformation(OPF(stream))
+        mi = OPF(stream).to_book_metadata()
         if not mi.language:
             mi.language = get_lang().replace('_', '-')
         self.oeb.metadata.add('language', mi.language)
-        if not mi.title:
-            mi.title = self.oeb.translate(__('Unknown'))
-        if not mi.authors:
-            mi.authors = [self.oeb.translate(__('Unknown'))]
         if not mi.book_producer:
-            mi.book_producer = '%(a)s (%(v)s) [http://%(a)s.kovidgoyal.net]'%\
+            mi.book_producer = '%(a)s (%(v)s) [http://%(a)s-ebook.com]'%\
                 dict(a=__appname__, v=__version__)
         meta_info_to_oeb_metadata(mi, self.oeb.metadata, self.logger)
-        self.oeb.metadata.add('identifier', str(uuid.uuid4()), id='uuid_id',
-                                scheme='uuid')
+        m = self.oeb.metadata
+        m.add('identifier', str(uuid.uuid4()), id='uuid_id', scheme='uuid')
         self.oeb.uid = self.oeb.metadata.identifier[-1]
+        if not m.title:
+            m.add('title', self.oeb.translate(__('Unknown')))
+        has_aut = False
+        for x in m.creator:
+            if getattr(x, 'role', '').lower() in ('', 'aut'):
+                has_aut = True
+                break
+        if not has_aut:
+            m.add('creator', self.oeb.translate(__('Unknown')), role='aut')
+
 
     def _manifest_prune_invalid(self):
         '''

src/calibre/ebooks/oeb/transforms/cover.py

@@ -31,12 +31,14 @@ class CoverManager(object):
                 </style>
             </head>
             <body>
-                <svg version="1.1" xmlns="http://www.w3.org/2000/svg"
-                    xmlns:xlink="http://www.w3.org/1999/xlink"
-                    width="100%%" height="100%%" viewBox="__viewbox__"
-                    preserveAspectRatio="__ar__">
-                    <image width="__width__" height="__height__" xlink:href="%s"/>
-                </svg>
+                <div>
+                    <svg version="1.1" xmlns="http://www.w3.org/2000/svg"
+                        xmlns:xlink="http://www.w3.org/1999/xlink"
+                        width="100%%" height="100%%" viewBox="__viewbox__"
+                        preserveAspectRatio="__ar__">
+                        <image width="__width__" height="__height__" xlink:href="%s"/>
+                    </svg>
+                </div>
             </body>
         </html>
         ''')

src/calibre/ebooks/oeb/transforms/metadata.py

@@ -12,33 +12,33 @@ from calibre import guess_type
 
 def meta_info_to_oeb_metadata(mi, m, log):
     from calibre.ebooks.oeb.base import OPF
-    if mi.title:
+    if not mi.is_null('title'):
         m.clear('title')
         m.add('title', mi.title)
     if mi.title_sort:
         if not m.title:
             m.add('title', mi.title_sort)
         m.title[0].file_as = mi.title_sort
-    if mi.authors:
+    if not mi.is_null('authors'):
         m.filter('creator', lambda x : x.role.lower() in ['aut', ''])
         for a in mi.authors:
             attrib = {'role':'aut'}
             if mi.author_sort:
                 attrib[OPF('file-as')] = mi.author_sort
             m.add('creator', a, attrib=attrib)
-    if mi.book_producer:
+    if not mi.is_null('book_producer'):
         m.filter('contributor', lambda x : x.role.lower() == 'bkp')
         m.add('contributor', mi.book_producer, role='bkp')
-    if mi.comments:
+    if not mi.is_null('comments'):
         m.clear('description')
         m.add('description', mi.comments)
-    if mi.publisher:
+    if not mi.is_null('publisher'):
         m.clear('publisher')
         m.add('publisher', mi.publisher)
-    if mi.series:
+    if not mi.is_null('series'):
         m.clear('series')
         m.add('series', mi.series)
-    if mi.isbn:
+    if not mi.is_null('isbn'):
         has = False
         for x in m.identifier:
             if x.scheme.lower() == 'isbn':
@@ -46,29 +46,29 @@ def meta_info_to_oeb_metadata(mi, m, log):
                 has = True
         if not has:
             m.add('identifier', mi.isbn, scheme='ISBN')
-    if mi.language:
+    if not mi.is_null('language'):
         m.clear('language')
         m.add('language', mi.language)
-    if mi.series_index is not None:
+    if not mi.is_null('series_index'):
         m.clear('series_index')
         m.add('series_index', mi.format_series_index())
-    if mi.rating is not None:
+    if not mi.is_null('rating'):
         m.clear('rating')
         m.add('rating', '%.2f'%mi.rating)
-    if mi.tags:
+    if not mi.is_null('tags'):
         m.clear('subject')
         for t in mi.tags:
             m.add('subject', t)
-    if mi.pubdate is not None:
+    if not mi.is_null('pubdate'):
         m.clear('date')
         m.add('date', isoformat(mi.pubdate))
-    if mi.timestamp is not None:
+    if not mi.is_null('timestamp'):
         m.clear('timestamp')
         m.add('timestamp', isoformat(mi.timestamp))
-    if mi.rights is not None:
+    if not mi.is_null('rights'):
         m.clear('rights')
         m.add('rights', mi.rights)
-    if mi.publication_type is not None:
+    if not mi.is_null('publication_type'):
         m.clear('publication_type')
         m.add('publication_type', mi.publication_type)
     if not m.timestamp:

src/calibre/ebooks/oeb/transforms/rescale.py

@@ -39,7 +39,7 @@ class RescaleImages(object):
             if item.media_type.startswith('image'):
                 ext = item.media_type.split('/')[-1].upper()
                 if ext == 'JPG': ext = 'JPEG'
-                if ext not in ('PNG', 'JPEG'):
+                if ext not in ('PNG', 'JPEG', 'GIF'):
                     ext = 'JPEG'
 
                 raw = item.data

src/calibre/ebooks/rtf/rtfml.py

@@ -10,13 +10,6 @@ Transform OEB content into RTF markup
 
 import os
 import re
-
-try:
-    from PIL import Image
-    Image
-except ImportError:
-    import Image
-
 import cStringIO
 
 from lxml import etree
@@ -26,6 +19,7 @@ from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace, \
 from calibre.ebooks.oeb.stylizer import Stylizer
 from calibre.ebooks.metadata import authors_to_string
 from calibre.utils.filenames import ascii_text
+from calibre.utils.magick.draw import save_cover_data_to, identify_data
 
 TAGS = {
     'b': '\\b',
@@ -153,10 +147,8 @@ class RTFMLizer(object):
         return text
 
     def image_to_hexstring(self, data):
-        im = Image.open(cStringIO.StringIO(data))
-        data = cStringIO.StringIO()
-        im.convert('RGB').save(data, 'JPEG')
-        data = data.getvalue()
+        data = save_cover_data_to(data, 'cover.jpg', return_data=True)
+        width, height = identify_data(data)[:2]
 
         raw_hex = ''
         for char in data:
@@ -173,7 +165,7 @@ class RTFMLizer(object):
             col += 1
             hex_string += char
 
-        return (hex_string, im.size[0], im.size[1])
+        return (hex_string, width, height)
 
     def clean_text(self, text):
         # Remove excess spaces at beginning and end of lines

src/calibre/gui2/__init__.py

@@ -1,8 +1,9 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 """ The GUI """
-import os, sys, Queue
+import os, sys, Queue, threading
 from threading import RLock
+from urllib import unquote
 
 from PyQt4.Qt import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, \
                          QByteArray, QTranslator, QCoreApplication, QThread, \
@@ -311,11 +312,14 @@ class FunctionDispatcher(QObject):
         if not queued:
             typ = Qt.AutoConnection if queued is None else Qt.DirectConnection
         self.dispatch_signal.connect(self.dispatch, type=typ)
+        self.q = Queue.Queue()
+        self.lock = threading.Lock()
 
     def __call__(self, *args, **kwargs):
-        q = Queue.Queue()
-        self.dispatch_signal.emit(q, args, kwargs)
-        return q.get()
+        with self.lock:
+            self.dispatch_signal.emit(self.q, args, kwargs)
+            res = self.q.get()
+        return res
 
     def dispatch(self, q, args, kwargs):
         try:
@@ -502,6 +506,11 @@ class FileDialog(QObject):
             fs = QFileDialog.getOpenFileNames(parent, title, initial_dir, ftext, "")
             for f in fs:
                 f = unicode(f)
+                if not f: continue
+                if not os.path.exists(f):
+                    # QFileDialog for some reason quotes spaces
+                    # on linux if there is more than one space in a row
+                    f = unquote(f)
                 if f and os.path.exists(f):
                     self.selected_files.append(f)
         else:

src/calibre/gui2/actions/__init__.py

@@ -131,6 +131,7 @@ class InterfaceAction(QObject):
         Called whenever the current library is changed.
 
         :param db: The LibraryDatabase corresponding to the current library.
+
         '''
         pass
 
@@ -148,6 +149,7 @@ class InterfaceAction(QObject):
         long periods of time.
 
         :return: False to halt the shutdown. You are responsible for telling
-        the user why the shutdown was halted.
+                 the user why the shutdown was halted.
+
         '''
         return True

src/calibre/gui2/actions/add.py

@@ -230,17 +230,18 @@ class AddAction(InterfaceAction):
         self._files_added(paths, names, infos, on_card=on_card)
         # set the in-library flags, and as a consequence send the library's
         # metadata for this book to the device. This sets the uuid to the
-        # correct value.
+        # correct value. Note that set_books_in_library might sync_booklists
         self.gui.set_books_in_library(booklists=[model.db], reset=True)
-        model.reset()
+        self.gui.refresh_ondevice()
 
-    def add_books_from_device(self, view):
-        rows = view.selectionModel().selectedRows()
-        if not rows or len(rows) == 0:
-            d = error_dialog(self.gui, _('Add to library'), _('No book selected'))
-            d.exec_()
-            return
-        paths = [p for p in view._model.paths(rows) if p is not None]
+    def add_books_from_device(self, view, paths=None):
+        if paths is None:
+            rows = view.selectionModel().selectedRows()
+            if not rows or len(rows) == 0:
+                d = error_dialog(self.gui, _('Add to library'), _('No book selected'))
+                d.exec_()
+                return
+            paths = [p for p in view.model().paths(rows) if p is not None]
         ve = self.gui.device_manager.device.VIRTUAL_BOOK_EXTENSIONS
         def ext(x):
             ans = os.path.splitext(x)[1]
@@ -261,7 +262,7 @@ class AddAction(InterfaceAction):
             return
         from calibre.gui2.add import Adder
         self.__adder_func = partial(self._add_from_device_adder, on_card=None,
-                                                    model=view._model)
+                                                    model=view.model())
         self._adder = Adder(self.gui, self.gui.library_view.model().db,
                 self.Dispatcher(self.__adder_func), spare_server=self.gui.spare_server)
         self._adder.add(paths)

src/calibre/gui2/actions/choose_library.py

@@ -8,16 +8,17 @@ __docformat__ = 'restructuredtext en'
 import os, shutil
 from functools import partial
 
-from PyQt4.Qt import QMenu, Qt, QInputDialog
+from PyQt4.Qt import QMenu, Qt, QInputDialog, QThread, pyqtSignal, QProgressDialog
 
 from calibre import isbytestring
 from calibre.constants import filesystem_encoding
 from calibre.utils.config import prefs
 from calibre.gui2 import gprefs, warning_dialog, Dispatcher, error_dialog, \
-    question_dialog
+    question_dialog, info_dialog
 from calibre.gui2.actions import InterfaceAction
+from calibre.gui2.dialogs.check_library import CheckLibraryDialog
 
-class LibraryUsageStats(object):
+class LibraryUsageStats(object): # {{{
 
     def __init__(self):
         self.stats = {}
@@ -73,7 +74,73 @@ class LibraryUsageStats(object):
         if stats is not None:
             self.stats[newloc] = stats
         self.write_stats()
+# }}}
 
+# Check Integrity {{{
+
+class VacThread(QThread):
+
+    check_done = pyqtSignal(object, object)
+    callback   = pyqtSignal(object, object)
+
+    def __init__(self, parent, db):
+        QThread.__init__(self, parent)
+        self.db = db
+        self._parent = parent
+
+    def run(self):
+        err = bad = None
+        try:
+            bad = self.db.check_integrity(self.callbackf)
+        except:
+            import traceback
+            err = traceback.format_exc()
+        self.check_done.emit(bad, err)
+
+    def callbackf(self, progress, msg):
+        self.callback.emit(progress, msg)
+
+
+class CheckIntegrity(QProgressDialog):
+
+    def __init__(self, db, parent=None):
+        QProgressDialog.__init__(self, parent)
+        self.db = db
+        self.setCancelButton(None)
+        self.setMinimum(0)
+        self.setMaximum(100)
+        self.setWindowTitle(_('Checking database integrity'))
+        self.setAutoReset(False)
+        self.setValue(0)
+
+        self.vthread = VacThread(self, db)
+        self.vthread.check_done.connect(self.check_done,
+                type=Qt.QueuedConnection)
+        self.vthread.callback.connect(self.callback, type=Qt.QueuedConnection)
+        self.vthread.start()
+
+    def callback(self, progress, msg):
+        self.setLabelText(msg)
+        self.setValue(int(100*progress))
+
+    def check_done(self, bad, err):
+        if err:
+            error_dialog(self, _('Error'),
+                    _('Failed to check database integrity'),
+                    det_msg=err, show=True)
+        elif bad:
+            titles = [self.db.title(x, index_is_id=True) for x in bad]
+            det_msg = '\n'.join(titles)
+            warning_dialog(self, _('Some inconsistencies found'),
+                    _('The following books had formats listed in the '
+                        'database that are not actually available. '
+                        'The entries for the formats have been removed. '
+                        'You should check them manually. This can '
+                        'happen if you manipulate the files in the '
+                        'library folder directly.'), det_msg=det_msg, show=True)
+        self.reset()
+
+# }}}
 
 class ChooseLibraryAction(InterfaceAction):
 
@@ -115,6 +182,31 @@ class ChooseLibraryAction(InterfaceAction):
                     type=Qt.QueuedConnection)
             self.choose_menu.addAction(ac)
 
+        self.rename_separator = self.choose_menu.addSeparator()
+
+        self.maintenance_menu = QMenu(_('Library Maintenance'))
+        ac = self.create_action(spec=(_('Library metadata backup status'),
+                        'lt.png', None, None), attr='action_backup_status')
+        ac.triggered.connect(self.backup_status, type=Qt.QueuedConnection)
+        self.maintenance_menu.addAction(ac)
+        ac = self.create_action(spec=(_('Start backing up metadata of all books'),
+                        'lt.png', None, None), attr='action_backup_metadata')
+        ac.triggered.connect(self.mark_dirty, type=Qt.QueuedConnection)
+        self.maintenance_menu.addAction(ac)
+        ac = self.create_action(spec=(_('Check library'), 'lt.png',
+                                      None, None), attr='action_check_library')
+        ac.triggered.connect(self.check_library, type=Qt.QueuedConnection)
+        self.maintenance_menu.addAction(ac)
+        ac = self.create_action(spec=(_('Check database integrity'), 'lt.png',
+                                      None, None), attr='action_check_database')
+        ac.triggered.connect(self.check_database, type=Qt.QueuedConnection)
+        self.maintenance_menu.addAction(ac)
+        ac = self.create_action(spec=(_('Recover database'), 'lt.png',
+                                    None, None), attr='action_restore_database')
+        ac.triggered.connect(self.restore_database, type=Qt.QueuedConnection)
+        self.maintenance_menu.addAction(ac)
+        self.choose_menu.addMenu(self.maintenance_menu)
+
     def library_name(self):
         db = self.gui.library_view.model().db
         path = db.library_path
@@ -139,23 +231,32 @@ class ChooseLibraryAction(InterfaceAction):
         self.qs_locations = [i[1] for i in locations]
         self.rename_menu.clear()
         self.delete_menu.clear()
+        quick_actions, rename_actions, delete_actions = [], [], []
         for name, loc in locations:
-            self.quick_menu.addAction(name, Dispatcher(partial(self.switch_requested,
+            ac = self.quick_menu.addAction(name, Dispatcher(partial(self.switch_requested,
                 loc)))
-            self.rename_menu.addAction(name, Dispatcher(partial(self.rename_requested,
+            quick_actions.append(ac)
+            ac = self.rename_menu.addAction(name, Dispatcher(partial(self.rename_requested,
                 name, loc)))
-            self.delete_menu.addAction(name, Dispatcher(partial(self.delete_requested,
+            rename_actions.append(ac)
+            ac = self.delete_menu.addAction(name, Dispatcher(partial(self.delete_requested,
                 name, loc)))
+            delete_actions.append(ac)
 
+        qs_actions = []
         for i, x in enumerate(locations[:len(self.switch_actions)]):
             name, loc = x
             ac = self.switch_actions[i]
             ac.setText(name)
             ac.setVisible(True)
+            qs_actions.append(ac)
 
         self.quick_menu_action.setVisible(bool(locations))
         self.rename_menu_action.setVisible(bool(locations))
         self.delete_menu_action.setVisible(bool(locations))
+        self.gui.location_manager.set_switch_actions(quick_actions,
+                rename_actions, delete_actions, qs_actions,
+                self.action_choose)
 
 
     def location_selected(self, loc):
@@ -206,6 +307,47 @@ class ChooseLibraryAction(InterfaceAction):
         self.stats.remove(location)
         self.build_menus()
 
+    def backup_status(self, location):
+        dirty_text = 'no'
+        try:
+            dirty_text = \
+                  unicode(self.gui.library_view.model().db.dirty_queue_length())
+        except:
+            dirty_text = _('none')
+        info_dialog(self.gui, _('Backup status'), '<p>'+
+                _('Book metadata files remaining to be written: %s') % dirty_text,
+                show=True)
+
+    def mark_dirty(self):
+        db = self.gui.library_view.model().db
+        db.dirtied(list(db.data.iterallids()))
+        info_dialog(self.gui, _('Backup metadata'),
+            _('Metadata will be backed up while calibre is running, at the '
+              'rate of approximately 1 book per second.'), show=True)
+
+    def check_library(self):
+        db = self.gui.library_view.model().db
+        d = CheckLibraryDialog(self.gui.parent(), db)
+        d.exec_()
+
+    def check_database(self, *args):
+        m = self.gui.library_view.model()
+        m.stop_metadata_backup()
+        try:
+            d = CheckIntegrity(m.db, self.gui)
+            d.exec_()
+        finally:
+            m.start_metadata_backup()
+
+    def restore_database(self):
+        info_dialog(self.gui, _('Recover database'), '<p>'+
+            _(
+              'This command rebuilds your calibre database from the information '
+              'stored by calibre in the OPF files.<p>'
+              'This function is not currently available in the GUI. You can '
+              'recover your database using the \'calibredb restore_database\' '
+              'command line function.'
+              ), show=True)
 
     def switch_requested(self, location):
         if not self.change_library_allowed():
@@ -245,11 +387,6 @@ class ChooseLibraryAction(InterfaceAction):
         c.exec_()
 
     def change_library_allowed(self):
-        if self.gui.device_connected:
-            warning_dialog(self.gui, _('Not allowed'),
-                    _('You cannot change libraries when a device is'
-                        ' connected.'), show=True)
-            return False
         if self.gui.job_manager.has_jobs():
             warning_dialog(self.gui, _('Not allowed'),
                     _('You cannot change libraries while jobs'

src/calibre/gui2/actions/convert.py

@@ -181,5 +181,6 @@ class ConvertAction(InterfaceAction):
         self.gui.tags_view.recount()
         if self.gui.current_view() is self.gui.library_view:
             current = self.gui.library_view.currentIndex()
-            self.gui.library_view.model().current_changed(current, QModelIndex())
+            if current.isValid():
+                self.gui.library_view.model().current_changed(current, QModelIndex())
 

src/calibre/gui2/actions/copy_to_library.py

@@ -18,14 +18,16 @@ from calibre.utils.config import prefs, tweaks
 
 class Worker(Thread):
 
-    def __init__(self, ids, db, loc, progress, done):
+    def __init__(self, ids, db, loc, progress, done, delete_after):
         Thread.__init__(self)
         self.ids = ids
+        self.processed = set([])
         self.db = db
         self.loc = loc
         self.error = None
         self.progress = progress
         self.done = done
+        self.delete_after = delete_after
 
     def run(self):
         try:
@@ -67,10 +69,12 @@ class Worker(Thread):
                         self.add_formats(identical_book, paths, newdb, replace=False)
             if not added:
                 newdb.import_book(mi, paths, notify=False, import_hooks=False,
-                        apply_import_tags=tweaks['add_new_book_tags_when_importing_books'])
+                    apply_import_tags=tweaks['add_new_book_tags_when_importing_books'],
+                    preserve_uuid=self.delete_after)
                 co = self.db.conversion_options(x, 'PIPE')
                 if co is not None:
                     newdb.set_conversion_options(x, 'PIPE', co)
+            self.processed.add(x)
 
 
 class CopyToLibraryAction(InterfaceAction):
@@ -107,9 +111,13 @@ class CopyToLibraryAction(InterfaceAction):
         for name, loc in locations:
             self.menu.addAction(name, partial(self.copy_to_library,
                 loc))
+            self.menu.addAction(name + ' ' + _('(delete after copy)'),
+                    partial(self.copy_to_library,  loc, delete_after=True))
+            self.menu.addSeparator()
+
         self.qaction.setVisible(bool(locations))
 
-    def copy_to_library(self, loc):
+    def copy_to_library(self, loc, delete_after=False):
         rows = self.gui.library_view.selectionModel().selectedRows()
         if not rows or len(rows) == 0:
             return error_dialog(self.gui, _('Cannot copy'),
@@ -128,7 +136,8 @@ class CopyToLibraryAction(InterfaceAction):
             self.pd.set_msg(_('Copying') + ' ' + title)
             self.pd.set_value(idx)
 
-        self.worker = Worker(ids, db, loc, Dispatcher(progress), Dispatcher(self.pd.accept))
+        self.worker = Worker(ids, db, loc, Dispatcher(progress),
+                             Dispatcher(self.pd.accept), delete_after)
         self.worker.start()
 
         self.pd.exec_()
@@ -140,7 +149,16 @@ class CopyToLibraryAction(InterfaceAction):
         else:
             self.gui.status_bar.show_message(_('Copied %d books to %s') %
                     (len(ids), loc), 2000)
-
+            if delete_after and self.worker.processed:
+                v = self.gui.library_view
+                ci = v.currentIndex()
+                row = None
+                if ci.isValid():
+                    row = ci.row()
+
+                v.model().delete_books_by_id(self.worker.processed)
+                self.gui.iactions['Remove Books'].library_ids_deleted(
+                        self.worker.processed, row)
 
 
 

src/calibre/gui2/actions/delete.py

@@ -149,6 +149,18 @@ class DeleteAction(InterfaceAction):
         self.gui.library_view.model().current_changed(self.gui.library_view.currentIndex(),
                 self.gui.library_view.currentIndex())
 
+
+    def library_ids_deleted(self, ids_deleted, current_row=None):
+        view = self.gui.library_view
+        for v in (self.gui.memory_view, self.gui.card_a_view, self.gui.card_b_view):
+            if v is None:
+                continue
+            v.model().clear_ondevice(ids_deleted)
+        if current_row is not None:
+            ci = view.model().index(current_row, 0)
+            if ci.isValid():
+                view.set_current_row(current_row)
+
     def delete_books(self, *args):
         '''
         Delete selected books from device or library.
@@ -168,14 +180,7 @@ class DeleteAction(InterfaceAction):
             if ci.isValid():
                 row = ci.row()
             ids_deleted = view.model().delete_books(rows)
-            for v in (self.gui.memory_view, self.gui.card_a_view, self.gui.card_b_view):
-                if v is None:
-                    continue
-                v.model().clear_ondevice(ids_deleted)
-            if row is not None:
-                ci = view.model().index(row, 0)
-                if ci.isValid():
-                    view.set_current_row(row)
+            self.library_ids_deleted(ids_deleted, row)
         else:
             if not confirm('<p>'+_('The selected books will be '
                                    '<b>permanently deleted</b> '

src/calibre/gui2/actions/edit_metadata.py

@@ -8,15 +8,14 @@ __docformat__ = 'restructuredtext en'
 import os
 from functools import partial
 
-from PyQt4.Qt import Qt, QTimer, QMenu
+from PyQt4.Qt import Qt, QMenu
 
-from calibre.gui2 import error_dialog, config, warning_dialog
+from calibre.gui2 import error_dialog, config
 from calibre.gui2.dialogs.metadata_single import MetadataSingleDialog
 from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.dialogs.tag_list_editor import TagListEditor
 from calibre.gui2.actions import InterfaceAction
-from calibre.gui2.dialogs.progress import BlockingBusy
 
 class EditMetadataAction(InterfaceAction):
 
@@ -84,52 +83,33 @@ class EditMetadataAction(InterfaceAction):
 
     def do_download_metadata(self, ids, covers=True, set_metadata=True,
             set_social_metadata=None):
-        db = self.gui.library_view.model().db
+        m = self.gui.library_view.model()
+        db = m.db
         if set_social_metadata is None:
             get_social_metadata = config['get_social_metadata']
         else:
             get_social_metadata = set_social_metadata
-        from calibre.gui2.metadata import DownloadMetadata
-        self._download_book_metadata = DownloadMetadata(db, ids,
-                get_covers=covers, set_metadata=set_metadata,
-                get_social_metadata=get_social_metadata)
-        self._download_book_metadata.start()
+        from calibre.gui2.metadata import DoDownload
         if set_social_metadata is not None and set_social_metadata:
             x = _('social metadata')
         else:
             x = _('covers') if covers and not set_metadata else _('metadata')
-        self._book_metadata_download_check = QTimer(self.gui)
-        self._book_metadata_download_check.timeout.connect(self.book_metadata_download_check,
-                type=Qt.QueuedConnection)
-        self._book_metadata_download_check.start(100)
-        self._bb_dialog = BlockingBusy(_('Downloading %s for %d book(s)')%(x,
-            len(ids)), parent=self.gui)
-        self._bb_dialog.exec_()
-
-    def book_metadata_download_check(self):
-        if self._download_book_metadata.is_alive():
-            return
-        self._book_metadata_download_check.stop()
-        self._bb_dialog.accept()
+        title = _('Downloading %s for %d book(s)')%(x, len(ids))
+        self._download_book_metadata = DoDownload(self.gui, title, db, ids,
+                get_covers=covers, set_metadata=set_metadata,
+                get_social_metadata=get_social_metadata)
+        m.stop_metadata_backup()
+        try:
+            self._download_book_metadata.exec_()
+        finally:
+            m.start_metadata_backup()
         cr = self.gui.library_view.currentIndex().row()
         x = self._download_book_metadata
-        self._download_book_metadata = None
-        if x.exception is None:
+        if x.updated:
             self.gui.library_view.model().refresh_ids(
                 x.updated, cr)
             if self.gui.cover_flow:
                 self.gui.cover_flow.dataChanged()
-            if x.failures:
-                details = ['%s: %s'%(title, reason) for title,
-                        reason in x.failures.values()]
-                details = '%s\n'%('\n'.join(details))
-                warning_dialog(self.gui, _('Failed to download some metadata'),
-                    _('Failed to download metadata for the following:'),
-                    det_msg=details).exec_()
-        else:
-            err = _('Failed to download metadata:')
-            error_dialog(self.gui, _('Error'), err, det_msg=x.tb).exec_()
-
 
     def edit_metadata(self, checked, bulk=None):
         '''
@@ -184,12 +164,13 @@ class EditMetadataAction(InterfaceAction):
         self.gui.tags_view.blockSignals(True)
         try:
             changed = MetadataBulkDialog(self.gui, rows,
-                self.gui.library_view.model().db).changed
+                self.gui.library_view.model()).changed
         finally:
             self.gui.tags_view.blockSignals(False)
         if changed:
-            self.gui.library_view.model().resort(reset=False)
-            self.gui.library_view.model().research()
+            m = self.gui.library_view.model()
+            m.resort(reset=False)
+            m.research()
             self.gui.tags_view.recount()
             if self.gui.cover_flow:
                 self.gui.cover_flow.dataChanged()

src/calibre/gui2/add.py

@@ -138,7 +138,7 @@ class DBAdder(Thread): # {{{
             self.critical[name] = open(opf, 'rb').read().decode('utf-8', 'replace')
         else:
             try:
-                mi = MetaInformation(OPF(opf))
+                mi = OPF(opf).to_book_metadata()
             except:
                 import traceback
                 mi = MetaInformation('', [_('Unknown')])
@@ -152,7 +152,8 @@ class DBAdder(Thread): # {{{
             mi.application_id = None
         if self.db is not None:
             if cover:
-                cover = open(cover, 'rb').read()
+                with open(cover, 'rb') as f:
+                    cover = f.read()
             orig_formats = formats
             formats = [f for f in formats if not f.lower().endswith('.opf')]
             if prefs['add_formats_to_existing']:
@@ -381,11 +382,7 @@ class Adder(QObject): # {{{
 
 # }}}
 
-###############################################################################
-############################## END ADDER ######################################
-###############################################################################
-
-class Saver(QObject):
+class Saver(QObject): # {{{
 
     def __init__(self, parent, db, callback, rows, path, opts,
             spare_server=None):
@@ -446,4 +443,5 @@ class Saver(QObject):
         self.pd.set_msg(_('Saved')+' '+title)
         if not ok:
             self.failures.add((title, tb))
+# }}}
 

src/calibre/gui2/book_details.py

@@ -28,6 +28,8 @@ WEIGHTS[_('Tags')] = 4
 
 def render_rows(data):
     keys = data.keys()
+    # First sort by name. The WEIGHTS sort will preserve this sub-order
+    keys.sort(cmp=lambda x, y: cmp(x.lower(), y.lower()))
     keys.sort(cmp=lambda x, y: cmp(WEIGHTS[x], WEIGHTS[y]))
     rows = []
     for key in keys:

src/calibre/gui2/convert/metadata.py

@@ -21,7 +21,10 @@ from calibre.gui2.convert import Widget
 def create_opf_file(db, book_id):
     mi = db.get_metadata(book_id, index_is_id=True)
     mi.application_id = uuid.uuid4()
+    old_cover = mi.cover
+    mi.cover = None
     raw = metadata_to_opf(mi)
+    mi.cover = old_cover
     opf_file = PersistentTemporaryFile('.opf')
     opf_file.write(raw)
     opf_file.close()

src/calibre/gui2/custom_column_widgets.py

@@ -299,7 +299,9 @@ class Series(Base):
         val, s_index = self.gui_val
         val = self.normalize_ui_val(val)
         if val != self.initial_val or s_index != self.initial_index:
-            if s_index == 0.0:
+            if val == '':
+                val = s_index = None
+            elif s_index == 0.0:
                 if tweaks['series_index_auto_increment'] == 'next':
                     s_index = self.db.get_next_cc_series_num_for(val,
                                                              num=self.col_id)
@@ -348,7 +350,11 @@ def populate_metadata_page(layout, db, book_id, bulk=False, two_column=False, pa
     ans = []
     column = row = comments_row = 0
     for col in cols:
+        if not x[col]['editable']:
+            continue
         dt = x[col]['datatype']
+        if dt == 'composite':
+            continue
         if dt == 'comments':
             continue
         w = widget_factory(dt, col)
@@ -448,9 +454,25 @@ class BulkSeries(BulkBase):
         self.name_widget = w
         self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), w]
 
-        self.widgets.append(QLabel(_('Automatically number books in this series'), parent))
-        self.idx_widget=QCheckBox(parent)
-        self.widgets.append(self.idx_widget)
+        self.widgets.append(QLabel('', parent))
+        w = QWidget(parent)
+        layout = QHBoxLayout(w)
+        layout.setContentsMargins(0, 0, 0, 0)
+        self.remove_series = QCheckBox(parent)
+        self.remove_series.setText(_('Remove series'))
+        layout.addWidget(self.remove_series)
+        self.idx_widget = QCheckBox(parent)
+        self.idx_widget.setText(_('Automatically number books'))
+        layout.addWidget(self.idx_widget)
+        self.force_number = QCheckBox(parent)
+        self.force_number.setText(_('Force numbers to start with '))
+        layout.addWidget(self.force_number)
+        self.series_start_number = QSpinBox(parent)
+        self.series_start_number.setMinimum(1)
+        self.series_start_number.setProperty("value", 1)
+        layout.addWidget(self.series_start_number)
+        layout.addItem(QSpacerItem(20, 10, QSizePolicy.Expanding, QSizePolicy.Minimum))
+        self.widgets.append(w)
 
     def initialize(self, book_id):
         self.idx_widget.setChecked(False)
@@ -461,17 +483,26 @@ class BulkSeries(BulkBase):
     def getter(self):
         n = unicode(self.name_widget.currentText()).strip()
         i = self.idx_widget.checkState()
-        return n, i
+        f = self.force_number.checkState()
+        s = self.series_start_number.value()
+        r = self.remove_series.checkState()
+        return n, i, f, s, r
 
     def commit(self, book_ids, notify=False):
-        val, update_indices = self.gui_val
-        val = self.normalize_ui_val(val)
-        if val != '':
+        val, update_indices, force_start, at_value, clear = self.gui_val
+        val = None if clear else self.normalize_ui_val(val)
+        if clear or val != '':
             extras = []
             next_index = self.db.get_next_cc_series_num_for(val, num=self.col_id)
             for book_id in book_ids:
+                if clear:
+                    extras.append(None)
+                    continue
                 if update_indices:
-                    if tweaks['series_index_auto_increment'] == 'next':
+                    if force_start:
+                        s_index = at_value
+                        at_value += 1
+                    elif tweaks['series_index_auto_increment'] == 'next':
                         s_index = next_index
                         next_index += 1
                     else:
@@ -479,6 +510,8 @@ class BulkSeries(BulkBase):
                 else:
                     s_index = self.db.get_custom_extra(book_id, num=self.col_id,
                                                        index_is_id=True)
+                    if s_index is None:
+                        s_index = 1.0
                 extras.append(s_index)
             self.db.set_custom_bulk(book_ids, val, extras=extras,
                                    num=self.col_id, notify=notify)

src/calibre/gui2/device.py

@@ -23,7 +23,7 @@ from calibre.gui2 import config, error_dialog, Dispatcher, dynamic, \
                         warning_dialog, \
                         question_dialog, info_dialog, choose_dir
 from calibre.ebooks.metadata import authors_to_string
-from calibre import preferred_encoding, prints
+from calibre import preferred_encoding, prints, force_unicode
 from calibre.utils.filenames import ascii_filename
 from calibre.devices.errors import FreeSpaceError
 from calibre.utils.smtp import compose_mail, sendmail, extract_email_address, \
@@ -34,6 +34,8 @@ from calibre.ebooks.metadata.meta import set_metadata
 from calibre.constants import DEBUG
 from calibre.utils.config import prefs, tweaks
 from calibre.utils.magick.draw import thumbnail
+from calibre.library.save_to_disk import plugboard_any_device_value, \
+                                         plugboard_any_format_value
 # }}}
 
 class DeviceJob(BaseJob): # {{{
@@ -102,6 +104,28 @@ class DeviceJob(BaseJob): # {{{
 
     # }}}
 
+def find_plugboard(device_name, format, plugboards):
+    cpb = None
+    if format in plugboards:
+        cpb = plugboards[format]
+    elif plugboard_any_format_value in plugboards:
+        cpb = plugboards[plugboard_any_format_value]
+    if cpb is not None:
+        if device_name in cpb:
+            cpb = cpb[device_name]
+        elif plugboard_any_device_value in cpb:
+            cpb = cpb[plugboard_any_device_value]
+        else:
+            cpb = None
+    if DEBUG:
+        prints('Device using plugboard', format, device_name, cpb)
+    return cpb
+
+def device_name_for_plugboards(device_class):
+    if hasattr(device_class, 'DEVICE_PLUGBOARD_NAME'):
+        return device_class.DEVICE_PLUGBOARD_NAME
+    return device_class.__class__.__name__
+
 class DeviceManager(Thread): # {{{
 
     def __init__(self, connected_slot, job_manager, open_feedback_slot, sleep_time=2):
@@ -308,7 +332,10 @@ class DeviceManager(Thread): # {{{
         self.device.sync_booklists(booklists, end_session=False)
         return self.device.card_prefix(end_session=False), self.device.free_space()
 
-    def sync_booklists(self, done, booklists):
+    def sync_booklists(self, done, booklists, plugboards):
+        if hasattr(self.connected_device, 'set_plugboards') and \
+                callable(self.connected_device.set_plugboards):
+            self.connected_device.set_plugboards(plugboards, find_plugboard)
         return self.create_job(self._sync_booklists, done, args=[booklists],
                         description=_('Send metadata to device'))
 
@@ -317,19 +344,30 @@ class DeviceManager(Thread): # {{{
                                args=[booklist, on_card],
                         description=_('Send collections to device'))
 
-    def _upload_books(self, files, names, on_card=None, metadata=None):
+    def _upload_books(self, files, names, on_card=None, metadata=None, plugboards=None):
         '''Upload books to device: '''
+        if hasattr(self.connected_device, 'set_plugboards') and \
+                callable(self.connected_device.set_plugboards):
+            self.connected_device.set_plugboards(plugboards, find_plugboard)
         if metadata and files and len(metadata) == len(files):
             for f, mi in zip(files, metadata):
                 if isinstance(f, unicode):
                     ext = f.rpartition('.')[-1].lower()
+                    cpb = find_plugboard(
+                            device_name_for_plugboards(self.connected_device),
+                            ext, plugboards)
                     if ext:
                         try:
                             if DEBUG:
                                 prints('Setting metadata in:', mi.title, 'at:',
                                         f, file=sys.__stdout__)
                             with open(f, 'r+b') as stream:
-                                set_metadata(stream, mi, stream_type=ext)
+                                if cpb:
+                                    newmi = mi.deepcopy_metadata()
+                                    newmi.template_to_attribute(mi, cpb)
+                                else:
+                                    newmi = mi
+                                set_metadata(stream, newmi, stream_type=ext)
                         except:
                             if DEBUG:
                                 prints(traceback.format_exc(), file=sys.__stdout__)
@@ -338,12 +376,12 @@ class DeviceManager(Thread): # {{{
                                         metadata=metadata, end_session=False)
 
     def upload_books(self, done, files, names, on_card=None, titles=None,
-                     metadata=None):
+                     metadata=None, plugboards=None):
         desc = _('Upload %d books to device')%len(names)
         if titles:
             desc += u':' + u', '.join(titles)
         return self.create_job(self._upload_books, done, args=[files, names],
-                kwargs={'on_card':on_card,'metadata':metadata}, description=desc)
+                kwargs={'on_card':on_card,'metadata':metadata,'plugboards':plugboards}, description=desc)
 
     def add_books_to_metadata(self, locations, metadata, booklists):
         self.device.add_books_to_metadata(locations, metadata, booklists)
@@ -721,14 +759,16 @@ class DeviceMixin(object): # {{{
                 self.device_manager.device.__class__.get_gui_name()+\
                         _(' detected.'), 3000)
             self.device_connected = device_kind
-            self.refresh_ondevice_info (device_connected = True, reset_only = True)
+            self.library_view.set_device_connected(self.device_connected)
+            self.refresh_ondevice (reset_only = True)
         else:
             self.device_connected = None
             self.status_bar.device_disconnected()
             if self.current_view() != self.library_view:
                 self.book_details.reset_info()
             self.location_manager.update_devices()
-            self.refresh_ondevice_info(device_connected=False)
+            self.library_view.set_device_connected(self.device_connected)
+            self.refresh_ondevice()
 
     def info_read(self, job):
         '''
@@ -760,9 +800,9 @@ class DeviceMixin(object): # {{{
         self.card_b_view.set_editable(self.device_manager.device.CAN_SET_METADATA)
         self.sync_news()
         self.sync_catalogs()
-        self.refresh_ondevice_info(device_connected = True)
+        self.refresh_ondevice()
 
-    def refresh_ondevice_info(self, device_connected, reset_only = False):
+    def refresh_ondevice(self, reset_only = False):
         '''
         Force the library view to refresh, taking into consideration new
         device books information
@@ -770,7 +810,7 @@ class DeviceMixin(object): # {{{
         self.book_on_device(None, reset=True)
         if reset_only:
             return
-        self.library_view.set_device_connected(device_connected)
+        self.library_view.model().refresh_ondevice()
 
     # }}}
 
@@ -803,7 +843,7 @@ class DeviceMixin(object): # {{{
         self.book_on_device(None, reset=True)
         # We need to reset the ondevice flags in the library. Use a big hammer,
         # so we don't need to worry about whether some succeeded or not.
-        self.refresh_ondevice_info(device_connected=True, reset_only=False)
+        self.refresh_ondevice(reset_only=False)
 
     def dispatch_sync_event(self, dest, delete, specific):
         rows = self.library_view.selectionModel().selectedRows()
@@ -939,12 +979,12 @@ class DeviceMixin(object): # {{{
         for jobname, exception, tb in results:
             title = jobname.partition(':')[-1]
             if exception is not None:
-                errors.append([title, exception, tb])
+                errors.append(list(map(force_unicode, [title, exception, tb])))
             else:
                 good.append(title)
         if errors:
-            errors = '\n'.join([
-                    '%s\n\n%s\n%s\n' %
+            errors = u'\n'.join([
+                    u'%s\n\n%s\n%s\n' %
                     (title, e, tb) for \
                             title, e, tb in errors
                     ])
@@ -1222,8 +1262,9 @@ class DeviceMixin(object): # {{{
         '''
         Upload metadata to device.
         '''
+        plugboards = self.library_view.model().db.prefs.get('plugboards', {})
         self.device_manager.sync_booklists(Dispatcher(self.metadata_synced),
-                                           self.booklists())
+                                           self.booklists(), plugboards)
 
     def metadata_synced(self, job):
         '''
@@ -1255,10 +1296,11 @@ class DeviceMixin(object): # {{{
         :param files: List of either paths to files or file like objects
         '''
         titles = [i.title for i in metadata]
+        plugboards = self.library_view.model().db.prefs.get('plugboards', {})
         job = self.device_manager.upload_books(
                 Dispatcher(self.books_uploaded),
                 files, names, on_card=on_card,
-                metadata=metadata, titles=titles
+                metadata=metadata, titles=titles, plugboards=plugboards
               )
         self.upload_memory[job] = (metadata, on_card, memory, files)
 
@@ -1300,7 +1342,7 @@ class DeviceMixin(object): # {{{
         if not self.set_books_in_library(self.booklists(), reset=True):
             self.upload_booklists()
         self.book_on_device(None, reset=True)
-        self.refresh_ondevice_info(device_connected = True)
+        self.refresh_ondevice()
 
         view = self.card_a_view if on_card == 'carda' else \
             self.card_b_view if on_card == 'cardb' else self.memory_view
@@ -1371,15 +1413,16 @@ class DeviceMixin(object): # {{{
 
         # Force a reset if the caches are not initialized
         if reset or not hasattr(self, 'db_book_title_cache'):
+            # Build a cache (map) of the library, so the search isn't On**2
+            self.db_book_title_cache = {}
+            self.db_book_uuid_cache = {}
             # It might be possible to get here without having initialized the
             # library view. In this case, simply give up
             try:
                 db = self.library_view.model().db
             except:
                 return False
-            # Build a cache (map) of the library, so the search isn't On**2
-            self.db_book_title_cache = {}
-            self.db_book_uuid_cache = {}
+
             for id in db.data.iterallids():
                 mi = db.get_metadata(id, index_is_id=True)
                 title = clean_string(mi.title)
@@ -1413,7 +1456,7 @@ class DeviceMixin(object): # {{{
                     if update_metadata:
                         book.smart_update(self.db_book_uuid_cache[book.uuid],
                                           replace_metadata=True)
-                    book.in_library = True
+                    book.in_library = 'UUID'
                     # ensure that the correct application_id is set
                     book.application_id = \
                         self.db_book_uuid_cache[book.uuid].application_id
@@ -1426,21 +1469,21 @@ class DeviceMixin(object): # {{{
                     # will match if any of the db_id, author, or author_sort
                     # also match.
                     if getattr(book, 'application_id', None) in d['db_ids']:
-                        book.in_library = True
                         # app_id already matches a db_id. No need to set it.
                         if update_metadata:
                             book.smart_update(d['db_ids'][book.application_id],
                                               replace_metadata=True)
+                        book.in_library = 'APP_ID'
                         continue
                     # Sonys know their db_id independent of the application_id
                     # in the metadata cache. Check that as well.
                     if getattr(book, 'db_id', None) in d['db_ids']:
-                        book.in_library = True
-                        book.application_id = \
-                                    d['db_ids'][book.db_id].application_id
                         if update_metadata:
                             book.smart_update(d['db_ids'][book.db_id],
                                               replace_metadata=True)
+                        book.in_library = 'DB_ID'
+                        book.application_id = \
+                                    d['db_ids'][book.db_id].application_id
                         continue
                     # We now know that the application_id is not right. Set it
                     # to None to prevent book_on_device from accidentally
@@ -1452,19 +1495,19 @@ class DeviceMixin(object): # {{{
                         # either can appear as the author
                         book_authors = clean_string(authors_to_string(book.authors))
                         if book_authors in d['authors']:
-                            book.in_library = True
-                            book.application_id = \
-                                    d['authors'][book_authors].application_id
                             if update_metadata:
                                 book.smart_update(d['authors'][book_authors],
                                                   replace_metadata=True)
-                        elif book_authors in d['author_sort']:
-                            book.in_library = True
+                            book.in_library = 'AUTHOR'
                             book.application_id = \
-                                d['author_sort'][book_authors].application_id
+                                    d['authors'][book_authors].application_id
+                        elif book_authors in d['author_sort']:
                             if update_metadata:
                                 book.smart_update(d['author_sort'][book_authors],
                                                   replace_metadata=True)
+                            book.in_library = 'AUTH_SORT'
+                            book.application_id = \
+                                d['author_sort'][book_authors].application_id
                 else:
                     # Book definitely not matched. Clear its application ID
                     book.application_id = None
@@ -1476,8 +1519,10 @@ class DeviceMixin(object): # {{{
 
         if update_metadata:
             if self.device_manager.is_device_connected:
+                plugboards = self.library_view.model().db.prefs.get('plugboards', {})
                 self.device_manager.sync_booklists(
-                                    Dispatcher(self.metadata_synced), booklists)
+                                    Dispatcher(self.metadata_synced), booklists,
+                                    plugboards)
         return update_metadata
     # }}}
 

src/calibre/gui2/device_drivers/configwidget.py

@@ -6,7 +6,9 @@ __docformat__ = 'restructuredtext en'
 
 from PyQt4.Qt import QWidget, QListWidgetItem, Qt, QVariant, SIGNAL
 
+from calibre.gui2 import error_dialog
 from calibre.gui2.device_drivers.configwidget_ui import Ui_ConfigWidget
+from calibre.utils.formatter import validation_formatter
 
 class ConfigWidget(QWidget, Ui_ConfigWidget):
 
@@ -77,3 +79,15 @@ class ConfigWidget(QWidget, Ui_ConfigWidget):
 
     def use_author_sort(self):
         return self.opt_use_author_sort.isChecked()
+
+    def validate(self):
+        tmpl = unicode(self.opt_save_template.text())
+        try:
+            validation_formatter.validate(tmpl)
+            return True
+        except Exception, err:
+            error_dialog(self, _('Invalid template'),
+                    '<p>'+_('The template %s is invalid:')%tmpl + \
+                    '<br>'+unicode(err), show=True)
+
+            return False

src/calibre/gui2/dialogs/check_library.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
+__docformat__ = 'restructuredtext en'
+__license__   = 'GPL v3'
+
+from PyQt4.Qt import QDialog, QVBoxLayout, QHBoxLayout, QTreeWidget, QLabel, \
+            QPushButton, QDialogButtonBox, QApplication, QTreeWidgetItem, \
+            QLineEdit
+
+from calibre.library.check_library import CheckLibrary, CHECKS
+
+class Item(QTreeWidgetItem):
+    pass
+
+class CheckLibraryDialog(QDialog):
+
+    def __init__(self, parent, db):
+        QDialog.__init__(self, parent)
+        self.db = db
+
+        self.setWindowTitle(_('Check Library'))
+
+        self._layout = QVBoxLayout(self)
+        self.setLayout(self._layout)
+
+        self.log = QTreeWidget(self)
+        self._layout.addWidget(self.log)
+
+        self.check = QPushButton(_('Run the check'))
+        self.check.setDefault(False)
+        self.check.clicked.connect(self.run_the_check)
+        self.copy = QPushButton(_('Copy to clipboard'))
+        self.copy.setDefault(False)
+        self.copy.clicked.connect(self.copy_to_clipboard)
+        self.ok = QPushButton('&Done')
+        self.ok.setDefault(True)
+        self.ok.clicked.connect(self.accept)
+        self.cancel = QPushButton('&Cancel')
+        self.cancel.setDefault(False)
+        self.cancel.clicked.connect(self.reject)
+        self.bbox = QDialogButtonBox(self)
+        self.bbox.addButton(self.copy, QDialogButtonBox.ActionRole)
+        self.bbox.addButton(self.check, QDialogButtonBox.ActionRole)
+        self.bbox.addButton(self.cancel, QDialogButtonBox.RejectRole)
+        self.bbox.addButton(self.ok, QDialogButtonBox.AcceptRole)
+
+        h = QHBoxLayout()
+        ln = QLabel(_('Names to ignore:'))
+        h.addWidget(ln)
+        self.name_ignores = QLineEdit()
+        self.name_ignores.setText(db.prefs.get('check_library_ignore_names', ''))
+        ln.setBuddy(self.name_ignores)
+        h.addWidget(self.name_ignores)
+        le = QLabel(_('Extensions to ignore'))
+        h.addWidget(le)
+        self.ext_ignores = QLineEdit()
+        self.ext_ignores.setText(db.prefs.get('check_library_ignore_extensions', ''))
+        le.setBuddy(self.ext_ignores)
+        h.addWidget(self.ext_ignores)
+        self._layout.addLayout(h)
+
+        self._layout.addWidget(self.bbox)
+        self.resize(750, 500)
+        self.bbox.setEnabled(True)
+
+        self.run_the_check()
+
+    def accept(self):
+        self.db.prefs['check_library_ignore_extensions'] = \
+                                            unicode(self.ext_ignores.text())
+        self.db.prefs['check_library_ignore_names'] = \
+                                            unicode(self.name_ignores.text())
+        QDialog.accept(self)
+
+    def box_to_list(self, txt):
+        return [f.strip() for f in txt.split(',') if f.strip()]
+
+    def run_the_check(self):
+        checker = CheckLibrary(self.db.library_path, self.db)
+        checker.scan_library(self.box_to_list(unicode(self.name_ignores.text())),
+                             self.box_to_list(unicode(self.ext_ignores.text())))
+
+        plaintext = []
+
+        def builder(tree, checker, check):
+            attr = check[0]
+            list = getattr(checker, attr, None)
+            if list is None:
+                return
+
+            h = check[1]
+            tl = Item([h])
+            for problem in list:
+                it = Item()
+                it.setText(0, problem[0])
+                it.setText(1, problem[1])
+                p = ', '.join(problem[2])
+                it.setText(2, p)
+                tl.addChild(it)
+                plaintext.append(','.join([h, problem[0], problem[1], p]))
+            tree.addTopLevelItem(tl)
+
+        t = self.log
+        t.clear()
+        t.setColumnCount(3);
+        t.setHeaderLabels([_('Name'), _('Path from library'), _('Additional Information')])
+        for check in CHECKS:
+            builder(t, checker, check)
+
+        t.setColumnWidth(0, 200)
+        t.setColumnWidth(1, 400)
+
+        self.text_results = '\n'.join(plaintext)
+
+    def copy_to_clipboard(self):
+        QApplication.clipboard().setText(self.text_results)
+
+
+if __name__ == '__main__':
+    app = QApplication([])
+    d = CheckLibraryDialog()
+    d.exec_()

src/calibre/gui2/dialogs/edit_authors_dialog.py

@@ -6,6 +6,7 @@ __license__   = 'GPL v3'
 from PyQt4.Qt import Qt, QDialog, QTableWidgetItem, QAbstractItemView
 
 from calibre.ebooks.metadata import author_to_author_sort
+from calibre.gui2 import error_dialog
 from calibre.gui2.dialogs.edit_authors_dialog_ui import Ui_EditAuthorsDialog
 
 class tableItem(QTableWidgetItem):
@@ -109,6 +110,12 @@ class EditAuthorsDialog(QDialog, Ui_EditAuthorsDialog):
         if col == 0:
             item = self.table.item(row, 0)
             aut  = unicode(item.text()).strip()
+            amper = aut.find('&')
+            if amper >= 0:
+                error_dialog(self.parent(), _('Invalid author name'),
+                        _('Author names cannot contain & characters.')).exec_()
+                aut = aut.replace('&', '%')
+                self.table.item(row, 0).setText(aut)
             c = self.table.item(row, 1)
             c.setText(author_to_author_sort(aut))
             item = c

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -3,57 +3,131 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
 '''Dialog to edit metadata in bulk'''
 
-from threading import Thread
 import re
 
-from PyQt4.Qt import QDialog, QGridLayout
+from PyQt4.Qt import Qt, QDialog, QGridLayout, QVBoxLayout, QFont, QLabel, \
+                     pyqtSignal
 from PyQt4 import QtGui
 
 from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
 from calibre.gui2.dialogs.tag_editor import TagEditor
-from calibre.ebooks.metadata import string_to_authors, \
-    authors_to_string
+from calibre.ebooks.metadata import string_to_authors, authors_to_string
 from calibre.gui2.custom_column_widgets import populate_metadata_page
-from calibre.gui2.dialogs.progress import BlockingBusy
-from calibre.gui2 import error_dialog, Dispatcher
+from calibre.gui2 import error_dialog
+from calibre.gui2.progress_indicator import ProgressIndicator
+from calibre.utils.config import dynamic
 
-class Worker(Thread):
+class MyBlockingBusy(QDialog):
+
+    do_one_signal = pyqtSignal()
+
+    phases = ['',
+              _('Title/Author'),
+              _('Standard metadata'),
+              _('Custom metadata'),
+              _('Search/Replace'),
+    ]
+
+    def __init__(self, msg, args, db, ids, cc_widgets, s_r_func,
+                 parent=None, window_title=_('Working')):
+        QDialog.__init__(self, parent)
+
+        self._layout = QVBoxLayout()
+        self.setLayout(self._layout)
+        self.msg_text = msg
+        self.msg = QLabel(msg+'        ') # Ensure dialog is wide enough
+        #self.msg.setWordWrap(True)
+        self.font = QFont()
+        self.font.setPointSize(self.font.pointSize() + 8)
+        self.msg.setFont(self.font)
+        self.pi = ProgressIndicator(self)
+        self.pi.setDisplaySize(100)
+        self._layout.addWidget(self.pi, 0, Qt.AlignHCenter)
+        self._layout.addSpacing(15)
+        self._layout.addWidget(self.msg, 0, Qt.AlignHCenter)
+        self.setWindowTitle(window_title)
+        self.resize(self.sizeHint())
+        self.start()
 
-    def __init__(self, args, db, ids, cc_widgets, callback):
-        Thread.__init__(self)
         self.args = args
         self.db = db
         self.ids = ids
         self.error = None
-        self.callback = callback
         self.cc_widgets = cc_widgets
+        self.s_r_func = s_r_func
+        self.do_one_signal.connect(self.do_one_safe, Qt.QueuedConnection)
 
-    def doit(self):
+    def start(self):
+        self.pi.startAnimation()
+
+    def stop(self):
+        self.pi.stopAnimation()
+
+    def accept(self):
+        self.stop()
+        return QDialog.accept(self)
+
+    def exec_(self):
+        self.current_index = 0
+        self.current_phase = 1
+        self.do_one_signal.emit()
+        return QDialog.exec_(self)
+
+    def do_one_safe(self):
+        try:
+            if self.current_index >= len(self.ids):
+                self.current_phase += 1
+                self.current_index = 0
+                if self.current_phase > 4:
+                    self.db.commit()
+                    return self.accept()
+            id = self.ids[self.current_index]
+            percent = int((self.current_index*100)/float(len(self.ids)))
+            self.msg.setText(self.msg_text.format(self.phases[self.current_phase],
+                                        percent))
+            self.do_one(id)
+        except Exception, err:
+            import traceback
+            try:
+                err = unicode(err)
+            except:
+                err = repr(err)
+            self.error = (err, traceback.format_exc())
+            return self.accept()
+
+    def do_one(self, id):
         remove, add, au, aus, do_aus, rating, pub, do_series, \
             do_autonumber, do_remove_format, remove_format, do_swap_ta, \
-            do_remove_conv, do_auto_author, series = self.args
+            do_remove_conv, do_auto_author, series, do_series_restart, \
+            series_start_value, do_title_case, clear_series = self.args
+
 
         # first loop: do author and title. These will commit at the end of each
         # operation, because each operation modifies the file system. We want to
         # try hard to keep the DB and the file system in sync, even in the face
         # of exceptions or forced exits.
-        for id in self.ids:
+        if self.current_phase == 1:
+            title_set = False
             if do_swap_ta:
                 title = self.db.title(id, index_is_id=True)
                 aum = self.db.authors(id, index_is_id=True)
                 if aum:
                     aum = [a.strip().replace('|', ',') for a in aum.split(',')]
                     new_title = authors_to_string(aum)
+                    if do_title_case:
+                        new_title = new_title.title()
                     self.db.set_title(id, new_title, notify=False)
+                    title_set = True
                 if title:
                     new_authors = string_to_authors(title)
                     self.db.set_authors(id, new_authors, notify=False)
-
+            if do_title_case and not title_set:
+                title = self.db.title(id, index_is_id=True)
+                self.db.set_title(id, title.title(), notify=False)
             if au:
                 self.db.set_authors(id, string_to_authors(au), notify=False)
-
-        # All of these just affect the DB, so we can tolerate a total rollback
-        for id in self.ids:
+        elif self.current_phase == 2:
+            # All of these just affect the DB, so we can tolerate a total rollback
             if do_auto_author:
                 x = self.db.author_sort_from_book(id, index_is_id=True)
                 if x:
@@ -68,8 +142,15 @@ class Worker(Thread):
             if pub:
                 self.db.set_publisher(id, pub, notify=False, commit=False)
 
+            if clear_series:
+                self.db.set_series(id, '', notify=False, commit=False)
+
             if do_series:
-                next = self.db.get_next_series_num_for(series)
+                if do_series_restart:
+                    next = series_start_value
+                    series_start_value += 1
+                else:
+                    next = self.db.get_next_series_num_for(series)
                 self.db.set_series(id, series, notify=False, commit=False)
                 num = next if do_autonumber and series else 1.0
                 self.db.set_series_index(id, num, notify=False, commit=False)
@@ -79,42 +160,44 @@ class Worker(Thread):
 
             if do_remove_conv:
                 self.db.delete_conversion_options(id, 'PIPE', commit=False)
-        self.db.commit()
-
-        for w in self.cc_widgets:
-            w.commit(self.ids)
-        self.db.bulk_modify_tags(self.ids, add=add, remove=remove,
-                notify=False)
-
-    def run(self):
-        try:
-            self.doit()
-        except Exception, err:
-            import traceback
-            try:
-                err = unicode(err)
-            except:
-                err = repr(err)
-            self.error = (err, traceback.format_exc())
-
-        self.callback()
+        elif self.current_phase == 3:
+            # both of these are fast enough to just do them all
+            for w in self.cc_widgets:
+                w.commit(self.ids)
+            self.db.bulk_modify_tags(self.ids, add=add, remove=remove,
+                                         notify=False)
+            self.current_index = len(self.ids)
+        elif self.current_phase == 4:
+            self.s_r_func(id)
+        # do the next one
+        self.current_index += 1
+        self.do_one_signal.emit()
 
 
 class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
 
-    s_r_functions = {
-                    ''          : lambda x: x,
-                    _('Lower Case') : lambda x: x.lower(),
-                    _('Upper Case')     : lambda x: x.upper(),
-                    _('Title Case')     : lambda x: x.title(),
-            }
+    s_r_functions = {       ''              : lambda x: x,
+                            _('Lower Case') : lambda x: x.lower(),
+                            _('Upper Case') : lambda x: x.upper(),
+                            _('Title Case') : lambda x: x.title(),
+                    }
 
-    def __init__(self, window, rows, db):
+    s_r_match_modes = [     _('Character match'),
+                            _('Regular Expression'),
+                      ]
+
+    s_r_replace_modes = [   _('Replace field'),
+                            _('Prepend to field'),
+                            _('Append to field'),
+                        ]
+
+    def __init__(self, window, rows, model):
         QDialog.__init__(self, window)
         Ui_MetadataBulkDialog.__init__(self)
         self.setupUi(self)
-        self.db = db
-        self.ids = [db.id(r) for r in rows]
+        self.model = model
+        self.db = model.db
+        self.ids = [self.db.id(r) for r in rows]
         self.box_title.setText('<p>' +
                 _('Editing meta information for <b>%d books</b>') %
                 len(rows))
@@ -135,8 +218,9 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
         self.series.currentIndexChanged[int].connect(self.series_changed)
         self.series.editTextChanged.connect(self.series_changed)
         self.tag_editor_button.clicked.connect(self.tag_editor)
+        self.autonumber_series.stateChanged[int].connect(self.auto_number_changed)
 
-        if len(db.custom_column_label_map) == 0:
+        if len(self.db.custom_field_keys(include_composites=False)) == 0:
             self.central_widget.removeTab(1)
         else:
             self.create_custom_column_editors()
@@ -148,86 +232,165 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
         self.search_for.initialize('bulk_edit_search_for')
         self.replace_with.initialize('bulk_edit_replace_with')
         self.test_text.initialize('bulk_edit_test_test')
-        fields = ['']
+        self.all_fields = ['']
+        self.writable_fields = ['']
         fm = self.db.field_metadata
         for f in fm:
             if (f in ['author_sort'] or (
-                fm[f]['datatype'] == 'text' or fm[f]['datatype'] == 'series')
+                fm[f]['datatype'] in ['text', 'series'])
                     and fm[f].get('search_terms', None)
                     and f not in ['formats', 'ondevice']):
-                fields.append(f)
-        fields.sort()
-        self.search_field.addItems(fields)
-        self.search_field.setMaxVisibleItems(min(len(fields), 20))
+                self.all_fields.append(f)
+                self.writable_fields.append(f)
+            if fm[f]['datatype'] == 'composite':
+                self.all_fields.append(f)
+        self.all_fields.sort()
+        self.writable_fields.sort()
+        self.search_field.setMaxVisibleItems(20)
+        self.destination_field.setMaxVisibleItems(20)
         offset = 10
-        self.s_r_number_of_books = min(7, len(self.ids))
+        self.s_r_number_of_books = min(10, len(self.ids))
         for i in range(1,self.s_r_number_of_books+1):
             w = QtGui.QLabel(self.tabWidgetPage3)
             w.setText(_('Book %d:')%i)
-            self.gridLayout1.addWidget(w, i+offset, 0, 1, 1)
+            self.testgrid.addWidget(w, i+offset, 0, 1, 1)
             w = QtGui.QLineEdit(self.tabWidgetPage3)
             w.setReadOnly(True)
             name = 'book_%d_text'%i
             setattr(self, name, w)
             self.book_1_text.setObjectName(name)
-            self.gridLayout1.addWidget(w, i+offset, 1, 1, 1)
+            self.testgrid.addWidget(w, i+offset, 1, 1, 1)
             w = QtGui.QLineEdit(self.tabWidgetPage3)
             w.setReadOnly(True)
             name = 'book_%d_result'%i
             setattr(self, name, w)
             self.book_1_text.setObjectName(name)
-            self.gridLayout1.addWidget(w, i+offset, 2, 1, 1)
+            self.testgrid.addWidget(w, i+offset, 2, 1, 1)
 
-        self.s_r_heading.setText('<p>'+
-                           _('Search and replace in text fields using '
-                             'regular expressions. The search text is an '
-                             'arbitrary python-compatible regular expression. '
-                             'The replacement text can contain backreferences '
-                             'to parenthesized expressions in the pattern. '
-                             'The search is not anchored, and can match and '
-                             'replace multiple times on the same string. See '
-                             '<a href="http://docs.python.org/library/re.html"> '
-                             'this reference</a> '
-                             'for more information, and in particular the \'sub\' '
-                             'function.') + '<p>' + _(
-                             'Note: <b>you can destroy your library</b> '
-                             'using this feature. Changes are permanent. There '
-                             'is no undo function. You are strongly encouraged '
-                             'to back up your library before proceeding.'))
+        self.main_heading = _(
+                 '<b>You can destroy your library using this feature.</b> '
+                 'Changes are permanent. There is no undo function. '
+                 ' This feature is experimental, and there may be bugs. '
+                 'You are strongly encouraged to back up your library '
+                 'before proceeding.<p>'
+                 'Search and replace in text fields using character matching '
+                 'or regular expressions. ')
+
+        self.character_heading = _(
+                 'In character mode, the field is searched for the entered '
+                 'search text. The text is replaced by the specified replacement '
+                 'text everywhere it is found in the specified field. After '
+                 'replacement is finished, the text can be changed to '
+                 'upper-case, lower-case, or title-case. If the case-sensitive '
+                 'check box is checked, the search text must match exactly. If '
+                 'it is unchecked, the search text will match both upper- and '
+                 'lower-case letters'
+                 )
+
+        self.regexp_heading = _(
+                 'In regular expression mode, the search text is an '
+                 'arbitrary python-compatible regular expression. The '
+                 'replacement text can contain backreferences to parenthesized '
+                 'expressions in the pattern. The search is not anchored, '
+                 'and can match and replace multiple times on the same string. '
+                 'The modification functions (lower-case etc) are applied to the '
+                 'matched text, not to the field as a whole. '
+                 'The destination box specifies the field where the result after '
+                 'matching and replacement is to be assigned. You can replace '
+                 'the text in the field, or prepend or append the matched text. '
+                 'See <a href="http://docs.python.org/library/re.html"> '
+                 'this reference</a> for more information on python\'s regular '
+                 'expressions, and in particular the \'sub\' function.'
+                 )
+
+        self.search_mode.addItems(self.s_r_match_modes)
+        self.search_mode.setCurrentIndex(dynamic.get('s_r_search_mode', 0))
+        self.replace_mode.addItems(self.s_r_replace_modes)
+        self.replace_mode.setCurrentIndex(0)
+
+        self.s_r_search_mode = 0
         self.s_r_error = None
         self.s_r_obj = None
 
         self.replace_func.addItems(sorted(self.s_r_functions.keys()))
-        self.search_field.currentIndexChanged[str].connect(self.s_r_field_changed)
+        self.search_mode.currentIndexChanged[int].connect(self.s_r_search_mode_changed)
+        self.search_field.currentIndexChanged[int].connect(self.s_r_search_field_changed)
+        self.destination_field.currentIndexChanged[str].connect(self.s_r_destination_field_changed)
+
+        self.replace_mode.currentIndexChanged[int].connect(self.s_r_paint_results)
         self.replace_func.currentIndexChanged[str].connect(self.s_r_paint_results)
         self.search_for.editTextChanged[str].connect(self.s_r_paint_results)
         self.replace_with.editTextChanged[str].connect(self.s_r_paint_results)
         self.test_text.editTextChanged[str].connect(self.s_r_paint_results)
+        self.comma_separated.stateChanged.connect(self.s_r_paint_results)
+        self.case_sensitive.stateChanged.connect(self.s_r_paint_results)
         self.central_widget.setCurrentIndex(0)
 
-    def s_r_field_changed(self, txt):
-        txt = unicode(txt)
+        self.search_for.completer().setCaseSensitivity(Qt.CaseSensitive)
+        self.replace_with.completer().setCaseSensitivity(Qt.CaseSensitive)
+
+        self.s_r_search_mode_changed(self.search_mode.currentIndex())
+
+    def s_r_get_field(self, mi, field):
+        if field:
+            fm = self.db.metadata_for_field(field)
+            val = mi.get(field, None)
+            if val is None:
+                val = []
+            elif not fm['is_multiple']:
+                val = [val]
+            elif field == 'authors':
+                val = [v.replace(',', '|') for v in val]
+        else:
+            val = []
+        return val
+
+    def s_r_search_field_changed(self, idx):
         for i in range(0, self.s_r_number_of_books):
-            if txt:
-                fm = self.db.field_metadata[txt]
-                id = self.ids[i]
-                val = self.db.get_property(id, index_is_id=True,
-                                           loc=fm['rec_index'])
-                if val is None:
-                    val = ''
-                if fm['is_multiple']:
-                    val = [t.strip() for t in val.split(fm['is_multiple']) if t.strip()]
-                    if val:
-                        val.sort(cmp=lambda x,y: cmp(x.lower(), y.lower()))
-                        val = val[0]
-                        if txt == 'authors':
-                            val = val.replace('|', ',')
-                    else:
-                        val = ''
-            else:
-                val = ''
             w = getattr(self, 'book_%d_text'%(i+1))
-            w.setText(val)
+            mi = self.db.get_metadata(self.ids[i], index_is_id=True)
+            src = unicode(self.search_field.currentText())
+            t = self.s_r_get_field(mi, src)
+            w.setText(''.join(t[0:1]))
+
+        if self.search_mode.currentIndex() == 0:
+            self.destination_field.setCurrentIndex(idx)
+        else:
+            self.s_r_paint_results(None)
+
+    def s_r_destination_field_changed(self, txt):
+        txt = unicode(txt)
+        self.comma_separated.setEnabled(True)
+        if txt:
+            fm = self.db.metadata_for_field(txt)
+            if fm['is_multiple']:
+                self.comma_separated.setEnabled(False)
+                self.comma_separated.setChecked(True)
+        self.s_r_paint_results(None)
+
+    def s_r_search_mode_changed(self, val):
+        self.search_field.clear()
+        self.destination_field.clear()
+        if val == 0:
+            self.search_field.addItems(self.writable_fields)
+            self.destination_field.addItems(self.writable_fields)
+            self.destination_field.setCurrentIndex(0)
+            self.destination_field.setVisible(False)
+            self.destination_field_label.setVisible(False)
+            self.replace_mode.setCurrentIndex(0)
+            self.replace_mode.setVisible(False)
+            self.replace_mode_label.setVisible(False)
+            self.comma_separated.setVisible(False)
+            self.s_r_heading.setText('<p>'+self.main_heading + self.character_heading)
+        else:
+            self.search_field.addItems(self.all_fields)
+            self.destination_field.addItems(self.writable_fields)
+            self.destination_field.setVisible(True)
+            self.destination_field_label.setVisible(True)
+            self.replace_mode.setVisible(True)
+            self.replace_mode_label.setVisible(True)
+            self.comma_separated.setVisible(True)
+            self.s_r_heading.setText('<p>'+self.main_heading + self.regexp_heading)
         self.s_r_paint_results(None)
 
     def s_r_set_colors(self):
@@ -242,17 +405,75 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             getattr(self, 'book_%d_result'%(i+1)).setText('')
 
     def s_r_func(self, match):
-        rf = self.s_r_functions[unicode(self.replace_func.currentText())]
-        rv = unicode(self.replace_with.text())
-        val = match.expand(rv)
-        return rf(val)
+        rfunc = self.s_r_functions[unicode(self.replace_func.currentText())]
+        rtext = unicode(self.replace_with.text())
+        rtext = match.expand(rtext)
+        return rfunc(rtext)
+
+    def s_r_do_regexp(self, mi):
+        src_field = unicode(self.search_field.currentText())
+        src = self.s_r_get_field(mi, src_field)
+        result = []
+        rfunc = self.s_r_functions[unicode(self.replace_func.currentText())]
+        for s in src:
+            t = self.s_r_obj.sub(self.s_r_func, s)
+            if self.search_mode.currentIndex() == 0:
+                t = rfunc(t)
+            result.append(t)
+        return result
+
+    def s_r_do_destination(self, mi, val):
+        src = unicode(self.search_field.currentText())
+        if src == '':
+            return ''
+        dest = unicode(self.destination_field.currentText())
+        if dest == '':
+            if self.db.metadata_for_field(src)['datatype'] == 'composite':
+                raise Exception(_('You must specify a destination when source is a composite field'))
+            dest = src
+        dest_mode = self.replace_mode.currentIndex()
+
+        if dest_mode != 0:
+            dest_val = mi.get(dest, '')
+            if dest_val is None:
+                dest_val = []
+            elif isinstance(dest_val, list):
+                if dest == 'authors':
+                    dest_val = [v.replace(',', '|') for v in dest_val]
+            else:
+                dest_val = [dest_val]
+        else:
+            dest_val = []
+
+        if len(val) > 0:
+            if src == 'authors':
+                val = [v.replace(',', '|') for v in val]
+        if dest_mode == 1:
+            val.extend(dest_val)
+        elif dest_mode == 2:
+            val[0:0] = dest_val
+        return val
+
+    def s_r_replace_mode_separator(self):
+        if self.comma_separated.isChecked():
+            return ','
+        return ''
 
     def s_r_paint_results(self, txt):
         self.s_r_error = None
         self.s_r_set_colors()
+
+        if self.case_sensitive.isChecked():
+            flags = 0
+        else:
+            flags = re.I
+
         try:
-            self.s_r_obj = re.compile(unicode(self.search_for.text()))
-        except re.error as e:
+            if self.search_mode.currentIndex() == 0:
+                self.s_r_obj = re.compile(re.escape(unicode(self.search_for.text())), flags)
+            else:
+                self.s_r_obj = re.compile(unicode(self.search_for.text()), flags)
+        except Exception as e:
             self.s_r_obj = None
             self.s_r_error = e
             self.s_r_set_colors()
@@ -261,66 +482,72 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
         try:
             self.test_result.setText(self.s_r_obj.sub(self.s_r_func,
                                      unicode(self.test_text.text())))
-        except re.error as e:
+        except Exception as e:
             self.s_r_error = e
             self.s_r_set_colors()
             return
 
         for i in range(0,self.s_r_number_of_books):
-            wt = getattr(self, 'book_%d_text'%(i+1))
+            mi = self.db.get_metadata(self.ids[i], index_is_id=True)
             wr = getattr(self, 'book_%d_result'%(i+1))
             try:
-                wr.setText(self.s_r_obj.sub(self.s_r_func, unicode(wt.text())))
-            except re.error as e:
+                result = self.s_r_do_regexp(mi)
+                t = self.s_r_do_destination(mi, result[0:1])
+                t = self.s_r_replace_mode_separator().join(t)
+                wr.setText(t)
+            except Exception as e:
                 self.s_r_error = e
                 self.s_r_set_colors()
                 break
 
-    def do_search_replace(self):
-        field = unicode(self.search_field.currentText())
-        if not field or not self.s_r_obj:
+    def do_search_replace(self, id):
+        source = unicode(self.search_field.currentText())
+        if not source or not self.s_r_obj:
             return
+        dest = unicode(self.destination_field.currentText())
+        if not dest:
+            dest = source
+        dfm = self.db.field_metadata[dest]
 
-        fm = self.db.field_metadata[field]
+        mi = self.db.get_metadata(id, index_is_id=True,)
+        val = mi.get(source)
+        if val is None:
+            return
+        val = self.s_r_do_regexp(mi)
+        val = self.s_r_do_destination(mi, val)
+        if dfm['is_multiple']:
+            if dfm['is_custom']:
+                # The standard tags and authors values want to be lists.
+                # All custom columns are to be strings
+                val = dfm['is_multiple'].join(val)
+            if dest == 'authors' and len(val) == 0:
+                error_dialog(self, _('Search/replace invalid'),
+                             _('Authors cannot be set to the empty string. '
+                               'Book title %s not processed')%mi.title,
+                             show=True)
+                return
+        else:
+            val = self.s_r_replace_mode_separator().join(val)
+            if dest == 'title' and len(val) == 0:
+                error_dialog(self, _('Search/replace invalid'),
+                             _('Title cannot be set to the empty string. '
+                               'Book title %s not processed')%mi.title,
+                             show=True)
+                return
 
-        def apply_pattern(val):
-            try:
-                return self.s_r_obj.sub(self.s_r_func, val)
-            except:
-                return val
-
-        for id in self.ids:
-            val = self.db.get_property(id, index_is_id=True,
-                                       loc=fm['rec_index'])
-            if val is None:
-                continue
-            if fm['is_multiple']:
-                res = []
-                for val in [t.strip() for t in val.split(fm['is_multiple'])]:
-                    v = apply_pattern(val).strip()
-                    if v:
-                        res.append(v)
-                val = res
-                if fm['is_custom']:
-                    # The standard tags and authors values want to be lists.
-                    # All custom columns are to be strings
-                    val = fm['is_multiple'].join(val)
-                elif field == 'authors':
-                    val = [v.replace('|', ',') for v in val]
+        if dfm['is_custom']:
+            extra = self.db.get_custom_extra(id, label=dfm['label'], index_is_id=True)
+            self.db.set_custom(id, val, label=dfm['label'], extra=extra,
+                               commit=False)
+        else:
+            if dest == 'comments':
+                setter = self.db.set_comment
             else:
-                val = apply_pattern(val)
-
-            if fm['is_custom']:
-                extra = self.db.get_custom_extra(id, label=fm['label'], index_is_id=True)
-                self.db.set_custom(id, val, label=fm['label'], extra=extra,
-                                   commit=False)
+                setter = getattr(self.db, 'set_'+dest)
+            if dest in ['title', 'authors']:
+                setter(id, val, notify=False)
             else:
-                if field == 'comments':
-                    setter = self.db.set_comment
-                else:
-                    setter = getattr(self.db, 'set_'+field)
                 setter(id, val, notify=False, commit=False)
-        self.db.commit()
 
     def create_custom_column_editors(self):
         w = self.central_widget.widget(1)
@@ -343,11 +570,11 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
 
     def initalize_authors(self):
         all_authors = self.db.all_authors()
-        all_authors.sort(cmp=lambda x, y : cmp(x[1], y[1]))
+        all_authors.sort(cmp=lambda x, y : cmp(x[1].lower(), y[1].lower()))
 
         for i in all_authors:
             id, name = i
-            name = authors_to_string([name.strip().replace('|', ',') for n in name.split(',')])
+            name = name.strip().replace('|', ',')
             self.authors.addItem(name)
         self.authors.setEditText('')
 
@@ -378,6 +605,16 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             self.tags.update_tags_cache(self.db.all_tags())
             self.remove_tags.update_tags_cache(self.db.all_tags())
 
+    def auto_number_changed(self, state):
+        if state:
+            self.series_numbering_restarts.setEnabled(True)
+            self.series_start_number.setEnabled(True)
+        else:
+            self.series_numbering_restarts.setEnabled(False)
+            self.series_numbering_restarts.setChecked(False)
+            self.series_start_number.setEnabled(False)
+            self.series_start_number.setValue(1)
+
     def accept(self):
         if len(self.ids) < 1:
             return QDialog.accept(self)
@@ -404,33 +641,42 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
         rating = self.rating.value()
         pub = unicode(self.publisher.text())
         do_series = self.write_series
+        clear_series = self.clear_series.isChecked()
         series = unicode(self.series.currentText()).strip()
         do_autonumber = self.autonumber_series.isChecked()
+        do_series_restart = self.series_numbering_restarts.isChecked()
+        series_start_value = self.series_start_number.value()
         do_remove_format = self.remove_format.currentIndex() > -1
         remove_format = unicode(self.remove_format.currentText())
         do_swap_ta = self.swap_title_and_author.isChecked()
         do_remove_conv = self.remove_conversion_settings.isChecked()
         do_auto_author = self.auto_author_sort.isChecked()
+        do_title_case = self.change_title_to_title_case.isChecked()
 
         args = (remove, add, au, aus, do_aus, rating, pub, do_series,
                 do_autonumber, do_remove_format, remove_format, do_swap_ta,
-                do_remove_conv, do_auto_author, series)
+                do_remove_conv, do_auto_author, series, do_series_restart,
+                series_start_value, do_title_case, clear_series)
 
-        bb = BlockingBusy(_('Applying changes to %d books. This may take a while.')
-                %len(self.ids), parent=self)
-        self.worker = Worker(args, self.db, self.ids,
+        bb = MyBlockingBusy(_('Applying changes to %d books.\nPhase {0} {1}%%.')
+                %len(self.ids), args, self.db, self.ids,
                 getattr(self, 'custom_column_widgets', []),
-                Dispatcher(bb.accept, parent=bb))
-        self.worker.start()
-        bb.exec_()
+                self.do_search_replace, parent=self)
 
-        if self.worker.error is not None:
+        # The metadata backup thread causes database commits
+        # which can slow down bulk editing of large numbers of books
+        self.model.stop_metadata_backup()
+        try:
+            bb.exec_()
+        finally:
+            self.model.start_metadata_backup()
+
+        if bb.error is not None:
             return error_dialog(self, _('Failed'),
-                    self.worker.error[0], det_msg=self.worker.error[1],
+                    bb.error[0], det_msg=bb.error[1],
                     show=True)
 
-        self.do_search_replace()
-
+        dynamic['s_r_search_mode'] = self.search_mode.currentIndex()
         self.db.clean()
         return QDialog.accept(self)
 

src/calibre/gui2/dialogs/metadata_bulk.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>679</width>
-    <height>685</height>
+    <width>752</width>
+    <height>715</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -225,23 +225,108 @@
           </widget>
          </item>
          <item row="7" column="1">
-          <widget class="EnComboBox" name="series">
-           <property name="toolTip">
-            <string>List of known series. You can add new series.</string>
-           </property>
-           <property name="whatsThis">
-            <string>List of known series. You can add new series.</string>
-           </property>
-           <property name="editable">
-            <bool>true</bool>
-           </property>
-           <property name="insertPolicy">
-            <enum>QComboBox::InsertAlphabetically</enum>
-           </property>
-           <property name="sizeAdjustPolicy">
-            <enum>QComboBox::AdjustToContents</enum>
-           </property>
-          </widget>
+          <layout class="QHBoxLayout" name="HLayout_34">
+           <item>
+            <widget class="EnComboBox" name="series">
+             <property name="toolTip">
+              <string>List of known series. You can add new series.</string>
+             </property>
+             <property name="whatsThis">
+              <string>List of known series. You can add new series.</string>
+             </property>
+             <property name="editable">
+              <bool>true</bool>
+             </property>
+             <property name="insertPolicy">
+              <enum>QComboBox::InsertAlphabetically</enum>
+             </property>
+             <property name="sizeAdjustPolicy">
+              <enum>QComboBox::AdjustToContents</enum>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QCheckBox" name="clear_series">
+             <property name="toolTip">
+              <string>If checked, the series will be cleared</string>
+             </property>
+             <property name="text">
+              <string>Clear series</string>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <spacer name="HSpacer_344">
+             <property name="orientation">
+              <enum>Qt::Horizontal</enum>
+             </property>
+             <property name="sizeHint" stdset="0">
+              <size>
+               <width>20</width>
+               <height>00</height>
+              </size>
+             </property>
+            </spacer>
+           </item>
+          </layout>
+         </item>
+         <item row="8" column="1" colspan="2">
+          <layout class="QHBoxLayout" name="HLayout_3">
+           <item>
+            <widget class="QCheckBox" name="autonumber_series">
+             <property name="toolTip">
+              <string>If not checked, the series number for the books will be set to 1.
+If checked, selected books will be automatically numbered, in the order
+you selected them. So if you selected Book A and then Book B,
+Book A will have series number 1 and Book B series number 2.</string>
+             </property>
+             <property name="text">
+              <string>Automatically number books in this series</string>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QCheckBox" name="series_numbering_restarts">
+             <property name="enabled">
+              <bool>false</bool>
+             </property>
+             <property name="toolTip">
+              <string>Series will normally be renumbered from the highest number in the database
+for that series. Checking this box will tell calibre to start numbering
+from the value in the box</string>
+             </property>
+             <property name="text">
+              <string>Force numbers to start with </string>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QSpinBox" name="series_start_number">
+             <property name="enabled">
+              <bool>false</bool>
+             </property>
+             <property name="minimum">
+              <number>1</number>
+             </property>
+             <property name="value">
+              <number>1</number>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <spacer name="HSpacer_34">
+             <property name="orientation">
+              <enum>Qt::Horizontal</enum>
+             </property>
+             <property name="sizeHint" stdset="0">
+              <size>
+               <width>20</width>
+               <height>10</height>
+              </size>
+             </property>
+            </spacer>
+           </item>
+          </layout>
          </item>
          <item row="9" column="0">
           <widget class="QLabel" name="label_5">
@@ -270,16 +355,14 @@
            </property>
           </widget>
          </item>
-         <item row="8" column="1">
-          <widget class="QCheckBox" name="autonumber_series">
-           <property name="toolTip">
-            <string>Selected books will be automatically numbered,
-in the order you selected them.
-So if you selected Book A and then Book B,
-Book A will have series number 1 and Book B series number 2.</string>
-           </property>
+         <item row="12" column="0" colspan="2">
+          <widget class="QCheckBox" name="change_title_to_title_case">
            <property name="text">
-            <string>Automatically number books in this series</string>
+            <string>Change title to title case</string>
+           </property>
+           <property name="toolTip">
+            <string>Force the title to be in title case. If both this and swap authors are checked,
+title and author are swapped before the title case is set</string>
            </property>
           </widget>
          </item>
@@ -295,7 +378,7 @@ Future conversion of these books will use the default settings.</string>
            </property>
           </widget>
          </item>
-         <item row="12" column="0" colspan="3">
+         <item row="15" column="0" colspan="3">
           <spacer name="verticalSpacer_2">
            <property name="orientation">
             <enum>Qt::Vertical</enum>
@@ -319,7 +402,7 @@ Future conversion of these books will use the default settings.</string>
         <attribute name="title">
          <string>&amp;Search and replace (experimental)</string>
         </attribute>
-        <layout class="QGridLayout" name="gridLayout">
+        <layout class="QGridLayout" name="vargrid">
          <property name="sizeConstraint">
           <enum>QLayout::SetMinimumSize</enum>
          </property>
@@ -351,6 +434,47 @@ Future conversion of these books will use the default settings.</string>
           </widget>
          </item>
          <item row="3" column="1">
+          <widget class="QComboBox" name="search_field">
+           <property name="toolTip">
+            <string>The name of the field that you want to search</string>
+           </property>
+          </widget>
+         </item>
+         <item row="3" column="2">
+          <layout class="QHBoxLayout" name="HLayout_3">
+           <item>
+            <widget class="QLabel" name="xlabel_24">
+             <property name="text">
+              <string>Search mode:</string>
+             </property>
+             <property name="buddy">
+              <cstring>search_mode</cstring>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QComboBox" name="search_mode">
+             <property name="toolTip">
+              <string>Choose whether to use basic text matching or advanced regular expression matching</string>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <spacer name="HSpacer_2">
+             <property name="orientation">
+              <enum>Qt::Horizontal</enum>
+             </property>
+             <property name="sizeHint" stdset="0">
+              <size>
+               <width>20</width>
+               <height>10</height>
+              </size>
+             </property>
+            </spacer>
+           </item>
+          </layout>
+         </item>
+         <item row="4" column="0">
           <widget class="QLabel" name="xlabel_2">
            <property name="text">
             <string>&amp;Search for:</string>
@@ -360,7 +484,33 @@ Future conversion of these books will use the default settings.</string>
            </property>
           </widget>
          </item>
-         <item row="3" column="2">
+         <item row="4" column="1">
+          <widget class="HistoryLineEdit" name="search_for">
+           <property name="toolTip">
+            <string>Enter the what you are looking for, either plain text or a regular expression, depending on the mode</string>
+           </property>
+           <property name="sizePolicy">
+            <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
+             <horstretch>100</horstretch>
+             <verstretch>0</verstretch>
+            </sizepolicy>
+           </property>
+          </widget>
+         </item>
+         <item row="4" column="2">
+          <widget class="QCheckBox" name="case_sensitive">
+           <property name="toolTip">
+            <string>Check this box if the search string must match exactly upper and lower case. Uncheck it if case is to be ignored</string>
+           </property>
+           <property name="text">
+            <string>Case sensitive</string>
+           </property>
+           <property name="checked">
+            <bool>true</bool>
+           </property>
+          </widget>
+         </item>
+         <item row="5" column="0">
           <widget class="QLabel" name="xlabel_4">
            <property name="text">
             <string>&amp;Replace with:</string>
@@ -370,29 +520,114 @@ Future conversion of these books will use the default settings.</string>
            </property>
           </widget>
          </item>
-         <item row="4" column="0">
-          <widget class="QComboBox" name="search_field"/>
-         </item>
-         <item row="4" column="1">
-          <widget class="HistoryLineEdit" name="search_for"/>
-         </item>
-         <item row="4" column="2">
-          <widget class="HistoryLineEdit" name="replace_with"/>
-         </item>
          <item row="5" column="1">
-          <widget class="QLabel" name="label_41">
-           <property name="text">
-            <string>Apply function &amp;after replace:</string>
-           </property>
-           <property name="buddy">
-            <cstring>replace_func</cstring>
+          <widget class="HistoryLineEdit" name="replace_with">
+           <property name="toolTip">
+            <string>The replacement text. The matched search text will be replaced with this string</string>
            </property>
           </widget>
          </item>
          <item row="5" column="2">
-          <widget class="QComboBox" name="replace_func"/>
+          <layout class="QHBoxLayout" name="verticalLayout">
+           <item>
+            <widget class="QLabel" name="label_41">
+             <property name="text">
+              <string>Apply function after replace:</string>
+             </property>
+             <property name="buddy">
+              <cstring>replace_func</cstring>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QComboBox" name="replace_func">
+             <property name="toolTip">
+              <string>Specify how the text is to be processed after matching and replacement. In character mode, the entire
+field is processed. In regular expression mode, only the matched text is processed</string>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <spacer name="HSpacer_1">
+             <property name="orientation">
+              <enum>Qt::Horizontal</enum>
+             </property>
+             <property name="sizeHint" stdset="0">
+              <size>
+               <width>20</width>
+               <height>10</height>
+              </size>
+             </property>
+            </spacer>
+           </item>
+          </layout>
+         </item>
+         <item row="6" column="0">
+          <widget class="QLabel" name="destination_field_label">
+           <property name="text">
+            <string>&amp;Destination field:</string>
+           </property>
+           <property name="buddy">
+            <cstring>destination_field</cstring>
+           </property>
+          </widget>
          </item>
          <item row="6" column="1">
+          <widget class="QComboBox" name="destination_field">
+           <property name="toolTip">
+            <string>The field that the text will be put into after all replacements. If blank, the source field is used.</string>
+           </property>
+          </widget>
+         </item>
+         <item row="6" column="2">
+          <layout class="QHBoxLayout" name="verticalLayout">
+           <item>
+            <widget class="QLabel" name="replace_mode_label">
+             <property name="text">
+              <string>Mode:</string>
+             </property>
+             <property name="buddy">
+              <cstring>replace_mode</cstring>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QComboBox" name="replace_mode">
+             <property name="toolTip">
+              <string>Specify how the text should be copied into the destination.</string>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QCheckBox" name="comma_separated">
+             <property name="toolTip">
+              <string>If the replace mode is prepend or append, then this box indicates whether a comma or
+nothing should be put between the original text and the inserted text</string>
+             </property>
+             <property name="text">
+              <string>use comma</string>
+             </property>
+             <property name="checked">
+              <bool>true</bool>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <spacer name="zHSpacer_1">
+             <property name="orientation">
+              <enum>Qt::Horizontal</enum>
+             </property>
+             <property name="sizeHint" stdset="0">
+              <size>
+               <width>20</width>
+               <height>10</height>
+              </size>
+             </property>
+            </spacer>
+           </item>
+          </layout>
+         </item>
+         <item row="7" column="1">
           <widget class="QLabel" name="xlabel_3">
            <property name="text">
             <string>Test &amp;text</string>
@@ -402,8 +637,8 @@ Future conversion of these books will use the default settings.</string>
            </property>
           </widget>
          </item>
-         <item row="6" column="2">
-          <widget class="QLabel" name="label_5">
+         <item row="7" column="2">
+          <widget class="QLabel" name="label_51">
            <property name="text">
             <string>Test re&amp;sult</string>
            </property>
@@ -412,19 +647,33 @@ Future conversion of these books will use the default settings.</string>
            </property>
           </widget>
          </item>
-         <item row="7" column="0">
-          <widget class="QLabel" name="label_31">
-           <property name="text">
-            <string>Your test:</string>
+         <item row="9" column="0" colspan="4">
+          <widget class="QScrollArea" name="scrollArea11">
+           <property name="frameShape">
+            <enum>QFrame::NoFrame</enum>
            </property>
+           <property name="widgetResizable">
+            <bool>true</bool>
+           </property>
+           <widget class="QWidget" name="gridLayoutWidget_2">
+            <layout class="QGridLayout" name="testgrid">
+             <item row="8" column="0">
+              <widget class="QLabel" name="label_31">
+               <property name="text">
+                <string>Your test:</string>
+               </property>
+              </widget>
+             </item>
+             <item row="8" column="1">
+              <widget class="HistoryLineEdit" name="test_text"/>
+             </item>
+             <item row="8" column="2">
+              <widget class="QLineEdit" name="test_result"/>
+             </item>
+            </layout>
+           </widget>
           </widget>
          </item>
-         <item row="7" column="1">
-          <widget class="HistoryLineEdit" name="test_text"/>
-         </item>
-         <item row="7" column="2">
-          <widget class="QLineEdit" name="test_result"/>
-         </item>
          <item row="20" column="1">
           <spacer name="verticalSpacer">
            <property name="orientation">
@@ -433,7 +682,7 @@ Future conversion of these books will use the default settings.</string>
            <property name="sizeHint" stdset="0">
             <size>
              <width>20</width>
-             <height>40</height>
+             <height>0</height>
             </size>
            </property>
           </spacer>

src/calibre/gui2/dialogs/metadata_single.py

@@ -308,7 +308,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
         im = Image()
         im.load(cdata)
         im.trim(10)
-        cdata = im.export('jpg')
+        cdata = im.export('png')
         pix = QPixmap()
         pix.loadFromData(cdata)
         self.cover.setPixmap(pix)

src/calibre/gui2/dialogs/metadata_single.ui

@@ -630,10 +630,16 @@ Using this button to create author sort will change author sort from red to gree
                         <property name="toolTip">
                          <string>Remove border (if any) from cover</string>
                         </property>
+                        <property name="text">
+                         <string>T&amp;rim</string>
+                        </property>
                         <property name="icon">
                          <iconset resource="../../../../resources/images.qrc">
                           <normaloff>:/images/trim.png</normaloff>:/images/trim.png</iconset>
                         </property>
+                        <property name="toolButtonStyle">
+                         <enum>Qt::ToolButtonTextBesideIcon</enum>
+                        </property>
                        </widget>
                       </item>
                       <item>

src/calibre/gui2/dialogs/scheduler.py

@@ -57,6 +57,10 @@ class SchedulerDialog(QDialog, Ui_Dialog):
 
         self.old_news.setValue(gconf['oldest_news'])
 
+    def keyPressEvent(self, ev):
+        if ev.key() not in (Qt.Key_Enter, Qt.Key_Return):
+            return QDialog.keyPressEvent(self, ev)
+
     def break_cycles(self):
         self.disconnect(self.recipe_model,  SIGNAL('searched(PyQt_PyObject)'),
                 self.search_done)

src/calibre/gui2/dialogs/tweak_epub.ui

@@ -10,7 +10,7 @@
     <x>0</x>
     <y>0</y>
     <width>382</width>
-    <height>242</height>
+    <height>265</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -32,7 +32,7 @@
       <string>&amp;Explode ePub</string>
      </property>
      <property name="icon">
-      <iconset>
+      <iconset resource="../../../../resources/images.qrc">
        <normaloff>:/images/wizard.png</normaloff>:/images/wizard.png</iconset>
      </property>
     </widget>
@@ -49,7 +49,7 @@
       <string>&amp;Rebuild ePub</string>
      </property>
      <property name="icon">
-      <iconset>
+      <iconset resource="../../../../resources/images.qrc">
        <normaloff>:/images/exec.png</normaloff>:/images/exec.png</iconset>
      </property>
     </widget>
@@ -63,7 +63,7 @@
       <string>&amp;Cancel</string>
      </property>
      <property name="icon">
-      <iconset>
+      <iconset resource="../../../../resources/images.qrc">
        <normaloff>:/images/window-close.png</normaloff>:/images/window-close.png</iconset>
      </property>
     </widget>
@@ -71,7 +71,7 @@
    <item row="0" column="0">
     <widget class="QLabel" name="label">
      <property name="text">
-      <string>Explode the ePub to display contents in a file browser window. To tweak individual files, right-click, then 'Open with...' your editor of choice. When tweaks are complete, close the file browser window. Rebuild the ePub, updating your calibre library.</string>
+      <string>&lt;p&gt;Explode the ePub to display contents in a file browser window. To tweak individual files, right-click, then 'Open with...' your editor of choice. When tweaks are complete, close the file browser window &lt;b&gt;and the editor windows you used to edit files in the epub&lt;/b&gt;.&lt;/p&gt;&lt;p&gt;Rebuild the ePub, updating your calibre library.&lt;/p&gt;</string>
      </property>
      <property name="wordWrap">
       <bool>true</bool>

src/calibre/gui2/dialogs/user_profiles.py

@@ -38,7 +38,10 @@ class CustomRecipeModel(QAbstractListModel):
         return False
 
     def rowCount(self, *args):
-        return len(self.recipe_model.custom_recipe_collection)
+        try:
+            return len(self.recipe_model.custom_recipe_collection)
+        except:
+            return 0
 
     def data(self, index, role):
         if role == Qt.DisplayRole:
@@ -100,6 +103,8 @@ class UserProfiles(ResizableDialog, Ui_Dialog):
 
     def break_cycles(self):
         self.recipe_model = self._model.recipe_model = None
+        self.available_profiles = None
+        self.model = self._model = None
 
     def remove_selected_items(self):
         indices = self.available_profiles.selectionModel().selectedRows()

src/calibre/gui2/dialogs/user_profiles.ui

@@ -375,7 +375,7 @@ p, li { white-space: pre-wrap; }
                  <item>
                   <widget class="QLabel" name="label_8">
                    <property name="text">
-                    <string>For help with writing advanced news recipes, please visit &lt;a href=&quot;http://__appname__.kovidgoyal.net/user_manual/news.html&quot;&gt;User Recipes&lt;/a&gt;</string>
+                    <string>For help with writing advanced news recipes, please visit &lt;a href=&quot;http://__appname__-ebook.com/user_manual/news.html&quot;&gt;User Recipes&lt;/a&gt;</string>
                    </property>
                    <property name="wordWrap">
                     <bool>true</bool>

src/calibre/gui2/layout.py

@@ -56,10 +56,11 @@ class LocationManager(QObject): # {{{
                 self._mem.append(a)
             else:
                 ac.setToolTip(tooltip)
+            ac.calibre_name = name
 
             return ac
 
-        ac('library', _('Library'), 'lt.png',
+        self.library_action = ac('library', _('Library'), 'lt.png',
                 _('Show books in calibre library'))
         ac('main', _('Device'), 'reader.png',
                 _('Show books in the main memory of the device'))
@@ -68,6 +69,24 @@ class LocationManager(QObject): # {{{
         ac('cardb', _('Card B'), 'sd.png',
                 _('Show books in storage card B'))
 
+    def set_switch_actions(self, quick_actions, rename_actions, delete_actions,
+            switch_actions, choose_action):
+        self.switch_menu = QMenu()
+        self.switch_menu.addAction(choose_action)
+        self.cs_menus = []
+        for t, acs in [(_('Quick switch'), quick_actions),
+                (_('Rename library'), rename_actions),
+                (_('Delete library'), delete_actions)]:
+            if acs:
+                self.cs_menus.append(QMenu(t))
+                for ac in acs:
+                    self.cs_menus[-1].addAction(ac)
+                self.switch_menu.addMenu(self.cs_menus[-1])
+        self.switch_menu.addSeparator()
+        for ac in switch_actions:
+            self.switch_menu.addAction(ac)
+        self.library_action.setMenu(self.switch_menu)
+
     def _location_selected(self, location, *args):
         if location != self.current_location and hasattr(self,
                 'location_'+location):
@@ -112,7 +131,6 @@ class LocationManager(QObject): # {{{
             ac.setWhatsThis(t)
             ac.setStatusTip(t)
 
-
     @property
     def has_device(self):
         return max(self.free) > -1
@@ -152,7 +170,7 @@ class SearchBar(QWidget): # {{{
         l.addWidget(x)
         x.setToolTip(_("Advanced search"))
 
-        self.label = x = QLabel('&Search:')
+        self.label = x = QLabel(_('&Search:'))
         l.addWidget(self.label)
         x.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Minimum)
 
@@ -197,14 +215,14 @@ class SearchBar(QWidget): # {{{
 
 # }}}
 
-class Spacer(QWidget):
+class Spacer(QWidget): # {{{
 
     def __init__(self, parent):
         QWidget.__init__(self, parent)
         self.l = QHBoxLayout()
         self.setLayout(self.l)
         self.l.addStretch(10)
-
+# }}}
 
 class ToolBar(QToolBar): # {{{
 
@@ -228,6 +246,7 @@ class ToolBar(QToolBar): # {{{
         self.added_actions = []
         self.build_bar()
         self.preferred_width = self.sizeHint().width()
+        self.setAcceptDrops(True)
 
     def apply_settings(self):
         sz = gprefs['toolbar_icon_size']
@@ -317,6 +336,59 @@ class ToolBar(QToolBar): # {{{
     def database_changed(self, db):
         pass
 
+    #support drag&drop from/to library from/to reader/card
+    def dragEnterEvent(self, event):
+        md = event.mimeData()
+        if md.hasFormat("application/calibre+from_library") or \
+           md.hasFormat("application/calibre+from_device"):
+            event.setDropAction(Qt.CopyAction)
+            event.accept()
+        else:
+            event.ignore()
+
+    def dragMoveEvent(self, event):
+        allowed = False
+        md = event.mimeData()
+        #Drop is only allowed in the location manager widget's different from the selected one
+        for ac in self.location_manager.available_actions:
+            w = self.widgetForAction(ac)
+            if w is not None:
+                if ( md.hasFormat("application/calibre+from_library") or \
+                     md.hasFormat("application/calibre+from_device") ) and \
+                        w.geometry().contains(event.pos()) and \
+                        isinstance(w, QToolButton) and not w.isChecked():
+                    allowed = True
+                    break
+        if allowed:
+            event.acceptProposedAction()
+        else:
+            event.ignore()
+
+    def dropEvent(self, event):
+        data = event.mimeData()
+
+        mime = 'application/calibre+from_library'
+        if data.hasFormat(mime):
+            ids = list(map(int, str(data.data(mime)).split()))
+            tgt = None
+            for ac in self.location_manager.available_actions:
+                w = self.widgetForAction(ac)
+                if w is not None and w.geometry().contains(event.pos()):
+                    tgt = ac.calibre_name
+            if tgt is not None:
+                if tgt == 'main':
+                    tgt = None
+                self.gui.sync_to_device(tgt, False, send_ids=ids)
+                event.accept()
+
+        mime = 'application/calibre+from_device'
+        if data.hasFormat(mime):
+            paths = [unicode(u.toLocalFile()) for u in data.urls()]
+            if paths:
+                self.gui.iactions['Add Books'].add_books_from_device(
+                        self.gui.current_view(), paths=paths)
+                event.accept()
+
 # }}}
 
 class MainWindowMixin(object): # {{{

src/calibre/gui2/library/delegates.py

@@ -15,10 +15,11 @@ from PyQt4.Qt import QColor, Qt, QModelIndex, QSize, \
                      QStyledItemDelegate, QCompleter, \
                      QComboBox
 
-from calibre.gui2 import UNDEFINED_QDATE
+from calibre.gui2 import UNDEFINED_QDATE, error_dialog
 from calibre.gui2.widgets import EnLineEdit, TagsLineEdit
 from calibre.utils.date import now, format_date
 from calibre.utils.config import tweaks
+from calibre.utils.formatter import validation_formatter
 from calibre.gui2.dialogs.comments_dialog import CommentsDialog
 
 class RatingDelegate(QStyledItemDelegate): # {{{
@@ -152,7 +153,7 @@ class TextDelegate(QStyledItemDelegate): # {{{
             complete_items = [i[1] for i in self.auto_complete_function()]
             completer = QCompleter(complete_items, self)
             completer.setCaseSensitivity(Qt.CaseInsensitive)
-            completer.setCompletionMode(QCompleter.InlineCompletion)
+            completer.setCompletionMode(QCompleter.PopupCompletion)
             editor.setCompleter(completer)
         return editor
 #}}}
@@ -303,6 +304,33 @@ class CcBoolDelegate(QStyledItemDelegate): # {{{
             val = 2 if val is None else 1 if not val else 0
         editor.setCurrentIndex(val)
 
+# }}}
+
+class CcTemplateDelegate(QStyledItemDelegate): # {{{
+    def __init__(self, parent):
+        '''
+        Delegate for custom_column bool data.
+        '''
+        QStyledItemDelegate.__init__(self, parent)
+
+    def createEditor(self, parent, option, index):
+        return EnLineEdit(parent)
+
+    def setModelData(self, editor, model, index):
+        val = unicode(editor.text())
+        try:
+            validation_formatter.validate(val)
+        except Exception, err:
+            error_dialog(self.parent(), _('Invalid template'),
+                    '<p>'+_('The template %s is invalid:')%val + \
+                    '<br>'+str(err), show=True)
+        model.setData(index, QVariant(val), Qt.EditRole)
+
+    def setEditorData(self, editor, index):
+        m = index.model()
+        val = m.custom_columns[m.column_map[index.column()]]['display']['composite_template']
+        editor.setText(val)
+
 
 # }}}
 

src/calibre/gui2/library/models.py

@@ -21,10 +21,10 @@ from calibre.utils.date import dt_factory, qt_to_dt, isoformat
 from calibre.ebooks.metadata.meta import set_metadata as _set_metadata
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \
-    REGEXP_MATCH, CoverCache
+    REGEXP_MATCH, CoverCache, MetadataBackup
 from calibre.library.cli import parse_series_string
 from calibre import strftime, isbytestring, prepare_string_for_xml
-from calibre.constants import filesystem_encoding
+from calibre.constants import filesystem_encoding, DEBUG
 from calibre.gui2.library import DEFAULT_SORT
 
 def human_readable(size, precision=1):
@@ -72,7 +72,7 @@ class BooksModel(QAbstractTableModel): # {{{
                         'publisher' : _("Publisher"),
                         'tags'      : _("Tags"),
                         'series'    : _("Series"),
-                        }
+    }
 
     def __init__(self, parent=None, buffer=40):
         QAbstractTableModel.__init__(self, parent)
@@ -89,6 +89,7 @@ class BooksModel(QAbstractTableModel): # {{{
         self.alignment_map = {}
         self.buffer_size = buffer
         self.cover_cache = None
+        self.metadata_backup = None
         self.bool_yes_icon = QIcon(I('ok.png'))
         self.bool_no_icon = QIcon(I('list_remove.png'))
         self.bool_blank_icon = QIcon(I('blank.png'))
@@ -120,6 +121,9 @@ class BooksModel(QAbstractTableModel): # {{{
 
     def set_device_connected(self, is_connected):
         self.device_connected = is_connected
+        self.refresh_ondevice()
+
+    def refresh_ondevice(self):
         self.db.refresh_ondevice()
         self.refresh() # does a resort()
         self.research()
@@ -129,7 +133,7 @@ class BooksModel(QAbstractTableModel): # {{{
 
     def set_database(self, db):
         self.db = db
-        self.custom_columns = self.db.field_metadata.get_custom_field_metadata()
+        self.custom_columns = self.db.field_metadata.custom_field_metadata()
         self.column_map = list(self.orig_headers.keys()) + \
                           list(self.custom_columns)
         def col_idx(name):
@@ -151,13 +155,28 @@ class BooksModel(QAbstractTableModel): # {{{
         self.database_changed.emit(db)
         if self.cover_cache is not None:
             self.cover_cache.stop()
+            # Would like to to a join here, but the thread might be waiting to
+            # do something on the GUI thread. Deadlock.
         self.cover_cache = CoverCache(db, FunctionDispatcher(self.db.cover))
         self.cover_cache.start()
+        self.stop_metadata_backup()
+        self.start_metadata_backup()
         def refresh_cover(event, ids):
             if event == 'cover' and self.cover_cache is not None:
                 self.cover_cache.refresh(ids)
         db.add_listener(refresh_cover)
 
+    def start_metadata_backup(self):
+        self.metadata_backup = MetadataBackup(self.db)
+        self.metadata_backup.start()
+
+    def stop_metadata_backup(self):
+        if getattr(self, 'metadata_backup', None) is not None:
+            self.metadata_backup.stop()
+            # Would like to to a join here, but the thread might be waiting to
+            # do something on the GUI thread. Deadlock.
+
+
     def refresh_ids(self, ids, current_row=-1):
         rows = self.db.refresh_ids(ids)
         if rows:
@@ -318,7 +337,11 @@ class BooksModel(QAbstractTableModel): # {{{
             data[_('Series')] = \
                 _('Book <font face="serif">%s</font> of %s.')%\
                     (sidx, prepare_string_for_xml(series))
-
+        mi = self.db.get_metadata(idx)
+        for key in mi.custom_field_keys():
+            name, val = mi.format_field(key)
+            if val:
+                data[name] = val
         return data
 
     def set_cache(self, idx):
@@ -338,13 +361,14 @@ class BooksModel(QAbstractTableModel): # {{{
             self.cover_cache.set_cache(ids)
 
     def current_changed(self, current, previous, emit_signal=True):
-        idx = current.row()
-        self.set_cache(idx)
-        data = self.get_book_display_info(idx)
-        if emit_signal:
-            self.new_bookdisplay_data.emit(data)
-        else:
-            return data
+        if current.isValid():
+            idx = current.row()
+            self.set_cache(idx)
+            data = self.get_book_display_info(idx)
+            if emit_signal:
+                self.new_bookdisplay_data.emit(data)
+            else:
+                return data
 
     def get_book_info(self, index):
         if isinstance(index, int):
@@ -367,7 +391,6 @@ class BooksModel(QAbstractTableModel): # {{{
         return ans
 
     def get_metadata(self, rows, rows_are_ids=False, full_metadata=False):
-        # Should this add the custom columns? It doesn't at the moment
         metadata, _full_metadata = [], []
         if not rows_are_ids:
             rows = [self.db.id(row.row()) for row in rows]
@@ -616,8 +639,9 @@ class BooksModel(QAbstractTableModel): # {{{
         for col in self.custom_columns:
             idx = self.custom_columns[col]['rec_index']
             datatype = self.custom_columns[col]['datatype']
-            if datatype in ('text', 'comments'):
-                self.dc[col] = functools.partial(text_type, idx=idx, mult=self.custom_columns[col]['is_multiple'])
+            if datatype in ('text', 'comments', 'composite'):
+                self.dc[col] = functools.partial(text_type, idx=idx,
+                                                 mult=self.custom_columns[col]['is_multiple'])
             elif datatype in ('int', 'float'):
                 self.dc[col] = functools.partial(number_type, idx=idx)
             elif datatype == 'datetime':
@@ -625,8 +649,8 @@ class BooksModel(QAbstractTableModel): # {{{
             elif datatype == 'bool':
                 self.dc[col] = functools.partial(bool_type, idx=idx)
                 self.dc_decorator[col] = functools.partial(
-                                            bool_type_decorator, idx=idx,
-                                            bool_cols_are_tristate=tweaks['bool_custom_columns_are_tristate'] == 'yes')
+                            bool_type_decorator, idx=idx,
+                            bool_cols_are_tristate=tweaks['bool_custom_columns_are_tristate'] == 'yes')
             elif datatype == 'rating':
                 self.dc[col] = functools.partial(rating_type, idx=idx)
             elif datatype == 'series':
@@ -675,6 +699,10 @@ class BooksModel(QAbstractTableModel): # {{{
             if role == Qt.DisplayRole:
                 return QVariant(self.headers[self.column_map[section]])
             return NONE
+        if DEBUG and role == Qt.ToolTipRole and orientation == Qt.Vertical:
+                col = self.db.field_metadata['uuid']['rec_index']
+                return QVariant(_('This book\'s UUID is "{0}"').format(self.db.data[section][col]))
+
         if role == Qt.DisplayRole: # orientation is vertical
             return QVariant(section+1)
         return NONE
@@ -692,7 +720,8 @@ class BooksModel(QAbstractTableModel): # {{{
         return flags
 
     def set_custom_column_data(self, row, colhead, value):
-        typ = self.custom_columns[colhead]['datatype']
+        cc = self.custom_columns[colhead]
+        typ = cc['datatype']
         label=self.db.field_metadata.key_to_label(colhead)
         s_index = None
         if typ in ('text', 'comments'):
@@ -718,8 +747,20 @@ class BooksModel(QAbstractTableModel): # {{{
                 val = qt_to_dt(val, as_utc=False)
         elif typ == 'series':
             val, s_index = parse_series_string(self.db, label, value.toString())
-        self.db.set_custom(self.db.id(row), val, extra=s_index,
+            if not val:
+                val = s_index = None
+        elif typ == 'composite':
+            tmpl = unicode(value.toString()).strip()
+            disp = cc['display']
+            disp['composite_template'] = tmpl
+            self.db.set_custom_column_metadata(cc['colnum'], display = disp)
+            self.refresh(reset=True)
+            return True
+
+        id = self.db.id(row)
+        self.db.set_custom(id, val, extra=s_index,
                            label=label, num=None, append=False, notify=True)
+        self.refresh_ids([id], current_row=row)
         return True
 
     def setData(self, index, value, role):
@@ -764,6 +805,7 @@ class BooksModel(QAbstractTableModel): # {{{
                     self.db.set_pubdate(id, qt_to_dt(val, as_utc=False))
                 else:
                     self.db.set(row, column, val)
+                self.refresh_ids([id], row)
             self.dataChanged.emit(index, index)
         return True
 
@@ -887,7 +929,7 @@ class DeviceBooksModel(BooksModel): # {{{
                 }
         self.marked_for_deletion = {}
         self.search_engine = OnDeviceSearch(self)
-        self.editable = True
+        self.editable = ['title', 'authors', 'collections']
         self.book_in_library = None
 
     def mark_for_deletion(self, job, rows, rows_are_ids=False):
@@ -933,13 +975,13 @@ class DeviceBooksModel(BooksModel): # {{{
         if self.map[index.row()] in self.indices_to_be_deleted():
             return Qt.ItemIsUserCheckable  # Can't figure out how to get the disabled flag in python
         flags = QAbstractTableModel.flags(self, index)
-        if index.isValid() and self.editable:
+        if index.isValid():
             cname = self.column_map[index.column()]
-            if cname in ('title', 'authors') or \
-                    (cname == 'collections' and \
-                     callable(getattr(self.db, 'supports_collections', None)) and \
-                     self.db.supports_collections() and \
-                     prefs['manage_device_metadata']=='manual'):
+            if cname in self.editable and \
+                     (cname != 'collections' or \
+                     (callable(getattr(self.db, 'supports_collections', None)) and \
+                      self.db.supports_collections() and \
+                      prefs['manage_device_metadata']=='manual')):
                 flags |= Qt.ItemIsEditable
         return flags
 
@@ -1046,19 +1088,28 @@ class DeviceBooksModel(BooksModel): # {{{
         self.db = db
         self.map = list(range(0, len(db)))
 
+    def cover(self, row):
+        item = self.db[self.map[row]]
+        cdata = item.thumbnail
+        img = QImage()
+        if cdata is not None:
+            if hasattr(cdata, 'image_path'):
+                img.load(cdata.image_path)
+            elif cdata:
+                if isinstance(cdata, (tuple, list)):
+                    img.loadFromData(cdata[-1])
+                else:
+                    img.loadFromData(cdata)
+        if img.isNull():
+            img = self.default_image
+        return img
+
     def current_changed(self, current, previous):
         data = {}
         item = self.db[self.map[current.row()]]
-        cdata = item.thumbnail
-        if cdata is not None:
-            img = QImage()
-            if hasattr(cdata, 'image_path'):
-                img.load(cdata.image_path)
-            else:
-                img.loadFromData(cdata)
-            if img.isNull():
-                img = self.default_image
-            data['cover'] = img
+        cover = self.cover(current.row())
+        if cover is not self.default_image:
+            data['cover'] = cover
         type = _('Unknown')
         ext = os.path.splitext(item.path)[1]
         if ext:
@@ -1159,6 +1210,8 @@ class DeviceBooksModel(BooksModel): # {{{
                 if tags:
                     tags.sort(cmp=lambda x,y: cmp(x.lower(), y.lower()))
                     return QVariant(', '.join(tags))
+            elif DEBUG and cname == 'inlibrary':
+                return QVariant(self.db[self.map[row]].in_library)
         elif role == Qt.ToolTipRole and index.isValid():
             if self.map[row] in self.indices_to_be_deleted():
                 return QVariant(_('Marked for deletion'))
@@ -1180,8 +1233,10 @@ class DeviceBooksModel(BooksModel): # {{{
         return NONE
 
     def headerData(self, section, orientation, role):
-        if role == Qt.ToolTipRole:
+        if role == Qt.ToolTipRole and orientation == Qt.Horizontal:
             return QVariant(_('The lookup/search name is "{0}"').format(self.column_map[section]))
+        if DEBUG and role == Qt.ToolTipRole and orientation == Qt.Vertical:
+            return QVariant(_('This book\'s UUID is "{0}"').format(self.db[self.map[section]].uuid))
         if role != Qt.DisplayRole:
             return NONE
         if orientation == Qt.Horizontal:
@@ -1220,7 +1275,14 @@ class DeviceBooksModel(BooksModel): # {{{
     def set_editable(self, editable):
         # Cannot edit if metadata is sent on connect. Reason: changes will
         # revert to what is in the library on next connect.
-        self.editable = editable and prefs['manage_device_metadata']!='on_connect'
+        if isinstance(editable, list):
+            self.editable = editable
+        elif editable:
+            self.editable = ['title', 'authors', 'collections']
+        else:
+            self.editable = []
+        if prefs['manage_device_metadata']=='on_connect':
+            self.editable = []
 
     def set_search_restriction(self, s):
         pass

src/calibre/gui2/library/views.py

@@ -9,16 +9,18 @@ import os
 from functools import partial
 
 from PyQt4.Qt import QTableView, Qt, QAbstractItemView, QMenu, pyqtSignal, \
-    QModelIndex, QIcon
+    QModelIndex, QIcon, QItemSelection, QMimeData, QDrag, QApplication, \
+    QPoint, QPixmap, QUrl, QImage, QPainter, QColor, QRect
 
 from calibre.gui2.library.delegates import RatingDelegate, PubDateDelegate, \
     TextDelegate, DateDelegate, TagsDelegate, CcTextDelegate, \
-    CcBoolDelegate, CcCommentsDelegate, CcDateDelegate
+    CcBoolDelegate, CcCommentsDelegate, CcDateDelegate, CcTemplateDelegate
 from calibre.gui2.library.models import BooksModel, DeviceBooksModel
 from calibre.utils.config import tweaks, prefs
 from calibre.gui2 import error_dialog, gprefs
 from calibre.gui2.library import DEFAULT_SORT
-
+from calibre.constants import filesystem_encoding
+from calibre import force_unicode
 
 class BooksView(QTableView): # {{{
 
@@ -31,6 +33,7 @@ class BooksView(QTableView): # {{{
         self.setDragEnabled(True)
         self.setDragDropOverwriteMode(False)
         self.setDragDropMode(self.DragDrop)
+        self.drag_start_pos = None
         self.setAlternatingRowColors(True)
         self.setSelectionBehavior(self.SelectRows)
         self.setShowGrid(False)
@@ -47,6 +50,7 @@ class BooksView(QTableView): # {{{
         self.cc_text_delegate = CcTextDelegate(self)
         self.cc_bool_delegate = CcBoolDelegate(self)
         self.cc_comments_delegate = CcCommentsDelegate(self)
+        self.cc_template_delegate = CcTemplateDelegate(self)
         self.display_parent = parent
         self._model = modelcls(self)
         self.setModel(self._model)
@@ -391,6 +395,8 @@ class BooksView(QTableView): # {{{
                     self.setItemDelegateForColumn(cm.index(colhead), self.cc_bool_delegate)
                 elif cc['datatype'] == 'rating':
                     self.setItemDelegateForColumn(cm.index(colhead), self.rating_delegate)
+                elif cc['datatype'] == 'composite':
+                    self.setItemDelegateForColumn(cm.index(colhead), self.cc_template_delegate)
             else:
                 dattr = colhead+'_delegate'
                 delegate = colhead if hasattr(self, dattr) else 'text'
@@ -419,10 +425,105 @@ class BooksView(QTableView): # {{{
         Accept a drop event and return a list of paths that can be read from
         and represent files with extensions.
         '''
-        if event.mimeData().hasFormat('text/uri-list'):
-            urls = [unicode(u.toLocalFile()) for u in event.mimeData().urls()]
+        md = event.mimeData()
+        if md.hasFormat('text/uri-list') and not \
+                md.hasFormat('application/calibre+from_library'):
+            urls = [unicode(u.toLocalFile()) for u in md.urls()]
             return [u for u in urls if os.path.splitext(u)[1] and os.access(u, os.R_OK)]
 
+    def drag_icon(self, cover, multiple):
+        cover = cover.scaledToHeight(120, Qt.SmoothTransformation)
+        if multiple:
+            base_width = cover.width()
+            base_height = cover.height()
+            base = QImage(base_width+21, base_height+21,
+                    QImage.Format_ARGB32_Premultiplied)
+            base.fill(QColor(255, 255, 255, 0).rgba())
+            p = QPainter(base)
+            rect = QRect(20, 0, base_width, base_height)
+            p.fillRect(rect, QColor('white'))
+            p.drawRect(rect)
+            rect.moveLeft(10)
+            rect.moveTop(10)
+            p.fillRect(rect, QColor('white'))
+            p.drawRect(rect)
+            rect.moveLeft(0)
+            rect.moveTop(20)
+            p.fillRect(rect, QColor('white'))
+            p.save()
+            p.setCompositionMode(p.CompositionMode_SourceAtop)
+            p.drawImage(rect.topLeft(), cover)
+            p.restore()
+            p.drawRect(rect)
+            p.end()
+            cover = base
+        return QPixmap.fromImage(cover)
+
+    def drag_data(self):
+        m = self.model()
+        db = m.db
+        rows = self.selectionModel().selectedRows()
+        selected = map(m.id, rows)
+        ids = ' '.join(map(str, selected))
+        md = QMimeData()
+        md.setData('application/calibre+from_library', ids)
+        fmt = prefs['output_format']
+
+        def url_for_id(i):
+            ans = db.format_abspath(i, fmt, index_is_id=True)
+            if ans is None:
+                fmts = db.formats(i, index_is_id=True)
+                if fmts:
+                    fmts = fmts.split(',')
+                else:
+                    fmts = []
+                for f in fmts:
+                    ans = db.format_abspath(i, f, index_is_id=True)
+                    if ans is not None:
+                        break
+            if ans is None:
+                ans = db.abspath(i, index_is_id=True)
+            return QUrl.fromLocalFile(ans)
+
+        md.setUrls([url_for_id(i) for i in selected])
+        drag = QDrag(self)
+        drag.setMimeData(md)
+        cover = self.drag_icon(m.cover(self.currentIndex().row()),
+                len(selected) > 1)
+        drag.setHotSpot(QPoint(-15, -15))
+        drag.setPixmap(cover)
+        return drag
+
+    def event_has_mods(self, event=None):
+        mods = event.modifiers() if event is not None else \
+                QApplication.keyboardModifiers()
+        return mods & Qt.ControlModifier or mods & Qt.ShiftModifier
+
+    def mousePressEvent(self, event):
+        if event.button() == Qt.LeftButton and not self.event_has_mods():
+            self.drag_start_pos = event.pos()
+        return QTableView.mousePressEvent(self, event)
+
+    def mouseMoveEvent(self, event):
+        if self.drag_start_pos is None:
+            return QTableView.mouseMoveEvent(self, event)
+
+        if self.event_has_mods():
+            self.drag_start_pos = None
+            return
+
+        if not (event.buttons() & Qt.LeftButton) or \
+                (event.pos() - self.drag_start_pos).manhattanLength() \
+                      < QApplication.startDragDistance():
+            return
+
+        index = self.indexAt(event.pos())
+        if not index.isValid():
+            return
+        drag = self.drag_data()
+        drag.exec_(Qt.CopyAction)
+        self.drag_start_pos = None
+
     def dragEnterEvent(self, event):
         if int(event.possibleActions() & Qt.CopyAction) + \
            int(event.possibleActions() & Qt.MoveAction) == 0:
@@ -485,29 +586,29 @@ class BooksView(QTableView): # {{{
         Select rows identified by identifiers. identifiers can be a set of ids,
         row numbers or QModelIndexes.
         '''
-        selmode = self.selectionMode()
-        self.setSelectionMode(QAbstractItemView.MultiSelection)
-        try:
-            rows = set([x.row() if hasattr(x, 'row') else x for x in
-                identifiers])
-            if using_ids:
-                rows = set([])
-                identifiers = set(identifiers)
-                m = self.model()
-                for row in range(m.rowCount(QModelIndex())):
-                    if m.id(row) in identifiers:
-                        rows.add(row)
-            if rows:
-                row = list(sorted(rows))[0]
-                if change_current:
-                    self.set_current_row(row, select=False)
-                if scroll:
-                    self.scroll_to_row(row)
-            self.clearSelection()
-            for r in rows:
-                self.selectRow(r)
-        finally:
-            self.setSelectionMode(selmode)
+        rows = set([x.row() if hasattr(x, 'row') else x for x in
+            identifiers])
+        if using_ids:
+            rows = set([])
+            identifiers = set(identifiers)
+            m = self.model()
+            for row in xrange(m.rowCount(QModelIndex())):
+                if m.id(row) in identifiers:
+                    rows.add(row)
+        rows = list(sorted(rows))
+        if rows:
+            row = rows[0]
+            if change_current:
+                self.set_current_row(row, select=False)
+            if scroll:
+                self.scroll_to_row(row)
+        sm = self.selectionModel()
+        sel = QItemSelection()
+        m = self.model()
+        max_col = m.columnCount(QModelIndex()) - 1
+        for row in rows:
+            sel.select(m.index(row, 0), m.index(row, max_col))
+        sm.select(sel, sm.ClearAndSelect)
 
     def close(self):
         self._model.close()
@@ -544,6 +645,21 @@ class DeviceBooksView(BooksView): # {{{
         self.setDragDropMode(self.NoDragDrop)
         self.setAcceptDrops(False)
 
+    def drag_data(self):
+        m = self.model()
+        rows = self.selectionModel().selectedRows()
+        paths = [force_unicode(p, enc=filesystem_encoding) for p in m.paths(rows) if p]
+        md = QMimeData()
+        md.setData('application/calibre+from_device', 'dummy')
+        md.setUrls([QUrl.fromLocalFile(p) for p in paths])
+        drag = QDrag(self)
+        drag.setMimeData(md)
+        cover = self.drag_icon(m.cover(self.currentIndex().row()), len(paths) >
+                1)
+        drag.setHotSpot(QPoint(-15, -15))
+        drag.setPixmap(cover)
+        return drag
+
     def contextMenuEvent(self, event):
         edit_collections = callable(getattr(self._model.db, 'supports_collections', None)) and \
             self._model.db.supports_collections() and \

src/calibre/gui2/main.py

@@ -233,8 +233,7 @@ class GuiRunner(QObject):
     def show_splash_screen(self):
         self.splash_pixmap = QPixmap()
         self.splash_pixmap.load(I('library.png'))
-        self.splash_screen = QSplashScreen(self.splash_pixmap,
-            Qt.SplashScreen)
+        self.splash_screen = QSplashScreen(self.splash_pixmap)
         self.splash_screen.showMessage(_('Starting %s: Loading books...') %
                 __appname__)
         self.splash_screen.show()

src/calibre/gui2/metadata.py

@@ -9,51 +9,59 @@ __docformat__ = 'restructuredtext en'
 import traceback
 from threading import Thread
 from Queue import Queue, Empty
+from functools import partial
 
+from PyQt4.Qt import QObject, Qt, pyqtSignal, QTimer, QDialog, \
+        QVBoxLayout, QTextBrowser, QLabel, QGroupBox, QDialogButtonBox
 
 from calibre.ebooks.metadata.fetch import search, get_social_metadata
-from calibre.gui2 import config
+from calibre.gui2 import config, error_dialog
+from calibre.gui2.dialogs.progress import ProgressDialog
 from calibre.ebooks.metadata.covers import download_cover
 from calibre.customize.ui import get_isbndb_key
-from calibre import prints
-from calibre.constants import DEBUG
 
 class Worker(Thread):
+    'Cover downloader'
 
     def __init__(self):
         Thread.__init__(self)
-        self.setDaemon(True)
+        self.daemon = True
         self.jobs = Queue()
         self.results = Queue()
 
     def run(self):
         while True:
-            mi = self.jobs.get()
+            id, mi = self.jobs.get()
             if not getattr(mi, 'isbn', False):
                 break
             try:
                 cdata, errors = download_cover(mi)
                 if cdata:
-                    self.results.put((mi.isbn, cdata))
-                elif DEBUG:
-                    prints('Cover download failed:', errors)
+                    self.results.put((id, mi, True, cdata))
+                else:
+                    msg = []
+                    for e in errors:
+                        if not e[0]:
+                            msg.append(e[-1] + ' - ' + e[1])
+                    self.results.put((id, mi, False, '\n'.join(msg)))
             except:
-                traceback.print_exc()
+                self.results.put((id, mi, False, traceback.format_exc()))
 
     def __enter__(self):
         self.start()
         return self
 
     def __exit__(self, *args):
-        self.jobs.put(False)
+        self.jobs.put((False, False))
 
 
 class DownloadMetadata(Thread):
+    'Metadata downloader'
 
     def __init__(self, db, ids, get_covers, set_metadata=True,
             get_social_metadata=True):
         Thread.__init__(self)
-        self.setDaemon(True)
+        self.daemon = True
         self.metadata = {}
         self.covers   = {}
         self.set_metadata = set_metadata
@@ -63,34 +71,42 @@ class DownloadMetadata(Thread):
         self.updated = set([])
         self.get_covers = get_covers
         self.worker = Worker()
+        self.results = Queue()
+        self.keep_going = True
         for id in ids:
             self.metadata[id] = db.get_metadata(id, index_is_id=True)
             self.metadata[id].rating = None
+        self.total = len(ids)
+        if self.get_covers:
+            self.total += len(ids)
+        self.fetched_metadata = {}
+        self.fetched_covers = {}
+        self.failures = {}
+        self.cover_failures = {}
+        self.exception = self.tb = None
 
     def run(self):
-        self.exception = self.tb = None
         try:
             self._run()
         except Exception, e:
             self.exception = e
-            import traceback
             self.tb = traceback.format_exc()
 
     def _run(self):
         self.key = get_isbndb_key()
         if not self.key:
             self.key = None
-        self.fetched_metadata = {}
-        self.failures = {}
         with self.worker:
             for id, mi in self.metadata.items():
+                if not self.keep_going:
+                    break
                 args = {}
                 if mi.isbn:
                     args['isbn'] = mi.isbn
                 else:
-                    if not mi.title or mi.title == _('Unknown'):
+                    if mi.is_null('title'):
                         self.failures[id] = \
-                                (str(id), _('Book has neither title nor ISBN'))
+                                _('Book has neither title nor ISBN')
                         continue
                     args['title'] = mi.title
                     if mi.authors and mi.authors[0] != _('Unknown'):
@@ -101,8 +117,11 @@ class DownloadMetadata(Thread):
                 if results:
                     fmi = results[0]
                     self.fetched_metadata[id] = fmi
-                    if fmi.isbn and self.get_covers:
-                        self.worker.jobs.put(fmi)
+                    if self.get_covers:
+                        if fmi.isbn:
+                            self.worker.jobs.put((id, fmi))
+                        else:
+                            self.results.put((id, 'cover', False, mi.title))
                     if (not config['overwrite_author_title_metadata']):
                         fmi.authors = mi.authors
                         fmi.author_sort = mi.author_sort
@@ -114,42 +133,169 @@ class DownloadMetadata(Thread):
                             mi.rating *= 2
                     if not self.get_social_metadata:
                         mi.tags = []
+                    self.results.put((id, 'metadata', True, mi.title))
                 else:
-                    self.failures[id] = (mi.title,
-                        _('No matches found for this book'))
+                    self.failures[id] = _('No matches found for this book')
+                    self.results.put((id, 'metadata', False, mi.title))
+                    self.results.put((id, 'cover', False, mi.title))
                 self.commit_covers()
 
         self.commit_covers(True)
-        for id in self.fetched_metadata:
-            mi = self.metadata[id]
-            if self.set_metadata:
-                self.db.set_metadata(id, mi)
-            if not self.set_metadata and self.get_social_metadata:
-                if mi.rating:
-                    self.db.set_rating(id, mi.rating)
-                if mi.tags:
-                    self.db.set_tags(id, mi.tags)
-                if mi.comments:
-                    self.db.set_comment(id, mi.comments)
-                if mi.series:
-                    self.db.set_series(id, mi.series)
-                    if mi.series_index is not None:
-                        self.db.set_series_index(id, mi.series_index)
-
-        self.updated = set(self.fetched_metadata)
-
 
     def commit_covers(self, all=False):
         if all:
-            self.worker.jobs.put(False)
+            self.worker.jobs.put((False, False))
         while True:
             try:
-                isbn, cdata = self.worker.results.get(False)
-                for id, mi in self.metadata.items():
-                    if mi.isbn == isbn:
-                        self.db.set_cover(id, cdata)
+                id, fmi, ok, cdata = self.worker.results.get_nowait()
+                if ok:
+                    self.fetched_covers[id] = cdata
+                    self.results.put((id, 'cover', ok, fmi.title))
+                else:
+                    self.results.put((id, 'cover', ok, fmi.title))
+                    try:
+                        self.cover_failures[id] = unicode(cdata)
+                    except:
+                        self.cover_failures[id] = repr(cdata)
             except Empty:
                 if not all or not self.worker.is_alive():
                     return
 
+class DoDownload(QObject):
+
+    idle_process = pyqtSignal()
+
+    def __init__(self, parent, title, db, ids, get_covers, set_metadata=True,
+            get_social_metadata=True):
+        QObject.__init__(self, parent)
+        self.pd = ProgressDialog(title, min=0, max=0, parent=parent)
+        self.pd.canceled_signal.connect(self.cancel)
+        self.idle_process.connect(self.do_one, type=Qt.QueuedConnection)
+        self.downloader = None
+        self.create = partial(DownloadMetadata, db, ids, get_covers,
+                set_metadata=set_metadata,
+                get_social_metadata=get_social_metadata)
+        self.timer = QTimer(self)
+        self.timer.timeout.connect(self.do_one, type=Qt.QueuedConnection)
+        self.db = db
+        self.updated = set([])
+        self.total = len(ids)
+
+    def exec_(self):
+        self.timer.start(50)
+        ret = self.pd.exec_()
+        if getattr(self.downloader, 'exception', None) is not None and \
+                ret == self.pd.Accepted:
+            error_dialog(self.parent(), _('Failed'),
+                    _('Failed to download metadata'), show=True)
+        else:
+            self.show_report()
+        return ret
+
+    def cancel(self, *args):
+        self.timer.stop()
+        self.downloader.keep_going = False
+        self.pd.reject()
+
+    def do_one(self):
+        if self.downloader is None:
+            self.downloader = self.create()
+            self.downloader.start()
+            self.pd.set_min(0)
+            self.pd.set_max(self.downloader.total)
+        try:
+            r = self.downloader.results.get_nowait()
+            self.handle_result(r)
+        except Empty:
+            pass
+        if not self.downloader.is_alive():
+            self.timer.stop()
+            while True:
+                try:
+                    r = self.downloader.results.get_nowait()
+                    self.handle_result(r)
+                except Empty:
+                    break
+            self.pd.accept()
+
+    def handle_result(self, r):
+        id_, typ, ok, title = r
+        what = _('cover') if typ == 'cover' else _('metadata')
+        which = _('Downloaded') if ok else _('Failed to get')
+        self.pd.set_msg(_('%s %s for: %s') % (which, what, title))
+        self.pd.value += 1
+        if ok:
+            self.updated.add(id_)
+            if typ == 'cover':
+                try:
+                    self.db.set_cover(id_,
+                            self.downloader.fetched_covers.pop(id_))
+                except:
+                    self.downloader.cover_failures[id_] = \
+                            traceback.format_exc()
+            else:
+                try:
+                    self.set_metadata(id_)
+                except:
+                    self.downloader.failures[id_] = \
+                            traceback.format_exc()
+
+    def set_metadata(self, id_):
+        mi = self.downloader.metadata[id_]
+        if self.downloader.set_metadata:
+            self.db.set_metadata(id_, mi)
+        if not self.downloader.set_metadata and self.downloader.get_social_metadata:
+            if mi.rating:
+                self.db.set_rating(id_, mi.rating)
+            if mi.tags:
+                self.db.set_tags(id_, mi.tags)
+            if mi.comments:
+                self.db.set_comment(id_, mi.comments)
+            if mi.series:
+                self.db.set_series(id_, mi.series)
+                if mi.series_index is not None:
+                    self.db.set_series_index(id_, mi.series_index)
+
+    def show_report(self):
+        f, cf = self.downloader.failures, self.downloader.cover_failures
+        report = []
+        if f:
+            report.append(
+                '<h3>Failed to download metadata for the following:</h3><ol>')
+            for id_, err in f.items():
+                mi = self.downloader.metadata[id_]
+                report.append('<li><b>%s</b><pre>%s</pre></li>' % (mi.title,
+                    unicode(err)))
+            report.append('</ol>')
+        if cf:
+            report.append(
+                '<h3>Failed to download cover for the following:</h3><ol>')
+            for id_, err in cf.items():
+                mi = self.downloader.metadata[id_]
+                report.append('<li><b>%s</b><pre>%s</pre></li>' % (mi.title,
+                    unicode(err)))
+            report.append('</ol>')
+
+        if len(self.updated) != self.total or report:
+            d = QDialog(self.parent())
+            bb = QDialogButtonBox(QDialogButtonBox.Ok, parent=d)
+            v1 = QVBoxLayout()
+            d.setLayout(v1)
+            d.setWindowTitle(_('Done'))
+            v1.addWidget(QLabel(_('Successfully downloaded metadata for %d out of %d books') %
+                (len(self.updated), self.total)))
+            gb = QGroupBox(_('Details'), self.parent())
+            v2 = QVBoxLayout()
+            gb.setLayout(v2)
+            b = QTextBrowser(self.parent())
+            v2.addWidget(b)
+            b.setHtml('\n'.join(report))
+            v1.addWidget(gb)
+            v1.addWidget(bb)
+            bb.accepted.connect(d.accept)
+            d.resize(800, 600)
+            d.exec_()
+
+
+
 

src/calibre/gui2/preferences/columns.py

@@ -21,7 +21,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
     def genesis(self, gui):
         self.gui = gui
         db = self.gui.library_view.model().db
-        self.custcols = copy.deepcopy(db.field_metadata.get_custom_field_metadata())
+        self.custcols = copy.deepcopy(db.field_metadata.custom_field_metadata())
 
         self.column_up.clicked.connect(self.up_column)
         self.column_down.clicked.connect(self.down_column)

src/calibre/gui2/preferences/create_custom_column.py

@@ -38,6 +38,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
                         'is_multiple':False},
                     8:{'datatype':'bool',
                         'text':_('Yes/No'), 'is_multiple':False},
+                    9:{'datatype':'composite',
+                        'text':_('Column built from other columns'), 'is_multiple':False},
                 }
 
     def __init__(self, parent, editing, standard_colheads, standard_colnames):
@@ -80,12 +82,15 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
         ct = c['datatype'] if not c['is_multiple'] else '*text'
         self.orig_column_number = c['colnum']
         self.orig_column_name = col
-        column_numbers = dict(map(lambda x:(self.column_types[x]['datatype'], x), self.column_types))
+        column_numbers = dict(map(lambda x:(self.column_types[x]['datatype'], x),
+                                  self.column_types))
         self.column_type_box.setCurrentIndex(column_numbers[ct])
         self.column_type_box.setEnabled(False)
         if ct == 'datetime':
             if c['display'].get('date_format', None):
                 self.date_format_box.setText(c['display'].get('date_format', ''))
+        elif ct == 'composite':
+            self.composite_box.setText(c['display'].get('composite_template', ''))
         self.datatype_changed()
         self.exec_()
 
@@ -94,9 +99,10 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             col_type = self.column_types[self.column_type_box.currentIndex()]['datatype']
         except:
             col_type = None
-        df_visible = col_type == 'datetime'
         for x in ('box', 'default_label', 'label'):
-            getattr(self, 'date_format_'+x).setVisible(df_visible)
+            getattr(self, 'date_format_'+x).setVisible(col_type == 'datetime')
+        for x in ('box', 'default_label', 'label'):
+            getattr(self, 'composite_'+x).setVisible(col_type == 'composite')
 
 
     def accept(self):
@@ -104,9 +110,11 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
         if not col:
             return self.simple_error('', _('No lookup name was provided'))
         if re.match('^\w*$', col) is None or not col[0].isalpha() or col.lower() != col:
-            return self.simple_error('', _('The lookup name must contain only lower case letters, digits and underscores, and start with a letter'))
+            return self.simple_error('', _('The lookup name must contain only '
+                    'lower case letters, digits and underscores, and start with a letter'))
         if col.endswith('_index'):
-            return self.simple_error('', _('Lookup names cannot end with _index, because these names are reserved for the index of a series column.'))
+            return self.simple_error('', _('Lookup names cannot end with _index, '
+                    'because these names are reserved for the index of a series column.'))
         col_heading = unicode(self.column_heading_box.text())
         col_type = self.column_types[self.column_type_box.currentIndex()]['datatype']
         if col_type == '*text':
@@ -118,14 +126,17 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             return self.simple_error('', _('No column heading was provided'))
         bad_col = False
         if col in self.parent.custcols:
-            if not self.editing_col or self.parent.custcols[col]['colnum'] != self.orig_column_number:
+            if not self.editing_col or \
+                    self.parent.custcols[col]['colnum'] != self.orig_column_number:
                 bad_col = True
         if bad_col:
             return self.simple_error('', _('The lookup name %s is already used')%col)
+
         bad_head = False
         for t in self.parent.custcols:
             if self.parent.custcols[t]['name'] == col_heading:
-                if not self.editing_col or self.parent.custcols[t]['colnum'] != self.orig_column_number:
+                if not self.editing_col or \
+                        self.parent.custcols[t]['colnum'] != self.orig_column_number:
                     bad_head = True
         for t in self.standard_colheads:
             if self.standard_colheads[t] == col_heading:
@@ -133,12 +144,18 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
         if bad_head:
             return self.simple_error('', _('The heading %s is already used')%col_heading)
 
-        date_format = {}
+        display_dict = {}
         if col_type == 'datetime':
             if self.date_format_box.text():
-                date_format = {'date_format':unicode(self.date_format_box.text())}
+                display_dict = {'date_format':unicode(self.date_format_box.text())}
             else:
-                date_format = {'date_format': None}
+                display_dict = {'date_format': None}
+
+        if col_type == 'composite':
+            if not self.composite_box.text():
+                return self.simple_error('', _('You must enter a template for'
+                    ' composite columns'))
+            display_dict = {'composite_template':unicode(self.composite_box.text())}
 
         db = self.parent.gui.library_view.model().db
         key = db.field_metadata.custom_field_prefix+col
@@ -148,8 +165,7 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
                     'label':col,
                     'name':col_heading,
                     'datatype':col_type,
-                    'editable':True,
-                    'display':date_format,
+                    'display':display_dict,
                     'normalized':None,
                     'colnum':None,
                     'is_multiple':is_multiple,
@@ -164,7 +180,7 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             item.setText(col_heading)
             self.parent.custcols[self.orig_column_name]['label'] = col
             self.parent.custcols[self.orig_column_name]['name'] = col_heading
-            self.parent.custcols[self.orig_column_name]['display'].update(date_format)
+            self.parent.custcols[self.orig_column_name]['display'].update(display_dict)
             self.parent.custcols[self.orig_column_name]['*edited'] = True
             self.parent.custcols[self.orig_column_name]['*must_restart'] = True
         QDialog.accept(self)

src/calibre/gui2/preferences/create_custom_column.ui

@@ -147,9 +147,59 @@
          </property>
         </widget>
        </item>
+       <item row="5" column="1">
+        <layout class="QHBoxLayout" name="horizontalLayout_4">
+         <item>
+          <widget class="QLineEdit" name="composite_box">
+           <property name="sizePolicy">
+            <sizepolicy hsizetype="Maximum" vsizetype="Fixed">
+             <horstretch>0</horstretch>
+             <verstretch>0</verstretch>
+            </sizepolicy>
+           </property>
+           <property name="toolTip">
+            <string>&lt;p&gt;Field template. Uses the same syntax as save templates.</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QLabel" name="composite_default_label">
+           <property name="toolTip">
+            <string>Similar to save templates. For example, {title} {isbn}</string>
+           </property>
+           <property name="text">
+            <string>Default: (nothing)</string>
+           </property>
+          </widget>
+         </item>
+        </layout>
+       </item>
+       <item row="5" column="0">
+        <widget class="QLabel" name="composite_label">
+         <property name="text">
+          <string>&amp;Template</string>
+         </property>
+         <property name="buddy">
+          <cstring>composite_box</cstring>
+         </property>
+        </widget>
+       </item>
+       <item row="10" column="0" colspan="3">
+        <spacer name="verticalSpacer_2">
+         <property name="orientation">
+          <enum>Qt::Vertical</enum>
+         </property>
+         <property name="sizeHint" stdset="0">
+          <size>
+           <width>20</width>
+           <height>40</height>
+          </size>
+         </property>
+        </spacer>
+       </item>
       </layout>
      </item>
-     <item row="3" column="0">
+     <item row="11" column="0">
       <widget class="QDialogButtonBox" name="button_box">
        <property name="orientation">
         <enum>Qt::Horizontal</enum>
@@ -184,6 +234,7 @@
   <tabstop>column_heading_box</tabstop>
   <tabstop>column_type_box</tabstop>
   <tabstop>date_format_box</tabstop>
+  <tabstop>composite_box</tabstop>
   <tabstop>button_box</tabstop>
  </tabstops>
  <resources/>

src/calibre/gui2/preferences/misc.py

@@ -5,80 +5,14 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-from PyQt4.Qt import QProgressDialog, QThread, Qt, pyqtSignal
 
 from calibre.gui2.preferences import ConfigWidgetBase, test_widget
 from calibre.gui2.preferences.misc_ui import Ui_Form
-from calibre.gui2 import error_dialog, config, warning_dialog, \
-        open_local_file, info_dialog
+from calibre.gui2 import error_dialog, config, open_local_file, info_dialog
 from calibre.constants import isosx
 
 # Check Integrity {{{
 
-class VacThread(QThread):
-
-    check_done = pyqtSignal(object, object)
-    callback   = pyqtSignal(object, object)
-
-    def __init__(self, parent, db):
-        QThread.__init__(self, parent)
-        self.db = db
-        self._parent = parent
-
-    def run(self):
-        err = bad = None
-        try:
-            bad = self.db.check_integrity(self.callbackf)
-        except:
-            import traceback
-            err = traceback.format_exc()
-        self.check_done.emit(bad, err)
-
-    def callbackf(self, progress, msg):
-        self.callback.emit(progress, msg)
-
-
-class CheckIntegrity(QProgressDialog):
-
-    def __init__(self, db, parent=None):
-        QProgressDialog.__init__(self, parent)
-        self.db = db
-        self.setCancelButton(None)
-        self.setMinimum(0)
-        self.setMaximum(100)
-        self.setWindowTitle(_('Checking database integrity'))
-        self.setAutoReset(False)
-        self.setValue(0)
-
-        self.vthread = VacThread(self, db)
-        self.vthread.check_done.connect(self.check_done,
-                type=Qt.QueuedConnection)
-        self.vthread.callback.connect(self.callback, type=Qt.QueuedConnection)
-        self.vthread.start()
-
-    def callback(self, progress, msg):
-        self.setLabelText(msg)
-        self.setValue(int(100*progress))
-
-    def check_done(self, bad, err):
-        if err:
-            error_dialog(self, _('Error'),
-                    _('Failed to check database integrity'),
-                    det_msg=err, show=True)
-        elif bad:
-            titles = [self.db.title(x, index_is_id=True) for x in bad]
-            det_msg = '\n'.join(titles)
-            warning_dialog(self, _('Some inconsistencies found'),
-                    _('The following books had formats listed in the '
-                        'database that are not actually available. '
-                        'The entries for the formats have been removed. '
-                        'You should check them manually. This can '
-                        'happen if you manipulate the files in the '
-                        'library folder directly.'), det_msg=det_msg, show=True)
-        self.reset()
-
-# }}}
-
 class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
     def genesis(self, gui):
@@ -87,7 +21,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('worker_limit', config, restart_required=True)
         r('enforce_cpu_limit', config, restart_required=True)
         self.device_detection_button.clicked.connect(self.debug_device_detection)
-        self.compact_button.clicked.connect(self.compact)
         self.button_open_config_dir.clicked.connect(self.open_config_dir)
         self.button_osx_symlinks.clicked.connect(self.create_symlinks)
         self.button_osx_symlinks.setVisible(isosx)
@@ -97,10 +30,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         d = DebugDevice(self)
         d.exec_()
 
-    def compact(self, *args):
-        d = CheckIntegrity(self.gui.library_view.model().db, self)
-        d.exec_()
-
     def open_config_dir(self, *args):
         from calibre.utils.config import config_dir
         open_local_file(config_dir)

src/calibre/gui2/preferences/misc.ui

@@ -77,13 +77,6 @@
      </property>
     </spacer>
    </item>
-   <item row="5" column="0" colspan="2">
-    <widget class="QPushButton" name="compact_button">
-     <property name="text">
-      <string>&amp;Check database integrity</string>
-     </property>
-    </widget>
-   </item>
    <item row="6" column="0">
     <spacer name="verticalSpacer_7">
      <property name="orientation">
@@ -124,7 +117,7 @@
      </property>
     </widget>
    </item>
-   <item row="10" column="0">
+   <item row="20" column="0">
     <spacer name="verticalSpacer_9">
      <property name="orientation">
       <enum>Qt::Vertical</enum>
@@ -132,7 +125,7 @@
      <property name="sizeHint" stdset="0">
       <size>
        <width>20</width>
-       <height>18</height>
+       <height>1000</height>
       </size>
      </property>
     </spacer>

src/calibre/gui2/preferences/plugboard.py

@@ -0,0 +1,323 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from PyQt4 import QtGui
+from PyQt4.Qt import Qt
+
+from calibre.gui2 import error_dialog
+from calibre.gui2.device import device_name_for_plugboards
+from calibre.gui2.preferences import ConfigWidgetBase, test_widget
+from calibre.gui2.preferences.plugboard_ui import Ui_Form
+from calibre.customize.ui import metadata_writers, device_plugins
+from calibre.library.save_to_disk import plugboard_any_format_value, \
+                        plugboard_any_device_value, plugboard_save_to_disk_value
+from calibre.utils.formatter import validation_formatter
+
+class ConfigWidget(ConfigWidgetBase, Ui_Form):
+
+    def genesis(self, gui):
+        self.gui = gui
+        self.db = gui.library_view.model().db
+        self.current_plugboards = self.db.prefs.get('plugboards',{})
+        self.current_device = None
+        self.current_format = None
+
+    def initialize(self):
+        def field_cmp(x, y):
+            if x.startswith('#'):
+                if y.startswith('#'):
+                    return cmp(x.lower(), y.lower())
+                else:
+                    return 1
+            elif y.startswith('#'):
+                return -1
+            else:
+                return cmp(x.lower(), y.lower())
+
+        ConfigWidgetBase.initialize(self)
+
+        if self.gui.device_manager.connected_device is not None:
+            self.device_label.setText(_('Device currently connected: ') +
+                    self.gui.device_manager.connected_device.__class__.__name__)
+        else:
+            self.device_label.setText(_('Device currently connected: None'))
+
+        self.devices = ['', 'APPLE', 'FOLDER_DEVICE']
+        for device in device_plugins():
+            n = device_name_for_plugboards(device)
+            if n not in self.devices:
+                self.devices.append(n)
+        self.devices.sort(cmp=lambda x, y: cmp(x.lower(), y.lower()))
+        self.devices.insert(1, plugboard_save_to_disk_value)
+        self.devices.insert(2, plugboard_any_device_value)
+        self.new_device.addItems(self.devices)
+
+        self.formats = ['']
+        for w in metadata_writers():
+            for f in w.file_types:
+                self.formats.append(f)
+        self.formats.append('device_db')
+        self.formats.sort()
+        self.formats.insert(1, plugboard_any_format_value)
+        self.new_format.addItems(self.formats)
+
+        self.dest_fields = ['',
+                            'authors', 'author_sort', 'language', 'publisher',
+                            'tags', 'title', 'title_sort']
+
+        self.source_widgets = []
+        self.dest_widgets = []
+        for i in range(0, len(self.dest_fields)-1):
+            w = QtGui.QLineEdit(self)
+            self.source_widgets.append(w)
+            self.fields_layout.addWidget(w, 5+i, 0, 1, 1)
+            w = QtGui.QComboBox(self)
+            self.dest_widgets.append(w)
+            self.fields_layout.addWidget(w, 5+i, 1, 1, 1)
+
+        self.edit_device.currentIndexChanged[str].connect(self.edit_device_changed)
+        self.edit_format.currentIndexChanged[str].connect(self.edit_format_changed)
+        self.new_device.currentIndexChanged[str].connect(self.new_device_changed)
+        self.new_format.currentIndexChanged[str].connect(self.new_format_changed)
+        self.existing_plugboards.itemClicked.connect(self.existing_pb_clicked)
+        self.ok_button.clicked.connect(self.ok_clicked)
+        self.del_button.clicked.connect(self.del_clicked)
+
+        self.refilling = False
+        self.refill_all_boxes()
+
+    def clear_fields(self, edit_boxes=False, new_boxes=False):
+        self.ok_button.setEnabled(False)
+        self.del_button.setEnabled(False)
+        for w in self.source_widgets:
+            w.clear()
+        for w in self.dest_widgets:
+            w.clear()
+        if edit_boxes:
+            self.edit_device.setCurrentIndex(0)
+            self.edit_format.setCurrentIndex(0)
+        if new_boxes:
+            self.new_device.setCurrentIndex(0)
+            self.new_format.setCurrentIndex(0)
+
+    def set_fields(self):
+        self.ok_button.setEnabled(True)
+        self.del_button.setEnabled(True)
+        for w in self.source_widgets:
+            w.clear()
+        for w in self.dest_widgets:
+            w.addItems(self.dest_fields)
+
+    def set_field(self, i, src, dst):
+        self.source_widgets[i].setText(src)
+        idx = self.dest_fields.index(dst)
+        self.dest_widgets[i].setCurrentIndex(idx)
+
+    def edit_device_changed(self, txt):
+        self.current_device = None
+        if txt == '':
+            self.clear_fields(new_boxes=False)
+            return
+        self.clear_fields(new_boxes=True)
+        self.current_device = unicode(txt)
+        fpb = self.current_plugboards.get(self.current_format, None)
+        if fpb is None:
+            print 'edit_device_changed: none format!'
+            return
+        dpb = fpb.get(self.current_device, None)
+        if dpb is None:
+            print 'edit_device_changed: none device!'
+            return
+        self.set_fields()
+        for i,op in enumerate(dpb):
+            self.set_field(i, op[0], op[1])
+        self.ok_button.setEnabled(True)
+        self.del_button.setEnabled(True)
+
+    def edit_format_changed(self, txt):
+        self.edit_device.setCurrentIndex(0)
+        self.current_device = None
+        self.current_format = None
+        if txt == '':
+            self.clear_fields(new_boxes=False)
+            return
+        self.clear_fields(new_boxes=True)
+        txt = unicode(txt)
+        fpb = self.current_plugboards.get(txt, None)
+        if fpb is None:
+            print 'edit_format_changed: none editable format!'
+            return
+        self.current_format = txt
+        devices = ['']
+        for d in fpb:
+            devices.append(d)
+        self.edit_device.clear()
+        self.edit_device.addItems(devices)
+
+    def new_device_changed(self, txt):
+        self.current_device = None
+        if txt == '':
+            self.clear_fields(edit_boxes=False)
+            return
+        self.clear_fields(edit_boxes=True)
+        self.current_device = unicode(txt)
+        error = False
+        if self.current_format == plugboard_any_format_value:
+            # user specified any format.
+            for f in self.current_plugboards:
+                devs = set(self.current_plugboards[f])
+                if self.current_device != plugboard_save_to_disk_value and \
+                        plugboard_any_device_value in devs:
+                    # specific format/any device in list. conflict.
+                    # note: any device does not match save_to_disk
+                    error = True
+                    break
+                if self.current_device in devs:
+                    # specific format/current device in list. conflict
+                    error = True
+                    break
+                if self.current_device == plugboard_any_device_value:
+                    # any device and a specific device already there. conflict
+                    error = True
+                    break
+        else:
+            # user specified specific format.
+            for f in self.current_plugboards:
+                devs = set(self.current_plugboards[f])
+                if f == plugboard_any_format_value and \
+                                self.current_device in devs:
+                    # any format/same device in list. conflict.
+                    error = True
+                    break
+                if f == self.current_format and self.current_device in devs:
+                    # current format/current device in list. conflict
+                    error = True
+                    break
+                if f == self.current_format and plugboard_any_device_value in devs:
+                    # current format/any device in list. conflict
+                    error = True
+                    break
+
+        if error:
+            error_dialog(self, '',
+                     _('That format and device already has a plugboard or '
+                       'conflicts with another plugboard.'),
+                     show=True)
+            self.new_device.setCurrentIndex(0)
+            return
+        self.set_fields()
+
+    def new_format_changed(self, txt):
+        self.current_format = None
+        self.current_device = None
+        self.new_device.setCurrentIndex(0)
+        if txt:
+            self.clear_fields(edit_boxes=True)
+            self.current_format = unicode(txt)
+        else:
+            self.clear_fields(edit_boxes=False)
+
+    def ok_clicked(self):
+        pb = []
+        for i in range(0, len(self.source_widgets)):
+            s = unicode(self.source_widgets[i].text())
+            if s:
+                d = self.dest_widgets[i].currentIndex()
+                if d != 0:
+                    try:
+                        validation_formatter.validate(s)
+                    except Exception, err:
+                        error_dialog(self, _('Invalid template'),
+                                '<p>'+_('The template %s is invalid:')%s + \
+                                '<br>'+str(err), show=True)
+                        return
+                    pb.append((s, self.dest_fields[d]))
+                else:
+                    error_dialog(self, _('Invalid destination'),
+                            '<p>'+_('The destination field cannot be blank'),
+                            show=True)
+                    return
+        if len(pb) == 0:
+            if self.current_format in self.current_plugboards:
+                fpb = self.current_plugboards[self.current_format]
+                if self.current_device in fpb:
+                    del fpb[self.current_device]
+                if len(fpb) == 0:
+                    del self.current_plugboards[self.current_format]
+        else:
+            if self.current_format not in self.current_plugboards:
+                self.current_plugboards[self.current_format] = {}
+            fpb = self.current_plugboards[self.current_format]
+            fpb[self.current_device] = pb
+        self.changed_signal.emit()
+        self.refill_all_boxes()
+
+    def del_clicked(self):
+        if self.current_format in self.current_plugboards:
+            fpb = self.current_plugboards[self.current_format]
+            if self.current_device in fpb:
+                del fpb[self.current_device]
+            if len(fpb) == 0:
+                del self.current_plugboards[self.current_format]
+        self.changed_signal.emit()
+        self.refill_all_boxes()
+
+    def existing_pb_clicked(self, Qitem):
+        item = Qitem.data(Qt.UserRole).toPyObject()
+        self.edit_format.setCurrentIndex(self.edit_format.findText(item[0]))
+        self.edit_device.setCurrentIndex(self.edit_device.findText(item[1]))
+
+    def refill_all_boxes(self):
+        if self.refilling:
+            return
+        self.refilling = True
+        self.current_device = None
+        self.current_format = None
+        self.clear_fields(new_boxes=True)
+        self.edit_format.clear()
+        self.edit_format.addItem('')
+        for format in self.current_plugboards:
+            self.edit_format.addItem(format)
+        self.edit_format.setCurrentIndex(0)
+        self.edit_device.clear()
+        self.ok_button.setEnabled(False)
+        self.del_button.setEnabled(False)
+        self.existing_plugboards.clear()
+        for f in self.formats:
+            if f not in self.current_plugboards:
+                continue
+            for d in self.devices:
+                if d not in self.current_plugboards[f]:
+                    continue
+                ops = []
+                for op in self.current_plugboards[f][d]:
+                    ops.append('([' + op[0] + '] -> ' + op[1] + ')')
+                txt = '%s:%s = %s\n'%(f, d, ', '.join(ops))
+                item = QtGui.QListWidgetItem(txt)
+                item.setData(Qt.UserRole, (f, d))
+                self.existing_plugboards.addItem(item)
+        self.refilling = False
+
+    def restore_defaults(self):
+        ConfigWidgetBase.restore_defaults(self)
+        self.current_plugboards = {}
+        self.refill_all_boxes()
+        self.changed_signal.emit()
+
+    def commit(self):
+        self.db.prefs.set('plugboards', self.current_plugboards)
+        return ConfigWidgetBase.commit(self)
+
+    def refresh_gui(self, gui):
+        pass
+
+
+if __name__ == '__main__':
+    from PyQt4.Qt import QApplication
+    app = QApplication([])
+    test_widget('Import/Export', 'plugboards')
+

src/calibre/gui2/preferences/plugboard.ui

@@ -0,0 +1,234 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Form</class>
+ <widget class="QWidget" name="Form">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>931</width>
+    <height>389</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Form</string>
+  </property>
+  <layout class="QGridLayout" name="gridLayout">
+   <item row="0" column="0" colspan="2">
+    <widget class="QLabel" name="label">
+     <property name="text">
+      <string>Here you can change the metadata calibre uses to update a book when saving to disk or sending to device.
+
+Use this dialog to define a 'plugboard' for a format (or all formats) and a device (or all devices). The plugboard specifies what template is connected to what field. The template is used to compute a value, and that value is assigned to the connected field.
+
+Often templates will contain simple references to composite columns, but this is not necessary. You can use any template in a source box that you can use elsewhere in calibre.
+
+One possible use for a plugboard is to alter the title to contain series information. Another would be to change the author sort, something that mobi users might do to force it to use the ';' that the kindle requires. A third would be to specify the language.</string>
+     </property>
+     <property name="textFormat">
+      <enum>Qt::PlainText</enum>
+     </property>
+     <property name="wordWrap">
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
+   <item row="1" column="0" colspan="2">
+    <widget class="Line" name="line">
+     <property name="orientation">
+      <enum>Qt::Horizontal</enum>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="0" colspan="2">
+    <widget class="QLabel" name="device_label"/>
+   </item>
+   <item row="3" column="0" colspan="2">
+    <widget class="Line" name="line">
+     <property name="orientation">
+      <enum>Qt::Horizontal</enum>
+     </property>
+    </widget>
+   </item>
+   <item row="4" column="0">
+    <layout class="QGridLayout" name="gridLayout_2">
+     <item row="0" column="1">
+      <widget class="QLabel" name="label_6">
+       <property name="text">
+        <string>Format (choose first)</string>
+       </property>
+       <property name="alignment">
+        <set>Qt::AlignCenter</set>
+       </property>
+      </widget>
+     </item>
+     <item row="0" column="2">
+      <widget class="QLabel" name="label_7">
+       <property name="text">
+        <string>Device (choose second)</string>
+       </property>
+       <property name="alignment">
+        <set>Qt::AlignCenter</set>
+       </property>
+      </widget>
+     </item>
+     <item row="1" column="0">
+      <widget class="QLabel" name="label_5">
+       <property name="text">
+        <string>Add new plugboard</string>
+       </property>
+      </widget>
+     </item>
+     <item row="1" column="1">
+      <widget class="QComboBox" name="new_format"/>
+     </item>
+     <item row="1" column="2">
+      <widget class="QComboBox" name="new_device"/>
+     </item>
+     <item row="2" column="0">
+      <widget class="QLabel" name="label_4">
+       <property name="text">
+        <string>Edit existing plugboard</string>
+       </property>
+      </widget>
+     </item>
+     <item row="2" column="1">
+      <widget class="QComboBox" name="edit_format"/>
+     </item>
+     <item row="2" column="2">
+      <widget class="QComboBox" name="edit_device"/>
+     </item>
+     <item row="3" column="0">
+      <widget class="QLabel" name="label_41">
+       <property name="text">
+        <string>Existing plugboards</string>
+       </property>
+       <property name="alignment">
+        <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignTop</set>
+       </property>
+      </widget>
+     </item>
+     <item row="3" column="1" colspan="2">
+      <widget class="QListWidget" name="existing_plugboards">
+       <property name="sizeIncrement">
+        <size>
+         <width>0</width>
+         <height>0</height>
+        </size>
+       </property>
+      </widget>
+     </item>
+     <item row="4" column="0">
+      <spacer name="verticalSpacer">
+       <property name="orientation">
+        <enum>Qt::Vertical</enum>
+       </property>
+       <property name="sizeHint" stdset="0">
+        <size>
+         <width>20</width>
+         <height>40</height>
+        </size>
+       </property>
+      </spacer>
+     </item>
+    </layout>
+   </item>
+   <item row="4" column="1">
+    <layout class="QGridLayout" name="fields_layout">
+     <item row="0" column="0">
+      <widget class="QLabel" name="label_2">
+       <property name="text">
+        <string>Source template</string>
+       </property>
+       <property name="alignment">
+        <set>Qt::AlignCenter</set>
+       </property>
+      </widget>
+     </item>
+     <item row="0" column="1">
+      <widget class="QLabel" name="label_3">
+       <property name="text">
+        <string>Destination field</string>
+       </property>
+       <property name="alignment">
+        <set>Qt::AlignCenter</set>
+       </property>
+      </widget>
+     </item>
+     <item row="21" column="0">
+      <spacer name="verticalSpacer_2">
+       <property name="orientation">
+        <enum>Qt::Vertical</enum>
+       </property>
+       <property name="sizeHint" stdset="0">
+        <size>
+         <width>20</width>
+         <height>40</height>
+        </size>
+       </property>
+      </spacer>
+     </item>
+     <item row="19" column="0">
+      <layout class="QHBoxLayout" name="horizontalLayout">
+       <item>
+        <spacer name="horizontalSpacer_2">
+         <property name="orientation">
+          <enum>Qt::Horizontal</enum>
+         </property>
+         <property name="sizeHint" stdset="0">
+          <size>
+           <width>40</width>
+           <height>20</height>
+          </size>
+         </property>
+        </spacer>
+       </item>
+       <item>
+        <widget class="QPushButton" name="ok_button">
+         <property name="text">
+          <string>Save plugboard</string>
+         </property>
+        </widget>
+       </item>
+       <item>
+        <spacer name="horizontalSpacer">
+         <property name="orientation">
+          <enum>Qt::Horizontal</enum>
+         </property>
+         <property name="sizeHint" stdset="0">
+          <size>
+           <width>40</width>
+           <height>20</height>
+          </size>
+         </property>
+        </spacer>
+       </item>
+       <item>
+        <widget class="QPushButton" name="del_button">
+         <property name="text">
+          <string>Delete plugboard</string>
+         </property>
+        </widget>
+       </item>
+       <item>
+        <spacer name="horizontalSpacer_3">
+         <property name="orientation">
+          <enum>Qt::Horizontal</enum>
+         </property>
+         <property name="sizeHint" stdset="0">
+          <size>
+           <width>40</width>
+           <height>20</height>
+          </size>
+         </property>
+        </spacer>
+       </item>
+      </layout>
+     </item>
+    </layout>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections/>
+</ui>

src/calibre/gui2/preferences/plugins.py

@@ -199,7 +199,11 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                     config_dialog.exec_()
 
                     if config_dialog.result() == QDialog.Accepted:
-                        plugin.save_settings(config_widget)
+                        if hasattr(config_widget, 'validate'):
+                            if config_widget.validate():
+                                plugin.save_settings(config_widget)
+                        else:
+                            plugin.save_settings(config_widget)
                         self._plugin_model.refresh_plugin(plugin)
                 else:
                     help_text = plugin.customization_help(gui=True)

src/calibre/gui2/preferences/save_template.py

@@ -10,8 +10,9 @@ from PyQt4.Qt import QWidget, pyqtSignal
 
 from calibre.gui2 import error_dialog
 from calibre.gui2.preferences.save_template_ui import Ui_Form
-from calibre.library.save_to_disk import FORMAT_ARG_DESCS, \
-        preprocess_template
+from calibre.library.save_to_disk import FORMAT_ARG_DESCS, preprocess_template
+from calibre.utils.formatter import validation_formatter
+
 
 class SaveTemplate(QWidget, Ui_Form):
 
@@ -26,8 +27,11 @@ class SaveTemplate(QWidget, Ui_Form):
         variables = sorted(FORMAT_ARG_DESCS.keys())
         rows = []
         for var in variables:
-            rows.append(u'<tr><td>%s</td><td>%s</td></tr>'%
+            rows.append(u'<tr><td>%s</td><td>&nbsp;</td><td>%s</td></tr>'%
                     (var, FORMAT_ARG_DESCS[var]))
+        rows.append(u'<tr><td>%s&nbsp;</td><td>&nbsp;</td><td>%s</td></tr>'%(
+            _('Any custom field'),
+            _('The lookup name of any custom field. These names begin with "#")')))
         table = u'<table>%s</table>'%(u'\n'.join(rows))
         self.template_variables.setText(table)
 
@@ -41,12 +45,14 @@ class SaveTemplate(QWidget, Ui_Form):
         self.changed_signal.emit()
 
     def validate(self):
+        '''
+        Do a syntax check on the format string. Doing a semantic check
+        (verifying that the fields exist) is not useful in the presence of
+        custom fields, because they may or may not exist.
+        '''
         tmpl = preprocess_template(self.opt_template.text())
-        fa = {}
-        for x in FORMAT_ARG_DESCS.keys():
-            fa[x]='random long string'
         try:
-            tmpl.format(**fa)
+            validation_formatter.validate(tmpl)
         except Exception, err:
             error_dialog(self, _('Invalid template'),
                     '<p>'+_('The template %s is invalid:')%tmpl + \

src/calibre/gui2/preferences/sending.py

@@ -22,6 +22,9 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
         r = self.register
 
+        for x in ('send_timefmt',):
+            r(x, self.proxy)
+
         choices = [(_('Manual management'), 'manual'),
                 (_('Only on send'), 'on_send'),
                 (_('Automatic management'), 'on_connect')]

src/calibre/gui2/preferences/sending.ui

@@ -80,7 +80,20 @@
      </property>
     </widget>
    </item>
-   <item row="2" column="0" colspan="3">
+   <item row="2" column="0">
+    <widget class="QLabel" name="label_2">
+     <property name="text">
+      <string>Format &amp;dates as:</string>
+     </property>
+     <property name="buddy">
+      <cstring>opt_send_timefmt</cstring>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="1">
+    <widget class="QLineEdit" name="opt_send_timefmt"/>
+   </item>
+   <item row="3" column="0" colspan="3">
     <widget class="QLabel" name="label_43">
      <property name="text">
       <string>Here you can control how calibre will save your books when you click the Send to Device button. This setting can be overriden for individual devices by customizing the device interface plugins in Preferences-&gt;Advanced-&gt;Plugins</string>
@@ -90,7 +103,7 @@
      </property>
     </widget>
    </item>
-   <item row="3" column="0" colspan="3">
+   <item row="4" column="0" colspan="3">
     <widget class="SaveTemplate" name="send_template" native="true"/>
    </item>
   </layout>

src/calibre/gui2/tag_view.py

@@ -20,6 +20,7 @@ from calibre.gui2 import config, NONE
 from calibre.library.field_metadata import TagsIcons
 from calibre.utils.search_query_parser import saved_searches
 from calibre.gui2 import error_dialog
+from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.dialogs.tag_categories import TagCategories
 from calibre.gui2.dialogs.tag_list_editor import TagListEditor
 from calibre.gui2.dialogs.edit_authors_dialog import EditAuthorsDialog
@@ -66,6 +67,7 @@ class TagsView(QTreeView): # {{{
     author_sort_edit    = pyqtSignal(object, object)
     tag_item_renamed    = pyqtSignal()
     search_item_renamed = pyqtSignal()
+    drag_drop_finished  = pyqtSignal(object)
 
     def __init__(self, parent=None):
         QTreeView.__init__(self, parent=None)
@@ -79,6 +81,9 @@ class TagsView(QTreeView): # {{{
         self.setHeaderHidden(True)
         self.setItemDelegate(TagDelegate(self))
         self.made_connections = False
+        self.setAcceptDrops(True)
+        self.setDragDropMode(self.DropOnly)
+        self.setDropIndicatorShown(True)
 
     def set_database(self, db, tag_match, sort_by):
         self.hidden_categories = config['tag_browser_hidden_categories']
@@ -104,6 +109,103 @@ class TagsView(QTreeView): # {{{
     def database_changed(self, event, ids):
         self.refresh_required.emit()
 
+    def dragEnterEvent(self, event):
+        md = event.mimeData()
+        if md.hasFormat("application/calibre+from_library"):
+            event.setDropAction(Qt.CopyAction)
+            event.accept()
+        else:
+            event.ignore()
+
+    def dragMoveEvent(self, event):
+        allowed = False
+        idx = self.indexAt(event.pos())
+        m = self.model()
+        p = m.parent(idx)
+        if idx.isValid() and p.isValid():
+            item = m.data(p, Qt.UserRole)
+            fm = self.db.metadata_for_field(item.category_key)
+            if item.category_key in \
+                    ('tags', 'series', 'authors', 'rating', 'publisher') or\
+                    (fm['is_custom'] and \
+                     fm['datatype'] in ['text', 'rating', 'series']):
+                allowed = True
+        if allowed:
+            event.acceptProposedAction()
+        else:
+            event.ignore()
+
+    def dropEvent(self, event):
+        idx = self.indexAt(event.pos())
+        m = self.model()
+        p = m.parent(idx)
+        if idx.isValid() and p.isValid():
+            item = m.data(p, Qt.UserRole)
+            if item.type == TagTreeItem.CATEGORY:
+                fm = self.db.metadata_for_field(item.category_key)
+                if item.category_key in \
+                        ('tags', 'series', 'authors', 'rating', 'publisher') or\
+                        (fm['is_custom'] and \
+                         fm['datatype'] in ['text', 'rating', 'series']):
+                    child = m.data(idx, Qt.UserRole)
+                    md = event.mimeData()
+                    mime = 'application/calibre+from_library'
+                    ids = list(map(int, str(md.data(mime)).split()))
+                    self.handle_drop(item, child, ids)
+                    event.accept()
+                    return
+        event.ignore()
+
+    def handle_drop(self, parent, child, ids):
+        # print 'Dropped ids:', ids, parent.category_key, child.tag.name
+        key = parent.category_key
+        if (key == 'authors' and len(ids) >= 5):
+            if not confirm('<p>'+_('Changing the authors for several books can '
+                           'take a while. Are you sure?')
+                        +'</p>', 'tag_browser_drop_authors', self):
+                return
+        elif len(ids) > 15:
+            if not confirm('<p>'+_('Changing the metadata for that many books '
+                           'can take a while. Are you sure?')
+                        +'</p>', 'tag_browser_many_changes', self):
+                return
+
+        fm = self.db.metadata_for_field(key)
+        is_multiple = fm['is_multiple']
+        val = child.tag.name
+        for id in ids:
+            mi = self.db.get_metadata(id, index_is_id=True)
+
+            # Prepare to ignore the author, unless it is changed. Title is
+            # always ignored -- see the call to set_metadata
+            set_authors = False
+
+            # Author_sort cannot change explicitly. Changing the author might
+            # change it.
+            mi.author_sort = None # Never will change by itself.
+
+            if key == 'authors':
+                mi.authors = [val]
+                set_authors=True
+            elif fm['datatype'] == 'rating':
+                mi.set(key, len(val) * 2)
+            elif fm['is_custom'] and fm['datatype'] == 'series':
+                mi.set(key, val, extra=1.0)
+            elif is_multiple:
+                new_val = mi.get(key, [])
+                if val in new_val:
+                    # Fortunately, only one field can change, so the continue
+                    # won't break anything
+                    continue
+                new_val.append(val)
+                mi.set(key, new_val)
+            else:
+                mi.set(key, val)
+            self.db.set_metadata(id, mi, set_title=False,
+                                 set_authors=set_authors, commit=False)
+        self.db.commit()
+        self.drag_drop_finished.emit(ids)
+
     @property
     def match_all(self):
         return self.tag_match and self.tag_match.currentIndex() > 0
@@ -326,6 +428,8 @@ class TagTreeItem(object): # {{{
         self.children.append(child)
 
     def data(self, role):
+        if role == Qt.UserRole:
+            return self
         if self.type == self.TAG:
             return self.tag_data(role)
         if self.type == self.CATEGORY:
@@ -505,7 +609,12 @@ class TagsModel(QAbstractItemModel): # {{{
         key = item.parent.category_key
         # make certain we know about the item's category
         if key not in self.db.field_metadata:
-            return
+            return False
+        if key == 'authors':
+            if val.find('&') >= 0:
+                error_dialog(self.tags_view, _('Invalid author name'),
+                        _('Author names cannot contain & characters.')).exec_()
+                return False
         if key == 'search':
             if val in saved_searches().names():
                 error_dialog(self.tags_view, _('Duplicate search name'),
@@ -539,8 +648,14 @@ class TagsModel(QAbstractItemModel): # {{{
     def headerData(self, *args):
         return NONE
 
-    def flags(self, *args):
-        return Qt.ItemIsEnabled|Qt.ItemIsSelectable|Qt.ItemIsEditable
+    def flags(self, index, *args):
+        ans = Qt.ItemIsEnabled|Qt.ItemIsSelectable|Qt.ItemIsEditable
+        if index.isValid() and self.parent(index).isValid():
+            ans |= Qt.ItemIsDropEnabled
+        return ans
+
+    def supportedDropActions(self):
+        return Qt.CopyAction|Qt.MoveAction
 
     def path_for_index(self, index):
         ans = []
@@ -671,6 +786,7 @@ class TagBrowserMixin(object): # {{{
         self.tags_view.author_sort_edit.connect(self.do_author_sort_edit)
         self.tags_view.tag_item_renamed.connect(self.do_tag_item_renamed)
         self.tags_view.search_item_renamed.connect(self.saved_searches_changed)
+        self.tags_view.drag_drop_finished.connect(self.drag_drop_finished)
         self.edit_categories.clicked.connect(lambda x:
                 self.do_user_categories_edit())
 
@@ -720,11 +836,11 @@ class TagBrowserMixin(object): # {{{
                 rename_func = partial(db.rename_custom_item, label=cc_label)
                 delete_func = partial(db.delete_custom_item_using_id, label=cc_label)
             if rename_func:
+                for item in to_delete:
+                    delete_func(item)
                 for text in to_rename:
                         for old_id in to_rename[text]:
                             rename_func(old_id, new_name=unicode(text))
-                for item in to_delete:
-                    delete_func(item)
 
             # Clean up everything, as information could have changed for many books.
             self.library_view.model().refresh()
@@ -752,6 +868,9 @@ class TagBrowserMixin(object): # {{{
             self.library_view.model().refresh()
             self.tags_view.recount()
 
+    def drag_drop_finished(self, ids):
+        self.library_view.model().refresh_ids(ids)
+
 # }}}
 
 class TagBrowserWidget(QWidget): # {{{

src/calibre/gui2/tools.py

@@ -217,9 +217,12 @@ def fetch_scheduled_recipe(arg):
     if 'output_profile' in ps:
         recs.append(('output_profile', ps['output_profile'],
             OptionRecommendation.HIGH))
-        if ps['output_profile'] == 'kindle':
-            recs.append(('no_inline_toc', True,
-                OptionRecommendation.HIGH))
+        # Disabled since apparently some people use
+        # K4PC and, surprise, surprise, it doesn't support
+        # indexed MOBIs.
+        #if ps['output_profile'] == 'kindle':
+        #    recs.append(('no_inline_toc', True,
+        #        OptionRecommendation.HIGH))
 
     lf = load_defaults('look_and_feel')
     if lf.get('base_font_size', 0.0) != 0.0:

src/calibre/gui2/ui.py

@@ -19,7 +19,7 @@ from PyQt4.Qt import Qt, SIGNAL, QTimer, \
                      QMessageBox, QHelpEvent
 
 from calibre import  prints
-from calibre.constants import __appname__, isosx
+from calibre.constants import __appname__, isosx, DEBUG
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.utils.config import prefs, dynamic
 from calibre.utils.ipc.server import Server
@@ -360,6 +360,10 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, # {{{
 
     def library_moved(self, newloc):
         if newloc is None: return
+        try:
+            olddb = self.library_view.model().db
+        except:
+            olddb = None
         db = LibraryDatabase2(newloc)
         self.library_path = newloc
         self.book_on_device(None, reset=True)
@@ -380,6 +384,19 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, # {{{
         self.apply_named_search_restriction('') # reset restriction to null
         self.saved_searches_changed() # reload the search restrictions combo box
         self.apply_named_search_restriction(db.prefs['gui_restriction'])
+        if olddb is not None:
+            try:
+                olddb.conn.close()
+            except:
+                import traceback
+                traceback.print_exc()
+        if self.device_connected:
+            self.set_books_in_library(self.booklists(), reset=True)
+            self.refresh_ondevice()
+            self.memory_view.reset()
+            self.card_a_view.reset()
+            self.card_b_view.reset()
+
 
     def set_window_title(self):
         self.setWindowTitle(__appname__ + u' - ||%s||'%self.iactions['Choose Library'].library_name())
@@ -533,6 +550,10 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, # {{{
             # Save the current field_metadata for applications like calibre2opds
             # Goes here, because if cf is valid, db is valid.
             db.prefs['field_metadata'] = db.field_metadata.all_metadata()
+            db.commit_dirty_cache()
+            if DEBUG and db.gm_count > 0:
+                print 'get_metadata cache: {0:d} calls, {1:4.2f}% misses'.format(
+                        db.gm_count, (db.gm_missed*100.0)/db.gm_count)
         for action in self.iactions.values():
             if not action.shutting_down():
                 return
@@ -548,6 +569,10 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, # {{{
         cc = self.library_view.model().cover_cache
         if cc is not None:
             cc.stop()
+        mb = self.library_view.model().metadata_backup
+        if mb is not None:
+            mb.stop()
+
         self.hide_windows()
         self.emailer.stop()
         try:
@@ -558,9 +583,11 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, # {{{
                     s.exit()
             except:
                 pass
-            time.sleep(2)
         except KeyboardInterrupt:
             pass
+        time.sleep(2)
+        if mb is not None:
+            mb.flush()
         self.hide_windows()
         return True
 

src/calibre/gui2/update.py

@@ -3,13 +3,14 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import traceback
 
-from PyQt4.Qt import QThread, pyqtSignal, Qt, QUrl
+from PyQt4.Qt import QThread, pyqtSignal, Qt, QUrl, QDialog, QGridLayout, \
+        QLabel, QCheckBox, QDialogButtonBox, QIcon, QPixmap
 import mechanize
 
 from calibre.constants import __appname__, __version__, iswindows, isosx
 from calibre import browser
 from calibre.utils.config import prefs
-from calibre.gui2 import config, dynamic, question_dialog, open_url
+from calibre.gui2 import config, dynamic, open_url
 
 URL = 'http://status.calibre-ebook.com/latest'
 
@@ -37,6 +38,53 @@ class CheckForUpdates(QThread):
                 traceback.print_exc()
             self.sleep(self.INTERVAL)
 
+class UpdateNotification(QDialog):
+
+    def __init__(self, version, parent=None):
+        QDialog.__init__(self, parent)
+        self.resize(400, 250)
+        self.l = QGridLayout()
+        self.setLayout(self.l)
+        self.logo = QLabel()
+        self.logo.setMaximumWidth(110)
+        self.logo.setPixmap(QPixmap(I('lt.png')).scaled(100, 100,
+            Qt.IgnoreAspectRatio, Qt.SmoothTransformation))
+        self.label = QLabel('<p>'+
+            _('%s has been updated to version <b>%s</b>. '
+            'See the <a href="http://calibre-ebook.com/whats-new'
+            '">new features</a>. Visit the download pa'
+            'ge?')%(__appname__, version))
+        self.label.setOpenExternalLinks(True)
+        self.label.setWordWrap(True)
+        self.setWindowTitle(_('Update available!'))
+        self.setWindowIcon(QIcon(I('lt.png')))
+        self.l.addWidget(self.logo, 0, 0)
+        self.l.addWidget(self.label, 0, 1)
+        self.cb = QCheckBox(
+            _('Show this notification for future updates'), self)
+        self.l.addWidget(self.cb, 1, 0, 1, -1)
+        self.cb.setChecked(config.get('new_version_notification'))
+        self.cb.stateChanged.connect(self.show_future)
+        self.bb = QDialogButtonBox(self)
+        b = self.bb.addButton(_('&Get update'), self.bb.AcceptRole)
+        b.setDefault(True)
+        b.setIcon(QIcon(I('arrow-down.png')))
+        self.bb.addButton(self.bb.Cancel)
+        self.l.addWidget(self.bb, 2, 0, 1, -1)
+        self.bb.accepted.connect(self.accept)
+        self.bb.rejected.connect(self.reject)
+        dynamic.set('update to version %s'%version, False)
+
+    def show_future(self, *args):
+        config.set('new_version_notification', bool(self.cb.isChecked()))
+
+    def accept(self):
+        url = 'http://calibre-ebook.com/download_'+\
+            ('windows' if iswindows else 'osx' if isosx else 'linux')
+        open_url(QUrl(url))
+
+        QDialog.accept(self)
+
 class UpdateMixin(object):
 
     def __init__(self, opts):
@@ -53,15 +101,8 @@ class UpdateMixin(object):
 
         if config.get('new_version_notification') and \
                 dynamic.get('update to version %s'%version, True):
-            if question_dialog(self, _('Update available'),
-                    _('%s has been updated to version %s. '
-                    'See the <a href="http://calibre-ebook.com/whats-new'
-                    '">new features</a>. Visit the download pa'
-                    'ge?')%(__appname__, version)):
-                url = 'http://calibre-ebook.com/download_'+\
-                    ('windows' if iswindows else 'osx' if isosx else 'linux')
-                open_url(QUrl(url))
-            dynamic.set('update to version %s'%version, False)
-
+            self._update_notification__ = UpdateNotification(version,
+                    parent=self)
+            self._update_notification__.show()
 
 

src/calibre/gui2/viewer/main.py

@@ -166,6 +166,8 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
     def __init__(self, pathtoebook=None, debug_javascript=False):
         MainWindow.__init__(self, None)
         self.setupUi(self)
+        self.show_toc_on_open = False
+        self.current_book_has_toc = False
         self.base_window_title = unicode(self.windowTitle())
         self.iterator          = None
         self.current_page      = None
@@ -214,11 +216,12 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         self.action_metadata.setCheckable(True)
         self.action_metadata.setShortcut(Qt.CTRL+Qt.Key_I)
         self.action_table_of_contents.setCheckable(True)
+        self.toc.setMinimumWidth(80)
         self.action_reference_mode.setCheckable(True)
         self.connect(self.action_reference_mode, SIGNAL('triggered(bool)'),
                      lambda x: self.view.reference_mode(x))
         self.connect(self.action_metadata, SIGNAL('triggered(bool)'), lambda x:self.metadata.setVisible(x))
-        self.connect(self.action_table_of_contents, SIGNAL('triggered(bool)'), lambda x:self.toc.setVisible(x))
+        self.connect(self.action_table_of_contents, SIGNAL('toggled(bool)'), lambda x:self.toc.setVisible(x))
         self.connect(self.action_copy, SIGNAL('triggered(bool)'), self.copy)
         self.connect(self.action_font_size_larger, SIGNAL('triggered(bool)'),
                      self.font_size_larger)
@@ -259,7 +262,6 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
             f = functools.partial(self.load_ebook, pathtoebook)
             QTimer.singleShot(50, f)
         self.view.setMinimumSize(100, 100)
-        self.splitter.setSizes([1, 300])
         self.toc.setCursor(Qt.PointingHandCursor)
         self.tool_bar.setContextMenuPolicy(Qt.PreventContextMenu)
         self.tool_bar2.setContextMenuPolicy(Qt.PreventContextMenu)
@@ -285,6 +287,12 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
     def save_state(self):
         state = str(self.saveState(self.STATE_VERSION))
         dynamic['viewer_toolbar_state'] = state
+        dynamic.set('viewer_window_geometry', self.saveGeometry())
+        if self.current_book_has_toc:
+            dynamic.set('viewer_toc_isvisible', bool(self.toc.isVisible()))
+        if self.toc.isVisible():
+            dynamic.set('viewer_splitter_state',
+                bytearray(self.splitter.saveState()))
 
     def restore_state(self):
         state = dynamic.get('viewer_toolbar_state', None)
@@ -609,10 +617,15 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
             title = self.iterator.opf.title
             if not title:
                 title = os.path.splitext(os.path.basename(pathtoebook))[0]
-            self.action_table_of_contents.setDisabled(not self.iterator.toc)
             if self.iterator.toc:
                 self.toc_model = TOC(self.iterator.toc)
                 self.toc.setModel(self.toc_model)
+                if self.show_toc_on_open:
+                    self.action_table_of_contents.setChecked(True)
+            else:
+                self.action_table_of_contents.setChecked(False)
+            self.action_table_of_contents.setDisabled(not self.iterator.toc)
+            self.current_book_has_toc = bool(self.iterator.toc)
             self.current_title = title
             self.setWindowTitle(self.base_window_title+' - '+title)
             self.pos.setMaximum(sum(self.iterator.pages))
@@ -656,22 +669,21 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         return self
 
     def __exit__(self, *args):
-        self.write_settings()
         if self.iterator is not None:
             self.save_current_position()
             self.iterator.__exit__(*args)
 
-    def write_settings(self):
-        dynamic.set('viewer_window_geometry', self.saveGeometry())
-
     def read_settings(self):
         c = config().parse()
-        wg = dynamic['viewer_window_geometry']
-        if wg is not None and c.remember_window_size:
-            self.restoreGeometry(wg)
-
-
-
+        self.splitter.setSizes([1, 300])
+        if c.remember_window_size:
+            wg = dynamic.get('viewer_window_geometry', None)
+            if wg is not None:
+                self.restoreGeometry(wg)
+            ss = dynamic.get('viewer_splitter_state', None)
+            if ss is not None:
+                self.splitter.restoreState(ss)
+            self.show_toc_on_open = dynamic.get('viewer_toc_isvisible', False)
 
 def config(defaults=None):
     desc = _('Options to control the ebook viewer')

src/calibre/gui2/wizard/__init__.py

@@ -73,6 +73,14 @@ class JetBook(Device):
     manufacturer = 'Ectaco'
     id = 'jetbook'
 
+class JetBookMini(Device):
+
+    output_profile = 'jetbook5'
+    output_format  = 'FB2'
+    name = 'JetBook Mini'
+    manufacturer = 'Ectaco'
+    id = 'jetbookmini'
+
 class KindleDX(Kindle):
 
     output_profile = 'kindle_dx'
@@ -584,12 +592,42 @@ class LibraryPage(QWizardPage, LibraryUI):
         qt_app.load_translations()
         self.emit(SIGNAL('retranslate()'))
         self.init_languages()
+        try:
+            if prefs['language'].lower().startswith('zh'):
+                from calibre.customize.ui import enable_plugin
+                for name in ('Douban Books', 'Douban.com covers'):
+                    enable_plugin(name)
+        except:
+            pass
+
+    def is_library_dir_suitable(self, x):
+        return LibraryDatabase2.exists_at(x) or not os.listdir(x)
+
+    def validatePage(self):
+        newloc = unicode(self.location.text())
+        if not self.is_library_dir_suitable(newloc):
+            self.show_library_dir_error(newloc)
+            return False
+        return True
 
     def change(self):
-        dir = choose_dir(self, 'database location dialog',
+        x = choose_dir(self, 'database location dialog',
                          _('Select location for books'))
-        if dir:
-            self.location.setText(dir)
+        if x:
+            if self.is_library_dir_suitable(x):
+                self.location.setText(x)
+            else:
+                self.show_library_dir_error(x)
+
+    def show_library_dir_error(self, x):
+        if not isinstance(x, unicode):
+            try:
+                x = x.decode(filesystem_encoding)
+            except:
+                x = unicode(repr(x))
+        error_dialog(self, _('Bad location'),
+            _('You must choose an empty folder for '
+                'the calibre library. %s is not empty.')%x, show=True)
 
     def initializePage(self):
         lp = prefs['library_path']