Sync to trunk.

resources/recipes/dallas.recipe

@@ -7,22 +7,29 @@ class DallasNews(BasicNewsRecipe):
     max_articles_per_feed = 25
 
     no_stylesheets = True
-    remove_tags_before = dict(name='h2', attrs={'class':'vitstoryheadline'})
+    use_embedded_content = False
-    remove_tags_after  = dict(name='div', attrs={'style':'width: 100%; clear: right'})
+    remove_tags_before = dict(name='h1')
-    remove_tags_after  = dict(name='div', attrs={'id':'article_tools_bottom'})
+    keep_only_tags = {'class':lambda x: x and 'article' in x}
     remove_tags = [
-       dict(name='iframe'),
+            {'class':['DMNSocialTools', 'article ', 'article first ', 'article premium']},
-       dict(name='div', attrs={'class':'biblockmore'}),
-       dict(name='div', attrs={'style':'width: 100%; clear: right'}),
-       dict(name='div', attrs={'id':'article_tools_bottom'}),
-       #dict(name='ul', attrs={'class':'articleTools'}),
     ]
 
     feeds          = [
-                      ('Latest News', 'http://www.dallasnews.com/newskiosk/rss/dallasnewslatestnews.xml'),
+                      ('Local News',
-                      ('Local News', 'http://www.dallasnews.com/newskiosk/rss/dallasnewslocalnews.xml'),
+                          'http://www.dallasnews.com/news/politics/local-politics/?rss'),
-		      ('Nation and World', 'http://www.dallasnews.com/newskiosk/rss/dallasnewsnationworld.xml'),
+                      ('National Politics',
-		      ('Politics', 'http://www.dallasnews.com/newskiosk/rss/dallasnewsnationalpolitics.xml'),
+                          'http://www.dallasnews.com/news/politics/national-politic/?rss'),
-		      ('Science', 'http://www.dallasnews.com/newskiosk/rss/dallasnewsscience.xml'),
+                      ('State Politics',
+                          'http://www.dallasnews.com/news/politics/state-politics/?rss'),
+                      ('Religion',
+                         'http://www.dallasnews.com/news/religion/?rss'),
+                      ('Crime',
+                          'http://www.dallasnews.com/news/crime/headlines/?rss'),
+                      ('Celebrity News',
+                          'http://www.dallasnews.com/entertainment/celebrity-news/?rss&listname=TopStories'),
+                      ('Nation',
+                          'http://www.dallasnews.com/news/nation-world/nation/?rss'),
+                      ('World',
+                          'http://www.dallasnews.com/news/nation-world/world/?rss'),
                     ]
 

resources/recipes/gulfnews.recipe

@@ -0,0 +1,64 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011, Darko Miletic <darko.miletic at gmail.com>'
+'''
+gulfnews.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class GulfNews(BasicNewsRecipe):
+    title                 = 'Gulf News'
+    __author__            = 'Darko Miletic'
+    description           = 'News from United Arab Emirrates, persian gulf and rest of the world'
+    publisher             = 'Al Nisr Publishing LLC'
+    category              = 'news, politics, UAE, world'
+    oldest_article        = 2
+    max_articles_per_feed = 200
+    no_stylesheets        = True
+    encoding              = 'utf8'
+    use_embedded_content  = False
+    language              = 'en'
+    remove_empty_feeds    = True
+    publication_type      = 'newsportal'
+    masthead_url          = 'http://gulfnews.com/media/img/gulf_news_logo.jpg'
+    extra_css             = """
+                               body{font-family: Arial,Helvetica,sans-serif }
+                               img{margin-bottom: 0.4em; display:block}
+                               h1{font-family: Georgia, 'Times New Roman', Times, serif}
+                               ol,ul{list-style: none}
+                               .synopsis{font-size: small}
+                               .details{font-size: x-small}
+                               .image{font-size: xx-small}
+                            """
+
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
+
+    remove_tags = [
+                     dict(name=['meta','link','object','embed'])
+                     ,dict(attrs={'class':['quickLinks','ratings']})
+                     ,dict(attrs={'id':'imageSelector'})
+                  ]
+    remove_attributes=['lang']
+    keep_only_tags=[
+                     dict(name='h1')
+                     ,dict(attrs={'class':['synopsis','details','image','article']})
+                   ]
+
+
+    feeds = [
+              (u'UAE News'      , u'http://gulfnews.com/cmlink/1.446094')
+             ,(u'Business'      , u'http://gulfnews.com/cmlink/1.446098')
+             ,(u'Entertainment' , u'http://gulfnews.com/cmlink/1.446095')
+             ,(u'Sport'         , u'http://gulfnews.com/cmlink/1.446096')
+             ,(u'Life'          , u'http://gulfnews.com/cmlink/1.446097')
+            ]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        return soup

resources/recipes/las_vegas_review.recipe

@@ -3,12 +3,17 @@ from calibre.web.feeds.news import BasicNewsRecipe
 class AdvancedUserRecipe1274742400(BasicNewsRecipe):
 
     title = u'Las Vegas Review Journal'
-    __author__ = 'Joel'
+    __author__ = 'Kovid Goyal'
     language = 'en'
 
     oldest_article = 7
 
     max_articles_per_feed = 100
+    keep_only_tags = [dict(id='content-main')]
+    remove_tags = [dict(id=['right-col-content', 'trending-topics']),
+            {'class':['ppy-outer']}
+            ]
+    no_stylesheets = True
 
     feeds = [
             (u'News', u'http://www.lvrj.com/news.rss'),

resources/recipes/lavanguardia.recipe

@@ -20,8 +20,8 @@ class LaVanguardia(BasicNewsRecipe):
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False
-    delay                 = 1
+    delay                 = 5
-    encoding              = 'cp1252'
+ #   encoding              = 'cp1252'
     language = 'es'
 
     direction             = 'ltr'
@@ -35,8 +35,8 @@ class LaVanguardia(BasicNewsRecipe):
     html2epub_options  = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"'
 
     feeds              = [
-                            (u'Ciudadanos'           , u'http://feeds.feedburner.com/lavanguardia/ciudadanos'   )
+                            (u'Portada'           , u'http://feeds.feedburner.com/lavanguardia/home'   )
-                           ,(u'Cultura'              , u'http://feeds.feedburner.com/lavanguardia/cultura'      )
+                          ,(u'Cultura'              , u'http://feeds.feedburner.com/lavanguardia/cultura'      )
                            ,(u'Deportes'             , u'http://feeds.feedburner.com/lavanguardia/deportes'     )
                            ,(u'Economia'             , u'http://feeds.feedburner.com/lavanguardia/economia'     )
                            ,(u'El lector opina'      , u'http://feeds.feedburner.com/lavanguardia/lectoropina'  )
@@ -45,17 +45,17 @@ class LaVanguardia(BasicNewsRecipe):
                            ,(u'Internet y tecnologia', u'http://feeds.feedburner.com/lavanguardia/internet'     )
                            ,(u'Motor'                , u'http://feeds.feedburner.com/lavanguardia/motor'        )
                            ,(u'Politica'             , u'http://feeds.feedburner.com/lavanguardia/politica'     )
-                           ,(u'Sucessos'             , u'http://feeds.feedburner.com/lavanguardia/sucesos'      )
+                           ,(u'Sucesos'             , u'http://feeds.feedburner.com/lavanguardia/sucesos'      )
                          ]
 
 
     keep_only_tags = [
-                       dict(name='div', attrs={'class':'element1_3'})
+                       dict(name='div', attrs={'class':'detalle  noticia'})
-                     ]
+                    ]
 
     remove_tags        = [
                              dict(name=['object','link','script'])
-                            ,dict(name='div', attrs={'class':['colC','peu']})
+                            ,dict(name='div', attrs={'class':['colC','peu','jstoolbar']})
                          ]
 
     remove_tags_after = [dict(name='div', attrs={'class':'text'})]
@@ -67,4 +67,3 @@ class LaVanguardia(BasicNewsRecipe):
         for item in soup.findAll(style=True):
             del item['style']
         return soup
-

src/calibre/devices/misc.py

@@ -106,7 +106,7 @@ class PDNOVEL(USBMS):
     WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = '__UMS_COMPOSITE'
     THUMBNAIL_HEIGHT = 130
 
-    EBOOK_DIR_MAIN = 'eBooks'
+    EBOOK_DIR_MAIN = EBOOK_DIR_CARD_A = 'eBooks'
     SUPPORTS_SUB_DIRS = False
     DELETE_EXTS = ['.jpg', '.jpeg', '.png']
 

src/calibre/devices/prs505/driver.py

@@ -98,6 +98,9 @@ class PRS505(USBMS):
 
     THUMBNAIL_HEIGHT = 200
 
+    MAX_PATH_LEN = 201 # 250 - (max(len(CACHE_THUMBNAIL), len(MEDIA_THUMBNAIL)) +
+                       # len('main_thumbnail.jpg') + 1)
+
     def windows_filter_pnp_id(self, pnp_id):
         return '_LAUNCHER' in pnp_id
 
@@ -201,10 +204,13 @@ class PRS505(USBMS):
                                 self._card_b_prefix if idx == 2 \
                                     else self._main_prefix
                 for book in bl:
-                    p = os.path.join(prefix, book.lpath)
+                    try:
-                    self._upload_cover(os.path.dirname(p),
+                        p = os.path.join(prefix, book.lpath)
-                                      os.path.splitext(os.path.basename(p))[0],
+                        self._upload_cover(os.path.dirname(p),
-                                      book, p)
+                                          os.path.splitext(os.path.basename(p))[0],
+                                          book, p)
+                    except:
+                        debug_print('FAILED to upload cover', p)
         else:
             debug_print('PRS505: NOT uploading covers in sync_booklists')
 
@@ -232,8 +238,7 @@ class PRS505(USBMS):
         try:
             self._upload_cover(path, filename, metadata, filepath)
         except:
-            import traceback
+            debug_print('FAILED to upload cover', filepath)
-            traceback.print_exc()
 
     def _upload_cover(self, path, filename, metadata, filepath):
         if metadata.thumbnail and metadata.thumbnail[-1]:

src/calibre/devices/usbms/device.py

@@ -98,6 +98,9 @@ class Device(DeviceConfig, DevicePlugin):
     # copy these back to the library
     BACKLOADING_ERROR_MESSAGE = None
 
+    #: The maximum length of paths created on the device
+    MAX_PATH_LEN = 250
+
     def reset(self, key='-1', log_packets=False, report_progress=None,
             detected_device=None):
         self._main_prefix = self._card_a_prefix = self._card_b_prefix = None
@@ -875,7 +878,7 @@ class Device(DeviceConfig, DevicePlugin):
 
     def create_upload_path(self, path, mdata, fname, create_dirs=True):
         path = os.path.abspath(path)
-        extra_components = []
+        maxlen = self.MAX_PATH_LEN
 
         special_tag = None
         if mdata.tags:
@@ -902,7 +905,7 @@ class Device(DeviceConfig, DevicePlugin):
         app_id = str(getattr(mdata, 'application_id', ''))
         # The db id will be in the created filename
         extra_components = get_components(template, mdata, fname,
-                timefmt=opts.send_timefmt, length=250-len(app_id)-1)
+                timefmt=opts.send_timefmt, length=maxlen-len(app_id)-1)
         if not extra_components:
             extra_components.append(sanitize(self.filename_callback(fname,
                 mdata)))
@@ -937,12 +940,11 @@ class Device(DeviceConfig, DevicePlugin):
             return ans
 
         extra_components = list(map(remove_trailing_periods, extra_components))
-        components = shorten_components_to(250 - len(path), extra_components)
+        components = shorten_components_to(maxlen - len(path), extra_components)
         components = self.sanitize_path_components(components)
         filepath = os.path.join(path, *components)
         filedir = os.path.dirname(filepath)
 
-
         if create_dirs and not os.path.exists(filedir):
             os.makedirs(filedir)
 

src/calibre/ebooks/conversion/cli.py

@@ -42,6 +42,12 @@ option.
 For full documentation of the conversion system see
 ''') + 'http://calibre-ebook.com/user_manual/conversion.html'
 
+HEURISTIC_OPTIONS = ['markup_chapter_headings',
+                      'italicize_common_cases', 'fix_indents',
+                      'html_unwrap_factor', 'unwrap_lines',
+                      'delete_blank_paragraphs', 'format_scene_breaks',
+                      'dehyphenate', 'renumber_headings']
+
 def print_help(parser, log):
     help = parser.format_help().encode(preferred_encoding, 'replace')
     log(help)
@@ -83,6 +89,8 @@ def option_recommendation_to_cli_option(add_option, rec):
     if opt.long_switch == 'verbose':
         attrs['action'] = 'count'
         attrs.pop('type', '')
+    if opt.name in HEURISTIC_OPTIONS and rec.recommended_value is True:
+        switches = ['--disable-'+opt.long_switch]
     add_option(Option(*switches, **attrs))
 
 def add_input_output_options(parser, plumber):
@@ -129,18 +137,15 @@ def add_pipeline_options(parser, plumber):
                       'asciiize',
                   ]
                   ),
-                  
+
               'HEURISTIC PROCESSING' : (
-                  _('Modify the document text and structure using common patterns.'),
+                  _('Modify the document text and structure using common'
-                  [
+                     ' patterns. Disabled by default. Use %s to enable. '
-                      'enable_heuristics', 'markup_chapter_headings',
+                     ' Individual actions can be disabled with the %s options.')
-                      'italicize_common_cases', 'fix_indents',
+                  % ('--enable-heuristics', '--disable-*'),
-                      'html_unwrap_factor', 'unwrap_lines',
+                  ['enable_heuristics'] + HEURISTIC_OPTIONS
-                      'delete_blank_paragraphs', 'format_scene_breaks',
-                      'dehyphenate', 'renumber_headings',
-                  ]
                   ),
-                  
+
               'SEARCH AND REPLACE' : (
                  _('Modify the document text and structure using user defined patterns.'),
                  [

src/calibre/ebooks/conversion/plumber.py

@@ -490,19 +490,19 @@ OptionRecommendation(name='enable_heuristics',
            'heuristic processing to take place.')),
 
 OptionRecommendation(name='markup_chapter_headings',
-    recommended_value=False, level=OptionRecommendation.LOW,
+    recommended_value=True, level=OptionRecommendation.LOW,
     help=_('Detect unformatted chapter headings and sub headings. Change '
            'them to h2 and h3 tags.  This setting will not create a TOC, '
            'but can be used in conjunction with structure detection to create '
            'one.')),
 
 OptionRecommendation(name='italicize_common_cases',
-    recommended_value=False, level=OptionRecommendation.LOW,
+    recommended_value=True, level=OptionRecommendation.LOW,
     help=_('Look for common words and patterns that denote '
            'italics and italicize them.')),
 
 OptionRecommendation(name='fix_indents',
-    recommended_value=False, level=OptionRecommendation.LOW,
+    recommended_value=True, level=OptionRecommendation.LOW,
     help=_('Turn indentation created from multiple non-breaking space entities '
            'into CSS indents.')),
 
@@ -515,28 +515,28 @@ OptionRecommendation(name='html_unwrap_factor',
             'be reduced')),
 
 OptionRecommendation(name='unwrap_lines',
-    recommended_value=False, level=OptionRecommendation.LOW,
+    recommended_value=True, level=OptionRecommendation.LOW,
     help=_('Unwrap lines using punctuation and other formatting clues.')),
 
 OptionRecommendation(name='delete_blank_paragraphs',
-    recommended_value=False, level=OptionRecommendation.LOW,
+    recommended_value=True, level=OptionRecommendation.LOW,
     help=_('Remove empty paragraphs from the document when they exist between '
            'every other paragraph')),
 
 OptionRecommendation(name='format_scene_breaks',
-    recommended_value=False, level=OptionRecommendation.LOW,
+    recommended_value=True, level=OptionRecommendation.LOW,
     help=_('Left aligned scene break markers are center aligned. '
            'Replace soft scene breaks that use multiple blank lines with'
            'horizontal rules.')),
 
 OptionRecommendation(name='dehyphenate',
-    recommended_value=False, level=OptionRecommendation.LOW,
+    recommended_value=True, level=OptionRecommendation.LOW,
     help=_('Analyze hyphenated words throughout the document.  The '
            'document itself is used as a dictionary to determine whether hyphens '
            'should be retained or removed.')),
 
 OptionRecommendation(name='renumber_headings',
-    recommended_value=False, level=OptionRecommendation.LOW,
+    recommended_value=True, level=OptionRecommendation.LOW,
     help=_('Looks for occurrences of sequential <h1> or <h2> tags. '
            'The tags are renumbered to prevent splitting in the middle '
            'of chapter headings.')),

src/calibre/ebooks/conversion/preprocess.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 
 import functools, re
 
-from calibre import entity_to_unicode
+from calibre import entity_to_unicode, as_unicode
 
 XMLDECL_RE    = re.compile(r'^\s*<[?]xml.*?[?]>')
 SVG_NS       = 'http://www.w3.org/2000/svg'
@@ -201,7 +201,7 @@ class Dehyphenator(object):
             lookupword = self.removesuffixes.sub('', dehyphenated)
         else:
             lookupword = dehyphenated
-        if len(firsthalf) > 3 and self.prefixes.match(firsthalf) is None:
+        if len(firsthalf) > 4 and self.prefixes.match(firsthalf) is None:
             lookupword = self.removeprefix.sub('', lookupword)
         if self.verbose > 2:
             self.log("lookup word is: "+str(lookupword)+", orig is: " + str(hyphenated))
@@ -459,11 +459,12 @@ class HTMLPreProcessor(object):
                 try:
                     search_re = re.compile(search_pattern)
                     replace_txt = getattr(self.extra_opts, replace, '')
-                    if replace_txt == None:
+                    if not replace_txt:
                         replace_txt = ''
                     rules.insert(0, (search_re, replace_txt))
                 except Exception as e:
-                    self.log.error('Failed to parse %s regexp because %s' % (search, e))
+                    self.log.error('Failed to parse %r regexp because %s' %
+                            (search, as_unicode(e)))
 
         end_rules = []
         # delete soft hyphens - moved here so it's executed after header/footer removal

src/calibre/ebooks/txt/input.py

@@ -71,21 +71,41 @@ class TXTInput(InputFormatPlugin):
         txt = txt.decode(ienc, 'replace')
 
         txt = _ent_pat.sub(xml_entity_to_unicode, txt)
+
+        # Normalize line endings
+        txt = normalize_line_endings(txt)
+
+        if options.formatting_type == 'auto':
+            options.formatting_type = detect_formatting_type(txt)
+
+        if options.formatting_type == 'heuristic':
+            setattr(options, 'enable_heuristics', True)
+            setattr(options, 'markup_chapter_headings', True)
+            setattr(options, 'italicize_common_cases', True)
+            setattr(options, 'fix_indents', True)
+            setattr(options, 'preserve_spaces', True)
+            setattr(options, 'delete_blank_paragraphs', True)
+            setattr(options, 'format_scene_breaks', True)
+            setattr(options, 'dehyphenate', True)
+
+        # Determine the paragraph type of the document.
+        if options.paragraph_type == 'auto':
+            options.paragraph_type = detect_paragraph_type(txt)
+            if options.paragraph_type == 'unknown':
+                log.debug('Could not reliably determine paragraph type using block')
+                options.paragraph_type = 'block'
+            else:
+                log.debug('Auto detected paragraph type as %s' % options.paragraph_type)
+
         # Preserve spaces will replace multiple spaces to a space
         # followed by the   entity.
         if options.preserve_spaces:
             txt = preserve_spaces(txt)
 
-        # Normalize line endings
-        txt = normalize_line_endings(txt)
-
         # Get length for hyphen removal and punctuation unwrap
         docanalysis = DocAnalysis('txt', txt)
         length = docanalysis.line_length(.5)
 
-        if options.formatting_type == 'auto':
-            options.formatting_type = detect_formatting_type(txt)
-
         if options.formatting_type == 'markdown':
             log.debug('Running text though markdown conversion...')
             try:
@@ -96,16 +116,8 @@ class TXTInput(InputFormatPlugin):
         elif options.formatting_type == 'textile':
             log.debug('Running text though textile conversion...')
             html = convert_textile(txt)
-        else:
-            # Determine the paragraph type of the document.
-            if options.paragraph_type == 'auto':
-                options.paragraph_type = detect_paragraph_type(txt)
-                if options.paragraph_type == 'unknown':
-                    log.debug('Could not reliably determine paragraph type using block')
-                    options.paragraph_type = 'block'
-                else:
-                    log.debug('Auto detected paragraph type as %s' % options.paragraph_type)
 
+        else:
             # Dehyphenate
             dehyphenator = Dehyphenator(options.verbose, log=self.log)
             txt = dehyphenator(txt,'txt', length)
@@ -129,15 +141,6 @@ class TXTInput(InputFormatPlugin):
             flow_size = getattr(options, 'flow_size', 0)
             html = convert_basic(txt, epub_split_size_kb=flow_size)
 
-            if options.formatting_type == 'heuristic':
-                setattr(options, 'enable_heuristics', True)
-                setattr(options, 'markup_chapter_headings', True)
-                setattr(options, 'italicize_common_cases', True)
-                setattr(options, 'fix_indents', True)
-                setattr(options, 'delete_blank_paragraphs', True)
-                setattr(options, 'format_scene_breaks', True)
-                setattr(options, 'dehyphenate', True)
-
         from calibre.customize.ui import plugin_for_input_format
         html_input = plugin_for_input_format('html')
         for opt in html_input.options:

src/calibre/ebooks/txt/output.py

@@ -51,12 +51,12 @@ class TXTOutput(OutputFormatPlugin):
             recommended_value=False, level=OptionRecommendation.LOW,
             help=_('Do not remove links within the document. This is only ' \
             'useful when paired with the markdown-format option because' \
-            'links are always removed with plain text output.')),
+            ' links are always removed with plain text output.')),
         OptionRecommendation(name='keep_image_references',
             recommended_value=False, level=OptionRecommendation.LOW,
             help=_('Do not remove image references within the document. This is only ' \
             'useful when paired with the markdown-format option because' \
-            'image references are always removed with plain text output.')),
+            ' image references are always removed with plain text output.')),
      ])
 
     def convert(self, oeb_book, output_path, input_plugin, opts, log):

src/calibre/gui2/catalog/catalog_epub_mobi.ui

@@ -35,7 +35,7 @@
       </size>
      </property>
      <property name="toolTip">
-      <string>Sections to include in catalog.  All catalogs include 'Books by Author'.</string>
+      <string>Sections to include in catalog.</string>
      </property>
      <property name="title">
       <string>Included sections</string>
@@ -79,13 +79,13 @@
       <item row="0" column="0">
        <widget class="QCheckBox" name="generate_authors">
         <property name="enabled">
-         <bool>false</bool>
+         <bool>true</bool>
         </property>
         <property name="text">
          <string>Books by Author</string>
         </property>
         <property name="checked">
-         <bool>true</bool>
+         <bool>false</bool>
         </property>
        </widget>
       </item>

src/calibre/gui2/convert/bulk.py

@@ -94,7 +94,7 @@ class BulkConfig(Config):
             if not c: break
             self.stack.removeWidget(c)
 
-        widgets = [lf, hw, sr, ps, sd, toc]
+        widgets = [lf, hw, ps, sd, toc, sr]
         if output_widget is not None:
             widgets.append(output_widget)
         for w in widgets:

src/calibre/gui2/convert/heuristics.ui

@@ -17,11 +17,14 @@
    <item>
     <widget class="QLabel" name="label">
      <property name="text">
-      <string>&lt;b&gt;Heuristic processing&lt;/b&gt; means that calibre will scan your book for common patterns and fix them. As the name implies, this involves guesswork, which means that it could end up worsening the result of a conversion, if calibre guesses wrong. Therefore, it is disabled by default. Often, if a conversion does not turn out as you expect, turning on heuristics can improve matters.</string>
+      <string>&lt;b&gt;Heuristic processing&lt;/b&gt; means that calibre will scan your book for common patterns and fix them. As the name implies, this involves guesswork, which means that it could end up worsening the result of a conversion, if calibre guesses wrong. Therefore, it is disabled by default. Often, if a conversion does not turn out as you expect, turning on heuristics can improve matters. Read more about the various heuristic processing options in the &lt;a href=&quot;http://calibre-ebook.com/user_manual/conversion.html#heuristic-processing&quot;&gt;User Manual&lt;/a&gt;.</string>
      </property>
      <property name="wordWrap">
       <bool>true</bool>
      </property>
+     <property name="openExternalLinks">
+      <bool>true</bool>
+     </property>
     </widget>
    </item>
    <item>

src/calibre/gui2/convert/search_and_replace.py

@@ -12,9 +12,10 @@ from calibre.gui2 import error_dialog
 
 class SearchAndReplaceWidget(Widget, Ui_Form):
 
-    TITLE = _('Search &\nReplace')
+    TITLE = _('Search\n&\nReplace')
     HELP  = _('Modify the document text and structure using user defined patterns.')
     COMMIT_NAME = 'search_and_replace'
+    ICON = I('search.png')
 
     def __init__(self, parent, get_option, get_help, db=None, book_id=None):
         Widget.__init__(self, parent,
@@ -24,19 +25,19 @@ class SearchAndReplaceWidget(Widget, Ui_Form):
                 )
         self.db, self.book_id = db, book_id
         self.initialize_options(get_option, get_help, db, book_id)
-        self.opt_sr1_search.set_msg(_('Search Regular Expression'))
+        self.opt_sr1_search.set_msg(_('&Search Regular Expression'))
         self.opt_sr1_search.set_book_id(book_id)
         self.opt_sr1_search.set_db(db)
-        self.opt_sr2_search.set_msg(_('Search Regular Expression'))
+        self.opt_sr2_search.set_msg(_('&Search Regular Expression'))
         self.opt_sr2_search.set_book_id(book_id)
         self.opt_sr2_search.set_db(db)
-        self.opt_sr3_search.set_msg(_('Search Regular Expression'))
+        self.opt_sr3_search.set_msg(_('&Search Regular Expression'))
         self.opt_sr3_search.set_book_id(book_id)
         self.opt_sr3_search.set_db(db)
-        
+
     def break_cycles(self):
         Widget.break_cycles(self)
-        
+
         self.opt_sr1_search.break_cycles()
         self.opt_sr2_search.break_cycles()
         self.opt_sr3_search.break_cycles()
@@ -49,6 +50,6 @@ class SearchAndReplaceWidget(Widget, Ui_Form):
                 re.compile(pat)
             except Exception, err:
                 error_dialog(self, _('Invalid regular expression'),
-                             _('Invalid regular expression: %s')%err).exec_()
+                             _('Invalid regular expression: %s')%err, show=True)
                 return False
         return True

src/calibre/gui2/convert/search_and_replace.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>198</width>
+    <width>468</width>
-    <height>350</height>
+    <height>451</height>
    </rect>
   </property>
   <property name="sizePolicy">
@@ -23,7 +23,7 @@
    <property name="sizeConstraint">
     <enum>QLayout::SetDefaultConstraint</enum>
    </property>
-   <item row="0" column="0">
+   <item row="1" column="0">
     <widget class="QGroupBox" name="groupBox">
      <property name="sizePolicy">
       <sizepolicy hsizetype="Minimum" vsizetype="Preferred">
@@ -32,7 +32,7 @@
       </sizepolicy>
      </property>
      <property name="title">
-      <string>1.</string>
+      <string>First expression</string>
      </property>
      <layout class="QGridLayout" name="gridLayout_2">
       <property name="sizeConstraint">
@@ -57,7 +57,10 @@
          </sizepolicy>
         </property>
         <property name="text">
-         <string>Replacement Text</string>
+         <string>&amp;Replacement Text</string>
+        </property>
+        <property name="buddy">
+         <cstring>opt_sr1_replace</cstring>
         </property>
        </widget>
       </item>
@@ -74,7 +77,7 @@
      </layout>
     </widget>
    </item>
-   <item row="1" column="0">
+   <item row="2" column="0">
     <widget class="QGroupBox" name="groupBox_2">
      <property name="sizePolicy">
       <sizepolicy hsizetype="Minimum" vsizetype="Preferred">
@@ -83,7 +86,7 @@
       </sizepolicy>
      </property>
      <property name="title">
-      <string>2.</string>
+      <string>Second Expression</string>
      </property>
      <layout class="QGridLayout" name="gridLayout">
       <property name="sizeConstraint">
@@ -108,7 +111,10 @@
          </sizepolicy>
         </property>
         <property name="text">
-         <string>Replacement Text</string>
+         <string>&amp;Replacement Text</string>
+        </property>
+        <property name="buddy">
+         <cstring>opt_sr2_replace</cstring>
         </property>
        </widget>
       </item>
@@ -125,7 +131,7 @@
      </layout>
     </widget>
    </item>
-   <item row="2" column="0">
+   <item row="3" column="0">
     <widget class="QGroupBox" name="groupBox_3">
      <property name="sizePolicy">
       <sizepolicy hsizetype="Minimum" vsizetype="Preferred">
@@ -134,7 +140,7 @@
       </sizepolicy>
      </property>
      <property name="title">
-      <string>3.</string>
+      <string>Third expression</string>
      </property>
      <layout class="QGridLayout" name="gridLayout_3">
       <property name="sizeConstraint">
@@ -159,7 +165,10 @@
          </sizepolicy>
         </property>
         <property name="text">
-         <string>Replacement Text</string>
+         <string>&amp;Replacement Text</string>
+        </property>
+        <property name="buddy">
+         <cstring>opt_sr3_replace</cstring>
         </property>
        </widget>
       </item>
@@ -176,6 +185,19 @@
      </layout>
     </widget>
    </item>
+   <item row="0" column="0">
+    <widget class="QLabel" name="label">
+     <property name="text">
+      <string>&lt;p&gt;Search and replace uses &lt;i&gt;regular expressions&lt;/i&gt;. See the &lt;a href=&quot;http://calibre-ebook.com/user_manual/regexp.html&quot;&gt;regular expressions tutorial&lt;/a&gt; to get started with regular expressions. Also clicking the wizard buttons below will allow you to test your regular expression against the current input document.</string>
+     </property>
+     <property name="wordWrap">
+      <bool>true</bool>
+     </property>
+     <property name="openExternalLinks">
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
   </layout>
  </widget>
  <customwidgets>

src/calibre/gui2/convert/single.py

@@ -207,7 +207,7 @@ class Config(ResizableDialog, Ui_Dialog):
             if not c: break
             self.stack.removeWidget(c)
 
-        widgets = [self.mw, lf, hw, sr, ps, sd, toc]
+        widgets = [self.mw, lf, hw, ps, sd, toc, sr]
         if input_widget is not None:
             widgets.append(input_widget)
         if output_widget is not None:

src/calibre/gui2/convert/single.ui

@@ -100,7 +100,7 @@
       </size>
      </property>
      <property name="spacing">
-      <number>20</number>
+      <number>10</number>
      </property>
      <property name="wordWrap">
       <bool>true</bool>
@@ -129,8 +129,8 @@
        <rect>
         <x>0</x>
         <y>0</y>
-        <width>805</width>
+        <width>810</width>
-        <height>484</height>
+        <height>494</height>
        </rect>
       </property>
       <layout class="QVBoxLayout" name="verticalLayout_3">

src/calibre/gui2/convert/txt_output.py

@@ -23,9 +23,9 @@ class PluginWidget(Widget, Ui_Form):
         ['newline', 'max_line_length', 'force_max_line_length',
         'inline_toc', 'markdown_format', 'keep_links', 'keep_image_references',
         'txt_output_encoding'])
-        self.db, self.book_id = db, book_id        
+        self.db, self.book_id = db, book_id
         for x in get_option('newline').option.choices:
-            self.opt_newline.addItem(x)        
+            self.opt_newline.addItem(x)
         self.initialize_options(get_option, get_help, db, book_id)
 
         self.opt_markdown_format.stateChanged.connect(self.enable_markdown_format)
@@ -33,17 +33,14 @@ class PluginWidget(Widget, Ui_Form):
 
     def break_cycles(self):
         Widget.break_cycles(self)
-        
+
         try:
             self.opt_markdown_format.stateChanged.disconnect()
         except:
             pass
-        
+
     def enable_markdown_format(self, state):
-        if state == Qt.Checked:
+        state = state == Qt.Checked
-            state = True
-        else:
-            state = False
         self.opt_keep_links.setEnabled(state)
         self.opt_keep_image_references.setEnabled(state)
-        
+

src/calibre/gui2/convert/xexp_edit.ui

@@ -6,7 +6,7 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>434</width>
+    <width>430</width>
     <height>74</height>
    </rect>
   </property>
@@ -59,7 +59,7 @@
       <string>...</string>
      </property>
      <property name="icon">
-      <iconset>
+      <iconset resource="../../../../resources/images.qrc">
        <normaloff>:/images/wizard.png</normaloff>:/images/wizard.png</iconset>
      </property>
      <property name="iconSize">

src/calibre/gui2/custom_column_widgets.py

@@ -599,7 +599,7 @@ class BulkEnumeration(BulkBase, Enumeration):
         value = None
         ret_value = None
         dialog_shown = False
-        for book_id in book_ids:
+        for i,book_id in enumerate(book_ids):
             val = self.db.get_custom(book_id, num=self.col_id, index_is_id=True)
             if val and val not in self.col_metadata['display']['enum_values']:
                 if not dialog_shown:
@@ -610,7 +610,7 @@ class BulkEnumeration(BulkBase, Enumeration):
                             show=True, show_copy_button=False)
                     dialog_shown = True
                 ret_value = ' nochange '
-            elif value is not None and value != val:
+            elif (value is not None and value != val) or (val and i != 0):
                 ret_value = ' nochange '
             value = val
         if ret_value is None:

src/calibre/gui2/dialogs/metadata_single.py

@@ -724,7 +724,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
             au = _('Unknown')
         au = ' & '.join([a.strip().replace('|', ',') for a in au.split(',')])
         self.authors.setEditText(au)
-        
+
         self.authors.set_separator('&')
         self.authors.set_space_before_sep(True)
         self.authors.update_items_cache(self.db.all_author_names())

src/calibre/gui2/metadata/single.py

@@ -0,0 +1,785 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import textwrap, re, os
+
+from PyQt4.Qt import QDialogButtonBox, Qt, QTabWidget, QScrollArea, \
+    QVBoxLayout, QIcon, QToolButton, QWidget, QLabel, QGridLayout, \
+    QDoubleSpinBox, QListWidgetItem, QSize, pyqtSignal, QPixmap, \
+    QSplitter
+
+from calibre.gui2 import ResizableDialog, file_icon_provider, \
+        choose_files, error_dialog
+from calibre.utils.icu import sort_key
+from calibre.utils.config import tweaks
+from calibre.gui2.widgets import EnLineEdit, CompleteComboBox, \
+        EnComboBox, FormatList, ImageView
+from calibre.ebooks.metadata import title_sort, authors_to_string, \
+        string_to_authors
+from calibre.utils.date import local_tz
+from calibre import strftime
+from calibre.ebooks import BOOK_EXTENSIONS
+from calibre.customize.ui import run_plugins_on_import
+from calibre.utils.date import utcfromtimestamp
+
+
+'''
+The interface common to all widgets used to set basic metadata
+class BasicMetadataWidget(object):
+
+    LABEL = "label text"
+
+    def initialize(self, db, id_):
+        pass
+
+    def commit(self, db, id_):
+        return True
+
+    @dynamic_property
+    def current_val(self):
+        # Present in most but not all basic metadata widgets
+        def fget(self):
+            return None
+        def fset(self, val):
+            pass
+        return property(fget=fget, fset=fset)
+'''
+
+# Title {{{
+class TitleEdit(EnLineEdit):
+
+    TITLE_ATTR = 'title'
+    COMMIT = True
+    TOOLTIP = _('Change the title of this book')
+    LABEL = _('&Title:')
+
+    def __init__(self, parent):
+        self.dialog = parent
+        EnLineEdit.__init__(self, parent)
+        self.setToolTip(self.TOOLTIP)
+        self.setWhatsThis(self.TOOLTIP)
+
+    def get_default(self):
+        return _('Unknown')
+
+    def initialize(self, db, id_):
+        title = getattr(db, self.TITLE_ATTR)(id_, index_is_id=True)
+        self.current_val = title
+        self.original_val = self.current_val
+
+    def commit(self, db, id_):
+        title = self.current_val
+        if self.COMMIT:
+            getattr(db, 'set_', self.TITLE_ATTR)(id_, title, notify=False)
+        else:
+            getattr(db, 'set_', self.TITLE_ATTR)(id_, title, notify=False,
+                    commit=False)
+        return True
+
+    @dynamic_property
+    def current_val(self):
+
+        def fget(self):
+            title = unicode(self.text()).strip()
+            if not title:
+                title = self.get_default()
+            return title
+
+        def fset(self, val):
+            if hasattr(val, 'strip'):
+                val = val.strip()
+            if not val:
+                val = self.get_default()
+            self.setText(val)
+            self.setCursorPosition(0)
+
+        return property(fget=fget, fset=fset)
+
+class TitleSortEdit(TitleEdit):
+
+    TITLE_ATTR = 'title_sort'
+    COMMIT = False
+    TOOLTIP = _('Specify how this book should be sorted when by title.'
+            ' For example, The Exorcist might be sorted as Exorcist, The.')
+    LABEL = _('Title &sort:')
+
+    def __init__(self, parent, title_edit, autogen_button):
+        TitleEdit.__init__(self, parent)
+        self.title_edit = title_edit
+
+        base = self.TOOLTIP
+        ok_tooltip = '<p>' + textwrap.fill(base+'<br><br>'+
+                            _(' The green color indicates that the current '
+                              'title sort matches the current title'))
+        bad_tooltip = '<p>'+textwrap.fill(base + '<br><br>'+
+                _(' The red color warns that the current '
+                  'title sort does not match the current title. '
+                  'No action is required if this is what you want.'))
+        self.tooltips = (ok_tooltip, bad_tooltip)
+
+        self.title_edit.textChanged.connect(self.update_state)
+        self.textChanged.connect(self.update_state)
+
+        autogen_button.clicked.connect(self.auto_generate)
+        self.update_state()
+
+    def update_state(self, *args):
+        ts = title_sort(self.title_edit.current_val)
+        normal = ts == self.current_val
+        if normal:
+            col = 'rgb(0, 255, 0, 20%)'
+        else:
+            col = 'rgb(255, 0, 0, 20%)'
+        self.setStyleSheet('QLineEdit { color: black; '
+                              'background-color: %s; }'%col)
+        tt = self.tooltips[0 if normal else 1]
+        self.setToolTip(tt)
+        self.setWhatsThis(tt)
+
+    def auto_generate(self, *args):
+        self.current_val = title_sort(self.title_edit.current_val)
+
+# }}}
+
+# Authors {{{
+class AuthorsEdit(CompleteComboBox):
+
+    TOOLTIP = ''
+    LABEL = _('&Author(s):')
+
+    def __init__(self, parent):
+        self.dialog = parent
+        CompleteComboBox.__init__(self, parent)
+        self.setToolTip(self.TOOLTIP)
+        self.setWhatsThis(self.TOOLTIP)
+        self.setEditable(True)
+        self.setSizeAdjustPolicy(self.AdjustToMinimumContentsLengthWithIcon)
+
+    def get_default(self):
+        return _('Unknown')
+
+    def initialize(self, db, id_):
+        all_authors = db.all_authors()
+        all_authors.sort(key=lambda x : sort_key(x[1]))
+        for i in all_authors:
+            id, name = i
+            name = [name.strip().replace('|', ',') for n in name.split(',')]
+            self.addItem(authors_to_string(name))
+
+        self.set_separator('&')
+        self.set_space_before_sep(True)
+        self.update_items_cache(db.all_author_names())
+
+        au = db.authors(id_, index_is_id=True)
+        if not au:
+            au = _('Unknown')
+        self.current_val = [a.strip().replace('|', ',') for a in au.split(',')]
+        self.original_val = self.current_val
+
+    def commit(self, db, id_):
+        authors = self.current_val
+        db.set_authors(id_, authors, notify=False)
+        return True
+
+    @dynamic_property
+    def current_val(self):
+
+        def fget(self):
+            au = unicode(self.text()).strip()
+            if not au:
+                au = self.get_default()
+            return string_to_authors(au)
+
+        def fset(self, val):
+            if not val:
+                val = [self.get_default()]
+            self.setEditText(' & '.join([x.strip() for x in val]))
+            self.lineEdit().setCursorPosition(0)
+
+
+        return property(fget=fget, fset=fset)
+
+class AuthorSortEdit(EnLineEdit):
+
+    TOOLTIP = _('Specify how the author(s) of this book should be sorted. '
+            'For example Charles Dickens should be sorted as Dickens, '
+            'Charles.\nIf the box is colored green, then text matches '
+            'the individual author\'s sort strings. If it is colored '
+            'red, then the authors and this text do not match.')
+    LABEL = _('Author s&ort:')
+
+    def __init__(self, parent, authors_edit, autogen_button, db):
+        EnLineEdit.__init__(self, parent)
+        self.authors_edit = authors_edit
+        self.db = db
+
+        base = self.TOOLTIP
+        ok_tooltip = '<p>' + textwrap.fill(base+'<br><br>'+
+                _(' The green color indicates that the current '
+                    'author sort matches the current author'))
+        bad_tooltip = '<p>'+textwrap.fill(base + '<br><br>'+
+                _(' The red color indicates that the current '
+                    'author sort does not match the current author. '
+                    'No action is required if this is what you want.'))
+        self.tooltips = (ok_tooltip, bad_tooltip)
+
+        self.authors_edit.editTextChanged.connect(self.update_state)
+        self.textChanged.connect(self.update_state)
+
+        autogen_button.clicked.connect(self.auto_generate)
+        self.update_state()
+
+    @dynamic_property
+    def current_val(self):
+
+        def fget(self):
+            return unicode(self.text()).strip()
+
+        def fset(self, val):
+            if not val:
+                val = ''
+            self.setText(val.strip())
+            self.setCursorPosition(0)
+
+        return property(fget=fget, fset=fset)
+
+    def update_state(self, *args):
+        au = unicode(self.authors_edit.text())
+        au = re.sub(r'\s+et al\.$', '', au)
+        au = self.db.author_sort_from_authors(string_to_authors(au))
+
+        normal = au == self.current_val
+        if normal:
+            col = 'rgb(0, 255, 0, 20%)'
+        else:
+            col = 'rgb(255, 0, 0, 20%)'
+        self.setStyleSheet('QLineEdit { color: black; '
+                              'background-color: %s; }'%col)
+        tt = self.tooltips[0 if normal else 1]
+        self.setToolTip(tt)
+        self.setWhatsThis(tt)
+
+    def auto_generate(self, *args):
+        au = unicode(self.authors_edit.text())
+        au = re.sub(r'\s+et al\.$', '', au)
+        authors = string_to_authors(au)
+        self.current_val = self.db.author_sort_from_authors(authors)
+
+    def initialize(self, db, id_):
+        self.current_val = db.author_sort(id_, index_is_id=True)
+
+    def commit(self, db, id_):
+        aus = self.current_val
+        db.set_author_sort(id_, aus, notify=False, commit=False)
+        return True
+
+# }}}
+
+# Series {{{
+class SeriesEdit(EnComboBox):
+
+    TOOLTIP = _('List of known series. You can add new series.')
+    LABEL = _('&Series:')
+
+    def __init__(self, parent):
+        EnComboBox.__init__(self, parent)
+        self.dialog = parent
+        self.setSizeAdjustPolicy(
+                self.AdjustToMinimumContentsLengthWithIcon)
+        self.setToolTip(self.TOOLTIP)
+        self.setWhatsThis(self.TOOLTIP)
+        self.setEditable(True)
+
+    @dynamic_property
+    def current_val(self):
+
+        def fget(self):
+            return unicode(self.currentText()).strip()
+
+        def fset(self, val):
+            if not val:
+                val = ''
+            self.setEditText(val.strip())
+            self.setCursorPosition(0)
+
+        return property(fget=fget, fset=fset)
+
+    def initialize(self, db, id_):
+        all_series = db.all_series()
+        all_series.sort(key=lambda x : sort_key(x[1]))
+        series_id = db.series_id(id_, index_is_id=True)
+        idx, c = None, 0
+        for i in all_series:
+            id, name = i
+            if id == series_id:
+                idx = c
+            self.addItem(name)
+            c += 1
+
+        self.lineEdit().setText('')
+        if idx is not None:
+            self.setCurrentIndex(idx)
+        self.original_val = self.current_val
+
+    def commit(self, db, id_):
+        series = self.current_val
+        db.set_series(id_, series, notify=False, commit=True)
+        return True
+
+class SeriesIndexEdit(QDoubleSpinBox):
+
+    TOOLTIP = ''
+    LABEL = _('&Number:')
+
+    def __init__(self, parent, series_edit):
+        QDoubleSpinBox.__init__(self, parent)
+        self.dialog = parent
+        self.db = self.original_series_name = None
+        self.setMaximum(1000000)
+        self.series_edit = series_edit
+        series_edit.currentIndexChanged.connect(self.enable)
+        series_edit.editTextChanged.connect(self.enable)
+        series_edit.lineEdit().editingFinished.connect(self.increment)
+        self.enable()
+
+    def enable(self, *args):
+        self.setEnabled(bool(self.series_edit.current_val))
+
+    @dynamic_property
+    def current_val(self):
+
+        def fget(self):
+            return self.value()
+
+        def fset(self, val):
+            if val is None:
+                val = 1.0
+            val = float(val)
+            self.setValue(val)
+
+        return property(fget=fget, fset=fset)
+
+    def initialize(self, db, id_):
+        self.db = db
+        if self.series_edit.current_val:
+            val = db.series_index(id_, index_is_id=True)
+        else:
+            val = 1.0
+        self.current_val = val
+        self.original_val = self.current_val
+        self.original_series_name = self.series_edit.original_val
+
+    def commit(self, db, id_):
+        db.set_series_index(id_, self.current_val, notify=False, commit=False)
+        return True
+
+    def increment(self):
+        if self.db is not None:
+            try:
+                series = self.series_edit.current_val
+                if series and series != self.original_series_name:
+                    ns = 1.0
+                    if tweaks['series_index_auto_increment'] != 'const':
+                        ns = self.db.get_next_series_num_for(series)
+                    self.current_val = ns
+                    self.original_series_name = series
+            except:
+                import traceback
+                traceback.print_exc()
+
+
+# }}}
+
+class BuddyLabel(QLabel): # {{{
+
+    def __init__(self, buddy):
+        QLabel.__init__(self, buddy.LABEL)
+        self.setBuddy(buddy)
+        self.setAlignment(Qt.AlignRight|Qt.AlignVCenter)
+# }}}
+
+class Format(QListWidgetItem): # {{{
+
+    def __init__(self, parent, ext, size, path=None, timestamp=None):
+        self.path = path
+        self.ext = ext
+        self.size = float(size)/(1024*1024)
+        text = '%s (%.2f MB)'%(self.ext.upper(), self.size)
+        QListWidgetItem.__init__(self, file_icon_provider().icon_from_ext(ext),
+                                 text, parent, QListWidgetItem.UserType)
+        if timestamp is not None:
+            ts = timestamp.astimezone(local_tz)
+            t = strftime('%a, %d %b %Y [%H:%M:%S]', ts.timetuple())
+            text = _('Last modified: %s')%t
+            self.setToolTip(text)
+            self.setStatusTip(text)
+
+# }}}
+
+class FormatsManager(QWidget): # {{{
+
+    def __init__(self, parent):
+        QWidget.__init__(self, parent)
+        self.dialog = parent
+        self.changed = False
+
+        self.l = l = QGridLayout()
+        self.setLayout(l)
+        self.cover_from_format_button = QToolButton(self)
+        self.cover_from_format_button.setToolTip(
+                _('Set the cover for the book from the selected format'))
+        self.cover_from_format_button.setIcon(QIcon(I('book.png')))
+        self.cover_from_format_button.setIconSize(QSize(32, 32))
+
+        self.metadata_from_format_button = QToolButton(self)
+        self.metadata_from_format_button.setIcon(QIcon(I('edit_input.png')))
+        self.metadata_from_format_button.setIconSize(QSize(32, 32))
+        # TODO: Implement the *_from_format buttons
+
+        self.add_format_button = QToolButton(self)
+        self.add_format_button.setIcon(QIcon(I('add_book.png')))
+        self.add_format_button.setIconSize(QSize(32, 32))
+        self.add_format_button.clicked.connect(self.add_format)
+
+        self.remove_format_button = QToolButton(self)
+        self.remove_format_button.setIcon(QIcon(I('trash.png')))
+        self.remove_format_button.setIconSize(QSize(32, 32))
+        self.remove_format_button.clicked.connect(self.remove_format)
+
+        self.formats = FormatList(self)
+        self.formats.setAcceptDrops(True)
+        self.formats.formats_dropped.connect(self.formats_dropped)
+        self.formats.delete_format.connect(self.remove_format)
+        self.formats.itemDoubleClicked.connect(self.show_format)
+        self.formats.setDragDropMode(self.formats.DropOnly)
+        self.formats.setIconSize(QSize(32, 32))
+        self.formats.setMaximumWidth(200)
+
+        l.addWidget(self.cover_from_format_button,    0, 0, 1, 1)
+        l.addWidget(self.metadata_from_format_button, 2, 0, 1, 1)
+        l.addWidget(self.add_format_button,           0, 2, 1, 1)
+        l.addWidget(self.remove_format_button,        2, 2, 1, 1)
+        l.addWidget(self.formats,                     0, 1, 3, 1)
+
+
+
+    def initialize(self, db, id_):
+        self.changed = False
+        exts = db.formats(id_, index_is_id=True)
+        if exts:
+            exts = exts.split(',')
+            for ext in exts:
+                if not ext:
+                    ext = ''
+                size = db.sizeof_format(id_, ext, index_is_id=True)
+                timestamp = db.format_last_modified(id_, ext)
+                if size is None:
+                    continue
+                Format(self.formats, ext, size, timestamp=timestamp)
+
+    def commit(self, db, id_):
+        if not self.changed:
+            return True
+        old_extensions, new_extensions, paths = set(), set(), {}
+        for row in range(self.formats.count()):
+            fmt = self.formats.item(row)
+            ext, path = fmt.ext.lower(), fmt.path
+            if 'unknown' in ext.lower():
+                ext = None
+            if path:
+                new_extensions.add(ext)
+                paths[ext] = path
+            else:
+                old_extensions.add(ext)
+        for ext in new_extensions:
+            db.add_format(id_, ext, open(paths[ext], 'rb'), notify=False,
+                    index_is_id=True)
+        db_extensions = set([f.lower() for f in db.formats(id_,
+            index_is_id=True).split(',')])
+        extensions = new_extensions.union(old_extensions)
+        for ext in db_extensions:
+            if ext not in extensions:
+                db.remove_format(id_, ext, notify=False, index_is_id=True)
+
+        self.changed = False
+        return True
+
+    def add_format(self, *args):
+        files = choose_files(self, 'add formats dialog',
+                             _("Choose formats for ") +
+                             self.dialog.title.current_val,
+                             [(_('Books'), BOOK_EXTENSIONS)])
+        self._add_formats(files)
+
+    def _add_formats(self, paths):
+        added = False
+        if not paths:
+            return added
+        bad_perms = []
+        for _file in paths:
+            _file = os.path.abspath(_file)
+            if not os.access(_file, os.R_OK):
+                bad_perms.append(_file)
+                continue
+
+            nfile = run_plugins_on_import(_file)
+            if nfile is not None:
+                _file = nfile
+            stat = os.stat(_file)
+            size = stat.st_size
+            ext = os.path.splitext(_file)[1].lower().replace('.', '')
+            timestamp = utcfromtimestamp(stat.st_mtime)
+            for row in range(self.formats.count()):
+                fmt = self.formats.item(row)
+                if fmt.ext.lower() == ext:
+                    self.formats.takeItem(row)
+                    break
+            Format(self.formats, ext, size, path=_file, timestamp=timestamp)
+            self.changed = True
+            added = True
+        if bad_perms:
+            error_dialog(self, _('No permission'),
+                    _('You do not have '
+                'permission to read the following files:'),
+                det_msg='\n'.join(bad_perms), show=True)
+
+        return added
+
+    def formats_dropped(self, event, paths):
+        if self._add_formats(paths):
+            event.accept()
+
+    def remove_format(self, *args):
+        rows = self.formats.selectionModel().selectedRows(0)
+        for row in rows:
+            self.formats.takeItem(row.row())
+            self.changed = True
+
+    def show_format(self, item, *args):
+        fmt = item.ext
+        self.dialog.view_format.emit(fmt)
+
+# }}}
+
+class Cover(ImageView):
+
+    def __init__(self, parent):
+        ImageView.__init__(self, parent)
+        self._cdata = None
+        self.cover_changed.connect(self.set_pixmap_from_data)
+
+    def set_pixmap_from_data(self, data):
+        if not data:
+            self.current_val = None
+            return
+        orig = self.current_val
+        self.current_val = data
+        if self.current_val is None:
+            error_dialog(self, _('Invalid cover'),
+                    _('Could not change cover as the image is invalid.'),
+                    show=True)
+            self.current_val = orig
+
+    def initialize(self, db, id_):
+        self._cdata = None
+        self.current_val = db.cover(id_, index_is_id=True)
+        self.original_val = self.current_val
+
+    @property
+    def changed(self):
+        return self.current_val != self.original_val
+
+    @dynamic_property
+    def current_val(self):
+        def fget(self):
+            return self._cdata
+        def fset(self, cdata):
+            self._cdata = None
+            pm = QPixmap()
+            if cdata:
+                pm.loadFromData(cdata)
+            if pm.isNull():
+                pm = QPixmap(I('default_cover.png'))
+            else:
+                self._cdata = cdata
+            self.setPixmap(pm)
+            tt = _('This book has no cover')
+            if self._cdata:
+                tt = _('Cover size: %dx%d pixels') % \
+                (pm.width(), pm.height())
+            self.setToolTip(tt)
+
+        return property(fget=fget, fset=fset)
+
+    def commit(self, db, id_):
+        if self.changed:
+            if self.current_val:
+                db.set_cover(id_, self.current_val, notify=False, commit=False)
+            else:
+                db.remove_cover(id_, notify=False, commit=False)
+        return True
+
+
+
+class MetadataSingleDialog(ResizableDialog):
+
+    view_format = pyqtSignal(object)
+
+    def __init__(self, db, parent=None):
+        self.db = db
+        ResizableDialog.__init__(self, parent)
+
+    def setupUi(self, *args): # {{{
+        self.resize(990, 650)
+
+        self.button_box = QDialogButtonBox(
+                QDialogButtonBox.Ok|QDialogButtonBox.Cancel, Qt.Horizontal,
+                self)
+        self.button_box.accepted.connect(self.accept)
+        self.button_box.rejected.connect(self.reject)
+
+        self.scroll_area = QScrollArea(self)
+        self.scroll_area.setFrameShape(QScrollArea.NoFrame)
+        self.scroll_area.setWidgetResizable(True)
+        self.central_widget = QTabWidget(self)
+        self.scroll_area.setWidget(self.central_widget)
+
+        self.l = QVBoxLayout(self)
+        self.setLayout(self.l)
+        self.l.setMargin(0)
+        self.l.addWidget(self.scroll_area)
+        self.l.addWidget(self.button_box)
+
+        self.setWindowIcon(QIcon(I('edit_input.png')))
+        self.setWindowTitle(_('Edit Meta Information'))
+
+        self.create_basic_metadata_widgets()
+
+        self.do_layout()
+    # }}}
+
+    def create_basic_metadata_widgets(self): # {{{
+        self.basic_metadata_widgets = []
+        # Title
+        self.title = TitleEdit(self)
+        self.deduce_title_sort_button = QToolButton(self)
+        self.deduce_title_sort_button.setToolTip(
+            _('Automatically create the title sort entry based on the current '
+                'title entry.\nUsing this button to create title sort will '
+                'change title sort from red to green.'))
+        self.deduce_title_sort_button.setWhatsThis(
+                self.deduce_title_sort_button.toolTip())
+        self.title_sort = TitleSortEdit(self, self.title,
+                self.deduce_title_sort_button)
+        self.basic_metadata_widgets.extend([self.title, self.title_sort])
+
+        # Authors
+        self.authors = AuthorsEdit(self)
+        self.deduce_author_sort_button = QToolButton(self)
+        self.deduce_author_sort_button.setToolTip(_(
+        'Automatically create the author sort entry based on the current'
+        ' author entry.\n'
+        'Using this button to create author sort will change author sort from'
+        ' red to green.'))
+        self.author_sort = AuthorSortEdit(self, self.authors,
+                self.deduce_author_sort_button, db)
+        self.basic_metadata_widgets.extend([self.authors, self.author_sort])
+
+        self.swap_title_author_button = QToolButton(self)
+        self.swap_title_author_button.setIcon(QIcon(I('swap.png')))
+        self.swap_title_author_button.setToolTip(_(
+            'Swap the author and title'))
+        self.swap_title_author_button.clicked.connect(self.swap_title_author)
+
+        self.series = SeriesEdit(self)
+        self.remove_unused_series_button = QToolButton(self)
+        self.remove_unused_series_button.setToolTip(
+               _('Remove unused series (Series that have no books)') )
+        self.remove_unused_series_button.clicked.connect(self.remove_unused_series)
+        self.series_index = SeriesIndexEdit(self, self.series)
+        self.basic_metadata_widgets.extend([self.series, self.series_index])
+
+        self.formats_manager = FormatsManager(self)
+        self.basic_metadata_widgets.append(self.formats_manager)
+
+        self.cover = Cover(self)
+        self.basic_metadata_widgets.append(self.cover)
+
+    # }}}
+
+    def do_layout(self): # {{{
+        self.central_widget.clear()
+        self.tabs = []
+        self.labels = []
+        self.tabs.append(QWidget(self))
+        self.central_widget.addTab(self.tabs[0], _("&Basic metadata"))
+        self.tabs[0].l = l = QVBoxLayout()
+        self.tabs[0].tl = tl = QGridLayout()
+        self.tabs[0].setLayout(l)
+        l.addLayout(tl)
+
+        def create_row(row, one, two, three, col=1, icon='forward.png'):
+            ql = BuddyLabel(one)
+            tl.addWidget(ql, row, col+0, 1, 1)
+            self.labels.append(ql)
+            tl.addWidget(one, row, col+1, 1, 1)
+            if two is not None:
+                tl.addWidget(two, row, col+2, 1, 1)
+                two.setIcon(QIcon(I(icon)))
+            ql = BuddyLabel(three)
+            tl.addWidget(ql, row, col+3, 1, 1)
+            self.labels.append(ql)
+            tl.addWidget(three, row, col+4, 1, 1)
+
+        tl.addWidget(self.swap_title_author_button, 0, 0, 2, 1)
+
+        create_row(0, self.title, self.deduce_title_sort_button, self.title_sort)
+        create_row(1, self.authors, self.deduce_author_sort_button, self.author_sort)
+        create_row(2, self.series, self.remove_unused_series_button,
+                self.series_index, icon='trash.png')
+
+        tl.addWidget(self.formats_manager, 0, 6, 3, 1)
+
+        self.splitter = QSplitter(Qt.Horizontal, self)
+        self.splitter.addWidget(self.cover)
+        l.addWidget(self.splitter)
+    # }}}
+
+    def __call__(self, id_, has_next=False, has_previous=False):
+        # TODO: Next and previous buttons
+        self.book_id = id_
+        for widget in self.basic_metadata_widgets:
+            widget.initialize(self.db, id_)
+
+    def swap_title_author(self, *args):
+        title = self.title.current_val
+        self.title.current_val = authors_to_string(self.authors.current_val)
+        self.authors.current_val = string_to_authors(title)
+        self.title_sort.auto_generate()
+        self.author_sort.auto_generate()
+
+    def remove_unused_series(self, *args):
+        self.db.remove_unused_series()
+        idx = self.series.current_val
+        self.series.clear()
+        self.series.initialize(self.db, self.book_id)
+        if idx:
+            for i in range(self.series.count()):
+                if unicode(self.series.itemText(i)) == idx:
+                    self.series.setCurrentIndex(i)
+                    break
+
+
+if __name__ == '__main__':
+    from PyQt4.Qt import QApplication
+    app = QApplication([])
+    from calibre.library import db
+    db = db()
+    d = MetadataSingleDialog(db)
+    d(db.data[0][0])
+    d.exec_()
+

src/calibre/gui2/preferences/conversion.py

@@ -85,8 +85,8 @@ class CommonOptions(Base):
 
     def load_conversion_widgets(self):
         self.conversion_widgets = [LookAndFeelWidget, HeuristicsWidget,
-                SearchAndReplaceWidget, PageSetupWidget,
+                PageSetupWidget,
-                StructureDetectionWidget, TOCWidget]
+                StructureDetectionWidget, TOCWidget, SearchAndReplaceWidget,]
 
 class InputOptions(Base):
 

src/calibre/gui2/widgets.py

@@ -123,6 +123,8 @@ IMAGE_EXTENSIONS = ['jpg', 'jpeg', 'gif', 'png', 'bmp']
 
 class FormatList(QListWidget):
     DROPABBLE_EXTENSIONS = BOOK_EXTENSIONS
+    formats_dropped = pyqtSignal(object, object)
+    delete_format = pyqtSignal()
 
     @classmethod
     def paths_from_event(cls, event):
@@ -146,15 +148,14 @@ class FormatList(QListWidget):
     def dropEvent(self, event):
         paths = self.paths_from_event(event)
         event.setDropAction(Qt.CopyAction)
-        self.emit(SIGNAL('formats_dropped(PyQt_PyObject,PyQt_PyObject)'),
+        self.formats_dropped.emit(event, paths)
-                event, paths)
 
     def dragMoveEvent(self, event):
         event.acceptProposedAction()
 
     def keyPressEvent(self, event):
         if event.key() == Qt.Key_Delete:
-            self.emit(SIGNAL('delete_format()'))
+            self.delete_format.emit()
         else:
             return QListWidget.keyPressEvent(self, event)
 
@@ -162,6 +163,7 @@ class FormatList(QListWidget):
 class ImageView(QWidget):
 
     BORDER_WIDTH = 1
+    cover_changed = pyqtSignal(object)
 
     def __init__(self, parent=None):
         QWidget.__init__(self, parent)
@@ -201,8 +203,7 @@ class ImageView(QWidget):
             if not pmap.isNull():
                 self.setPixmap(pmap)
                 event.accept()
-                self.emit(SIGNAL('cover_changed(PyQt_PyObject)'), open(path,
+                self.cover_changed.emit(open(path, 'rb').read())
-                    'rb').read())
                 break
 
     def dragMoveEvent(self, event):
@@ -271,7 +272,7 @@ class ImageView(QWidget):
             pmap = cb.pixmap(cb.Selection)
         if not pmap.isNull():
             self.setPixmap(pmap)
-            self.emit(SIGNAL('cover_changed(PyQt_PyObject)'),
+            self.cover_changed.emit(
                     pixmap_to_data(pmap))
     # }}}
 

src/calibre/library/catalog.py

@@ -29,7 +29,6 @@ FIELDS = ['all', 'author_sort', 'authors', 'comments',
           'series_index', 'series', 'size', 'tags', 'timestamp', 'title',
           'uuid']
 
-
 #Allowed fields for template
 TEMPLATE_ALLOWED_FIELDS = [ 'author_sort', 'authors', 'id', 'isbn', 'pubdate',
     'publisher', 'series_index', 'series', 'tags', 'timestamp', 'title', 'uuid' ]
@@ -605,43 +604,42 @@ class EPUB_MOBI(CatalogPlugin):
                               "Default: '%default'\n"
                               "Applies to: ePub, MOBI output formats")),
                    Option('--generate-authors',
-                          default=True,
+                          default=False,
                           dest='generate_authors',
                           action = 'store_true',
-                          help=_("Include 'Authors' section in catalog."
+                          help=_("Include 'Authors' section in catalog.\n"
-                          "This switch is ignored - Books By Author section is always generated."
                           "Default: '%default'\n"
                           "Applies to: ePub, MOBI output formats")),
                    Option('--generate-descriptions',
-                          default=True,
+                          default=False,
                           dest='generate_descriptions',
                           action = 'store_true',
-                          help=_("Include book descriptions in catalog.\n"
+                          help=_("Include 'Descriptions' section in catalog.\n"
                           "Default: '%default'\n"
                           "Applies to: ePub, MOBI output formats")),
                    Option('--generate-genres',
-                          default=True,
+                          default=False,
                           dest='generate_genres',
                           action = 'store_true',
                           help=_("Include 'Genres' section in catalog.\n"
                           "Default: '%default'\n"
                           "Applies to: ePub, MOBI output formats")),
                    Option('--generate-titles',
-                          default=True,
+                          default=False,
                           dest='generate_titles',
                           action = 'store_true',
                           help=_("Include 'Titles' section in catalog.\n"
                           "Default: '%default'\n"
                           "Applies to: ePub, MOBI output formats")),
                    Option('--generate-series',
-                          default=True,
+                          default=False,
                           dest='generate_series',
                           action = 'store_true',
                           help=_("Include 'Series' section in catalog.\n"
                           "Default: '%default'\n"
                           "Applies to: ePub, MOBI output formats")),
                    Option('--generate-recently-added',
-                          default=True,
+                          default=False,
                           dest='generate_recently_added',
                           action = 'store_true',
                           help=_("Include 'Recently Added' section in catalog.\n"
@@ -976,7 +974,7 @@ class EPUB_MOBI(CatalogPlugin):
             self.__thumbWidth = 0
             self.__thumbHeight = 0
             self.__title = opts.catalog_title
-            self.__totalSteps = 8.0
+            self.__totalSteps = 6.0
             self.__useSeriesPrefixInTitlesSection = False
             self.__verbose = opts.verbose
 
@@ -1014,17 +1012,21 @@ class EPUB_MOBI(CatalogPlugin):
                                              (self.__archive_path, float(cached_thumb_width)))
 
             # Tweak build steps based on optional sections:  1 call for HTML, 1 for NCX
+            incremental_jobs = 0
+            if self.opts.generate_authors:
+                incremental_jobs += 2
             if self.opts.generate_titles:
-                self.__totalSteps += 2
+                incremental_jobs += 2
             if self.opts.generate_recently_added:
-                self.__totalSteps += 2
+                incremental_jobs += 2
                 if self.generateRecentlyRead:
-                    self.__totalSteps += 2
+                    incremental_jobs += 2
             if self.opts.generate_series:
-                self.__totalSteps += 2
+                incremental_jobs += 2
             if self.opts.generate_descriptions:
                 # +1 thumbs
-                self.__totalSteps += 3
+                incremental_jobs += 3
+            self.__totalSteps += incremental_jobs
 
             # Load section list templates
             templates = []
@@ -1358,13 +1360,21 @@ class EPUB_MOBI(CatalogPlugin):
             if self.opts.generate_descriptions:
                 self.generateThumbnails()
                 self.generateHTMLDescriptions()
-            self.generateHTMLByAuthor()
+            if self.opts.generate_authors:
+                self.generateHTMLByAuthor()
             if self.opts.generate_titles:
                 self.generateHTMLByTitle()
             if self.opts.generate_series:
                 self.generateHTMLBySeries()
             if self.opts.generate_genres:
                 self.generateHTMLByTags()
+                # If this is the only Section, and there are no genres, bail
+                if self.opts.section_list == ['Genres'] and not self.genres:
+                    error_msg = _("No Genres found to catalog.\nCheck 'Excluded genres'\nin E-book options.\n")
+                    self.opts.log.error(error_msg)
+                    self.error.append(_('No books available to catalog'))
+                    self.error.append(error_msg)
+                    return False
             if self.opts.generate_recently_added:
                 self.generateHTMLByDateAdded()
                 if self.generateRecentlyRead:
@@ -1372,7 +1382,8 @@ class EPUB_MOBI(CatalogPlugin):
 
             self.generateOPF()
             self.generateNCXHeader()
-            self.generateNCXByAuthor("Authors")
+            if self.opts.generate_authors:
+                self.generateNCXByAuthor("Authors")
             if self.opts.generate_titles:
                 self.generateNCXByTitle("Titles")
             if self.opts.generate_series:
@@ -1508,7 +1519,6 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
                 for tag in exclude_tags:
                     search_terms.append("tag:=%s" % tag)
                 search_phrase = "not (%s)" % " or ".join(search_terms)
-
             # If a list of ids are provided, don't use search_text
             if self.opts.ids:
                 self.opts.search_text = search_phrase
@@ -1879,7 +1889,8 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
                 # Link to author
                 emTag = Tag(soup, "em")
                 aTag = Tag(soup, "a")
-                aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(book['author']))
+                if self.opts.generate_authors:
+                    aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(book['author']))
                 aTag.insert(0, NavigableString(book['author']))
                 emTag.insert(0,aTag)
                 pBookTag.insert(ptc, emTag)
@@ -2149,7 +2160,8 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
                             pAuthorTag = Tag(soup, "p")
                             pAuthorTag['class'] = "author_index"
                             aTag = Tag(soup, "a")
-                            aTag['name'] = "%s" % self.generateAuthorAnchor(current_author)
+                            if self.opts.generate_authors:
+                                aTag['name'] = "%s" % self.generateAuthorAnchor(current_author)
                             aTag.insert(0,NavigableString(current_author))
                             pAuthorTag.insert(0,aTag)
                             divTag.insert(dtc,pAuthorTag)
@@ -2276,7 +2288,8 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
                         # Link to author
                         emTag = Tag(soup, "em")
                         aTag = Tag(soup, "a")
-                        aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author']))
+                        if self.opts.generate_authors:
+                            aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author']))
                         aTag.insert(0, NavigableString(new_entry['author']))
                         emTag.insert(0,aTag)
                         pBookTag.insert(ptc, emTag)
@@ -2425,7 +2438,8 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
                         # Link to author
                         emTag = Tag(soup, "em")
                         aTag = Tag(soup, "a")
-                        aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author']))
+                        if self.opts.generate_authors:
+                            aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author']))
                         aTag.insert(0, NavigableString(new_entry['author']))
                         emTag.insert(0,aTag)
                         pBookTag.insert(ptc, emTag)
@@ -2473,7 +2487,8 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
                         # Link to author
                         emTag = Tag(soup, "em")
                         aTag = Tag(soup, "a")
-                        aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author']))
+                        if self.opts.generate_authors:
+                            aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author']))
                         aTag.insert(0, NavigableString(new_entry['author']))
                         emTag.insert(0,aTag)
                         pBookTag.insert(ptc, emTag)
@@ -2692,7 +2707,8 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
 
                 # Link to author
                 aTag = Tag(soup, "a")
-                aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor",
+                if self.opts.generate_authors:
+                    aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor",
                                                 self.generateAuthorAnchor(escape(' & '.join(book['authors']))))
                 aTag.insert(0, NavigableString(' & '.join(book['authors'])))
                 pBookTag.insert(ptc, aTag)
@@ -3074,10 +3090,34 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
             textTag.insert(0, NavigableString(self.title))
             navLabelTag.insert(0, textTag)
             navPointTag.insert(0, navLabelTag)
-            contentTag = Tag(soup, 'content')
+
-            #contentTag['src'] = "content/book_%d.html" % int(self.booksByTitle[0]['id'])
+            if self.opts.generate_authors:
-            contentTag['src'] = "content/ByAlphaAuthor.html"
+                contentTag = Tag(soup, 'content')
-            navPointTag.insert(1, contentTag)
+                contentTag['src'] = "content/ByAlphaAuthor.html"
+                navPointTag.insert(1, contentTag)
+            elif self.opts.generate_titles:
+                contentTag = Tag(soup, 'content')
+                contentTag['src'] = "content/ByAlphaTitle.html"
+                navPointTag.insert(1, contentTag)
+            elif self.opts.generate_series:
+                contentTag = Tag(soup, 'content')
+                contentTag['src'] = "content/BySeries.html"
+                navPointTag.insert(1, contentTag)
+            elif self.opts.generate_genres:
+                contentTag = Tag(soup, 'content')
+                contentTag['src'] = "content/ByGenres.html"
+                navPointTag.insert(1, contentTag)
+            elif self.opts.generate_recently_added:
+                contentTag = Tag(soup, 'content')
+                contentTag['src'] = "content/ByDateAdded.html"
+                navPointTag.insert(1, contentTag)
+            else:
+                sort_descriptions_by = self.booksByAuthor if self.opts.sort_descriptions_by_author \
+                                                          else self.booksByTitle
+                contentTag = Tag(soup, 'content')
+                contentTag['src'] = "content/book_%d.html" % int(sort_descriptions_by[0]['id'])
+                navPointTag.insert(1, contentTag)
+
             cmiTag = Tag(soup, '%s' % 'calibre:meta-img')
             cmiTag['name'] = "mastheadImage"
             cmiTag['src'] = "images/mastheadImage.gif"
@@ -4140,7 +4180,8 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
                     pAuthorTag = Tag(soup, "p")
                     pAuthorTag['class'] = "author_index"
                     aTag = Tag(soup, "a")
-                    aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(book['author']))
+                    if self.opts.generate_authors:
+                        aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(book['author']))
                     aTag.insert(0, book['author'])
                     pAuthorTag.insert(0,aTag)
                     divTag.insert(dtc,pAuthorTag)
@@ -4371,7 +4412,8 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
 
             # Insert the author link (always)
             aTag = body.find('a', attrs={'class':'author'})
-            aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor",
+            if self.opts.generate_authors:
+                aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor",
                                             self.generateAuthorAnchor(book['author']))
 
             if publisher == ' ':
@@ -4860,6 +4902,8 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
 
         opts.basename = "Catalog"
         opts.cli_environment = not hasattr(opts,'sync')
+
+        # Hard-wired to always sort descriptions by author, with series after non-series
         opts.sort_descriptions_by_author = True
 
         build_log = []
@@ -4898,14 +4942,13 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
         if opts_dict['ids']:
             build_log.append(" book count: %d" % len(opts_dict['ids']))
 
-        '''
         sections_list = []
         if opts.generate_authors:
             sections_list.append('Authors')
-        '''
-        sections_list = ['Authors']
         if opts.generate_titles:
             sections_list.append('Titles')
+        if opts.generate_series:
+            sections_list.append('Series')
         if opts.generate_genres:
             sections_list.append('Genres')
         if opts.generate_recently_added:
@@ -4913,7 +4956,21 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
         if opts.generate_descriptions:
             sections_list.append('Descriptions')
 
+        if not sections_list:
+            if opts.cli_environment:
+                opts.log.warn('*** No Section switches specified, enabling all Sections ***')
+                opts.generate_authors = True
+                opts.generate_titles = True
+                opts.generate_series = True
+                opts.generate_genres = True
+                opts.generate_recently_added = True
+                opts.generate_descriptions = True
+                sections_list = ['Authors','Titles','Series','Genres','Recently Added','Descriptions']
+            else:
+                opts.log.warn('\n*** No enabled Sections, terminating catalog generation ***')
+                return ["No Included Sections","No enabled Sections.\nCheck E-book options tab\n'Included sections'\n"]
         build_log.append(u" Sections: %s" % ', '.join(sections_list))
+        opts.section_list = sections_list
 
         # Limit thumb_width to 1.0" - 2.0"
         try:
@@ -4948,6 +5005,7 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
 
         # Launch the Catalog builder
         catalog = self.CatalogBuilder(db, opts, self, report_progress=notification)
+
         if opts.verbose:
             log.info(" Begin catalog source generation")
         catalog.createDirectoryStructure()

src/calibre/manual/conversion.rst

@@ -260,20 +260,20 @@ The Output profile also controls the screen size. This will cause, for example,
 Heuristic Processing
 ---------------------
 
-Heuristic Processing provides a variety of functions which can be used that try to detect and correct 
+Heuristic Processing provides a variety of functions which can be used to try and detect and correct 
 common problems in poorly formatted input documents.  Use these functions if your input document suffers 
-from bad formatting. Because these functions rely on common patterns, be aware that in some cases an 
+from poor formatting. Because these functions rely on common patterns, be aware that in some cases an 
 option may lead to worse results, so use with care.  As an example, several of these options will
-remove all non-breaking-space entities.
+remove all non-breaking-space entities, or may include false positive matches relating to the function.
 
-:guilabel:`Preprocess input`
+:guilabel:`Enable heuristic processing`
-    This option activates various activates |app|'s Heuristic Processing stage of the conversion pipeline.
+    This option activates |app|'s Heuristic Processing stage of the conversion pipeline.
     This must be enabled in order for various sub-functions to be applied
 
 :guilabel:`Unwrap lines`
     Enabling this option will cause |app| to attempt to detect and correct hard line breaks that exist 
-    within a document using punctuation clues and line length.  |app| will first attempt to detect whether 
+    within a document using punctuation clues and line length. |app| will first attempt to detect whether 
-    hard line breaks exist, if they do not appear to exist |app| will not attempt to unwrap lines.  The 
+    hard line breaks exist, if they do not appear to exist |app| will not attempt to unwrap lines. The 
     line-unwrap factor can be reduced if you want to 'force' |app| to unwrap lines.
 
 :guilabel:`Line-unwrap factor`
@@ -283,22 +283,22 @@ remove all non-breaking-space entities.
     correction, then this value should be reduced to somewhere between 0.1 and 0.2.
     
 :guilabel:`Detect and markup unformatted chapter headings and sub headings`
-    If your document does not have Chapter Markers and titles formatted differently from the rest of the text,
+    If your document does not have chapter headings and titles formatted differently from the rest of the text,
-    |app| can use this option to attempt detection them and surround them with heading tags. <h2> tags are used 
+    |app| can use this option to attempt detection them and surround them with heading tags. <h2> tags are used 
-    for chapter headings; &lt;h3&gt; tags are used for any titles that are detected.  
+    for chapter headings; <h3> tags are used for any titles that are detected.  
     
     This function will not create a TOC, but in many cases it will cause |app|'s default chapter detection settings 
-    to correctly detect chapters and build a TOC.  Adjust the Xpath under Structure Detection if a TOC is not automatically
+    to correctly detect chapters and build a TOC.  Adjust the XPath under Structure Detection if a TOC is not automatically
     created.  If there are no other headings used in the document then setting "//h:h2" under Structure Detection would
     be the easiest way to create a TOC for the document.
     
-    The inserted headings are not formatted, to apply formatting use the 'extra_css' option under
+    The inserted headings are not formatted, to apply formatting use the :guilabel:`Extra CSS` option under
     the Look and Feel conversion settings.  For example, to center heading tags, use the following::
 
         h2, h3 { text-align: center }
 
-:guilabel:`Renumber sequences of &lt;h1&gt; or &lt;h2&gt; tags`
+:guilabel:`Renumber sequences of <h1> or <h2> tags`
-    Some publishers format chapter headings using multiple &lt;h1&gt; or &lt;h2&gt; tags sequentially.  
+    Some publishers format chapter headings using multiple <h1> or <h2> tags sequentially.  
     |app|'s default conversion settings will cause such titles to be split into two pieces.  This option 
     will re-number the heading tags to prevent splitting.
 
@@ -331,21 +331,23 @@ remove all non-breaking-space entities.
     Some documents use a convention of defining text indents using non-breaking space entities.  When this option is enabled |app| will
     attempt to detect this sort of formatting and convert them to a 3% text indent using css.
 
-.. search-replace:
+.. _search-replace:
 
 Search & Replace
 ---------------------
 
-These options are useful primarily for conversion of PDF documents. Often, the conversion leaves
+These options are useful primarily for conversion of PDF documents or OCR conversions, though they can 
-behind page headers and footers in the text. These options use regular expressions to try and detect
+also be used to fix many document specific problems. As an example, some conversions can leaves behind page  
-the headers and footers and remove them. Remember that they operate on the intermediate XHTML produced
+headers and footers in the text. These options use regular expressions to try and detect headers, footers, 
-by the conversion pipeline. There is also a wizard to help you customize the regular expressions for
+or other arbitrary text and remove or replace them. Remember that they operate on the intermediate XHTML produced 
-your document.  These options can also be used for generic search and replace of any content by additionally 
+by the conversion pipeline. There is a wizard to help you customize the regular expressions for 
-specifying a replacement expression.
+your document.  Click the magic wand beside the expression box, and click the 'Test' button after composing 
+your search expression.  Successful matches will be highlighted in Yellow.
 
-The search works by using a python regular expression. All matched text is simply removed from
+The search works by using a python regular expression. All matched text is simply removed from 
-the document or replaced using the replacement pattern. You can learn more about regular expressions and 
+the document or replaced using the replacement pattern. The replacement pattern is optional, if left blank 
-their syntax at http://docs.python.org/library/re.html.
+then text matching the search pattern will be deleted from the document.  You can learn more about regular expressions  
+and their syntax at :ref:`regexptutorial`.
 
 .. _structure-detection:
 

src/calibre/manual/faq.rst

@@ -107,10 +107,10 @@ My device is not being detected by |app|?
 Follow these steps to find the problem:
 
     * Make sure that you are connecting only a single device to your computer at a time. Do not have another |app| supported device like an iPhone/iPad etc. at the same time.
-    * Make sure you are running the latest version of |app|. The latest version can always be downloaded from `http://calibre-ebook.com/download`_.
+    * Make sure you are running the latest version of |app|. The latest version can always be downloaded from `the calibre website <http://calibre-ebook.com/download>`_.
-    * Ensure your operating system is seeing the device. That is, the device should be mounted as a disk that you can access using Windows explorer or whatever the file management program on your computer is
+    * Ensure your operating system is seeing the device. That is, the device should be mounted as a disk that you can access using Windows explorer or whatever the file management program on your computer is.
-    * In calibre, go to Preferences->Plugins->Device Interface plugin and make sure the plugin for your device is enabled.
+    * In calibre, go to Preferences->Plugins->Device Interface plugin and make sure the plugin for your device is enabled, the plugin icon next to it should be green when it is enabled.
-    * If all the above steps fail, go to Preferences->Miscellaneous and click debug device detection with your device attached and post the output as a ticket on `http://bugs.calibre-ebook.com`_.
+    * If all the above steps fail, go to Preferences->Miscellaneous and click debug device detection with your device attached and post the output as a ticket on `the calibre bug tracker <http://bugs.calibre-ebook.com>`_.
 
 How does |app| manage collections on my SONY reader?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

src/calibre/manual/regexp.rst

@@ -21,7 +21,7 @@ This is, inevitably, going to be somewhat technical- after all, regular expressi
 Where in |app| can you use regular expressions?
 ---------------------------------------------------
 
-There are a few places |app| uses regular expressions. There's the header/footer removal in conversion options, metadata detection from filenames in the import settings and, since last version, there's the option to use regular expressions to search and replace in metadata of multiple books.
+There are a few places |app| uses regular expressions. There's the Search & Replace in conversion options, metadata detection from filenames in the import settings and Search & Replace when editing the metadata of books in bulk.
 
 What on earth *is* a regular expression?
 ------------------------------------------------
@@ -94,7 +94,7 @@ I think I'm beginning to understand these regular expressions now... how do I us
 Conversions
 ^^^^^^^^^^^^^^
 
-Let's begin with the conversion settings, which is really neat. In the structure detection part, you can input a regexp (short for regular expression) that describes the header or footer string that will be removed during the conversion. The neat part is the wizard. Click on the wizard staff and you get a preview of what |app| "sees" during the conversion process. Scroll down to the header or footer you want to remove, select and copy it, paste it into the regexp field on top of the window. If there are variable parts, like page numbers or so, use sets and quantifiers to cover those, and while you're at it, remember to escape special characters, if there are some. Hit the button labeled :guilabel:`Test` and |app| highlights the parts it would remove were you to use the regexp. Once you're satisfied, hit OK and convert. Be careful if your conversion source has tags like this example::
+Let's begin with the conversion settings, which is really neat. In the Search and Replace part, you can input a regexp (short for regular expression) that describes the string that will be replaced during the conversion. The neat part is the wizard. Click on the wizard staff and you get a preview of what |app| "sees" during the conversion process. Scroll down to the string you want to remove, select and copy it, paste it into the regexp field on top of the window. If there are variable parts, like page numbers or so, use sets and quantifiers to cover those, and while you're at it, remember to escape special characters, if there are some. Hit the button labeled :guilabel:`Test` and |app| highlights the parts it would replace were you to use the regexp. Once you're satisfied, hit OK and convert. Be careful if your conversion source has tags like this example::
 
     Maybe, but the cops feel like you do, Anita. What's one more dead vampire?
     New laws don't change that. </p>
@@ -104,7 +104,7 @@ Let's begin with the conversion settings, which is really neat. In the structure
     <p class="calibre4"> It had only been two years since Addison v. Clark.
     The court case gave us a revised version of what life was
     
-(shamelessly ripped out of `this thread <http://www.mobileread.com/forums/showthread.php?t=75594">`_). You'd have to remove some of the tags as well. In this example, I'd recommend beginning with the tag ``<b class="calibre2">``, now you have to end with the corresponding closing tag (opening tags are ``<tag>``, closing tags are ``</tag>``), which is simply the next ``</b>`` in this case. (Refer to a good HTML manual or ask in the forum if you are unclear on this point.) The opening tag can be described using ``<b.*?>``, the closing tag using ``</b>``, thus we could remove everything between those tags using ``<b.*?>.*?</b>``. But using this expression would be a bad idea, because it removes everything enclosed by <b>- tags (which, by the way, render the enclosed text in bold print), and it's a fair bet that we'll remove portions of the book in this way. Instead, include the beginning of the enclosed string as well, making the regular expression ``<b.*?>\s*Generated\s+by\s+ABC\s+Amber\s+LIT.*?</b>`` The ``\s`` with quantifiers are included here instead of explicitly using the spaces as seen in the string to catch any variations of the string that might occur. Remember to check what |app| will remove to make sure you don't remove any portions you want to keep if you test a new expression. If you only check one occurrence, you might miss a mismatch somewhere else in the text. Also note that should you accidentally remove more or fewer tags than you actually wanted to, |app| tries to repair the damaged code after doing the header/footer removal.
+(shamelessly ripped out of `this thread <http://www.mobileread.com/forums/showthread.php?t=75594">`_). You'd have to remove some of the tags as well. In this example, I'd recommend beginning with the tag ``<b class="calibre2">``, now you have to end with the corresponding closing tag (opening tags are ``<tag>``, closing tags are ``</tag>``), which is simply the next ``</b>`` in this case. (Refer to a good HTML manual or ask in the forum if you are unclear on this point.) The opening tag can be described using ``<b.*?>``, the closing tag using ``</b>``, thus we could remove everything between those tags using ``<b.*?>.*?</b>``. But using this expression would be a bad idea, because it removes everything enclosed by <b>- tags (which, by the way, render the enclosed text in bold print), and it's a fair bet that we'll remove portions of the book in this way. Instead, include the beginning of the enclosed string as well, making the regular expression ``<b.*?>\s*Generated\s+by\s+ABC\s+Amber\s+LIT.*?</b>`` The ``\s`` with quantifiers are included here instead of explicitly using the spaces as seen in the string to catch any variations of the string that might occur. Remember to check what |app| will remove to make sure you don't remove any portions you want to keep if you test a new expression. If you only check one occurrence, you might miss a mismatch somewhere else in the text. Also note that should you accidentally remove more or fewer tags than you actually wanted to, |app| tries to repair the damaged code after doing the removal.
 
 Adding books
 ^^^^^^^^^^^^^^^^

src/calibre/utils/filenames.py

@@ -42,30 +42,44 @@ def supports_long_names(path):
     else:
         return True
 
-def shorten_components_to(length, components):
+def shorten_component(s, by_what):
+    l = len(s)
+    if l < by_what:
+        return s
+    l = (l - by_what)//2
+    if l <= 0:
+        return s
+    return s[:l] + s[-l:]
+
+def shorten_components_to(length, components, more_to_take=0):
     filepath = os.sep.join(components)
-    extra = len(filepath) - length
+    extra = len(filepath) - (length - more_to_take)
     if extra < 1:
         return components
-    delta = int(ceil(extra/float(len(components))))
+    deltas = []
-    ans = []
     for x in components:
+        pct = len(x)/float(len(filepath))
+        deltas.append(int(ceil(pct*extra)))
+    ans = []
+
+    for i, x in enumerate(components):
+        delta = deltas[i]
         if delta > len(x):
             r = x[0] if x is components[-1] else ''
         else:
             if x is components[-1]:
                 b, e = os.path.splitext(x)
                 if e == '.': e = ''
-                r = b[:-delta]+e
+                r = shorten_component(b, delta)+e
                 if r.startswith('.'): r = x[0]+r
             else:
-                r = x[:-delta]
+                r = shorten_component(x, delta)
             r = r.strip()
             if not r:
                 r = x.strip()[0] if x.strip() else 'x'
         ans.append(r)
     if len(os.sep.join(ans)) > length:
-        return shorten_components_to(length, ans)
+        return shorten_components_to(length, components, more_to_take+2)
     return ans
 
 def find_executable_in_path(name, path=None):

src/calibre/utils/formatter_functions.py

@@ -75,7 +75,7 @@ class FormatterFunction(object):
             exc_type, exc_value, exc_traceback = sys.exc_info()
             info = ': '.join(traceback.format_exception(exc_type, exc_value,
                                         exc_traceback)[-2:]).replace('\n', '')
-            return _('Exception ' + info)
+            return _('Exception ') + info
 
 all_builtin_functions = []
 class BuiltinFormatterFunction(FormatterFunction):