Sync to trunk.

Changelog.yaml

@@ -4,6 +4,120 @@
 # for important features/bug fixes.
 # Also, each release can have new and improved recipes.
 
+- version: 0.7.40
+  date: 2011-01-14
+
+  new features:
+    - title: "A new 'highlight matches' search mode" 
+      description: >
+        "There is now a checkbox next to the search bar named 'Highlight'. If you check it, searching will highlight
+        all matched books instead of filtering the book list to all matched books."
+
+    - title: "RTF Input: Improved support for conversion of images. The bug where some images were shrunk should no longer happen"
+
+    - title: "Template language: Allow you to create your own formatting functions. Accessible via Preferences->Advanced->Template functions"
+
+    - title: "News download: Convert various HTML 5 tags into <div> to support readers that cannot handle HTML 5 tags"
+
+    - title: "RTF metadata: Add support for publisher and tags."
+      tickets: [6657]
+
+    - title: "BibTeX catalog: Add support for custom columns"
+
+    - title: "TXT Input: Support for textile markup"
+
+    - title: "Various minor tweaks to improve usability of Preferences->Plugins"
+
+    - title: "TXT Output: Convert <hr> to scene break marker."
+
+    - title: "Support for the Archos 70"
+
+    - title: "SONY Driver: Add an option to automatically refresh the covers on every connect. Accessible via: Preferences->Plugins->Device interface plugins"
+
+    - title: "Add access to the larger template editor from plugboards via context menu."
+
+    - title: "Speed improvement when connecting a large library to a device"
+
+    - title: "Speedup when searching on multiple words in a large library"
+
+    - title: "TXT Input: Add a heauristic formatting processor"
+
+
+  bug fixes:
+    - title: "Fix bug that caused automatic news removal to remove any book that has a tag that contains the word 'news' instead of only books that have the tag News"
+
+    - title: "Refactor the downloading social metadata message box to allow canceling."
+      tickets: [8234]
+
+    - title: "Kobo drive does not deal with Null value in DateCreated column"
+      tickets: [8308]
+
+    - title: "MOBI Input: Fix regression that caused images placed inside svg tags to be discarded"
+
+    - title: "Fix selecting Tablet output profile would actually select the Samsung Galaxy S profile"
+
+    - title: "Catalog generation: Fix a condition that could cause TOCs to not be properly generated in MOBI format catalogs"
+      tickets: [8295]
+
+    - title: "Zip file reading: Be more tolerant when a zip file has a damaged file directory"
+
+    - title: "RTF Input: Various code cleanups. Go back to trying to handle unicode mappings without pre-processing. This will mean that some RTF files that used to convert, won't anymore. Please open tickets and attach them."
+      tickets: [8171]
+
+    - title: "ImageMagick: When identifying an image don't read the entire image"
+
+    - title: "FB2 Output: Add cover to FB2 metadata."
+
+    - title: "Fix inability to customize builting recipe when more than one recipe has the same name"
+      tickets: [8281]
+
+    - title: "RTF Input: Fix regression that broke the Preprocess HTML option"
+
+    - title: "Fix XSS vulnerability in content server."
+      tickets: [7980]
+
+    - title: "TXT Output: Clean up and produce consistant output. Spacing around headings. Headings are not indented when using the remove paragraph spacing option."
+
+    - title: "Catalog generation: Handle invalid covers gracefully"
+
+    - title: "Email settings: Before displaying the email test dialog warn the user that it will expose their email password"
+
+    - title: "PDB Output: Fix regression that caused some PDB files to not work with other software"
+      tickets: [8231]
+
+  improved recipes:
+    - Financial Times UK
+    - Globe and Mail
+    - Wired Daily
+    - MIT Technology Review
+    - MSNBC
+    - expansion.com
+    - New York Times
+    - Heraldo de Aragon
+    - Exiled online
+
+  new recipes:
+    - title: "Yakima Herald and Tri-City Herald"
+      author: "Laura Gjovaag"
+
+    - title: "Wichita Eagle"
+      author: "Jason Cameron"
+
+    - title: "Pressthink and Zero Hedge"
+      author: "Darko Miletic"
+
+    - title: "tyzden"
+      author: "zemiak"
+
+    - title: "El Correo"
+      author: "desUBIKado"
+
+    - title: "Cicero"
+      author: "mad"
+
+    - title: "El Publico"
+      author: "Gerardo Diez"
+
 - version: 0.7.38
   date: 2011-01-07
 

resources/recipes/financial_times_uk.recipe

@@ -1,5 +1,5 @@
 __license__   = 'GPL v3'
-__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2010-2011, Darko Miletic <darko.miletic at gmail.com>'
 '''
 ft.com
 '''
@@ -52,12 +52,9 @@ class FinancialTimes(BasicNewsRecipe):
                 .copyright{font-size: x-small}
                 """
 
-    def parse_index(self):
+    def get_artlinks(self, elem):
         articles = []
-        soup = self.index_to_soup(self.INDEX)
+        for item in elem.findAll('a',href=True):
-        wide = soup.find('div',attrs={'class':'wide'})
-        if wide:
-            for item in wide.findAll('a',href=True):
             url   = self.PREFIX + item['href']
             title = self.tag_to_string(item)
             date = strftime(self.timefmt)
@@ -67,7 +64,26 @@ class FinancialTimes(BasicNewsRecipe):
                              ,'url'        :url
                              ,'description':''
                             })
-        return [('FT UK edition',articles)]
+        return articles
+                
+    def parse_index(self):
+        feeds = []
+        soup = self.index_to_soup(self.INDEX)
+        wide = soup.find('div',attrs={'class':'wide'})
+        if not wide:
+           return feeds
+        strest = wide.findAll('h3', attrs={'class':'section'})
+        if not strest:
+           return feeds
+        st = wide.find('h4',attrs={'class':'section-no-arrow'})
+        if st:
+           strest.insert(0,st)
+        for item in strest:
+            ftitle   = self.tag_to_string(item)
+            self.report_progress(0, _('Fetching feed')+' %s...'%(ftitle))            
+            feedarts = self.get_artlinks(item.parent.ul)
+            feeds.append((ftitle,feedarts))
+        return feeds
 
     def preprocess_html(self, soup):
         return self.adeify_images(soup)

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.7.38'
+__version__   = '0.7.40'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 
 import re

src/calibre/gui2/actions/edit_metadata.py

@@ -99,7 +99,7 @@ class EditMetadataAction(InterfaceAction):
             x = _('social metadata')
         else:
             x = _('covers') if covers and not set_metadata else _('metadata')
-        title = _('Downloading %s for %d book(s)')%(x, len(ids))
+        title = _('Downloading {0} for {1} book(s)').format(x, len(ids))
         self._download_book_metadata = DoDownload(self.gui, title, db, ids,
                 get_covers=covers, set_metadata=set_metadata,
                 get_social_metadata=get_social_metadata)

src/calibre/gui2/actions/view.py

@@ -92,7 +92,12 @@ class ViewAction(InterfaceAction):
         formats = [list(f.upper().split(',')) if f else None for f in formats]
         all_fmts = set([])
         for x in formats:
+            if x:
                 for f in x: all_fmts.add(f)
+        if not all_fmts:
+            error_dialog(self.gui,  _('Format unavailable'),
+                    _('Selected books have no formats'), show=True)
+            return
         d = ChooseFormatDialog(self.gui, _('Choose the format to view'),
                 list(sorted(all_fmts)))
         if d.exec_() == d.Accepted:

src/calibre/gui2/catalog/catalog_epub_mobi.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>650</width>
-    <height>582</height>
+    <height>596</height>
    </rect>
   </property>
   <property name="sizePolicy">
@@ -41,41 +41,54 @@
       <string>Included sections</string>
      </property>
      <layout class="QGridLayout" name="gridLayout_2">
-      <item row="0" column="0">
+      <item row="0" column="1">
-       <widget class="QCheckBox" name="generate_titles">
-        <property name="text">
-         <string>Books by &amp;Title</string>
-        </property>
-       </widget>
-      </item>
-      <item row="4" column="0">
-       <widget class="QCheckBox" name="generate_series">
-        <property name="text">
-         <string>Books by &amp;Series</string>
-        </property>
-       </widget>
-      </item>
-      <item row="0" column="2">
-       <widget class="QCheckBox" name="generate_recently_added">
-        <property name="text">
-         <string>Recently &amp;Added</string>
-        </property>
-       </widget>
-      </item>
-      <item row="5" column="0">
        <widget class="QCheckBox" name="generate_genres">
         <property name="text">
          <string>Books by &amp;Genre</string>
         </property>
        </widget>
       </item>
-      <item row="4" column="2">
+      <item row="4" column="1">
+       <widget class="QCheckBox" name="generate_recently_added">
+        <property name="text">
+         <string>Recently &amp;Added</string>
+        </property>
+       </widget>
+      </item>
+      <item row="5" column="1">
        <widget class="QCheckBox" name="generate_descriptions">
         <property name="text">
          <string>&amp;Descriptions</string>
         </property>
        </widget>
       </item>
+      <item row="5" column="0">
+       <widget class="QCheckBox" name="generate_series">
+        <property name="text">
+         <string>Books by &amp;Series</string>
+        </property>
+       </widget>
+      </item>
+      <item row="4" column="0">
+       <widget class="QCheckBox" name="generate_titles">
+        <property name="text">
+         <string>Books by &amp;Title</string>
+        </property>
+       </widget>
+      </item>
+      <item row="0" column="0">
+       <widget class="QCheckBox" name="generate_authors">
+        <property name="enabled">
+         <bool>false</bool>
+        </property>
+        <property name="text">
+         <string>Books by Author</string>
+        </property>
+        <property name="checked">
+         <bool>true</bool>
+        </property>
+       </widget>
+      </item>
      </layout>
     </widget>
    </item>
@@ -94,14 +107,10 @@
       </size>
      </property>
      <property name="toolTip">
-      <string>&lt;!DOCTYPE HTML PUBLIC &quot;-//W3C//DTD HTML 4.0//EN&quot; &quot;http://www.w3.org/TR/REC-html40/strict.dtd&quot;&gt;
+      <string>&lt;p&gt;Default pattern 
-&lt;html&gt;&lt;head&gt;&lt;meta name=&quot;qrichtext&quot; content=&quot;1&quot; /&gt;&lt;style type=&quot;text/css&quot;&gt;
+\[.+\]
-p, li { white-space: pre-wrap; }
+excludes tags of the form [tag], 
-&lt;/style&gt;&lt;/head&gt;&lt;body style=&quot; font-family:'Lucida Grande'; font-size:13pt; font-weight:400; font-style:normal;&quot;&gt;
+e.g., [Project Gutenberg]&lt;/p&gt;</string>
-&lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;Default pattern &lt;/p&gt;
-&lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-family:'Courier New,courier';&quot;&gt;\[.+\]&lt;/span&gt;&lt;/p&gt;
-&lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;excludes tags of the form [&lt;span style=&quot; font-family:'Courier New,courier';&quot;&gt;tag&lt;/span&gt;], &lt;/p&gt;
-&lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;e.g., [Project Gutenberg]&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
      </property>
      <property name="title">
       <string>Excluded genres</string>
@@ -239,12 +248,8 @@ p, li { white-space: pre-wrap; }
            </size>
           </property>
           <property name="toolTip">
-           <string>&lt;!DOCTYPE HTML PUBLIC &quot;-//W3C//DTD HTML 4.0//EN&quot; &quot;http://www.w3.org/TR/REC-html40/strict.dtd&quot;&gt;
+           <string>&lt;p&gt;Comma-separated list of tags to exclude.
-&lt;html&gt;&lt;head&gt;&lt;meta name=&quot;qrichtext&quot; content=&quot;1&quot; /&gt;&lt;style type=&quot;text/css&quot;&gt;
+Default: ~,Catalog</string>
-p, li { white-space: pre-wrap; }
-&lt;/style&gt;&lt;/head&gt;&lt;body style=&quot; font-family:'Lucida Grande'; font-size:13pt; font-weight:400; font-style:normal;&quot;&gt;
-&lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-size:12pt;&quot;&gt;Comma-separated list of tags to exclude.&lt;/span&gt;&lt;/p&gt;
-&lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-size:12pt;&quot;&gt;Default:&lt;/span&gt;&lt;span style=&quot; font-family:'Courier New,courier'; font-size:12pt;&quot;&gt; ~,Catalog&lt;/span&gt;&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
           </property>
          </widget>
         </item>
@@ -332,7 +337,7 @@ p, li { white-space: pre-wrap; }
       </size>
      </property>
      <property name="toolTip">
-      <string>Matching books will be displayed with ✓</string>
+      <string>Matching books will be displayed with a check mark</string>
      </property>
      <property name="title">
       <string>Read books</string>
@@ -471,7 +476,7 @@ p, li { white-space: pre-wrap; }
         <item>
          <widget class="QLineEdit" name="wishlist_tag">
           <property name="toolTip">
-           <string>Books tagged as Wishlist items will be displayed with ✕</string>
+           <string>Books tagged as Wishlist items will be displayed with an X</string>
           </property>
          </widget>
         </item>

src/calibre/gui2/preferences/social.py

@@ -62,6 +62,7 @@ class SocialMetadata(QDialog):
             return
         if not self.worker.is_alive():
             self.accept()
+            return
         QTimer.singleShot(50, self.update)
 
     def accept(self):

src/calibre/gui2/preferences/template_functions.py

@@ -25,37 +25,49 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         template function is written in python. It takes information from the
         book, processes it in some way, then returns a string result. Functions
         defined here are usable in templates in the same way that builtin
-        functions are usable. The function must be named evaluate, and must
+        functions are usable. The function must be named <b>evaluate</b>, and
-        have the signature shown below.</p>
+        must have the signature shown below.</p>
-        <p><code>evaluate(self, formatter, kwargs, mi, locals, your_arguments)
+        <p><code>evaluate(self, formatter, kwargs, mi, locals, your parameters)
         &rarr; returning a unicode string</code></p>
-        <p>The arguments to evaluate are:
+        <p>The parameters of the evaluate function are:
         <ul>
-        <li><b>formatter:</b> the instance of the formatter being used to
+        <li><b>formatter</b>: the instance of the formatter being used to
         evaluate the current template. You can use this to do recursive
         template evaluation.</li>
-        <li><b>kwargs:</b> a dictionary of metadata. Field values are in this
+        <li><b>kwargs</b>: a dictionary of metadata. Field values are in this
-        dictionary. mi: a Metadata instance. Used to get field information.
+        dictionary.
+        <li><b>mi</b>: a Metadata instance. Used to get field information.
         This parameter can be None in some cases, such as when evaluating
         non-book templates.</li>
-        <li><b>locals:</b> the local variables assigned to by the current
+        <li><b>locals</b>: the local variables assigned to by the current
         template program.</li>
-        <li><b>Your_arguments</b> must be one or more parameter (number
+        <li><b>your parameters</b>: You must supply one or more formal
-        matching the arg count box), or the value *args for a variable number
+        parameters. The number must match the arg count box, unless arg count is
-        of arguments. These are values passed into the function. One argument
+        -1 (variable number or arguments), in which case the last argument must
-        is required, and is usually the value of the field being operated upon.
+        be *args. At least one argument is required, and is usually the value of
-        Note that when writing in basic template mode, the user does not
+        the field being operated upon. Note that when writing in basic template
-        provide this first argument. Instead it is the value of the field the
+        mode, the user does not provide this first argument. Instead it is
-        function is operating upon.</li>
+        supplied by the formatter.</li>
         </ul></p>
         <p>
-        The following example function looks for various values in the tags
+        The following example function checks the value of the field. If the
-        metadata field, returning those values that appear in tags.
+        field is not empty, the field's value is returned, otherwise the value
+        EMPTY is returned.
         <pre>
+        name: my_ifempty
+        arg count: 1
+        doc: my_ifempty(val) -- return val if it is not empty, otherwise the string 'EMPTY'
+        program code:
         def evaluate(self, formatter, kwargs, mi, locals, val):
-            awards=['allbooks', 'PBook', 'ggff']
+            if val:
-            return ', '.join([t for t in kwargs.get('tags') if t in awards])
+                return val
-        </pre>
+            else:
+                return 'EMPTY'</pre>
+        This function can be called in any of the three template program modes:
+        <ul>
+        <li>single-function mode: {tags:my_ifempty()}</li>
+        <li>template program mode: {tags:'my_ifempty($)'}</li>
+        <li>general program mode: program: my_ifempty(field('tags'))</li>
         </p>
         ''')
         self.textBrowser.setHtml(help_text)
@@ -67,14 +79,22 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.build_function_names_box()
         self.function_name.currentIndexChanged[str].connect(self.function_index_changed)
         self.function_name.editTextChanged.connect(self.function_name_edited)
+        self.argument_count.valueChanged.connect(self.enable_replace_button)
+        self.documentation.textChanged.connect(self.enable_replace_button)
+        self.program.textChanged.connect(self.enable_replace_button)
         self.create_button.clicked.connect(self.create_button_clicked)
         self.delete_button.clicked.connect(self.delete_button_clicked)
         self.create_button.setEnabled(False)
         self.delete_button.setEnabled(False)
+        self.replace_button.setEnabled(False)
         self.clear_button.clicked.connect(self.clear_button_clicked)
+        self.replace_button.clicked.connect(self.replace_button_clicked)
         self.program.setTabStopWidth(20)
         self.highlighter = PythonHighlighter(self.program.document())
 
+    def enable_replace_button(self):
+        self.replace_button.setEnabled(self.delete_button.isEnabled())
+
     def clear_button_clicked(self):
         self.build_function_names_box()
         self.program.clear()
@@ -112,6 +132,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
             self.create_button.setEnabled(True)
             self.delete_button.setEnabled(False)
             self.build_function_names_box(set_to=name)
+            self.program.setReadOnly(False)
         else:
             error_dialog(self.gui, _('Template functions'),
                          _('Function not defined'), show=True)
@@ -143,6 +164,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.documentation.setReadOnly(False)
         self.argument_count.setReadOnly(False)
         self.create_button.setEnabled(True)
+        self.replace_button.setEnabled(False)
+        self.program.setReadOnly(False)
 
     def function_index_changed(self, txt):
         txt = unicode(txt)
@@ -156,15 +179,21 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         func = self.funcs[txt]
         self.argument_count.setValue(func.arg_count)
         self.documentation.setText(func.doc)
+        self.program.setPlainText(func.program_text)
         if txt in self.builtins:
             self.documentation.setReadOnly(True)
             self.argument_count.setReadOnly(True)
-            self.program.clear()
+            self.program.setReadOnly(True)
             self.delete_button.setEnabled(False)
         else:
             self.program.setPlainText(func.program_text)
             self.delete_button.setEnabled(True)
+            self.program.setReadOnly(False)
+        self.replace_button.setEnabled(False)
 
+    def replace_button_clicked(self):
+        self.delete_button_clicked()
+        self.create_button_clicked()
     def refresh_gui(self, gui):
         pass
 

src/calibre/gui2/preferences/template_functions.ui

@@ -38,7 +38,7 @@
        <item row="0" column="1">
         <widget class="QComboBox" name="function_name">
          <property name="toolTip">
-          <string>Enter the name of the function to create</string>
+          <string>Enter the name of the function to create.</string>
          </property>
          <property name="editable">
           <bool>true</bool>
@@ -48,7 +48,7 @@
        <item row="1" column="0">
         <widget class="QLabel" name="label_3">
          <property name="toolTip">
-          <string></string>
+          <string/>
          </property>
          <property name="text">
           <string>Arg &amp;count:</string>
@@ -100,6 +100,13 @@
            </property>
           </widget>
          </item>
+         <item>
+          <widget class="QPushButton" name="replace_button">
+           <property name="text">
+            <string>&amp;Replace</string>
+           </property>
+          </widget>
+         </item>
          <item>
           <widget class="QPushButton" name="create_button">
            <property name="text">
@@ -144,8 +151,7 @@
     </layout>
    </item>
    <item row="0" column="0">
-    <widget class="QTextBrowser" name="textBrowser">
+    <widget class="QTextBrowser" name="textBrowser"/>
-    </widget>
    </item>
   </layout>
  </widget>

src/calibre/library/catalog.py

@@ -599,8 +599,16 @@ class EPUB_MOBI(CatalogPlugin):
                           default=('~,'+_('Catalog')),
                           dest='exclude_tags',
                           action = None,
-                          help=_("Comma-separated list of tag words indicating book should be excluded from output.  Case-insensitive.\n"
+                          help=_("Comma-separated list of tag words indicating book should be excluded from output."
-                          "--exclude-tags=skip will match 'skip this book' and 'Skip will like this'.\n"
+                              "For example: 'skip' will match 'skip this book' and 'Skip will like this'."
+                              "Default: '%default'\n"
+                              "Applies to: ePub, MOBI output formats")),
+                   Option('--generate-authors',
+                          default=True,
+                          dest='generate_authors',
+                          action = 'store_true',
+                          help=_("Include 'Authors' section in catalog."
+                          "This switch is ignored - Books By Author section is always generated."
                           "Default: '%default'\n"
                           "Applies to: ePub, MOBI output formats")),
                    Option('--generate-descriptions',
@@ -1602,11 +1610,12 @@ class EPUB_MOBI(CatalogPlugin):
                 if author != current_author and i:
                     # Warn, exit if friendly matches previous, but sort doesn't
                     if author[0] == current_author[0]:
-                        error_msg = _("\nWarning: inconsistent Author Sort values for Author '%s', ") % author[0]
+                        error_msg = _('''
-                        error_msg += _("unable to continue building catalog.\n")
+\n*** Metadata error ***
-                        error_msg += _("Select all books by '%s', apply same Author Sort value in Edit Metadata dialog, ") % author[0]
+Inconsistent Author Sort values for Author '{0}', unable to continue building catalog.
-                        error_msg += _("then rebuild the catalog.\n")
+Select all books by '{0}', apply correct Author Sort value in Edit Metadata dialog,
-                        error_msg += _("Terminating catalog generation.\n")
+then rebuild the catalog.
+*** Terminating catalog generation ***\n''').format(author[0])
 
                         self.opts.log.warn(error_msg)
                         return False
@@ -4971,12 +4980,16 @@ class EPUB_MOBI(CatalogPlugin):
             build_log.append(" book count: %d" % len(opts_dict['ids']))
 
         sections_list = ['Authors']
+        '''
+        if opts.generate_authors:
+            sections_list.append('Authors')
+        '''
         if opts.generate_titles:
             sections_list.append('Titles')
-        if opts.generate_recently_added:
-            sections_list.append('Recently Added')
         if opts.generate_genres:
             sections_list.append('Genres')
+        if opts.generate_recently_added:
+            sections_list.append('Recently Added')
         if opts.generate_descriptions:
             sections_list.append('Descriptions')
 

src/calibre/library/cli.py

@@ -986,8 +986,8 @@ def command_restore_database(args, dbpath):
         return 1
 
     if not opts.really_do_it:
-        prints(_('You must provide the --really-do-it option to do a'
+        prints(_('You must provide the %s option to do a'
-            ' recovery'), end='\n\n')
+            ' recovery')%'--really-do-it', end='\n\n')
         parser.print_help()
         return 1
 

src/calibre/utils/formatter.py

@@ -96,7 +96,7 @@ class _Parser(object):
                     # classic assignment statement
                     self.consume()
                     cls = funcs['assign']
-                    return cls.eval(self.parent, self.parent.kwargs,
+                    return cls.eval_(self.parent, self.parent.kwargs,
                                     self.parent.book, self.parent.locals, id, self.expr())
                 return self.parent.locals.get(id, _('unknown id ') + id)
             # We have a function.
@@ -130,7 +130,7 @@ class _Parser(object):
                 cls = funcs[id]
                 if cls.arg_count != -1 and len(args) != cls.arg_count:
                     self.error('incorrect number of arguments for function {}'.format(id))
-                return cls.eval(self.parent, self.parent.kwargs,
+                return cls.eval_(self.parent, self.parent.kwargs,
                                 self.parent.book, self.parent.locals, *args)
             else:
                 f = self.parent.functions[id]
@@ -284,14 +284,13 @@ class TemplateFormatter(string.Formatter):
                     else:
                         args = self.arg_parser.scan(fmt[p+1:])[0]
                         args = [self.backslash_comma_to_comma.sub(',', a) for a in args]
-                    if (func.arg_count == 1 and (len(args) != 0)) or \
+                    if (func.arg_count == 1 and (len(args) != 1 or args[0])) or \
                             (func.arg_count > 1 and func.arg_count != len(args)+1):
-                        print args
                         raise ValueError('Incorrect number of arguments for function '+ fmt[0:p])
                     if func.arg_count == 1:
-                        val = func.eval(self, self.kwargs, self.book, self.locals, val).strip()
+                        val = func.eval_(self, self.kwargs, self.book, self.locals, val).strip()
                     else:
-                        val = func.eval(self, self.kwargs, self.book, self.locals,
+                        val = func.eval_(self, self.kwargs, self.book, self.locals,
                                         val, *args).strip()
         if val:
             val = self._do_format(val, dispfmt)

src/calibre/utils/formatter_functions.py

@@ -8,7 +8,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import re, traceback
+import inspect, re, traceback, sys
 
 from calibre.utils.titlecase import titlecase
 from calibre.utils.icu import capitalize, strcmp
@@ -58,13 +58,10 @@ class FormatterFunction(object):
     name = 'no name provided'
     arg_count = 0
 
-    def __init__(self):
-        formatter_functions.register_builtin(self)
-
     def evaluate(self, formatter, kwargs, mi, locals, *args):
         raise NotImplementedError()
 
-    def eval(self, formatter, kwargs, mi, locals, *args):
+    def eval_(self, formatter, kwargs, mi, locals, *args):
         try:
             ret = self.evaluate(formatter, kwargs, mi, locals, *args)
             if isinstance(ret, (str, unicode)):
@@ -75,9 +72,24 @@ class FormatterFunction(object):
                 return ','.join(list)
         except:
             traceback.print_exc()
-            return _('Function threw exception' + traceback.format_exc())
+            exc_type, exc_value, exc_traceback = sys.exc_info()
+            info = ': '.join(traceback.format_exception(exc_type, exc_value,
+                                        exc_traceback)[-2:]).replace('\n', '')
+            return _('Exception ' + info)
 
-class BuiltinStrcmp(FormatterFunction):
+
+class BuiltinFormatterFunction(FormatterFunction):
+    def __init__(self):
+        formatter_functions.register_builtin(self)
+        eval_func = inspect.getmembers(self.__class__,
+                        lambda x: inspect.ismethod(x) and x.__name__ == 'evaluate')
+        try:
+            lines = [l[4:] for l in inspect.getsourcelines(eval_func[0][1])[0]]
+        except:
+            lines = []
+        self.program_text = ''.join(lines)
+
+class BuiltinStrcmp(BuiltinFormatterFunction):
     name = 'strcmp'
     arg_count = 5
     doc = _('strcmp(x, y, lt, eq, gt) -- does a case-insensitive comparison of x '
@@ -92,7 +104,7 @@ class BuiltinStrcmp(FormatterFunction):
             return eq
         return gt
 
-class BuiltinCmp(FormatterFunction):
+class BuiltinCmp(BuiltinFormatterFunction):
     name = 'cmp'
     arg_count = 5
     doc =   _('cmp(x, y, lt, eq, gt) -- compares x and y after converting both to '
@@ -107,7 +119,7 @@ class BuiltinCmp(FormatterFunction):
             return eq
         return gt
 
-class BuiltinStrcat(FormatterFunction):
+class BuiltinStrcat(BuiltinFormatterFunction):
     name = 'strcat'
     arg_count = -1
     doc = _('strcat(a, b, ...) -- can take any number of arguments. Returns a '
@@ -120,7 +132,7 @@ class BuiltinStrcat(FormatterFunction):
             res += args[i]
         return res
 
-class BuiltinAdd(FormatterFunction):
+class BuiltinAdd(BuiltinFormatterFunction):
     name = 'add'
     arg_count = 2
     doc = _('add(x, y) -- returns x + y. Throws an exception if either x or y are not numbers.')
@@ -130,7 +142,7 @@ class BuiltinAdd(FormatterFunction):
         y = float(y if y else 0)
         return unicode(x + y)
 
-class BuiltinSubtract(FormatterFunction):
+class BuiltinSubtract(BuiltinFormatterFunction):
     name = 'subtract'
     arg_count = 2
     doc = _('subtract(x, y) -- returns x - y. Throws an exception if either x or y are not numbers.')
@@ -140,7 +152,7 @@ class BuiltinSubtract(FormatterFunction):
         y = float(y if y else 0)
         return unicode(x - y)
 
-class BuiltinMultiply(FormatterFunction):
+class BuiltinMultiply(BuiltinFormatterFunction):
     name = 'multiply'
     arg_count = 2
     doc = _('multiply(x, y) -- returns x * y. Throws an exception if either x or y are not numbers.')
@@ -150,7 +162,7 @@ class BuiltinMultiply(FormatterFunction):
         y = float(y if y else 0)
         return unicode(x * y)
 
-class BuiltinDivide(FormatterFunction):
+class BuiltinDivide(BuiltinFormatterFunction):
     name = 'divide'
     arg_count = 2
     doc = _('divide(x, y) -- returns x / y. Throws an exception if either x or y are not numbers.')
@@ -160,7 +172,7 @@ class BuiltinDivide(FormatterFunction):
         y = float(y if y else 0)
         return unicode(x / y)
 
-class BuiltinTemplate(FormatterFunction):
+class BuiltinTemplate(BuiltinFormatterFunction):
     name = 'template'
     arg_count = 1
     doc = _('template(x) -- evaluates x as a template. The evaluation is done '
@@ -168,17 +180,17 @@ class BuiltinTemplate(FormatterFunction):
             'the caller and the template evaluation. Because the { and } '
             'characters are special, you must use [[ for the { character and '
             ']] for the } character; they are converted automatically. '
-            'For example, ``template(\'[[title_sort]]\') will evaluate the '
+            'For example, template(\'[[title_sort]]\') will evaluate the '
             'template {title_sort} and return its value.')
 
     def evaluate(self, formatter, kwargs, mi, locals, template):
         template = template.replace('[[', '{').replace(']]', '}')
         return formatter.safe_format(template, kwargs, 'TEMPLATE', mi)
 
-class BuiltinEval(FormatterFunction):
+class BuiltinEval(BuiltinFormatterFunction):
     name = 'eval'
     arg_count = 1
-    doc = _('eval(template)`` -- evaluates the template, passing the local '
+    doc = _('eval(template) -- evaluates the template, passing the local '
             'variables (those \'assign\'ed to) instead of the book metadata. '
             ' This permits using the template processor to construct complex '
             'results from local variables.')
@@ -188,7 +200,7 @@ class BuiltinEval(FormatterFunction):
         template = template.replace('[[', '{').replace(']]', '}')
         return eval_formatter.safe_format(template, locals, 'EVAL', None)
 
-class BuiltinAssign(FormatterFunction):
+class BuiltinAssign(BuiltinFormatterFunction):
     name = 'assign'
     arg_count = 2
     doc = _('assign(id, val) -- assigns val to id, then returns val. '
@@ -198,7 +210,7 @@ class BuiltinAssign(FormatterFunction):
         locals[target] = value
         return value
 
-class BuiltinPrint(FormatterFunction):
+class BuiltinPrint(BuiltinFormatterFunction):
     name = 'print'
     arg_count = -1
     doc = _('print(a, b, ...) -- prints the arguments to standard output. '
@@ -209,7 +221,7 @@ class BuiltinPrint(FormatterFunction):
         print args
         return None
 
-class BuiltinField(FormatterFunction):
+class BuiltinField(BuiltinFormatterFunction):
     name = 'field'
     arg_count = 1
     doc = _('field(name) -- returns the metadata field named by name')
@@ -217,7 +229,7 @@ class BuiltinField(FormatterFunction):
     def evaluate(self, formatter, kwargs, mi, locals, name):
         return formatter.get_value(name, [], kwargs)
 
-class BuiltinSubstr(FormatterFunction):
+class BuiltinSubstr(BuiltinFormatterFunction):
     name = 'substr'
     arg_count = 3
     doc = _('substr(str, start, end) -- returns the start\'th through the end\'th '
@@ -230,7 +242,7 @@ class BuiltinSubstr(FormatterFunction):
     def evaluate(self, formatter, kwargs, mi, locals, str_, start_, end_):
         return str_[int(start_): len(str_) if int(end_) == 0 else int(end_)]
 
-class BuiltinLookup(FormatterFunction):
+class BuiltinLookup(BuiltinFormatterFunction):
     name = 'lookup'
     arg_count = -1
     doc = _('lookup(val, pattern, field, pattern, field, ..., else_field) -- '
@@ -257,7 +269,7 @@ class BuiltinLookup(FormatterFunction):
                 return formatter.vformat('{'+args[i+1].strip() + '}', [], kwargs)
             i += 2
 
-class BuiltinTest(FormatterFunction):
+class BuiltinTest(BuiltinFormatterFunction):
     name = 'test'
     arg_count = 3
     doc = _('test(val, text if not empty, text if empty) -- return `text if not '
@@ -269,7 +281,7 @@ class BuiltinTest(FormatterFunction):
         else:
             return value_not_set
 
-class BuiltinContains(FormatterFunction):
+class BuiltinContains(BuiltinFormatterFunction):
     name = 'contains'
     arg_count = 4
     doc = _('contains(val, pattern, text if match, text if not match) -- checks '
@@ -284,13 +296,13 @@ class BuiltinContains(FormatterFunction):
         else:
             return value_if_not
 
-class BuiltinSwitch(FormatterFunction):
+class BuiltinSwitch(BuiltinFormatterFunction):
     name = 'switch'
     arg_count = -1
     doc = _('switch(val, pattern, value, pattern, value, ..., else_value) -- '
-            'for each ``pattern, value`` pair, checks if the field matches '
+            'for each `pattern, value` pair, checks if the field matches '
-            'the regular expression ``pattern`` and if so, returns that '
+            'the regular expression `pattern` and if so, returns that '
-            'value. If no pattern matches, then else_value is returned. '
+            '`value`. If no pattern matches, then else_value is returned. '
             'You can have as many `pattern, value` pairs as you want')
 
     def evaluate(self, formatter, kwargs, mi, locals, val, *args):
@@ -304,7 +316,7 @@ class BuiltinSwitch(FormatterFunction):
                 return args[i+1]
             i += 2
 
-class BuiltinRe(FormatterFunction):
+class BuiltinRe(BuiltinFormatterFunction):
     name = 're'
     arg_count = 3
     doc = _('re(val, pattern, replacement) -- return the field after applying '
@@ -315,10 +327,10 @@ class BuiltinRe(FormatterFunction):
     def evaluate(self, formatter, kwargs, mi, locals, val, pattern, replacement):
         return re.sub(pattern, replacement, val)
 
-class BuiltinEvaluate(FormatterFunction):
+class BuiltinIfempty(BuiltinFormatterFunction):
-    name = 'evaluate'
+    name = 'ifempty'
     arg_count = 2
-    doc = _('evaluate(val, text if empty) -- return val if val is not empty, '
+    doc = _('ifempty(val, text if empty) -- return val if val is not empty, '
             'otherwise return `text if empty`')
 
     def evaluate(self, formatter, kwargs, mi, locals, val, value_if_empty):
@@ -327,7 +339,7 @@ class BuiltinEvaluate(FormatterFunction):
         else:
             return value_if_empty
 
-class BuiltinShorten(FormatterFunction):
+class BuiltinShorten(BuiltinFormatterFunction):
     name = 'shorten'
     arg_count = 4
     doc = _('shorten(val, left chars, middle text, right chars) -- Return a '
@@ -352,7 +364,7 @@ class BuiltinShorten(FormatterFunction):
         else:
             return val
 
-class BuiltinCount(FormatterFunction):
+class BuiltinCount(BuiltinFormatterFunction):
     name = 'count'
     arg_count = 2
     doc = _('count(val, separator) -- interprets the value as a list of items '
@@ -363,7 +375,7 @@ class BuiltinCount(FormatterFunction):
     def evaluate(self, formatter, kwargs, mi, locals, val, sep):
         return unicode(len(val.split(sep)))
 
-class BuiltinListitem(FormatterFunction):
+class BuiltinListitem(BuiltinFormatterFunction):
     name = 'list_item'
     arg_count = 3
     doc = _('list_item(val, index, separator) -- interpret the value as a list of '
@@ -383,7 +395,7 @@ class BuiltinListitem(FormatterFunction):
         except:
             return ''
 
-class BuiltinUppercase(FormatterFunction):
+class BuiltinUppercase(BuiltinFormatterFunction):
     name = 'uppercase'
     arg_count = 1
     doc = _('uppercase(val) -- return value of the field in upper case')
@@ -391,7 +403,7 @@ class BuiltinUppercase(FormatterFunction):
     def evaluate(self, formatter, kwargs, mi, locals, val):
         return val.upper()
 
-class BuiltinLowercase(FormatterFunction):
+class BuiltinLowercase(BuiltinFormatterFunction):
     name = 'lowercase'
     arg_count = 1
     doc = _('lowercase(val) -- return value of the field in lower case')
@@ -399,7 +411,7 @@ class BuiltinLowercase(FormatterFunction):
     def evaluate(self, formatter, kwargs, mi, locals, val):
         return val.lower()
 
-class BuiltinTitlecase(FormatterFunction):
+class BuiltinTitlecase(BuiltinFormatterFunction):
     name = 'titlecase'
     arg_count = 1
     doc = _('titlecase(val) -- return value of the field in title case')
@@ -407,7 +419,7 @@ class BuiltinTitlecase(FormatterFunction):
     def evaluate(self, formatter, kwargs, mi, locals, val):
         return titlecase(val)
 
-class BuiltinCapitalize(FormatterFunction):
+class BuiltinCapitalize(BuiltinFormatterFunction):
     name = 'capitalize'
     arg_count = 1
     doc = _('capitalize(val) -- return value of the field capitalized')
@@ -423,7 +435,7 @@ builtin_contains    = BuiltinContains()
 builtin_count       = BuiltinCount()
 builtin_divide      = BuiltinDivide()
 builtin_eval        = BuiltinEval()
-builtin_evaluate    = BuiltinEvaluate()
+builtin_ifempty     = BuiltinIfempty()
 builtin_field       = BuiltinField()
 builtin_list_item   = BuiltinListitem()
 builtin_lookup      = BuiltinLookup()

src/calibre/web/fetch/simple.py

@@ -493,7 +493,7 @@ def option_parser(usage=_('%prog URL\n\nWhere URL is for example http://google.c
     parser.add_option('--match-regexp', default=[], action='append', dest='match_regexps',
                       help=_('Only links that match this regular expression will be followed. This option can be specified multiple times, in which case as long as a link matches any one regexp, it will be followed. By default all links are followed.'))
     parser.add_option('--filter-regexp', default=[], action='append', dest='filter_regexps',
-                      help=_('Any link that matches this regular expression will be ignored. This option can be specified multiple times, in which case as long as any regexp matches a link, it will be ignored.By default, no links are ignored. If both --filter-regexp and --match-regexp are specified, then --filter-regexp is applied first.'))
+                      help=_('Any link that matches this regular expression will be ignored. This option can be specified multiple times, in which case as long as any regexp matches a link, it will be ignored.By default, no links are ignored. If both filter regexp and match regexp are specified, then filter regexp is applied first.'))
     parser.add_option('--dont-download-stylesheets', action='store_true', default=False,
                       help=_('Do not download CSS stylesheets.'), dest='no_stylesheets')
     parser.add_option('--verbose', help=_('Show detailed output information. Useful for debugging'),