merge from trunk

Changelog.yaml

@@ -4,6 +4,120 @@
 # for important features/bug fixes.
 # Also, each release can have new and improved recipes.
 
+- version: 0.7.40
+  date: 2011-01-14
+
+  new features:
+    - title: "A new 'highlight matches' search mode" 
+      description: >
+        "There is now a checkbox next to the search bar named 'Highlight'. If you check it, searching will highlight
+        all matched books instead of filtering the book list to all matched books."
+
+    - title: "RTF Input: Improved support for conversion of images. The bug where some images were shrunk should no longer happen"
+
+    - title: "Template language: Allow you to create your own formatting functions. Accessible via Preferences->Advanced->Template functions"
+
+    - title: "News download: Convert various HTML 5 tags into <div> to support readers that cannot handle HTML 5 tags"
+
+    - title: "RTF metadata: Add support for publisher and tags."
+      tickets: [6657]
+
+    - title: "BibTeX catalog: Add support for custom columns"
+
+    - title: "TXT Input: Support for textile markup"
+
+    - title: "Various minor tweaks to improve usability of Preferences->Plugins"
+
+    - title: "TXT Output: Convert <hr> to scene break marker."
+
+    - title: "Support for the Archos 70"
+
+    - title: "SONY Driver: Add an option to automatically refresh the covers on every connect. Accessible via: Preferences->Plugins->Device interface plugins"
+
+    - title: "Add access to the larger template editor from plugboards via context menu."
+
+    - title: "Speed improvement when connecting a large library to a device"
+
+    - title: "Speedup when searching on multiple words in a large library"
+
+    - title: "TXT Input: Add a heauristic formatting processor"
+
+
+  bug fixes:
+    - title: "Fix bug that caused automatic news removal to remove any book that has a tag that contains the word 'news' instead of only books that have the tag News"
+
+    - title: "Refactor the downloading social metadata message box to allow canceling."
+      tickets: [8234]
+
+    - title: "Kobo drive does not deal with Null value in DateCreated column"
+      tickets: [8308]
+
+    - title: "MOBI Input: Fix regression that caused images placed inside svg tags to be discarded"
+
+    - title: "Fix selecting Tablet output profile would actually select the Samsung Galaxy S profile"
+
+    - title: "Catalog generation: Fix a condition that could cause TOCs to not be properly generated in MOBI format catalogs"
+      tickets: [8295]
+
+    - title: "Zip file reading: Be more tolerant when a zip file has a damaged file directory"
+
+    - title: "RTF Input: Various code cleanups. Go back to trying to handle unicode mappings without pre-processing. This will mean that some RTF files that used to convert, won't anymore. Please open tickets and attach them."
+      tickets: [8171]
+
+    - title: "ImageMagick: When identifying an image don't read the entire image"
+
+    - title: "FB2 Output: Add cover to FB2 metadata."
+
+    - title: "Fix inability to customize builting recipe when more than one recipe has the same name"
+      tickets: [8281]
+
+    - title: "RTF Input: Fix regression that broke the Preprocess HTML option"
+
+    - title: "Fix XSS vulnerability in content server."
+      tickets: [7980]
+
+    - title: "TXT Output: Clean up and produce consistant output. Spacing around headings. Headings are not indented when using the remove paragraph spacing option."
+
+    - title: "Catalog generation: Handle invalid covers gracefully"
+
+    - title: "Email settings: Before displaying the email test dialog warn the user that it will expose their email password"
+
+    - title: "PDB Output: Fix regression that caused some PDB files to not work with other software"
+      tickets: [8231]
+
+  improved recipes:
+    - Financial Times UK
+    - Globe and Mail
+    - Wired Daily
+    - MIT Technology Review
+    - MSNBC
+    - expansion.com
+    - New York Times
+    - Heraldo de Aragon
+    - Exiled online
+
+  new recipes:
+    - title: "Yakima Herald and Tri-City Herald"
+      author: "Laura Gjovaag"
+
+    - title: "Wichita Eagle"
+      author: "Jason Cameron"
+
+    - title: "Pressthink and Zero Hedge"
+      author: "Darko Miletic"
+
+    - title: "tyzden"
+      author: "zemiak"
+
+    - title: "El Correo"
+      author: "desUBIKado"
+
+    - title: "Cicero"
+      author: "mad"
+
+    - title: "El Publico"
+      author: "Gerardo Diez"
+
 - version: 0.7.38
   date: 2011-01-07
 

resources/recipes/financial_times_uk.recipe

@@ -1,5 +1,5 @@
 __license__   = 'GPL v3'
-__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2010-2011, Darko Miletic <darko.miletic at gmail.com>'
 '''
 ft.com
 '''
@@ -52,22 +52,38 @@ class FinancialTimes(BasicNewsRecipe):
                 .copyright{font-size: x-small}
                 """
 
-    def parse_index(self):
+    def get_artlinks(self, elem):
         articles = []
+        for item in elem.findAll('a',href=True):
+            url   = self.PREFIX + item['href']
+            title = self.tag_to_string(item)
+            date = strftime(self.timefmt)
+            articles.append({
+                              'title'      :title
+                             ,'date'       :date
+                             ,'url'        :url
+                             ,'description':''
+                            })
+        return articles
+                
+    def parse_index(self):
+        feeds = []
         soup = self.index_to_soup(self.INDEX)
         wide = soup.find('div',attrs={'class':'wide'})
-        if wide:
-            for item in wide.findAll('a',href=True):
-                url   = self.PREFIX + item['href']
-                title = self.tag_to_string(item)
-                date = strftime(self.timefmt)
-                articles.append({
-                                  'title'      :title
-                                 ,'date'       :date
-                                 ,'url'        :url
-                                 ,'description':''
-                                })
-        return [('FT UK edition',articles)]
+        if not wide:
+           return feeds
+        strest = wide.findAll('h3', attrs={'class':'section'})
+        if not strest:
+           return feeds
+        st = wide.find('h4',attrs={'class':'section-no-arrow'})
+        if st:
+           strest.insert(0,st)
+        for item in strest:
+            ftitle   = self.tag_to_string(item)
+            self.report_progress(0, _('Fetching feed')+' %s...'%(ftitle))            
+            feedarts = self.get_artlinks(item.parent.ul)
+            feeds.append((ftitle,feedarts))
+        return feeds
 
     def preprocess_html(self, soup):
         return self.adeify_images(soup)

resources/recipes/pressthink.recipe

@@ -0,0 +1,61 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011, Darko Miletic <darko.miletic at gmail.com>'
+'''
+pressthink.org
+'''
+
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class PressThink(BasicNewsRecipe):
+    title                  = 'PressThink'
+    __author__             = 'Darko Miletic'
+    description            = 'Ghost of democracy in the media machine'
+    oldest_article         = 60
+    max_articles_per_feed  = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    encoding               = 'utf8'
+    publisher              = 'Arthur L. Carter Journalism Institute'
+    category               = 'news, USA, world, economy, politics, media'
+    language               = 'en'
+    publication_type       = 'blog'
+    extra_css              = """
+                                body{ font-family: Helvetica,Arial,sans-serif }
+                                img{display: block; margin-bottom: 0.5em}
+                                h6{font-size: 1.1em; font-weight: bold}
+                                .post-author{font-family: Georgia,serif}
+                                .post-title{color: #AB0000}
+                                .says{color: gray}
+                                .comment {
+                                    border-bottom: 1px dotted #555555;
+                                    border-top: 1px dotted #DDDDDD;
+                                    margin-left: 10px;
+                                    min-height: 100px;
+                                    padding: 15px 0 20px;
+                                }
+                             """
+
+    conversion_options = {
+                             'comments' : description
+                            ,'tags'     : category
+                            ,'language' : language
+                            ,'publisher': publisher
+                         }
+
+    remove_tags       = [dict(name=['form','iframe','embed','object','link','base','table','meta'])]
+    keep_only_tags    = [dict(attrs={'class':['post-title','post-author','entry','postmetadata alt','commentlist']})]
+
+    feeds = [(u'Articles', u'http://pressthink.org/feed/')]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll('img', alt=False):
+            item['alt'] = 'image'
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
+

resources/recipes/tri_city_herald.recipe

@@ -0,0 +1,25 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class TriCityHeraldRecipe(BasicNewsRecipe):
+    title          = u'Tri-City Herald'
+    description    = 'The Tri-City Herald Mid-Columbia.'
+    language = 'en'
+    __author__     = 'Laura Gjovaag'
+    oldest_article        = 1.5
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    remove_javascript     = True
+    keep_only_tags = [
+                      dict(name='div', attrs={'id':'story_header'}),
+                      dict(name='img', attrs={'class':'imageCycle'}),
+                      dict(name='div', attrs={'id':['cycleImageCaption', 'story_body']})
+                      ]
+    remove_tags    = [
+                      dict(name='div', attrs={'id':'story_mlt'}),
+                      dict(name='a', attrs={'id':'commentCount'}),
+                      dict(name=['script', 'noscript', 'style'])]
+    extra_css      = 'h1{font: bold 140%;} #cycleImageCaption{font: monospace 60%}'
+
+    feeds          = [
+                      (u'Tri-City Herald Mid-Columbia', u'http://www.tri-cityherald.com/901/index.rss')
+                     ]

resources/recipes/wichita_eagle.recipe

@@ -0,0 +1,29 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1294938721(BasicNewsRecipe):
+     title            = u'Wichita Eagle'
+     language = 'en'
+     __author__ = 'Jason Cameron'
+     description  = 'Daily news from the Wichita Eagle'
+     oldest_article = 1
+     max_articles_per_feed = 30
+     keep_only_tags = [dict(name='div', attrs={'id':'wide'})]
+     feeds          = [
+                                 (u'Local News',
+ u'http://www.kansas.com/news/local/index.rss'),
+                                 (u'National News',
+ u'http://www.kansas.com/news/nation-world/index.rss'),
+                                 (u'Sports',
+ u'http://www.kansas.com/sports/index.rss'),
+                                 (u'Opinion',
+ u'http://www.kansas.com/opinion/index.rss'),
+                                 (u'Life',
+ u'http://www.kansas.com/living/index.rss'),
+                                 (u'Entertainment',
+ u'http://www.kansas.com/entertainment/index.rss')
+                           ]
+
+     def print_version(self, url):
+         urlparts = url.split('/')
+         newadd = urlparts[5]+'/v-print'
+         return url.replace(url, newadd.join(url.split(urlparts[5])))

resources/recipes/yakima_herald.recipe

@@ -0,0 +1,21 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class YakimaHeraldRepublicRecipe(BasicNewsRecipe):
+    title          = u'Yakima Herald-Republic'
+    description    = 'The Yakima Herald-Republic.'
+    language = 'en'
+    __author__     = 'Laura Gjovaag'
+    oldest_article        = 1.5
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    remove_javascript     = True
+    keep_only_tags = [
+                      dict(name='div', attrs={'id':['searchleft', 'headline_credit']}),
+                      dict(name='div', attrs={'class':['photo', 'cauthor', 'photocredit']}),
+                      dict(name='div', attrs={'id':['content_body', 'footerleft']})
+                      ]
+    extra_css = '.cauthor {font: monospace 60%;} .photocredit {font: monospace 60%}'
+
+    feeds          = [
+                      (u'Yakima Herald Online', u'http://feeds.feedburner.com/yhronlinenews'),
+                     ]

setup/upload.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, re, cStringIO, base64, httplib, subprocess, hashlib, shutil
+import os, re, cStringIO, base64, httplib, subprocess, hashlib, shutil, time
 from subprocess import check_call
 from tempfile import NamedTemporaryFile, mkdtemp
 
@@ -160,7 +160,7 @@ class UploadToGoogleCode(Command):
 
         return 'multipart/form-data; boundary=%s' % BOUNDARY, CRLF.join(body)
 
-    def upload(self, fname, desc, labels=[]):
+    def upload(self, fname, desc, labels=[], retry=0):
         form_fields = [('summary', desc)]
         form_fields.extend([('label', l.strip()) for l in labels])
 
@@ -183,6 +183,10 @@ class UploadToGoogleCode(Command):
 
         print 'Failed to upload with code %d and reason: %s'%(resp.status,
                 resp.reason)
+        if retry < 1:
+            print 'Retrying in 5 seconds....'
+            time.sleep(5)
+            return self.upload(fname, desc, labels=labels, retry=retry+1)
         raise Exception('Failed to upload '+fname)
 
 

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.7.38'
+__version__   = '0.7.40'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 
 import re

src/calibre/customize/builtins.py

@@ -705,13 +705,17 @@ class ActionTweakEpub(InterfaceActionBase):
     name = 'Tweak ePub'
     actual_plugin = 'calibre.gui2.actions.tweak_epub:TweakEpubAction'
 
+class ActionNextMatch(InterfaceActionBase):
+    name = 'Next Match'
+    actual_plugin = 'calibre.gui2.actions.next_match:NextMatchAction'
+
 plugins += [ActionAdd, ActionFetchAnnotations, ActionGenerateCatalog,
         ActionConvert, ActionDelete, ActionEditMetadata, ActionView,
         ActionFetchNews, ActionSaveToDisk, ActionShowBookDetails,
         ActionRestart, ActionOpenFolder, ActionConnectShare,
         ActionSendToDevice, ActionHelp, ActionPreferences, ActionSimilarBooks,
         ActionAddToLibrary, ActionEditCollections, ActionChooseLibrary,
-        ActionCopyToLibrary, ActionTweakEpub]
+        ActionCopyToLibrary, ActionTweakEpub, ActionNextMatch]
 
 # }}}
 
@@ -843,6 +847,17 @@ class Plugboard(PreferencesPlugin):
     config_widget = 'calibre.gui2.preferences.plugboard'
     description = _('Change metadata fields before saving/sending')
 
+class TemplateFunctions(PreferencesPlugin):
+    name = 'TemplateFunctions'
+    icon = I('template_funcs.png')
+    gui_name = _('Template Functions')
+    category = 'Advanced'
+    gui_category = _('Advanced')
+    category_order = 5
+    name_order = 4
+    config_widget = 'calibre.gui2.preferences.template_functions'
+    description = _('Create your own template functions')
+
 class Email(PreferencesPlugin):
     name = 'Email'
     icon = I('mail.png')
@@ -904,6 +919,6 @@ class Misc(PreferencesPlugin):
 
 plugins += [LookAndFeel, Behavior, Columns, Toolbar, InputOptions,
         CommonOptions, OutputOptions, Adding, Saving, Sending, Plugboard,
-        Email, Server, Plugins, Tweaks, Misc]
+        Email, Server, Plugins, Tweaks, Misc, TemplateFunctions]
 
 #}}}

src/calibre/customize/profiles.py

@@ -441,7 +441,7 @@ class TabletOutput(iPadOutput):
 
 class SamsungGalaxy(TabletOutput):
     name = 'Samsung Galaxy'
-    shortname = 'galaxy'
+    short_name = 'galaxy'
     description = _('Intended for the Samsung Galaxy and similar tablet devices with '
             'a resolution of 600x1280')
     screen_size = comic_screen_size = (600, 1280)

src/calibre/devices/kobo/books.py

@@ -27,7 +27,7 @@ class Book(Book_):
 
         self.size = size # will be set later if None
 
-        if ContentType == '6':
+        if ContentType == '6' and date is not None:
             self.datetime = time.strptime(date, "%Y-%m-%dT%H:%M:%S.%f")
         else:
             try:

src/calibre/ebooks/mobi/reader.py

@@ -632,9 +632,18 @@ class MobiReader(object):
                 attrib['class'] = cls
 
         for tag in svg_tags:
-            p = tag.getparent()
-            if hasattr(p, 'remove'):
-                p.remove(tag)
+            images = tag.xpath('descendant::img[@src]')
+            parent = tag.getparent()
+
+            if images and hasattr(parent, 'find'):
+                index = parent.index(tag)
+                for img in images:
+                    img.getparent().remove(img)
+                    img.tail = img.text = None
+                    parent.insert(index, img)
+
+            if hasattr(parent, 'remove'):
+                parent.remove(tag)
 
     def create_opf(self, htmlfile, guide=None, root=None):
         mi = getattr(self.book_header.exth, 'mi', self.embedded_mi)

src/calibre/gui2/actions/__init__.py

@@ -111,7 +111,10 @@ class InterfaceAction(QObject):
         action.setWhatsThis(text)
         action.setAutoRepeat(False)
         if shortcut:
-            action.setShortcut(shortcut)
+            if isinstance(shortcut, list):
+                action.setShortcuts(shortcut)
+            else:
+                action.setShortcut(shortcut)
         setattr(self, attr, action)
         return action
 
@@ -170,6 +173,14 @@ class InterfaceAction(QObject):
         '''
         pass
 
+    def gui_layout_complete(self):
+        '''
+        Called once per action when the layout of the main GUI is
+        completed. If your action needs to make changes to the layout, they
+        should be done here, rather than in :meth:`initialization_complete`.
+        '''
+        pass
+
     def initialization_complete(self):
         '''
         Called once per action when the initialization of the main GUI is

src/calibre/gui2/actions/catalog.py

@@ -57,7 +57,7 @@ class GenerateCatalogAction(InterfaceAction):
         if job.result:
             # Search terms nulled catalog results
             return error_dialog(self.gui, _('No books found'),
-                    _("No books to catalog\nCheck exclusion criteria"),
+                    _("No books to catalog\nCheck job details"),
                     show=True)
         if job.failed:
             return self.gui.job_exception(job)

src/calibre/gui2/actions/edit_metadata.py

@@ -99,7 +99,7 @@ class EditMetadataAction(InterfaceAction):
             x = _('social metadata')
         else:
             x = _('covers') if covers and not set_metadata else _('metadata')
-        title = _('Downloading %s for %d book(s)')%(x, len(ids))
+        title = _('Downloading {0} for {1} book(s)').format(x, len(ids))
         self._download_book_metadata = DoDownload(self.gui, title, db, ids,
                 get_covers=covers, set_metadata=set_metadata,
                 get_social_metadata=get_social_metadata)

src/calibre/gui2/actions/next_match.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from calibre.gui2.actions import InterfaceAction
+
+class NextMatchAction(InterfaceAction):
+    name = 'Move to next highlighted book'
+    action_spec = (_('Move to next match'), 'arrow-down.png',
+            _('Move to next highlighted match'), [_('N'), _('F3')])
+    dont_add_to = frozenset(['toolbar-device', 'context-menu-device'])
+    action_type = 'current'
+
+    def genesis(self):
+        '''
+        Setup this plugin. Only called once during initialization. self.gui is
+        available. The action secified by :attr:`action_spec` is available as
+        ``self.qaction``.
+        '''
+        self.can_move = None
+        self.qaction.triggered.connect(self.move_forward)
+        self.create_action(spec=(_('Move to previous item'), 'arrow-up.png',
+              _('Move to previous highlighted item'), [_('Shift+N'),
+                  _('Shift+F3')]), attr='p_action')
+        self.gui.addAction(self.p_action)
+        self.p_action.triggered.connect(self.move_backward)
+
+    def gui_layout_complete(self):
+        self.gui.search_highlight_only.setVisible(True)
+
+    def location_selected(self, loc):
+        self.can_move = loc == 'library'
+        try:
+            self.gui.search_highlight_only.setVisible(self.can_move)
+        except:
+            import traceback
+            traceback.print_exc()
+
+    def move_forward(self):
+        if self.can_move is None:
+            self.can_move = self.gui.current_view() is self.gui.library_view
+            self.gui.search_highlight_only.setVisible(self.can_move)
+
+        if self.can_move:
+            self.gui.current_view().move_highlighted_row(forward=True)
+
+    def move_backward(self):
+        if self.can_move is None:
+            self.can_move = self.gui.current_view() is self.gui.library_view
+            self.gui.search_highlight_only.setVisible(self.can_move)
+
+        if self.can_move:
+            self.gui.current_view().move_highlighted_row(forward=False)

src/calibre/gui2/actions/view.py

@@ -92,7 +92,12 @@ class ViewAction(InterfaceAction):
         formats = [list(f.upper().split(',')) if f else None for f in formats]
         all_fmts = set([])
         for x in formats:
-            for f in x: all_fmts.add(f)
+            if x:
+                for f in x: all_fmts.add(f)
+        if not all_fmts:
+            error_dialog(self.gui,  _('Format unavailable'),
+                    _('Selected books have no formats'), show=True)
+            return
         d = ChooseFormatDialog(self.gui, _('Choose the format to view'),
                 list(sorted(all_fmts)))
         if d.exec_() == d.Accepted:

src/calibre/gui2/catalog/catalog_epub_mobi.py

@@ -6,67 +6,18 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-
 from calibre.ebooks.conversion.config import load_defaults
 from calibre.gui2 import gprefs
 
 from catalog_epub_mobi_ui import Ui_Form
-from PyQt4.Qt import QWidget, QLineEdit
+from PyQt4.Qt import QCheckBox, QComboBox, QDoubleSpinBox, QLineEdit, \
+                     QRadioButton, QWidget
 
 class PluginWidget(QWidget,Ui_Form):
 
     TITLE = _('E-book options')
     HELP  = _('Options specific to')+' EPUB/MOBI '+_('output')
 
-    CheckBoxControls = [
-                        'generate_titles',
-                        'generate_series',
-                        'generate_genres',
-                        'generate_recently_added',
-                        'generate_descriptions',
-                        'include_hr'
-                        ]
-    ComboBoxControls = [
-                        'read_source_field',
-                        'exclude_source_field',
-                        'header_note_source_field',
-                        'merge_source_field'
-                        ]
-    LineEditControls = [
-                        'exclude_genre',
-                        'exclude_pattern',
-                        'exclude_tags',
-                        'read_pattern',
-                        'wishlist_tag'
-                        ]
-    RadioButtonControls = [
-                        'merge_before',
-                        'merge_after'
-                        ]
-    SpinBoxControls = [
-                        'thumb_width'
-                        ]
-
-    OPTION_FIELDS = zip(CheckBoxControls,
-                        [True for i in CheckBoxControls],
-                        ['check_box' for i in CheckBoxControls])
-    OPTION_FIELDS += zip(ComboBoxControls,
-                        [None for i in ComboBoxControls],
-                        ['combo_box' for i in ComboBoxControls])
-    OPTION_FIELDS += zip(RadioButtonControls,
-                        [None for i in RadioButtonControls],
-                        ['radio_button' for i in RadioButtonControls])
-
-    # LineEditControls
-    OPTION_FIELDS += zip(['exclude_genre'],['\[.+\]'],['line_edit'])
-    OPTION_FIELDS += zip(['exclude_pattern'],[None],['line_edit'])
-    OPTION_FIELDS += zip(['exclude_tags'],['~,'+_('Catalog')],['line_edit'])
-    OPTION_FIELDS += zip(['read_pattern'],['+'],['line_edit'])
-    OPTION_FIELDS += zip(['wishlist_tag'],['Wishlist'],['line_edit'])
-
-    # SpinBoxControls
-    OPTION_FIELDS += zip(['thumb_width'],[1.00],['spin_box'])
-
     # Output synced to the connected device?
     sync_enabled = True
 
@@ -76,8 +27,69 @@ class PluginWidget(QWidget,Ui_Form):
     def __init__(self, parent=None):
         QWidget.__init__(self, parent)
         self.setupUi(self)
+        self._initControlArrays()
+
+    def _initControlArrays(self):
+
+        CheckBoxControls = []
+        ComboBoxControls = []
+        DoubleSpinBoxControls = []
+        LineEditControls = []
+        RadioButtonControls = []
+
+        for item in self.__dict__:
+            if type(self.__dict__[item]) is QCheckBox:
+                CheckBoxControls.append(str(self.__dict__[item].objectName()))
+            elif type(self.__dict__[item]) is QComboBox:
+                ComboBoxControls.append(str(self.__dict__[item].objectName()))
+            elif type(self.__dict__[item]) is QDoubleSpinBox:
+                DoubleSpinBoxControls.append(str(self.__dict__[item].objectName()))
+            elif type(self.__dict__[item]) is QLineEdit:
+                LineEditControls.append(str(self.__dict__[item].objectName()))
+            elif type(self.__dict__[item]) is QRadioButton:
+                RadioButtonControls.append(str(self.__dict__[item].objectName()))
+
+        option_fields = zip(CheckBoxControls,
+                            [True for i in CheckBoxControls],
+                            ['check_box' for i in CheckBoxControls])
+        option_fields += zip(ComboBoxControls,
+                            [None for i in ComboBoxControls],
+                            ['combo_box' for i in ComboBoxControls])
+        option_fields += zip(RadioButtonControls,
+                            [None for i in RadioButtonControls],
+                            ['radio_button' for i in RadioButtonControls])
+
+        # LineEditControls
+        option_fields += zip(['exclude_genre'],['\[.+\]'],['line_edit'])
+        option_fields += zip(['exclude_pattern'],[None],['line_edit'])
+        option_fields += zip(['exclude_tags'],['~,'+_('Catalog')],['line_edit'])
+        option_fields += zip(['read_pattern'],['+'],['line_edit'])
+        option_fields += zip(['wishlist_tag'],['Wishlist'],['line_edit'])
+
+        # SpinBoxControls
+        option_fields += zip(['thumb_width'],[1.00],['spin_box'])
+
+        self.OPTION_FIELDS = option_fields
 
     def initialize(self, name, db):
+        '''
+
+        CheckBoxControls (c_type: check_box):
+            ['generate_titles','generate_series','generate_genres',
+                            'generate_recently_added','generate_descriptions','include_hr']
+        ComboBoxControls (c_type: combo_box):
+            ['read_source_field','exclude_source_field','header_note_source_field',
+                            'merge_source_field']
+        LineEditControls (c_type: line_edit):
+            ['exclude_genre','exclude_pattern','exclude_tags','read_pattern',
+                            'wishlist_tag']
+        RadioButtonControls (c_type: radio_button):
+            ['merge_before','merge_after']
+        SpinBoxControls (c_type: spin_box):
+            ['thumb_width']
+
+        '''
+
         self.name = name
         self.db = db
         self.populateComboBoxes()
@@ -135,7 +147,7 @@ class PluginWidget(QWidget,Ui_Form):
     def options(self):
         # Save/return the current options
         # exclude_genre stores literally
-        # generate_titles, generate_recently_added, numbers_as_text stores as True/False
+        # generate_titles, generate_recently_added store as True/False
         # others store as lists
 
         opts_dict = {}

src/calibre/gui2/catalog/catalog_epub_mobi.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>650</width>
-    <height>582</height>
+    <height>596</height>
    </rect>
   </property>
   <property name="sizePolicy">
@@ -41,41 +41,54 @@
       <string>Included sections</string>
      </property>
      <layout class="QGridLayout" name="gridLayout_2">
-      <item row="0" column="0">
-       <widget class="QCheckBox" name="generate_titles">
-        <property name="text">
-         <string>Books by &amp;Title</string>
-        </property>
-       </widget>
-      </item>
-      <item row="4" column="0">
-       <widget class="QCheckBox" name="generate_series">
-        <property name="text">
-         <string>Books by &amp;Series</string>
-        </property>
-       </widget>
-      </item>
-      <item row="0" column="2">
-       <widget class="QCheckBox" name="generate_recently_added">
-        <property name="text">
-         <string>Recently &amp;Added</string>
-        </property>
-       </widget>
-      </item>
-      <item row="5" column="0">
+      <item row="0" column="1">
        <widget class="QCheckBox" name="generate_genres">
         <property name="text">
          <string>Books by &amp;Genre</string>
         </property>
        </widget>
       </item>
-      <item row="4" column="2">
+      <item row="4" column="1">
+       <widget class="QCheckBox" name="generate_recently_added">
+        <property name="text">
+         <string>Recently &amp;Added</string>
+        </property>
+       </widget>
+      </item>
+      <item row="5" column="1">
        <widget class="QCheckBox" name="generate_descriptions">
         <property name="text">
          <string>&amp;Descriptions</string>
         </property>
        </widget>
       </item>
+      <item row="5" column="0">
+       <widget class="QCheckBox" name="generate_series">
+        <property name="text">
+         <string>Books by &amp;Series</string>
+        </property>
+       </widget>
+      </item>
+      <item row="4" column="0">
+       <widget class="QCheckBox" name="generate_titles">
+        <property name="text">
+         <string>Books by &amp;Title</string>
+        </property>
+       </widget>
+      </item>
+      <item row="0" column="0">
+       <widget class="QCheckBox" name="generate_authors">
+        <property name="enabled">
+         <bool>false</bool>
+        </property>
+        <property name="text">
+         <string>Books by Author</string>
+        </property>
+        <property name="checked">
+         <bool>true</bool>
+        </property>
+       </widget>
+      </item>
      </layout>
     </widget>
    </item>
@@ -94,14 +107,10 @@
       </size>
      </property>
      <property name="toolTip">
-      <string>&lt;!DOCTYPE HTML PUBLIC &quot;-//W3C//DTD HTML 4.0//EN&quot; &quot;http://www.w3.org/TR/REC-html40/strict.dtd&quot;&gt;
-&lt;html&gt;&lt;head&gt;&lt;meta name=&quot;qrichtext&quot; content=&quot;1&quot; /&gt;&lt;style type=&quot;text/css&quot;&gt;
-p, li { white-space: pre-wrap; }
-&lt;/style&gt;&lt;/head&gt;&lt;body style=&quot; font-family:'Lucida Grande'; font-size:13pt; font-weight:400; font-style:normal;&quot;&gt;
-&lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;Default pattern &lt;/p&gt;
-&lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-family:'Courier New,courier';&quot;&gt;\[.+\]&lt;/span&gt;&lt;/p&gt;
-&lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;excludes tags of the form [&lt;span style=&quot; font-family:'Courier New,courier';&quot;&gt;tag&lt;/span&gt;], &lt;/p&gt;
-&lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;e.g., [Project Gutenberg]&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
+      <string>&lt;p&gt;Default pattern 
+\[.+\]
+excludes tags of the form [tag], 
+e.g., [Project Gutenberg]&lt;/p&gt;</string>
      </property>
      <property name="title">
       <string>Excluded genres</string>
@@ -239,12 +248,8 @@ p, li { white-space: pre-wrap; }
            </size>
           </property>
           <property name="toolTip">
-           <string>&lt;!DOCTYPE HTML PUBLIC &quot;-//W3C//DTD HTML 4.0//EN&quot; &quot;http://www.w3.org/TR/REC-html40/strict.dtd&quot;&gt;
-&lt;html&gt;&lt;head&gt;&lt;meta name=&quot;qrichtext&quot; content=&quot;1&quot; /&gt;&lt;style type=&quot;text/css&quot;&gt;
-p, li { white-space: pre-wrap; }
-&lt;/style&gt;&lt;/head&gt;&lt;body style=&quot; font-family:'Lucida Grande'; font-size:13pt; font-weight:400; font-style:normal;&quot;&gt;
-&lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-size:12pt;&quot;&gt;Comma-separated list of tags to exclude.&lt;/span&gt;&lt;/p&gt;
-&lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-size:12pt;&quot;&gt;Default:&lt;/span&gt;&lt;span style=&quot; font-family:'Courier New,courier'; font-size:12pt;&quot;&gt; ~,Catalog&lt;/span&gt;&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
+           <string>&lt;p&gt;Comma-separated list of tags to exclude.
+Default: ~,Catalog</string>
           </property>
          </widget>
         </item>
@@ -332,7 +337,7 @@ p, li { white-space: pre-wrap; }
       </size>
      </property>
      <property name="toolTip">
-      <string>Matching books will be displayed with ✓</string>
+      <string>Matching books will be displayed with a check mark</string>
      </property>
      <property name="title">
       <string>Read books</string>
@@ -471,7 +476,7 @@ p, li { white-space: pre-wrap; }
         <item>
          <widget class="QLineEdit" name="wishlist_tag">
           <property name="toolTip">
-           <string>Books tagged as Wishlist items will be displayed with ✕</string>
+           <string>Books tagged as Wishlist items will be displayed with an X</string>
           </property>
          </widget>
         </item>

src/calibre/gui2/dialogs/drm_error.ui

@@ -13,6 +13,10 @@
   <property name="windowTitle">
    <string>This book is DRMed</string>
   </property>
+  <property name="windowIcon">
+   <iconset resource="../../../../resources/images.qrc">
+    <normaloff>:/images/document-encrypt.png</normaloff>:/images/document-encrypt.png</iconset>
+  </property>
   <layout class="QGridLayout" name="gridLayout">
    <item row="0" column="0">
     <widget class="QLabel" name="label">

src/calibre/gui2/dialogs/metadata_single.py

@@ -790,7 +790,13 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
                         if d.opt_get_social_metadata.isChecked():
                             d2 = SocialMetadata(book, self)
                             d2.exec_()
-                            if d2.exceptions:
+                            if d2.timed_out:
+                                warning_dialog(self, _('Timed out'),
+                                    _('The download of social'
+                                    ' metadata timed out, the servers are'
+                                    ' probably busy. Try again later.'),
+                                    show=True)
+                            elif d2.exceptions:
                                 det = '\n'.join([x[0]+'\n\n'+x[-1]+'\n\n\n' for
                                     x in d2.exceptions])
                                 warning_dialog(self, _('There were errors'),

src/calibre/gui2/layout.py

@@ -196,9 +196,11 @@ class SearchBar(QWidget): # {{{
 
         x = parent.search_highlight_only = QCheckBox()
         x.setText(_('&Highlight'))
-        x.setToolTip(_('Highlight matched books in the book list, instead '
-            'of restricting the book list to the matches.'))
+        x.setToolTip('<p>'+_('When searching, highlight matched books, instead '
+            'of restricting the book list to the matches.<p> You can use the '
+            'N or F3 keys to go to the next match.'))
         l.addWidget(x)
+        x.setVisible(False)
 
         x = parent.saved_search = SavedSearchBox(self)
         x.setMaximumSize(QSize(150, 16777215))

src/calibre/gui2/library/models.py

@@ -93,8 +93,9 @@ class BooksModel(QAbstractTableModel): # {{{
         self.bool_no_icon = QIcon(I('list_remove.png'))
         self.bool_blank_icon = QIcon(I('blank.png'))
         self.device_connected = False
-        self.rows_matching = set()
-        self.lowest_row_matching = None
+        self.ids_to_highlight = []
+        self.ids_to_highlight_set = set()
+        self.current_highlighted_idx = None
         self.highlight_only = False
         self.read_config()
 
@@ -130,6 +131,9 @@ class BooksModel(QAbstractTableModel): # {{{
         self.book_on_device = func
 
     def set_database(self, db):
+        self.ids_to_highlight = []
+        self.ids_to_highlight_set = set()
+        self.current_highlighted_idx = None
         self.db = db
         self.custom_columns = self.db.field_metadata.custom_field_metadata()
         self.column_map = list(self.orig_headers.keys()) + \
@@ -237,21 +241,55 @@ class BooksModel(QAbstractTableModel): # {{{
         if self.last_search:
             self.research()
 
+    def get_current_highlighted_id(self):
+        if len(self.ids_to_highlight) == 0 or self.current_highlighted_idx is None:
+            return None
+        try:
+            return self.ids_to_highlight[self.current_highlighted_idx]
+        except:
+            return None
+
+    def get_next_highlighted_id(self, current_row, forward):
+        if len(self.ids_to_highlight) == 0 or self.current_highlighted_idx is None:
+            return None
+        if current_row is None:
+            row_ = self.current_highlighted_idx
+        else:
+            row_ = current_row
+        while True:
+            row_ += 1 if forward else -1
+            if row_ < 0:
+                row_ = self.count() - 1;
+            elif row_ >= self.count():
+                row_ = 0
+            if self.id(row_) in self.ids_to_highlight_set:
+                break
+        try:
+            self.current_highlighted_idx = self.ids_to_highlight.index(self.id(row_))
+        except:
+            # This shouldn't happen ...
+            return None
+        return self.get_current_highlighted_id()
+
     def search(self, text, reset=True):
         try:
             if self.highlight_only:
                 self.db.search('')
                 if not text:
-                    self.rows_matching = set()
-                    self.lowest_row_matching = None
+                    self.ids_to_highlight = []
+                    self.ids_to_highlight_set = set()
+                    self.current_highlighted_idx = None
                 else:
-                    self.rows_matching = self.db.search(text, return_matches=True)
-                    if self.rows_matching:
-                        self.lowest_row_matching = self.db.row(self.rows_matching[0])
-                    self.rows_matching = set(self.rows_matching)
+                    self.ids_to_highlight = self.db.search(text, return_matches=True)
+                    self.ids_to_highlight_set = set(self.ids_to_highlight)
+                    if self.ids_to_highlight:
+                        self.current_highlighted_idx = 0
+                    else:
+                        self.current_highlighted_idx = None
             else:
-                self.rows_matching = set()
-                self.lowest_row_matching = None
+                self.ids_to_highlight = []
+                self.ids_to_highlight_set = set()
+                self.current_highlighted_idx = None
                 self.db.search(text)
         except ParseException as e:
             self.searched.emit(e.msg)
@@ -674,7 +712,7 @@ class BooksModel(QAbstractTableModel): # {{{
         if role in (Qt.DisplayRole, Qt.EditRole):
             return self.column_to_dc_map[col](index.row())
         elif role == Qt.BackgroundColorRole:
-            if self.id(index) in self.rows_matching:
+            if self.id(index) in self.ids_to_highlight_set:
                 return QColor('lightgreen')
         elif role == Qt.DecorationRole:
             if self.column_to_dc_decorator_map[col] is not None:

src/calibre/gui2/library/views.py

@@ -680,10 +680,21 @@ class BooksView(QTableView): # {{{
     def set_editable(self, editable, supports_backloading):
         self._model.set_editable(editable)
 
+    def move_highlighted_row(self, forward):
+        rows = self.selectionModel().selectedRows()
+        if len(rows) > 0:
+            current_row = rows[0].row()
+        else:
+            current_row = None
+        id_to_select = self._model.get_next_highlighted_id(current_row, forward)
+        if id_to_select is not None:
+            self.select_rows([id_to_select], using_ids=True)
+
     def search_proxy(self, txt):
         self._model.search(txt)
-        if self._model.lowest_row_matching is not None:
-            self.select_rows([self._model.lowest_row_matching], using_ids=False)
+        id_to_select = self._model.get_current_highlighted_id()
+        if id_to_select is not None:
+            self.select_rows([id_to_select], using_ids=True)
         self.setFocus(Qt.OtherFocusReason)
 
     def connect_to_search_box(self, sb, search_done):

src/calibre/gui2/main.py

@@ -34,6 +34,9 @@ path_to_ebook to the database.
                       help=_('Log debugging information to console'))
     parser.add_option('--no-update-check', default=False, action='store_true',
             help=_('Do not check for updates'))
+    parser.add_option('--ignore-plugins', default=False, action='store_true',
+            help=_('Ignore custom plugins, useful if you installed a plugin'
+                ' that is preventing calibre from starting'))
     return parser
 
 def init_qt(args):

src/calibre/gui2/preferences/social.py

@@ -6,16 +6,19 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
+import time
+from threading import Thread
 
 from PyQt4.Qt import QDialog, QDialogButtonBox, Qt, QLabel, QVBoxLayout, \
-        SIGNAL, QThread
+        QTimer
 
 from calibre.ebooks.metadata import MetaInformation
 
-class Worker(QThread):
+class Worker(Thread):
 
-    def __init__(self, mi, parent):
-        QThread.__init__(self, parent)
+    def __init__(self, mi):
+        Thread.__init__(self)
+        self.daemon = True
         self.mi = MetaInformation(mi)
         self.exceptions = []
 
@@ -25,10 +28,12 @@ class Worker(QThread):
 
 class SocialMetadata(QDialog):
 
+    TIMEOUT = 300 # seconds
+
     def __init__(self, mi, parent):
         QDialog.__init__(self, parent)
 
-        self.bbox = QDialogButtonBox(QDialogButtonBox.Ok, Qt.Horizontal, self)
+        self.bbox = QDialogButtonBox(QDialogButtonBox.Cancel, Qt.Horizontal, self)
         self.mi = mi
         self.layout = QVBoxLayout(self)
         self.label = QLabel(_('Downloading social metadata, please wait...'), self)
@@ -36,15 +41,30 @@ class SocialMetadata(QDialog):
         self.layout.addWidget(self.label)
         self.layout.addWidget(self.bbox)
 
-        self.worker = Worker(mi, self)
-        self.connect(self.worker, SIGNAL('finished()'), self.accept)
-        self.connect(self.bbox, SIGNAL('rejected()'), self.reject)
+        self.worker = Worker(mi)
+        self.bbox.rejected.connect(self.reject)
         self.worker.start()
+        self.start_time = time.time()
+        self.timed_out = False
+        self.rejected = False
+        QTimer.singleShot(50, self.update)
 
     def reject(self):
-        self.disconnect(self.worker, SIGNAL('finished()'), self.accept)
+        self.rejected = True
         QDialog.reject(self)
 
+    def update(self):
+        if self.rejected:
+            return
+        if time.time() - self.start_time > self.TIMEOUT:
+            self.timed_out = True
+            self.reject()
+            return
+        if not self.worker.is_alive():
+            self.accept()
+            return
+        QTimer.singleShot(50, self.update)
+
     def accept(self):
         self.mi.tags = self.worker.mi.tags
         self.mi.rating = self.worker.mi.rating

src/calibre/gui2/preferences/template_functions.py

@@ -0,0 +1,216 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import traceback
+
+from calibre.gui2 import error_dialog
+from calibre.gui2.preferences import ConfigWidgetBase, test_widget
+from calibre.gui2.preferences.template_functions_ui import Ui_Form
+from calibre.gui2.widgets import PythonHighlighter
+from calibre.utils.formatter_functions import formatter_functions, compile_user_function
+
+
+class ConfigWidget(ConfigWidgetBase, Ui_Form):
+
+    def genesis(self, gui):
+        self.gui = gui
+        self.db = gui.library_view.model().db
+        self.current_plugboards = self.db.prefs.get('plugboards',{})
+        help_text = _('''
+        <p>Here you can add and remove functions used in template processing. A
+        template function is written in python. It takes information from the
+        book, processes it in some way, then returns a string result. Functions
+        defined here are usable in templates in the same way that builtin
+        functions are usable. The function must be named <b>evaluate</b>, and
+        must have the signature shown below.</p>
+        <p><code>evaluate(self, formatter, kwargs, mi, locals, your parameters)
+        &rarr; returning a unicode string</code></p>
+        <p>The parameters of the evaluate function are:
+        <ul>
+        <li><b>formatter</b>: the instance of the formatter being used to
+        evaluate the current template. You can use this to do recursive
+        template evaluation.</li>
+        <li><b>kwargs</b>: a dictionary of metadata. Field values are in this
+        dictionary.
+        <li><b>mi</b>: a Metadata instance. Used to get field information.
+        This parameter can be None in some cases, such as when evaluating
+        non-book templates.</li>
+        <li><b>locals</b>: the local variables assigned to by the current
+        template program.</li>
+        <li><b>your parameters</b>: You must supply one or more formal
+        parameters. The number must match the arg count box, unless arg count is
+        -1 (variable number or arguments), in which case the last argument must
+        be *args. At least one argument is required, and is usually the value of
+        the field being operated upon. Note that when writing in basic template
+        mode, the user does not provide this first argument. Instead it is
+        supplied by the formatter.</li>
+        </ul></p>
+        <p>
+        The following example function checks the value of the field. If the
+        field is not empty, the field's value is returned, otherwise the value
+        EMPTY is returned.
+        <pre>
+        name: my_ifempty
+        arg count: 1
+        doc: my_ifempty(val) -- return val if it is not empty, otherwise the string 'EMPTY'
+        program code:
+        def evaluate(self, formatter, kwargs, mi, locals, val):
+            if val:
+                return val
+            else:
+                return 'EMPTY'</pre>
+        This function can be called in any of the three template program modes:
+        <ul>
+        <li>single-function mode: {tags:my_ifempty()}</li>
+        <li>template program mode: {tags:'my_ifempty($)'}</li>
+        <li>general program mode: program: my_ifempty(field('tags'))</li>
+        </p>
+        ''')
+        self.textBrowser.setHtml(help_text)
+
+    def initialize(self):
+        self.funcs = formatter_functions.get_functions()
+        self.builtins = formatter_functions.get_builtins()
+
+        self.build_function_names_box()
+        self.function_name.currentIndexChanged[str].connect(self.function_index_changed)
+        self.function_name.editTextChanged.connect(self.function_name_edited)
+        self.argument_count.valueChanged.connect(self.enable_replace_button)
+        self.documentation.textChanged.connect(self.enable_replace_button)
+        self.program.textChanged.connect(self.enable_replace_button)
+        self.create_button.clicked.connect(self.create_button_clicked)
+        self.delete_button.clicked.connect(self.delete_button_clicked)
+        self.create_button.setEnabled(False)
+        self.delete_button.setEnabled(False)
+        self.replace_button.setEnabled(False)
+        self.clear_button.clicked.connect(self.clear_button_clicked)
+        self.replace_button.clicked.connect(self.replace_button_clicked)
+        self.program.setTabStopWidth(20)
+        self.highlighter = PythonHighlighter(self.program.document())
+
+    def enable_replace_button(self):
+        self.replace_button.setEnabled(self.delete_button.isEnabled())
+
+    def clear_button_clicked(self):
+        self.build_function_names_box()
+        self.program.clear()
+        self.documentation.clear()
+        self.argument_count.clear()
+        self.create_button.setEnabled(False)
+        self.delete_button.setEnabled(False)
+
+    def build_function_names_box(self, scroll_to='', set_to=''):
+        self.function_name.blockSignals(True)
+        func_names = sorted(self.funcs)
+        self.function_name.clear()
+        self.function_name.addItem('')
+        self.function_name.addItems(func_names)
+        self.function_name.setCurrentIndex(0)
+        if set_to:
+            self.function_name.setEditText(set_to)
+            self.create_button.setEnabled(True)
+        self.function_name.blockSignals(False)
+        if scroll_to:
+            idx = self.function_name.findText(scroll_to)
+            if idx >= 0:
+                self.function_name.setCurrentIndex(idx)
+                if scroll_to not in self.builtins:
+                    self.delete_button.setEnabled(True)
+
+    def delete_button_clicked(self):
+        name = unicode(self.function_name.currentText())
+        if name in self.builtins:
+            error_dialog(self.gui, _('Template functions'),
+                         _('You cannot delete a built-in function'), show=True)
+        if name in self.funcs:
+            del self.funcs[name]
+            self.changed_signal.emit()
+            self.create_button.setEnabled(True)
+            self.delete_button.setEnabled(False)
+            self.build_function_names_box(set_to=name)
+            self.program.setReadOnly(False)
+        else:
+            error_dialog(self.gui, _('Template functions'),
+                         _('Function not defined'), show=True)
+
+    def create_button_clicked(self):
+        self.changed_signal.emit()
+        name = unicode(self.function_name.currentText())
+        if name in self.funcs:
+            error_dialog(self.gui, _('Template functions'),
+                         _('Name already used'), show=True)
+            return
+        if self.argument_count.value() == 0:
+            error_dialog(self.gui, _('Template functions'),
+                         _('Argument count must be -1 or greater than zero'),
+                         show=True)
+            return
+        try:
+            prog = unicode(self.program.toPlainText())
+            cls = compile_user_function(name, unicode(self.documentation.toPlainText()),
+                                        self.argument_count.value(), prog)
+            self.funcs[name] = cls
+            self.build_function_names_box(scroll_to=name)
+        except:
+            error_dialog(self.gui, _('Template functions'),
+                         _('Exception while compiling function'), show=True,
+                         det_msg=traceback.format_exc())
+
+    def function_name_edited(self, txt):
+        self.documentation.setReadOnly(False)
+        self.argument_count.setReadOnly(False)
+        self.create_button.setEnabled(True)
+        self.replace_button.setEnabled(False)
+        self.program.setReadOnly(False)
+
+    def function_index_changed(self, txt):
+        txt = unicode(txt)
+        self.create_button.setEnabled(False)
+        if not txt:
+            self.argument_count.clear()
+            self.documentation.clear()
+            self.documentation.setReadOnly(False)
+            self.argument_count.setReadOnly(False)
+            return
+        func = self.funcs[txt]
+        self.argument_count.setValue(func.arg_count)
+        self.documentation.setText(func.doc)
+        self.program.setPlainText(func.program_text)
+        if txt in self.builtins:
+            self.documentation.setReadOnly(True)
+            self.argument_count.setReadOnly(True)
+            self.program.setReadOnly(True)
+            self.delete_button.setEnabled(False)
+        else:
+            self.program.setPlainText(func.program_text)
+            self.delete_button.setEnabled(True)
+            self.program.setReadOnly(False)
+        self.replace_button.setEnabled(False)
+
+    def replace_button_clicked(self):
+        self.delete_button_clicked()
+        self.create_button_clicked()
+    def refresh_gui(self, gui):
+        pass
+
+    def commit(self):
+        formatter_functions.reset_to_builtins()
+        pref_value = []
+        for f in self.funcs:
+            if f in self.builtins:
+                continue
+            func = self.funcs[f]
+            formatter_functions.register_function(func)
+            pref_value.append((func.name, func.doc, func.arg_count, func.program_text))
+        self.db.prefs.set('user_template_functions', pref_value)
+
+
+if __name__ == '__main__':
+    from PyQt4.Qt import QApplication
+    app = QApplication([])
+    test_widget('Advanced', 'TemplateFunctions')
+

src/calibre/gui2/preferences/template_functions.ui

@@ -0,0 +1,160 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Form</class>
+ <widget class="QWidget" name="Form">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>798</width>
+    <height>672</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Form</string>
+  </property>
+  <layout class="QGridLayout" name="gridLayout">
+   <item row="1" column="0" colspan="2">
+    <widget class="Line" name="line">
+     <property name="orientation">
+      <enum>Qt::Horizontal</enum>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="0">
+    <layout class="QHBoxLayout" name="horizontalLayout">
+     <item>
+      <layout class="QGridLayout" name="gridLayout_3">
+       <item row="0" column="0">
+        <widget class="QLabel" name="label_2">
+         <property name="text">
+          <string>&amp;Function:</string>
+         </property>
+         <property name="buddy">
+          <cstring>function_name</cstring>
+         </property>
+        </widget>
+       </item>
+       <item row="0" column="1">
+        <widget class="QComboBox" name="function_name">
+         <property name="toolTip">
+          <string>Enter the name of the function to create.</string>
+         </property>
+         <property name="editable">
+          <bool>true</bool>
+         </property>
+        </widget>
+       </item>
+       <item row="1" column="0">
+        <widget class="QLabel" name="label_3">
+         <property name="toolTip">
+          <string/>
+         </property>
+         <property name="text">
+          <string>Arg &amp;count:</string>
+         </property>
+         <property name="buddy">
+          <cstring>argument_count</cstring>
+         </property>
+        </widget>
+       </item>
+       <item row="1" column="1">
+        <widget class="QSpinBox" name="argument_count">
+         <property name="toolTip">
+          <string>Set this to -1 if the function takes a variable number of arguments</string>
+         </property>
+         <property name="minimum">
+          <number>-1</number>
+         </property>
+        </widget>
+       </item>
+       <item row="2" column="1">
+        <widget class="QTextEdit" name="documentation"/>
+       </item>
+       <item row="2" column="0">
+        <widget class="QLabel" name="label_4">
+         <property name="text">
+          <string>&amp;Documentation:</string>
+         </property>
+         <property name="alignment">
+          <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignTop</set>
+         </property>
+         <property name="buddy">
+          <cstring>documentation</cstring>
+         </property>
+        </widget>
+       </item>
+       <item row="3" column="1">
+        <layout class="QHBoxLayout" name="horizontalLayout_2">
+         <item>
+          <widget class="QPushButton" name="clear_button">
+           <property name="text">
+            <string>&amp;Clear</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QPushButton" name="delete_button">
+           <property name="text">
+            <string>&amp;Delete</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QPushButton" name="replace_button">
+           <property name="text">
+            <string>&amp;Replace</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QPushButton" name="create_button">
+           <property name="text">
+            <string>C&amp;reate</string>
+           </property>
+          </widget>
+         </item>
+        </layout>
+       </item>
+      </layout>
+     </item>
+     <item>
+      <layout class="QVBoxLayout" name="horizontalLayout1">
+       <item>
+        <widget class="QLabel" name="label_4">
+         <property name="text">
+          <string>&amp;Program Code: (be sure to follow python indenting rules)</string>
+         </property>
+         <property name="buddy">
+          <cstring>program</cstring>
+         </property>
+        </widget>
+       </item>
+       <item>
+        <widget class="QPlainTextEdit" name="program">
+         <property name="minimumSize">
+          <size>
+           <width>400</width>
+           <height>0</height>
+          </size>
+         </property>
+         <property name="documentTitle">
+          <string notr="true"/>
+         </property>
+         <property name="tabStopWidth">
+          <number>30</number>
+         </property>
+        </widget>
+       </item>
+      </layout>
+     </item>
+    </layout>
+   </item>
+   <item row="0" column="0">
+    <widget class="QTextBrowser" name="textBrowser"/>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections/>
+</ui>

src/calibre/gui2/ui.py

@@ -103,6 +103,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         self.gui_debug = gui_debug
         acmap = OrderedDict()
         for action in interface_actions():
+            if opts.ignore_plugins and action.plugin_path is not None:
+                continue
             try:
                 ac = action.load_actual_plugin(self)
             except:
@@ -256,6 +258,14 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
                 self.height())
         self.resize(self.width(), self._calculated_available_height)
 
+        for ac in self.iactions.values():
+            try:
+                ac.gui_layout_complete()
+            except:
+                import traceback
+                traceback.print_exc()
+                if ac.plugin_path is None:
+                    raise
 
         if config['autolaunch_server']:
             self.start_content_server()
@@ -269,7 +279,13 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         self.set_window_title()
 
         for ac in self.iactions.values():
-            ac.initialization_complete()
+            try:
+                ac.initialization_complete()
+            except:
+                import traceback
+                traceback.print_exc()
+                if ac.plugin_path is None:
+                    raise
 
         if show_gui and self.gui_debug is not None:
             info_dialog(self, _('Debug mode'), '<p>' +

src/calibre/library/catalog.py

@@ -599,8 +599,16 @@ class EPUB_MOBI(CatalogPlugin):
                           default=('~,'+_('Catalog')),
                           dest='exclude_tags',
                           action = None,
-                          help=_("Comma-separated list of tag words indicating book should be excluded from output.  Case-insensitive.\n"
-                          "--exclude-tags=skip will match 'skip this book' and 'Skip will like this'.\n"
+                          help=_("Comma-separated list of tag words indicating book should be excluded from output."
+                              "For example: 'skip' will match 'skip this book' and 'Skip will like this'."
+                              "Default: '%default'\n"
+                              "Applies to: ePub, MOBI output formats")),
+                   Option('--generate-authors',
+                          default=True,
+                          dest='generate_authors',
+                          action = 'store_true',
+                          help=_("Include 'Authors' section in catalog."
+                          "This switch is ignored - Books By Author section is always generated."
                           "Default: '%default'\n"
                           "Applies to: ePub, MOBI output formats")),
                    Option('--generate-descriptions',
@@ -1338,7 +1346,8 @@ class EPUB_MOBI(CatalogPlugin):
             if self.booksByTitle is None:
                 if not self.fetchBooksByTitle():
                     return False
-            self.fetchBooksByAuthor()
+            if not self.fetchBooksByAuthor():
+                return False
             self.fetchBookmarks()
             if self.opts.generate_descriptions:
                 self.generateHTMLDescriptions()
@@ -1536,18 +1545,6 @@ class EPUB_MOBI(CatalogPlugin):
                                 notes = ' · '.join(notes)
                         elif field_md['datatype'] == 'datetime':
                             notes = format_date(notes,'dd MMM yyyy')
-                        elif field_md['datatype'] == 'composite':
-                            m = re.match(r'\[(.+)\]$', notes)
-                            if m is not None:
-                                # Sniff for special pseudo-list string "[<item, item>]"
-                                bracketed_content = m.group(1)
-                                if ',' in bracketed_content:
-                                    # Recast the comma-separated items as a list
-                                    items = bracketed_content.split(',')
-                                    items = [i.strip() for i in items]
-                                    notes = ' &middot; '.join(items)
-                                else:
-                                    notes = bracketed_content
                         this_title['notes'] = {'source':field_md['name'],
                                                    'content':notes}
 
@@ -1568,7 +1565,10 @@ class EPUB_MOBI(CatalogPlugin):
                 return False
 
         def fetchBooksByAuthor(self):
-            # Generate a list of titles sorted by author from the database
+            '''
+            Generate a list of titles sorted by author from the database
+            return = Success
+            '''
 
             self.updateProgressFullStep("Sorting database")
 
@@ -1608,10 +1608,17 @@ class EPUB_MOBI(CatalogPlugin):
                     multiple_authors = True
 
                 if author != current_author and i:
-                    # Warn if friendly matches previous, but sort doesn't
+                    # Warn, exit if friendly matches previous, but sort doesn't
                     if author[0] == current_author[0]:
-                        self.opts.log.warn("Warning: multiple entries for Author '%s' with differing Author Sort metadata:" % author[0])
-                        self.opts.log.warn(" '%s' != '%s'" % (author[1], current_author[1]))
+                        error_msg = _('''
+\n*** Metadata error ***
+Inconsistent Author Sort values for Author '{0}', unable to continue building catalog.
+Select all books by '{0}', apply correct Author Sort value in Edit Metadata dialog,
+then rebuild the catalog.
+*** Terminating catalog generation ***\n''').format(author[0])
+
+                        self.opts.log.warn(error_msg)
+                        return False
 
                     # New author, save the previous author/sort/count
                     unique_authors.append((current_author[0], icu_title(current_author[1]),
@@ -1637,6 +1644,7 @@ class EPUB_MOBI(CatalogPlugin):
                        author[2])).encode('utf-8'))
 
             self.authors = unique_authors
+            return True
 
         def fetchBookmarks(self):
             '''
@@ -1751,8 +1759,6 @@ class EPUB_MOBI(CatalogPlugin):
                 # Generate the header from user-customizable template
                 soup = self.generateHTMLDescriptionHeader(title)
 
-
-
                 # Write the book entry to contentdir
                 outfile = open("%s/book_%d.html" % (self.contentDir, int(title['id'])), 'w')
                 outfile.write(soup.prettify())
@@ -3250,7 +3256,7 @@ class EPUB_MOBI(CatalogPlugin):
             # Loop over the series titles, find start of each letter, add description_preview_count books
             # Special switch for using different title list
             title_list = self.booksBySeries
-            current_letter = self.letter_or_symbol(title_list[0]['series'][0])
+            current_letter = self.letter_or_symbol(self.generateSortTitle(title_list[0]['series'])[0])
             title_letters = [current_letter]
             current_series_list = []
             current_series = ""
@@ -4362,7 +4368,7 @@ class EPUB_MOBI(CatalogPlugin):
                 _soup = BeautifulSoup('')
                 genresTag = Tag(_soup,'p')
                 gtc = 0
-                for (i, tag) in enumerate(book.get('tags', [])):
+                for (i, tag) in enumerate(sorted(book.get('tags', []))):
                     aTag = Tag(_soup,'a')
                     if self.opts.generate_genres:
                         aTag['href'] = "Genre_%s.html" % re.sub("\W","",tag.lower())
@@ -4381,6 +4387,7 @@ class EPUB_MOBI(CatalogPlugin):
                     formats.append(format.rpartition('.')[2].upper())
                 formats = ' &middot; '.join(formats)
 
+            # Date of publication
             pubdate = book['date']
             pubmonth, pubyear = pubdate.split(' ')
 
@@ -4973,12 +4980,16 @@ class EPUB_MOBI(CatalogPlugin):
             build_log.append(" book count: %d" % len(opts_dict['ids']))
 
         sections_list = ['Authors']
+        '''
+        if opts.generate_authors:
+            sections_list.append('Authors')
+        '''
         if opts.generate_titles:
             sections_list.append('Titles')
-        if opts.generate_recently_added:
-            sections_list.append('Recently Added')
         if opts.generate_genres:
             sections_list.append('Genres')
+        if opts.generate_recently_added:
+            sections_list.append('Recently Added')
         if opts.generate_descriptions:
             sections_list.append('Descriptions')
 

src/calibre/library/cli.py

@@ -986,8 +986,8 @@ def command_restore_database(args, dbpath):
         return 1
 
     if not opts.really_do_it:
-        prints(_('You must provide the --really-do-it option to do a'
-            ' recovery'), end='\n\n')
+        prints(_('You must provide the %s option to do a'
+            ' recovery')%'--really-do-it', end='\n\n')
         parser.print_help()
         return 1
 

src/calibre/library/database2.py

@@ -38,6 +38,7 @@ from calibre.utils.search_query_parser import saved_searches, set_saved_searches
 from calibre.ebooks import BOOK_EXTENSIONS, check_ebook_format
 from calibre.utils.magick.draw import save_cover_data_to
 from calibre.utils.recycle_bin import delete_file, delete_tree
+from calibre.utils.formatter_functions import load_user_template_functions
 
 
 copyfile = os.link if hasattr(os, 'link') else shutil.copyfile
@@ -185,6 +186,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         migrate_preference('saved_searches', {})
         set_saved_searches(self, 'saved_searches')
 
+        load_user_template_functions(self.prefs.get('user_template_functions', []))
+
         self.conn.executescript('''
         DROP TRIGGER IF EXISTS author_insert_trg;
         CREATE TEMP TRIGGER author_insert_trg

src/calibre/manual/gui.rst

@@ -478,6 +478,10 @@ Calibre has several keyboard shortcuts to save you time and mouse movement. Thes
       - Focus the search bar
     * - :kbd:`Shift+Ctrl+F`
       - Open the advanced search dialog
+    * - :kbd:`N or F3`
+      - Find the next book that matches the current search (only works if the highlight checkbox next to the search bar is checked)
+    * - :kbd:`Shift+N or Shift+F3`
+      - Find the next book that matches the current search (only works if the highlight checkbox next to the search bar is checked)
     * - :kbd:`Ctrl+D` 
       - Download metadata and shortcuts
     * - :kbd:`Ctrl+R` 

src/calibre/manual/template_lang.rst

@@ -308,6 +308,12 @@ The following program produces the same results as the original recipe, using on
 
 It would be possible to do the above with no custom columns by putting the program into the template box of the plugboard. However, to do so, all comments must be removed because the plugboard text box does not support multi-line editing. It is debatable whether the gain of not having the custom column is worth the vast increase in difficulty caused by the program being one giant line.
 
+
+User-defined Template Functions
+-------------------------------
+
+You can add your own functions to the template processor. Such functions are written in python, and can be used in any of the three template programming modes. The functions are added by going to Preferences -> Advanced -> Template Functions. Instructions are shown in that dialog.
+
 Special notes for save/send templates
 -------------------------------------
 

src/calibre/utils/formatter.py

@@ -4,12 +4,14 @@ Created on 23 Sep 2010
 @author: charles
 '''
 
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
 import re, string, traceback
-from functools import partial
 
 from calibre.constants import DEBUG
-from calibre.utils.titlecase import titlecase
-from calibre.utils.icu import capitalize, strcmp
+from calibre.utils.formatter_functions import formatter_functions
 
 class _Parser(object):
     LEX_OP  = 1
@@ -18,100 +20,13 @@ class _Parser(object):
     LEX_NUM = 4
     LEX_EOF = 5
 
-    def _python(self, func):
-        locals = {}
-        exec func in locals
-        if 'evaluate' not in locals:
-            self.error('no evaluate function in python')
-        try:
-            result = locals['evaluate'](self.parent.kwargs)
-            if isinstance(result, (float, int)):
-                result = unicode(result)
-            elif isinstance(result, list):
-                result = ','.join(result)
-            elif isinstance(result, str):
-                result = unicode(result)
-            return result
-        except Exception as e:
-            self.error('python function threw exception: ' + e.msg)
-
-
-    def _strcmp(self, x, y, lt, eq, gt):
-        v = strcmp(x, y)
-        if v < 0:
-            return lt
-        if v == 0:
-            return eq
-        return gt
-
-    def _cmp(self, x, y, lt, eq, gt):
-        x = float(x if x else 0)
-        y = float(y if y else 0)
-        if x < y:
-            return lt
-        if x == y:
-            return eq
-        return gt
-
-    def _assign(self, target, value):
-        self.variables[target] = value
-        return value
-
-    def _concat(self, *args):
-        i = 0
-        res = ''
-        for i in range(0, len(args)):
-            res += args[i]
-        return res
-
-    def _math(self, x, y, op=None):
-        ops = {
-               '+': lambda x, y: x + y,
-               '-': lambda x, y: x - y,
-               '*': lambda x, y: x * y,
-               '/': lambda x, y: x / y,
-            }
-        x = float(x if x else 0)
-        y = float(y if y else 0)
-        return unicode(ops[op](x, y))
-
-    def _template(self, template):
-        template = template.replace('[[', '{').replace(']]', '}')
-        return self.parent.safe_format(template, self.parent.kwargs, 'TEMPLATE',
-                                       self.parent.book)
-
-    def _eval(self, template):
-        template = template.replace('[[', '{').replace(']]', '}')
-        return eval_formatter.safe_format(template, self.variables, 'EVAL', None)
-
-    def _print(self, *args):
-        print args
-        return None
-
-    local_functions = {
-            'add'      : (2, partial(_math, op='+')),
-            'assign'   : (2, _assign),
-            'cmp'      : (5, _cmp),
-            'divide'   : (2, partial(_math, op='/')),
-            'eval'     : (1, _eval),
-            'field'    : (1, lambda s, x: s.parent.get_value(x, [], s.parent.kwargs)),
-            'multiply' : (2, partial(_math, op='*')),
-            'print'    : (-1, _print),
-            'python'   : (1, _python),
-            'strcat'   : (-1, _concat),
-            'strcmp'   : (5, _strcmp),
-            'substr'   : (3, lambda s, x, y, z: x[int(y): len(x) if int(z) == 0 else int(z)]),
-            'subtract' : (2, partial(_math, op='-')),
-            'template' : (1, _template)
-    }
-
     def __init__(self, val, prog, parent):
         self.lex_pos = 0
         self.prog = prog[0]
         if prog[1] != '':
             self.error(_('failed to scan program. Invalid input {0}').format(prog[1]))
         self.parent = parent
-        self.variables = {'$':val}
+        self.parent.locals = {'$':val}
 
     def error(self, message):
         m = 'Formatter: ' + message + _(' near ')
@@ -173,18 +88,22 @@ class _Parser(object):
 
     def expr(self):
         if self.token_is_id():
+            funcs = formatter_functions.get_functions()
             # We have an identifier. Determine if it is a function
             id = self.token()
             if not self.token_op_is_a('('):
                 if self.token_op_is_a('='):
                     # classic assignment statement
                     self.consume()
-                    return self._assign(id, self.expr())
-                return self.variables.get(id, _('unknown id ') + id)
+                    cls = funcs['assign']
+                    return cls.eval_(self.parent, self.parent.kwargs,
+                                    self.parent.book, self.parent.locals, id, self.expr())
+                return self.parent.locals.get(id, _('unknown id ') + id)
             # We have a function.
             # Check if it is a known one. We do this here so error reporting is
             # better, as it can identify the tokens near the problem.
-            if id not in self.parent.functions and id not in self.local_functions:
+
+            if id not in funcs:
                 self.error(_('unknown function {0}').format(id))
             # Eat the paren
             self.consume()
@@ -207,11 +126,12 @@ class _Parser(object):
                 self.error(_('missing closing parenthesis'))
 
             # Evaluate the function
-            if id in self.local_functions:
-                f = self.local_functions[id]
-                if f[0] != -1 and len(args) != f[0]:
+            if id in funcs:
+                cls = funcs[id]
+                if cls.arg_count != -1 and len(args) != cls.arg_count:
                     self.error('incorrect number of arguments for function {}'.format(id))
-                return f[1](self, *args)
+                return cls.eval_(self.parent, self.parent.kwargs,
+                                self.parent.book, self.parent.locals, *args)
             else:
                 f = self.parent.functions[id]
                 if f[0] != -1 and len(args) != f[0]+1:
@@ -241,91 +161,7 @@ class TemplateFormatter(string.Formatter):
         self.book = None
         self.kwargs = None
         self.program_cache = {}
-
-    def _lookup(self, val, *args):
-        if len(args) == 2: # here for backwards compatibility
-            if val:
-                return self.vformat('{'+args[0].strip()+'}', [], self.kwargs)
-            else:
-                return self.vformat('{'+args[1].strip()+'}', [], self.kwargs)
-        if (len(args) % 2) != 1:
-            raise ValueError(_('lookup requires either 2 or an odd number of arguments'))
-        i = 0
-        while i < len(args):
-            if i + 1 >= len(args):
-                return self.vformat('{' + args[i].strip() + '}', [], self.kwargs)
-            if re.search(args[i], val):
-                return self.vformat('{'+args[i+1].strip() + '}', [], self.kwargs)
-            i += 2
-
-    def _test(self, val, value_if_set, value_not_set):
-        if val:
-            return value_if_set
-        else:
-            return value_not_set
-
-    def _contains(self, val, test, value_if_present, value_if_not):
-        if re.search(test, val):
-            return value_if_present
-        else:
-            return value_if_not
-
-    def _switch(self, val, *args):
-        if (len(args) % 2) != 1:
-            raise ValueError(_('switch requires an odd number of arguments'))
-        i = 0
-        while i < len(args):
-            if i + 1 >= len(args):
-                return args[i]
-            if re.search(args[i], val):
-                return args[i+1]
-            i += 2
-
-    def _re(self, val, pattern, replacement):
-        return re.sub(pattern, replacement, val)
-
-    def _ifempty(self, val, value_if_empty):
-        if val:
-            return val
-        else:
-            return value_if_empty
-
-    def _shorten(self, val, leading, center_string, trailing):
-        l = max(0, int(leading))
-        t = max(0, int(trailing))
-        if len(val) > l + len(center_string) + t:
-            return val[0:l] + center_string + ('' if t == 0 else val[-t:])
-        else:
-            return val
-
-    def _count(self, val, sep):
-        return unicode(len(val.split(sep)))
-
-    def _list_item(self, val, index, sep):
-        if not val:
-            return ''
-        index = int(index)
-        val = val.split(sep)
-        try:
-            return val[index]
-        except:
-            return ''
-
-    functions = {
-                    'uppercase'     : (0, lambda s,x: x.upper()),
-                    'lowercase'     : (0, lambda s,x: x.lower()),
-                    'titlecase'     : (0, lambda s,x: titlecase(x)),
-                    'capitalize'    : (0, lambda s,x: capitalize(x)),
-                    'contains'      : (3, _contains),
-                    'count'         : (1, _count),
-                    'ifempty'       : (1, _ifempty),
-                    'list_item'     : (2, _list_item),
-                    'lookup'        : (-1, _lookup),
-                    're'            : (2, _re),
-                    'shorten'       : (3, _shorten),
-                    'switch'        : (-1, _switch),
-                    'test'          : (2, _test)
-        }
+        self.locals = {}
 
     def _do_format(self, val, fmt):
         if not fmt or not val:
@@ -436,23 +272,26 @@ class TemplateFormatter(string.Formatter):
                 else:
                     dispfmt = fmt[0:colon]
                     colon += 1
-                if fmt[colon:p] in self.functions:
+
+                funcs = formatter_functions.get_functions()
+                if fmt[colon:p] in funcs:
                     field = fmt[colon:p]
-                    func = self.functions[field]
-                    if func[0] == 1:
+                    func = funcs[field]
+                    if func.arg_count == 2:
                         # only one arg expected. Don't bother to scan. Avoids need
                         # for escaping characters
                         args = [fmt[p+1:-1]]
                     else:
                         args = self.arg_parser.scan(fmt[p+1:])[0]
                         args = [self.backslash_comma_to_comma.sub(',', a) for a in args]
-                    if (func[0] == 0 and (len(args) != 1 or args[0])) or \
-                            (func[0] > 0 and func[0] != len(args)):
+                    if (func.arg_count == 1 and (len(args) != 1 or args[0])) or \
+                            (func.arg_count > 1 and func.arg_count != len(args)+1):
                         raise ValueError('Incorrect number of arguments for function '+ fmt[0:p])
-                    if func[0] == 0:
-                        val = func[1](self, val).strip()
+                    if func.arg_count == 1:
+                        val = func.eval_(self, self.kwargs, self.book, self.locals, val).strip()
                     else:
-                        val = func[1](self, val, *args).strip()
+                        val = func.eval_(self, self.kwargs, self.book, self.locals,
+                                        val, *args).strip()
         if val:
             val = self._do_format(val, dispfmt)
         if not val:
@@ -472,6 +311,7 @@ class TemplateFormatter(string.Formatter):
         self.kwargs = kwargs
         self.book = book
         self.composite_values = {}
+        self.locals = {}
         try:
             ans = self.vformat(fmt, [], kwargs).strip()
         except Exception, e:

src/calibre/utils/formatter_functions.py

@@ -0,0 +1,487 @@
+'''
+Created on 13 Jan 2011
+
+@author: charles
+'''
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import inspect, re, traceback, sys
+
+from calibre.utils.titlecase import titlecase
+from calibre.utils.icu import capitalize, strcmp
+
+
+class FormatterFunctions(object):
+
+    def __init__(self):
+        self.builtins = {}
+        self.functions = {}
+
+    def register_builtin(self, func_class):
+        if not isinstance(func_class, FormatterFunction):
+            raise ValueError('Class %s is not an instance of FormatterFunction'%(
+                                    func_class.__class__.__name__))
+        name = func_class.name
+        if name in self.functions:
+            raise ValueError('Name %s already used'%name)
+        self.builtins[name] = func_class
+        self.functions[name] = func_class
+
+    def register_function(self, func_class):
+        if not isinstance(func_class, FormatterFunction):
+            raise ValueError('Class %s is not an instance of FormatterFunction'%(
+                                    func_class.__class__.__name__))
+        name = func_class.name
+        if name in self.functions:
+            raise ValueError('Name %s already used'%name)
+        self.functions[name] = func_class
+
+    def get_builtins(self):
+        return self.builtins
+
+    def get_functions(self):
+        return self.functions
+
+    def reset_to_builtins(self):
+        self.functions = dict([t for t in self.builtins.items()])
+
+formatter_functions = FormatterFunctions()
+
+
+
+class FormatterFunction(object):
+
+    doc = _('No documentation provided')
+    name = 'no name provided'
+    arg_count = 0
+
+    def evaluate(self, formatter, kwargs, mi, locals, *args):
+        raise NotImplementedError()
+
+    def eval_(self, formatter, kwargs, mi, locals, *args):
+        try:
+            ret = self.evaluate(formatter, kwargs, mi, locals, *args)
+            if isinstance(ret, (str, unicode)):
+                return ret
+            if isinstance(ret, (int, float, bool)):
+                return unicode(ret)
+            if isinstance(ret, list):
+                return ','.join(list)
+        except:
+            traceback.print_exc()
+            exc_type, exc_value, exc_traceback = sys.exc_info()
+            info = ': '.join(traceback.format_exception(exc_type, exc_value,
+                                        exc_traceback)[-2:]).replace('\n', '')
+            return _('Exception ' + info)
+
+
+class BuiltinFormatterFunction(FormatterFunction):
+    def __init__(self):
+        formatter_functions.register_builtin(self)
+        eval_func = inspect.getmembers(self.__class__,
+                        lambda x: inspect.ismethod(x) and x.__name__ == 'evaluate')
+        try:
+            lines = [l[4:] for l in inspect.getsourcelines(eval_func[0][1])[0]]
+        except:
+            lines = []
+        self.program_text = ''.join(lines)
+
+class BuiltinStrcmp(BuiltinFormatterFunction):
+    name = 'strcmp'
+    arg_count = 5
+    doc = _('strcmp(x, y, lt, eq, gt) -- does a case-insensitive comparison of x '
+            'and y as strings. Returns lt if x < y. Returns eq if x == y. '
+            'Otherwise returns gt.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):
+        v = strcmp(x, y)
+        if v < 0:
+            return lt
+        if v == 0:
+            return eq
+        return gt
+
+class BuiltinCmp(BuiltinFormatterFunction):
+    name = 'cmp'
+    arg_count = 5
+    doc =   _('cmp(x, y, lt, eq, gt) -- compares x and y after converting both to '
+            'numbers. Returns lt if x < y. Returns eq if x == y. Otherwise returns gt.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):
+        x = float(x if x else 0)
+        y = float(y if y else 0)
+        if x < y:
+            return lt
+        if x == y:
+            return eq
+        return gt
+
+class BuiltinStrcat(BuiltinFormatterFunction):
+    name = 'strcat'
+    arg_count = -1
+    doc = _('strcat(a, b, ...) -- can take any number of arguments. Returns a '
+            'string formed by concatenating all the arguments')
+
+    def evaluate(self, formatter, kwargs, mi, locals, *args):
+        i = 0
+        res = ''
+        for i in range(0, len(args)):
+            res += args[i]
+        return res
+
+class BuiltinAdd(BuiltinFormatterFunction):
+    name = 'add'
+    arg_count = 2
+    doc = _('add(x, y) -- returns x + y. Throws an exception if either x or y are not numbers.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, x, y):
+        x = float(x if x else 0)
+        y = float(y if y else 0)
+        return unicode(x + y)
+
+class BuiltinSubtract(BuiltinFormatterFunction):
+    name = 'subtract'
+    arg_count = 2
+    doc = _('subtract(x, y) -- returns x - y. Throws an exception if either x or y are not numbers.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, x, y):
+        x = float(x if x else 0)
+        y = float(y if y else 0)
+        return unicode(x - y)
+
+class BuiltinMultiply(BuiltinFormatterFunction):
+    name = 'multiply'
+    arg_count = 2
+    doc = _('multiply(x, y) -- returns x * y. Throws an exception if either x or y are not numbers.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, x, y):
+        x = float(x if x else 0)
+        y = float(y if y else 0)
+        return unicode(x * y)
+
+class BuiltinDivide(BuiltinFormatterFunction):
+    name = 'divide'
+    arg_count = 2
+    doc = _('divide(x, y) -- returns x / y. Throws an exception if either x or y are not numbers.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, x, y):
+        x = float(x if x else 0)
+        y = float(y if y else 0)
+        return unicode(x / y)
+
+class BuiltinTemplate(BuiltinFormatterFunction):
+    name = 'template'
+    arg_count = 1
+    doc = _('template(x) -- evaluates x as a template. The evaluation is done '
+            'in its own context, meaning that variables are not shared between '
+            'the caller and the template evaluation. Because the { and } '
+            'characters are special, you must use [[ for the { character and '
+            ']] for the } character; they are converted automatically. '
+            'For example, template(\'[[title_sort]]\') will evaluate the '
+            'template {title_sort} and return its value.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, template):
+        template = template.replace('[[', '{').replace(']]', '}')
+        return formatter.safe_format(template, kwargs, 'TEMPLATE', mi)
+
+class BuiltinEval(BuiltinFormatterFunction):
+    name = 'eval'
+    arg_count = 1
+    doc = _('eval(template) -- evaluates the template, passing the local '
+            'variables (those \'assign\'ed to) instead of the book metadata. '
+            ' This permits using the template processor to construct complex '
+            'results from local variables.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, template):
+        from formatter import eval_formatter
+        template = template.replace('[[', '{').replace(']]', '}')
+        return eval_formatter.safe_format(template, locals, 'EVAL', None)
+
+class BuiltinAssign(BuiltinFormatterFunction):
+    name = 'assign'
+    arg_count = 2
+    doc = _('assign(id, val) -- assigns val to id, then returns val. '
+            'id must be an identifier, not an expression')
+
+    def evaluate(self, formatter, kwargs, mi, locals, target, value):
+        locals[target] = value
+        return value
+
+class BuiltinPrint(BuiltinFormatterFunction):
+    name = 'print'
+    arg_count = -1
+    doc = _('print(a, b, ...) -- prints the arguments to standard output. '
+            'Unless you start calibre from the command line (calibre-debug -g), '
+            'the output will go to a black hole.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, *args):
+        print args
+        return None
+
+class BuiltinField(BuiltinFormatterFunction):
+    name = 'field'
+    arg_count = 1
+    doc = _('field(name) -- returns the metadata field named by name')
+
+    def evaluate(self, formatter, kwargs, mi, locals, name):
+        return formatter.get_value(name, [], kwargs)
+
+class BuiltinSubstr(BuiltinFormatterFunction):
+    name = 'substr'
+    arg_count = 3
+    doc = _('substr(str, start, end) -- returns the start\'th through the end\'th '
+            'characters of str. The first character in str is the zero\'th '
+            'character. If end is negative, then it indicates that many '
+            'characters counting from the right. If end is zero, then it '
+            'indicates the last character. For example, substr(\'12345\', 1, 0) '
+            'returns \'2345\', and substr(\'12345\', 1, -1) returns \'234\'.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, str_, start_, end_):
+        return str_[int(start_): len(str_) if int(end_) == 0 else int(end_)]
+
+class BuiltinLookup(BuiltinFormatterFunction):
+    name = 'lookup'
+    arg_count = -1
+    doc = _('lookup(val, pattern, field, pattern, field, ..., else_field) -- '
+            'like switch, except the arguments are field (metadata) names, not '
+            'text. The value of the appropriate field will be fetched and used. '
+            'Note that because composite columns are fields, you can use this '
+            'function in one composite field to use the value of some other '
+            'composite field. This is extremely useful when constructing '
+            'variable save paths')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val, *args):
+        if len(args) == 2: # here for backwards compatibility
+            if val:
+                return formatter.vformat('{'+args[0].strip()+'}', [], kwargs)
+            else:
+                return formatter.vformat('{'+args[1].strip()+'}', [], kwargs)
+        if (len(args) % 2) != 1:
+            raise ValueError(_('lookup requires either 2 or an odd number of arguments'))
+        i = 0
+        while i < len(args):
+            if i + 1 >= len(args):
+                return formatter.vformat('{' + args[i].strip() + '}', [], kwargs)
+            if re.search(args[i], val):
+                return formatter.vformat('{'+args[i+1].strip() + '}', [], kwargs)
+            i += 2
+
+class BuiltinTest(BuiltinFormatterFunction):
+    name = 'test'
+    arg_count = 3
+    doc = _('test(val, text if not empty, text if empty) -- return `text if not '
+            'empty` if the field is not empty, otherwise return `text if empty`')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val, value_if_set, value_not_set):
+        if val:
+            return value_if_set
+        else:
+            return value_not_set
+
+class BuiltinContains(BuiltinFormatterFunction):
+    name = 'contains'
+    arg_count = 4
+    doc = _('contains(val, pattern, text if match, text if not match) -- checks '
+            'if field contains matches for the regular expression `pattern`. '
+            'Returns `text if match` if matches are found, otherwise it returns '
+            '`text if no match`')
+
+    def evaluate(self, formatter, kwargs, mi, locals,
+                 val, test, value_if_present, value_if_not):
+        if re.search(test, val):
+            return value_if_present
+        else:
+            return value_if_not
+
+class BuiltinSwitch(BuiltinFormatterFunction):
+    name = 'switch'
+    arg_count = -1
+    doc = _('switch(val, pattern, value, pattern, value, ..., else_value) -- '
+            'for each `pattern, value` pair, checks if the field matches '
+            'the regular expression `pattern` and if so, returns that '
+            '`value`. If no pattern matches, then else_value is returned. '
+            'You can have as many `pattern, value` pairs as you want')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val, *args):
+        if (len(args) % 2) != 1:
+            raise ValueError(_('switch requires an odd number of arguments'))
+        i = 0
+        while i < len(args):
+            if i + 1 >= len(args):
+                return args[i]
+            if re.search(args[i], val):
+                return args[i+1]
+            i += 2
+
+class BuiltinRe(BuiltinFormatterFunction):
+    name = 're'
+    arg_count = 3
+    doc = _('re(val, pattern, replacement) -- return the field after applying '
+            'the regular expression. All instances of `pattern` are replaced '
+            'with `replacement`. As in all of calibre, these are '
+            'python-compatible regular expressions')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val, pattern, replacement):
+        return re.sub(pattern, replacement, val)
+
+class BuiltinIfempty(BuiltinFormatterFunction):
+    name = 'ifempty'
+    arg_count = 2
+    doc = _('ifempty(val, text if empty) -- return val if val is not empty, '
+            'otherwise return `text if empty`')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val, value_if_empty):
+        if val:
+            return val
+        else:
+            return value_if_empty
+
+class BuiltinShorten(BuiltinFormatterFunction):
+    name = 'shorten'
+    arg_count = 4
+    doc = _('shorten(val, left chars, middle text, right chars) -- Return a '
+            'shortened version of the field, consisting of `left chars` '
+            'characters from the beginning of the field, followed by '
+            '`middle text`, followed by `right chars` characters from '
+            'the end of the string. `Left chars` and `right chars` must be '
+            'integers. For example, assume the title of the book is '
+            '`Ancient English Laws in the Times of Ivanhoe`, and you want '
+            'it to fit in a space of at most 15 characters. If you use '
+            '{title:shorten(9,-,5)}, the result will be `Ancient E-nhoe`. '
+            'If the field\'s length is less than left chars + right chars + '
+            'the length of `middle text`, then the field will be used '
+            'intact. For example, the title `The Dome` would not be changed.')
+
+    def evaluate(self, formatter, kwargs, mi, locals,
+                 val, leading, center_string, trailing):
+        l = max(0, int(leading))
+        t = max(0, int(trailing))
+        if len(val) > l + len(center_string) + t:
+            return val[0:l] + center_string + ('' if t == 0 else val[-t:])
+        else:
+            return val
+
+class BuiltinCount(BuiltinFormatterFunction):
+    name = 'count'
+    arg_count = 2
+    doc = _('count(val, separator) -- interprets the value as a list of items '
+            'separated by `separator`, returning the number of items in the '
+            'list. Most lists use a comma as the separator, but authors '
+            'uses an ampersand. Examples: {tags:count(,)}, {authors:count(&)}')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val, sep):
+        return unicode(len(val.split(sep)))
+
+class BuiltinListitem(BuiltinFormatterFunction):
+    name = 'list_item'
+    arg_count = 3
+    doc = _('list_item(val, index, separator) -- interpret the value as a list of '
+            'items separated by `separator`, returning the `index`th item. '
+            'The first item is number zero. The last item can be returned '
+            'using `list_item(-1,separator)`. If the item is not in the list, '
+            'then the empty value is returned. The separator has the same '
+            'meaning as in the count function.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val, index, sep):
+        if not val:
+            return ''
+        index = int(index)
+        val = val.split(sep)
+        try:
+            return val[index]
+        except:
+            return ''
+
+class BuiltinUppercase(BuiltinFormatterFunction):
+    name = 'uppercase'
+    arg_count = 1
+    doc = _('uppercase(val) -- return value of the field in upper case')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val):
+        return val.upper()
+
+class BuiltinLowercase(BuiltinFormatterFunction):
+    name = 'lowercase'
+    arg_count = 1
+    doc = _('lowercase(val) -- return value of the field in lower case')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val):
+        return val.lower()
+
+class BuiltinTitlecase(BuiltinFormatterFunction):
+    name = 'titlecase'
+    arg_count = 1
+    doc = _('titlecase(val) -- return value of the field in title case')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val):
+        return titlecase(val)
+
+class BuiltinCapitalize(BuiltinFormatterFunction):
+    name = 'capitalize'
+    arg_count = 1
+    doc = _('capitalize(val) -- return value of the field capitalized')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val):
+        return capitalize(val)
+
+builtin_add         = BuiltinAdd()
+builtin_assign      = BuiltinAssign()
+builtin_capitalize  = BuiltinCapitalize()
+builtin_cmp         = BuiltinCmp()
+builtin_contains    = BuiltinContains()
+builtin_count       = BuiltinCount()
+builtin_divide      = BuiltinDivide()
+builtin_eval        = BuiltinEval()
+builtin_ifempty     = BuiltinIfempty()
+builtin_field       = BuiltinField()
+builtin_list_item   = BuiltinListitem()
+builtin_lookup      = BuiltinLookup()
+builtin_lowercase   = BuiltinLowercase()
+builtin_multiply    = BuiltinMultiply()
+builtin_print       = BuiltinPrint()
+builtin_re          = BuiltinRe()
+builtin_shorten     = BuiltinShorten()
+builtin_strcat      = BuiltinStrcat()
+builtin_strcmp      = BuiltinStrcmp()
+builtin_substr      = BuiltinSubstr()
+builtin_subtract    = BuiltinSubtract()
+builtin_switch      = BuiltinSwitch()
+builtin_template    = BuiltinTemplate()
+builtin_test        = BuiltinTest()
+builtin_titlecase   = BuiltinTitlecase()
+builtin_uppercase   = BuiltinUppercase()
+
+class FormatterUserFunction(FormatterFunction):
+    def __init__(self, name, doc, arg_count, program_text):
+        self.name = name
+        self.doc = doc
+        self.arg_count = arg_count
+        self.program_text = program_text
+
+tabs = re.compile(r'^\t*')
+def compile_user_function(name, doc, arg_count, eval_func):
+    def replace_func(mo):
+        return  mo.group().replace('\t', '    ')
+
+    func = '    ' + '\n    '.join([tabs.sub(replace_func, line )
+                                   for line in eval_func.splitlines()])
+    prog = '''
+from calibre.utils.formatter_functions import FormatterUserFunction
+class UserFunction(FormatterUserFunction):
+''' + func
+    locals = {}
+    exec prog in locals
+    cls = locals['UserFunction'](name, doc, arg_count, eval_func)
+    return cls
+
+def load_user_template_functions(funcs):
+    formatter_functions.reset_to_builtins()
+    for func in funcs:
+        try:
+            cls = compile_user_function(*func)
+            formatter_functions.register_function(cls)
+        except:
+            traceback.print_exc()