sync to trunk

.bzrignore

@@ -4,6 +4,7 @@ src/calibre/plugins
 resources/images.qrc
 src/calibre/manual/.build/
 src/calibre/manual/cli/
+src/calibre/manual/template_ref.rst
 build
 dist
 docs

Changelog.old.yaml

@@ -478,7 +478,7 @@
       type: major
       description : >
         "You can now save your frequently used searches and access them with a single click. For details
-        see http://calibre-ebook.com/user_manual/gui.html#search-sort"
+        see http://manual.calibre-ebook.com/gui.html#search-sort"
 
     - title: "Add searching by date/published date"
       tickets: [5244]

Changelog.yaml

@@ -584,7 +584,7 @@
     - title: "FB2 Output: Option to set the FB2 genre explicitly."
       tickets: [743178]
 
-    - title: "Plugin developers: calibre now has a new plugin API, see http://calibre-ebook.com/user_manual/creating_plugins.html. Your existing plugins should continue to work, but it would be good to test them to make sure."
+    - title: "Plugin developers: calibre now has a new plugin API, see http://manual.calibre-ebook.com/creating_plugins.html. Your existing plugins should continue to work, but it would be good to test them to make sure."
   
   bug fixes:
     - title: "Fix text color in the search bar set to black instead of the system font color"
@@ -969,7 +969,7 @@
 
   new features:
     - title: "Tag Browser: Support the creation of nested User Categories"
-      description: "See http://calibre-ebook.com/user_manual/gui.html#tag-browser for details"
+      description: "See http://manual.calibre-ebook.com/gui.html#tag-browser for details"
       type: major
 
     - title: "Disable Kent District Library plugin to download series information. The website could not handle the load calibre's 2 million users put on it. You can manually re-enable it if you really want series information, but it is very slow"
@@ -3842,7 +3842,7 @@
       type: major
       description: >
         "You can now change the icons used in the User Interface and other static resources. Details on how to
-         do this are at: http://calibre-ebook.com/user_manual/customize.html#overriding-icons-templates-etcetera"
+         do this are at: http://manual.calibre-ebook.com/customize.html#overriding-icons-templates-etcetera"
 
     - title: "Split the 'Send to device' button into two buttons, 'Connect/share' and 'Send to device'. The new 'Send to device' button will now only be available when a device is connected."
 

INSTALL

@@ -3,7 +3,7 @@ calibre supports installation from source, only on Linux.
 Note that you *do not* need to install from source to hack on
 the calibre source code. To get started with calibre development,
 use a normal calibre install and follow the instructions at
-http://calibre-ebook.com/user_manual/develop.html
+http://manual.calibre-ebook.com/develop.html
 
 On Linux, there are two kinds of installation from source possible.
 Note that both kinds require lots of dependencies as well as a

README

@@ -6,8 +6,8 @@ reading. It is cross platform, running on Linux, Windows and OS X.
 
 For screenshots: https://calibre-ebook.com/demo
 
-For installation/usage instructions please see
+For usage instructions please see
-http://calibre-ebook.com/user_manual
+http://manual.calibre-ebook.com
 
 For source code access:
 bzr branch lp:calibre

recipes/alt_om_herning.recipe

@@ -0,0 +1,43 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011, Rasmus Lauritsen <rasmus at lauritsen.info>'
+'''
+aoh.dk
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class aoh_dk(BasicNewsRecipe):
+    title                 = 'Alt om Herning'
+    __author__            = 'Rasmus Lauritsen'
+    description           = 'Nyheder fra Herning om omegn'
+    publisher             = 'Mediehuset Herning Folkeblad'
+    category              = 'news, local, Denmark'
+    oldest_article        = 14
+    max_articles_per_feed = 50
+    no_stylesheets        = True
+    delay                 = 1
+    encoding              = 'utf8'
+    use_embedded_content  = False
+    language              = 'da'
+    extra_css             = """ body{font-family: Verdana,Arial,sans-serif }
+                                img{margin-bottom: 0.4em}
+                                .txtContent,.stamp{font-size: small}
+                                """
+
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
+
+    feeds = [(u'All news', u'http://aoh.dk/rss.xml')]
+
+    keep_only_tags    = [
+                       dict(name='h1')
+                       ,dict(name='span', attrs={'class':['frontpage_body']})
+                        ]
+
+    remove_tags        = [
+                            dict(name=['object','link'])
+                         ]

recipes/bbc_fast.recipe

@@ -1,27 +1,30 @@
 __license__   = 'GPL v3'
-__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2010 - 2011, Darko Miletic <darko.miletic at gmail.com>'
 '''
 news.bbc.co.uk
 '''
-import re
 from calibre.web.feeds.recipes import BasicNewsRecipe
 
 class BBC(BasicNewsRecipe):
     title                  = 'BBC News (fast)'
     __author__             = 'Darko Miletic, Starson17'
-    description            = 'News from UK. A much faster version that does not download pictures'
+    description            = 'Visit BBC News for up-to-the-minute news, breaking news, video, audio and feature stories. BBC News provides trusted World and UK news as well as local and regional perspectives. Also entertainment, business, science, technology and health news.'
     oldest_article         = 2
     max_articles_per_feed  = 100
     no_stylesheets         = True
-    #delay                  = 1
     use_embedded_content   = False
     encoding               = 'utf8'
     publisher              = 'BBC'
     category               = 'news, UK, world'
     language               = 'en_GB'
     publication_type       = 'newsportal'
-    extra_css              = ' body{ font-family: Verdana,Helvetica,Arial,sans-serif } .introduction{font-weight: bold} .story-feature{display: block; padding: 0; border: 1px solid; width: 40%; font-size: small} .story-feature h2{text-align: center; text-transform: uppercase} '
+    masthead_url           = 'http://news.bbcimg.co.uk/img/1_0_1/cream/hi/news/news-blocks.gif'
-    preprocess_regexps     = [(re.compile(r'<!--.*?-->', re.DOTALL), lambda m: '')]
+    extra_css              = """
+                                 body{ font-family: Verdana,Helvetica,Arial,sans-serif }
+                                 .introduction{font-weight: bold}
+                                 .story-feature{display: block; padding: 0; border: 1px solid; width: 40%; font-size: small}
+                                 .story-feature h2{text-align: center; text-transform: uppercase}
+                             """
     conversion_options = {
                              'comments'        : description
                             ,'tags'            : category
@@ -31,31 +34,54 @@ class BBC(BasicNewsRecipe):
                          }
 
     keep_only_tags    = [
-                       dict(name='div', attrs={'class':['layout-block-a layout-block']})
+                          dict(name='div', attrs={'class':['layout-block-a layout-block']})
-                       ,dict(attrs={'class':['story-body','storybody']})
+                         ,dict(attrs={'class':['story-body','storybody']})
+                         ,dict(attrs={'id':['meta-information','story-body']})
                         ]
 
     remove_tags = [
-                       dict(name='div', attrs={'class':['story-feature related narrow', 'share-help', 'embedded-hyper', \
+                       dict(name='div', attrs={'class':['story-feature related narrow', \
-                       'story-feature wide ', 'story-feature narrow']})
+                                                        'share-help', 'embedded-hyper', \
-                       , dict(name=['img'])
+                                                        'story-feature wide ', \
-                        ]
+                                                        'story-feature narrow', \
+                                                        'hidden','story-actions', \
+                                                        'embedded-hyper']})
+                       ,dict(name=['img','meta','link','object','embed','iframe','base'])
+                       ,dict(attrs={'class':['hidden','videoInStoryC']})
+                       ,dict(attrs={'id':['bbccom_sponsor_section','toggle-controls', \
+                                          'toggle-images','toggle-title']})
+                  ]
 
-    remove_attributes = ['width','height']
+    remove_attributes = ['width','height','xmlns:og','lang','clear']
 
     feeds          = [
-                      ('News Front Page', 'http://newsrss.bbc.co.uk/rss/newsonline_world_edition/front_page/rss.xml'),
+                      ('Top Stories'        , 'http://feeds.bbci.co.uk/news/rss.xml'                        ),
-                      ('Science/Nature', 'http://newsrss.bbc.co.uk/rss/newsonline_world_edition/science/nature/rss.xml'),
+                      ('Science/Environment', 'http://feeds.bbci.co.uk/news/science_and_environment/rss.xml'),
-                      ('Technology', 'http://newsrss.bbc.co.uk/rss/newsonline_world_edition/technology/rss.xml'),
+                      ('Technology'         , 'http://feeds.bbci.co.uk/news/technology/rss.xml'             ),
-                      ('Entertainment', 'http://newsrss.bbc.co.uk/rss/newsonline_world_edition/entertainment/rss.xml'),
+                      ('Entertainment/Arts' , 'http://feeds.bbci.co.uk/news/entertainment_and_arts/rss.xml' ),
-                      ('Magazine', 'http://newsrss.bbc.co.uk/rss/newsonline_world_edition/uk_news/magazine/rss.xml'),
+                      ('Magazine'           , 'http://feeds.bbci.co.uk/news/magazine/rss.xml'               ),
-                      ('Business', 'http://newsrss.bbc.co.uk/rss/newsonline_world_edition/business/rss.xml'),
+                      ('Business'           , 'http://feeds.bbci.co.uk/news/business/rss.xml'               ),
-                      ('Health', 'http://newsrss.bbc.co.uk/rss/newsonline_world_edition/health/rss.xml'),
+                      ('Politics'           , 'http://feeds.bbci.co.uk/news/politics/rss.xml'               ),
-                      ('Americas', 'http://newsrss.bbc.co.uk/rss/newsonline_world_edition/americas/rss.xml'),
+                      ('Health'             , 'http://feeds.bbci.co.uk/news/health/rss.xml'                 ),
-                      ('Europe', 'http://newsrss.bbc.co.uk/rss/newsonline_world_edition/europe/rss.xml'),
+                      ('US&Canada'          , 'http://feeds.bbci.co.uk/news/world/us_and_canada/rss.xml'    ),
-                      ('South Asia', 'http://newsrss.bbc.co.uk/rss/newsonline_world_edition/south_asia/rss.xml'),
+                      ('Latin America'      , 'http://feeds.bbci.co.uk/news/world/latin_america/rss.xml'    ),
-                      ('UK', 'http://newsrss.bbc.co.uk/rss/newsonline_world_edition/uk_news/rss.xml'),
+                      ('Europe'             , 'http://feeds.bbci.co.uk/news/world/europe/rss.xml'           ),
-                      ('Asia-Pacific', 'http://newsrss.bbc.co.uk/rss/newsonline_world_edition/asia-pacific/rss.xml'),
+                      ('South Asia'         , 'http://feeds.bbci.co.uk/news/world/south_asia/rss.xml'       ),
-                      ('Africa', 'http://newsrss.bbc.co.uk/rss/newsonline_world_edition/africa/rss.xml'),
+                      ('England'            , 'http://feeds.bbci.co.uk/news/england/rss.xml'                ),
+                      ('Asia-Pacific'       , 'http://feeds.bbci.co.uk/news/world/asia_pacific/rss.xml'     ),
+                      ('Africa'             , 'http://feeds.bbci.co.uk/news/world/africa/rss.xml'           )
                     ]
 
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll('left'):
+            item.name='span'
+        for item in soup.findAll('a'):
+            if item.string is not None:
+               str = item.string
+               item.replaceWith(str)
+            else:
+               str = self.tag_to_string(item)
+               item.replaceWith(str)
+        return soup

recipes/dilemaveche.recipe

@@ -1,55 +1,71 @@
-# -*- coding: utf-8 -*-
-#!/usr/bin/env  python
-
-__license__   = 'GPL v3'
-__copyright__ = u'2011, Silviu Cotoar\u0103'
-'''
-dilemaveche.ro
-'''
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class DilemaVeche(BasicNewsRecipe):
-    title                 = u'Dilema Veche'
+    title          = u'Dilema Veche' # apare vinerea, mai pe dupa-masa,depinde de Luiza cred (care se semneaza ca fiind creatorul fiecarui articol in feed-ul RSS)
-    __author__            = u'Silviu Cotoar\u0103'
+    __author__            = 'song2' # inspirat din scriptul pentru Le Monde. Inspired from the Le Monde script
-    description           = u'Sunt vechi, domnule!'
+    description           = '"Sint vechi, domnule!" (I.L. Caragiale)'
-    publisher             = u'Dilema Veche'
+    publisher             = 'Adevarul Holding'
-    oldest_article        = 50
+    oldest_article        = 7
-    language              = 'ro'
+    max_articles_per_feed = 200
-    max_articles_per_feed = 100
+    encoding              = 'utf8'
-    no_stylesheets        = True
+    language = 'ro'
-    use_embedded_content  = False
+    masthead_url = 'http://www.dilemaveche.ro/sites/all/themes/dilema/theme/dilema_two/layouter/dilema_two_homepage/logo.png'
-    category              = 'Ziare'
+    publication_type = 'magazine'
-    encoding              = 'utf-8'
+    feeds = [
-    cover_url             = 'http://www.dilemaveche.ro/sites/all/themes/dilema/theme/dilema_two/layouter/dilema_two_homepage/logo.png'
+                ('Editoriale si opinii - Situatiunea', 'http://www.dilemaveche.ro/taxonomy/term/37/0/feed'),
-
+                ('Editoriale si opinii - Pe ce lume traim', 'http://www.dilemaveche.ro/taxonomy/term/38/0/feed'),
-    conversion_options = {
+                ('Editoriale si opinii - Bordeie si obiceie', 'http://www.dilemaveche.ro/taxonomy/term/44/0/feed'),
-                             'comments'   : description
+                ('Editoriale si opinii - Talc Show', 'http://www.dilemaveche.ro/taxonomy/term/44/0/feed'),
-                            ,'tags'       : category
+                ('Tema saptamanii', 'http://www.dilemaveche.ro/taxonomy/term/19/0/feed'),
-                            ,'language'   : language
+                ('La zi in cultura - Dilema va recomanda', 'http://www.dilemaveche.ro/taxonomy/term/58/0/feed'),
-                            ,'publisher'  : publisher
+                ('La zi in cultura - Carte', 'http://www.dilemaveche.ro/taxonomy/term/14/0/feed'),
-                         }
+                ('La zi in cultura - Film', 'http://www.dilemaveche.ro/taxonomy/term/13/0/feed'),
-
+                ('La zi in cultura - Muzica', 'http://www.dilemaveche.ro/taxonomy/term/1341/0/feed'),
-    keep_only_tags = [
+                ('La zi in cultura - Arte performative', 'http://www.dilemaveche.ro/taxonomy/term/1342/0/feed'),
-                          dict(name='h1', attrs={'class':'art_title'})
+                ('La zi in cultura - Arte vizuale', 'http://www.dilemaveche.ro/taxonomy/term/1512/0/feed'),
-                        , dict(name='h1', attrs={'class':'art_title online'})
+                ('Societate - Ieri cu vedere spre azi', 'http://www.dilemaveche.ro/taxonomy/term/15/0/feed'),
-                        , dict(name='div', attrs={'class':'item'})
+                ('Societate - Din polul opus', 'http://www.dilemaveche.ro/taxonomy/term/41/0/feed'),
-                        , dict(name='div', attrs={'class':'art_content'})
+                ('Societate - Mass comedia', 'http://www.dilemaveche.ro/taxonomy/term/43/0/feed'),
-                     ]
+                ('Societate - La singular si la plural', 'http://www.dilemaveche.ro/taxonomy/term/42/0/feed'),
-
+                ('Oameni si idei - Educatie', 'http://www.dilemaveche.ro/taxonomy/term/46/0/feed'),
+                ('Oameni si idei - Polemici si dezbateri', 'http://www.dilemaveche.ro/taxonomy/term/48/0/feed'),
+                ('Oameni si idei - Stiinta si tehnologie', 'http://www.dilemaveche.ro/taxonomy/term/46/0/feed'),
+                ('Dileme on-line', 'http://www.dilemaveche.ro/taxonomy/term/005/0/feed')
+                 ]
+    remove_tags_before = dict(name='div',attrs={'class':'spacer_10'})
     remove_tags = [
-                         dict(name='div', attrs={'class':['article_details']})
+        dict(name='div', attrs={'class':'art_related_left'}),
-                        , dict(name='div', attrs={'class':['controale']})
+        dict(name='div', attrs={'class':'controale'}),
-                        , dict(name='div', attrs={'class':['art_related_left']})
+		dict(name='div', attrs={'class':'simple_overlay'}),
-                  ]
+    ]
+    remove_tags_after = [dict(id='facebookLike')]
+    remove_javascript = True
+    no_stylesheets        = True
+    remove_empty_feeds = True
+    extra_css             = """
+        body{font-family: Georgia,Times,serif }
+        img{margin-bottom: 0.4em; display:block}
+                            """
+    def get_cover_url(self):
+        cover_url = None
+        soup = self.index_to_soup('http://dilemaveche.ro')
+        link_item = soup.find('div',attrs={'class':'box_dr_pdf_picture'})
+        if link_item and link_item.a:
+           cover_url = link_item.a['href']
+        br = BasicNewsRecipe.get_browser()
+        try:
+            br.open(cover_url)
+        except: #daca nu gaseste pdf-ul
+            self.log("\nPDF indisponibil")
+            link_item = soup.find('div',attrs={'class':'box_dr_pdf_picture'})
+            if link_item and link_item.img:
+                cover_url = link_item.img['src']
+            br = BasicNewsRecipe.get_browser()
+            try:
+                 br.open(cover_url)
+            except: #daca nu gaseste nici imaginea mica mica
+                print('Mama lor de nenorociti! nu este nici pdf nici imagine')
+                cover_url ='http://www.dilemaveche.ro/sites/all/themes/dilema/theme/dilema_two/layouter/dilema_two_homepage/logo.png'
+        return cover_url
+    cover_margins = (10, 15, '#ffffff')
 
-    remove_tags_after = [
-                            dict(name='div', attrs={'class':['article_details']})
-                        ]
-
-    feeds          = [
-                        (u'Feeds', u'http://www.dilemaveche.ro/rss.xml')
-                     ]
-
-    def preprocess_html(self, soup):
-        return self.adeify_images(soup)

recipes/mediapart.recipe

@@ -71,7 +71,7 @@ class Mediapart(BasicNewsRecipe):
         br = BasicNewsRecipe.get_browser()
         if self.username is not None and self.password is not None:
             br.open('http://www.mediapart.fr/')
-            br.select_form(nr=1)
+            br.select_form(nr=0)
             br['name'] = self.username
             br['pass'] = self.password
             br.submit()

recipes/metro_uk.recipe

@@ -0,0 +1,29 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+class AdvancedUserRecipe1306097511(BasicNewsRecipe):
+    title          = u'Metro UK'
+
+    no_stylesheets = True
+    oldest_article = 1
+    max_articles_per_feed = 200
+
+    __author__ = 'Dave Asbury'
+    language = 'en_GB'
+    simultaneous_downloads= 3
+
+    masthead_url        = 'http://e-edition.metro.co.uk/images/metro_logo.gif'
+
+    keep_only_tags = [
+                    dict(attrs={'class':['img-cnt figure']}),
+    dict(attrs={'class':['art-img']}),
+                    dict(name='h1'),
+    dict(name='h2', attrs={'class':'h2'}),
+                    dict(name='div', attrs={'class':'art-lft'})
+    ]
+    remove_tags    = [dict(name='div', attrs={'class':[ 'metroCommentFormWrap',
+                                                                             'commentForm', 'metroCommentInnerWrap',
+                             'art-rgt','pluck-app pluck-comm','news m12 clrd clr-l p5t', 'flt-r' ]})]
+
+    feeds          = [
+        (u'News', u'http://www.metro.co.uk/rss/news/'), (u'Money', u'http://www.metro.co.uk/rss/money/'), (u'Sport', u'http://www.metro.co.uk/rss/sport/'), (u'Film', u'http://www.metro.co.uk/rss/metrolife/film/'), (u'Music', u'http://www.metro.co.uk/rss/metrolife/music/'), (u'TV', u'http://www.metro.co.uk/rss/tv/'), (u'Showbiz', u'http://www.metro.co.uk/rss/showbiz/'), (u'Weird News', u'http://www.metro.co.uk/rss/weird/'), (u'Travel', u'http://www.metro.co.uk/rss/travel/'), (u'Lifestyle', u'http://www.metro.co.uk/rss/lifestyle/'), (u'Books', u'http://www.metro.co.uk/rss/lifestyle/books/'), (u'Food', u'http://www.metro.co.uk/rss/lifestyle/restaurants/')]
+
+

recipes/observatorul_cultural.recipe

@@ -0,0 +1,64 @@
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+coverpage = None
+
+class ObservatorulCultural(BasicNewsRecipe):
+    title                 = u'Observatorul cultural'
+    __author__            = 'song2' #prelucrat dupa un script de http://www.thenowhereman.com
+    encoding = 'utf-8'
+    language = 'ro'
+    publication_type = 'magazine'
+    description = 'Spiritul critic in acţiune\n'
+    no_stylesheets        = True
+    remove_javascript     = True
+    masthead_url='http://www.observatorcultural.ro/userfiles/article/sigla%20Observator%20cultural_02231058.JPG'
+    keep_only_tags = [
+        dict(name='div', attrs={'class':'detaliuArticol'})]
+    remove_tags = [dict(name='div', attrs={'class':'comentariiArticol'}),
+         dict(name='div', attrs={'class':'postComment'}),
+         dict(name='div', attrs={'class':'utileArticol'}),
+         dict(name='p', attrs={'class':'butonComenteaza'}),
+         dict(name='h5'),
+         dict(name='div', attrs={'style':'margin-top: 0px; padding-top: 0px;'})
+         ]
+    def parse_index(self):
+        soup = self.index_to_soup('http://www.observatorcultural.ro/Arhiva*-archive.html')
+        issueTag = soup.find('a', href=re.compile("observatorcultural.ro\/Numarul"))
+        issueURL = issueTag['href']
+        print issueURL;
+        issueSoup = self.index_to_soup(issueURL)
+        feeds = []
+        stories = []
+        for categorie in issueSoup.findAll('dl',attrs={'class':'continutArhive'}):
+            categ=self.tag_to_string(categorie.find('dt'))
+            for story in categorie.findAll('dd'):
+                title=[]
+                for bucatele in story.findAll('a'):
+                    title.append(bucatele)
+                if len(title)==1: #daca articolul nu are autor
+                    stories.append({
+                        'title' : self.tag_to_string(title[0]),
+                        'url'   : title[0]['href'],
+                        'date'  : '',
+                        'author' : ''})
+                else: # daca articolul are autor len(title)=2
+                    stories.append({
+                        'title' : self.tag_to_string(title[1]),
+                        'url'   :title[1]['href'],
+                        'date'  : '',
+                        'author' : self.tag_to_string(title[0])})
+                    print(self.tag_to_string(title[0]))
+                if 'Editorial' in categ:
+                    global coverpage
+                    coverpage=title[1]['href']  # am luat link-ul spre editorial
+            feeds.append((categ,stories))
+            stories = []
+        print feeds
+        return feeds
+#procedura de luat coperta
+    def get_cover_url(self):
+        soup = self.index_to_soup(coverpage)
+        link_item = soup.find('a',attrs={'rel':'lightbox'}) # caut imaginea textului
+        a=''
+        cover_url = a.join(link_item.img['src'].split('_details_'))
+        return cover_url

recipes/version2.recipe

@@ -0,0 +1,64 @@
+import re
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Rasmus Lauritsen <rasmus at lauritsen.info>'
+'''
+version2.dk
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class version2(BasicNewsRecipe):
+    title                    = 'Version2.dk'
+    __author__         = 'Rasmus Lauritsen'
+    description          = 'IT News'
+    publisher             = 'version2.dk'
+    category              = 'news, IT, hardware, software, Denmark'
+    oldest_article       = 14
+    max_articles_per_feed = 50
+    no_stylesheets      = True
+    remove_empty_feeds    = True
+    use_embedded_content  = False
+    encoding              = 'iso-8859-1'
+    language              = 'da'
+
+    extra_css             = """
+                                body {font-family: "Verdana",Times,serif}
+                                .articleauthor{color: #9F9F9F;
+                                               font-family: Arial, sans-serif;
+                                               font-size: small;
+                                               text-transform: uppercase}
+                                .rubric,.dd,h6#credit{color: #CD0021;
+                                        font-family: Arial, sans-serif;
+                                        font-size: small;
+                                        text-transform: uppercase}
+                                .descender:first-letter{display: inline; font-size: xx-large; font-weight: bold}
+                                .dd,h6#credit{color: gray}
+                                .c{display: block}
+                                .caption,h2#articleintro{font-style: italic}
+                                .caption{font-size: small}
+                            """
+
+    preprocess_regexps = [ (re.compile(r'</?a[^>]*>'),lambda match: ''),
+                                       (re.compile(r'<span[^>]*article-link-id.*?<br\s*\/?><br\s*\/?>'), lambda match: '')]
+
+    keep_only_tags = [dict(name='div', attrs={'class':'article'})]
+
+    remove_tags = [
+	dict(name='p',attrs={'class':'meta links'}),
+   	dict(name='div',attrs={'class':'float-right'}),
+   	dict(name='span',attrs={'class':'article-link-id'})
+ 	 ]
+
+    feeds              = [
+                            (u'Seneste nyheder'               , u'http://www.version2.dk/feeds/nyheder')
+                           ,(u'Forretningssoftware'             , u'http://www.version2.dk/feeds/forretningssoftware')
+                           ,(u'Internet & styresystemer' , u'http://www.version2.dk/feeds/styresystemer')
+                           ,(u'It-arkitektur'           , u'http://www.version2.dk/feeds/it-arkitektur')
+                           ,(u'It-styring & outsourcing'             , u'http://www.version2.dk/feeds/it-styring')
+                           ,(u'Job & karriere'                     , u'http://www.version2.dk/feeds/karriere')
+                           ,(u'Mobil it & tele'                   , u'http://www.version2.dk/feeds/tele')
+                           ,(u'Server/storage & netværk'                 , u'http://www.version2.dk/feeds/server-storage')
+                           ,(u'Sikkerhed'                      , u'http://www.version2.dk/feeds/sikkerhed')
+                           ,(u'Softwareudvikling'                     , u'http://www.version2.dk/feeds/softwareudvikling')
+                          ]

resources/template-functions.json

@@ -1,26 +1,27 @@
 {
     "and": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    while i < len(args):\n        if not args[i]:\n            return ''\n        i += 1\n    return '1'\n", 
-    "contains": "def evaluate(self, formatter, kwargs, mi, locals,\n             val, test, value_if_present, value_if_not):\n    if re.search(test, val):\n        return value_if_present\n    else:\n        return value_if_not\n", 
+    "contains": "def evaluate(self, formatter, kwargs, mi, locals,\n             val, test, value_if_present, value_if_not):\n    if re.search(test, val, flags=re.I):\n        return value_if_present\n    else:\n        return value_if_not\n", 
     "divide": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x / y)\n", 
     "uppercase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return val.upper()\n", 
     "strcat": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    res = ''\n    for i in range(0, len(args)):\n        res += args[i]\n    return res\n", 
-    "in_list": "def evaluate(self, formatter, kwargs, mi, locals, val, sep, pat, fv, nfv):\n    l = [v.strip() for v in val.split(sep) if v.strip()]\n    for v in l:\n        if re.search(pat, v):\n            return fv\n    return nfv\n", 
+    "in_list": "def evaluate(self, formatter, kwargs, mi, locals, val, sep, pat, fv, nfv):\n    l = [v.strip() for v in val.split(sep) if v.strip()]\n    if l:\n        for v in l:\n            if re.search(pat, v, flags=re.I):\n                return fv\n    return nfv\n", 
     "multiply": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x * y)\n", 
     "ifempty": "def evaluate(self, formatter, kwargs, mi, locals, val, value_if_empty):\n    if val:\n        return val\n    else:\n        return value_if_empty\n", 
     "booksize": "def evaluate(self, formatter, kwargs, mi, locals):\n    if mi.book_size is not None:\n        try:\n            return str(mi.book_size)\n        except:\n            pass\n    return ''\n", 
     "select": "def evaluate(self, formatter, kwargs, mi, locals, val, key):\n    if not val:\n        return ''\n    vals = [v.strip() for v in val.split(',')]\n    for v in vals:\n        if v.startswith(key+':'):\n            return v[len(key)+1:]\n    return ''\n", 
     "strcmp": "def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):\n    v = strcmp(x, y)\n    if v < 0:\n        return lt\n    if v == 0:\n        return eq\n    return gt\n", 
     "first_non_empty": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    while i < len(args):\n        if args[i]:\n            return args[i]\n        i += 1\n    return ''\n", 
-    "re": "def evaluate(self, formatter, kwargs, mi, locals, val, pattern, replacement):\n    return re.sub(pattern, replacement, val)\n", 
+    "re": "def evaluate(self, formatter, kwargs, mi, locals, val, pattern, replacement):\n    return re.sub(pattern, replacement, val, flags=re.I)\n", 
     "subtract": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x - y)\n", 
     "list_item": "def evaluate(self, formatter, kwargs, mi, locals, val, index, sep):\n    if not val:\n        return ''\n    index = int(index)\n    val = val.split(sep)\n    try:\n        return val[index]\n    except:\n        return ''\n", 
     "shorten": "def evaluate(self, formatter, kwargs, mi, locals,\n             val, leading, center_string, trailing):\n    l = max(0, int(leading))\n    t = max(0, int(trailing))\n    if len(val) > l + len(center_string) + t:\n        return val[0:l] + center_string + ('' if t == 0 else val[-t:])\n    else:\n        return val\n", 
     "field": "def evaluate(self, formatter, kwargs, mi, locals, name):\n    return formatter.get_value(name, [], kwargs)\n", 
     "add": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x + y)\n", 
-    "lookup": "def evaluate(self, formatter, kwargs, mi, locals, val, *args):\n    if len(args) == 2: # here for backwards compatibility\n        if val:\n            return formatter.vformat('{'+args[0].strip()+'}', [], kwargs)\n        else:\n            return formatter.vformat('{'+args[1].strip()+'}', [], kwargs)\n    if (len(args) % 2) != 1:\n        raise ValueError(_('lookup requires either 2 or an odd number of arguments'))\n    i = 0\n    while i < len(args):\n        if i + 1 >= len(args):\n            return formatter.vformat('{' + args[i].strip() + '}', [], kwargs)\n        if re.search(args[i], val):\n            return formatter.vformat('{'+args[i+1].strip() + '}', [], kwargs)\n        i += 2\n", 
+    "lookup": "def evaluate(self, formatter, kwargs, mi, locals, val, *args):\n    if len(args) == 2: # here for backwards compatibility\n        if val:\n            return formatter.vformat('{'+args[0].strip()+'}', [], kwargs)\n        else:\n            return formatter.vformat('{'+args[1].strip()+'}', [], kwargs)\n    if (len(args) % 2) != 1:\n        raise ValueError(_('lookup requires either 2 or an odd number of arguments'))\n    i = 0\n    while i < len(args):\n        if i + 1 >= len(args):\n            return formatter.vformat('{' + args[i].strip() + '}', [], kwargs)\n        if re.search(args[i], val, flags=re.I):\n            return formatter.vformat('{'+args[i+1].strip() + '}', [], kwargs)\n        i += 2\n", 
     "template": "def evaluate(self, formatter, kwargs, mi, locals, template):\n    template = template.replace('[[', '{').replace(']]', '}')\n    return formatter.__class__().safe_format(template, kwargs, 'TEMPLATE', mi)\n", 
     "print": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    print args\n    return None\n", 
     "merge_lists": "def evaluate(self, formatter, kwargs, mi, locals, list1, list2, separator):\n    l1 = [l.strip() for l in list1.split(separator) if l.strip()]\n    l2 = [l.strip() for l in list2.split(separator) if l.strip()]\n    lcl1 = set([icu_lower(l) for l in l1])\n    res = []\n    for i in l1:\n        res.append(i)\n    for i in l2:\n        if icu_lower(i) not in lcl1:\n            res.append(i)\n    return ', '.join(sorted(res, key=sort_key))\n", 
+    "str_in_list": "def evaluate(self, formatter, kwargs, mi, locals, val, sep, str, fv, nfv):\n    l = [v.strip() for v in val.split(sep) if v.strip()]\n    c = [v.strip() for v in str.split(sep) if v.strip()]\n    if l:\n        for v in l:\n            for t in c:\n                if strcmp(t, v) == 0:\n                    return fv\n    return nfv\n", 
     "titlecase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return titlecase(val)\n", 
     "subitems": "def evaluate(self, formatter, kwargs, mi, locals, val, start_index, end_index):\n    if not val:\n        return ''\n    si = int(start_index)\n    ei = int(end_index)\n    items = [v.strip() for v in val.split(',')]\n    rv = set()\n    for item in items:\n        component = item.split('.')\n        try:\n            if ei == 0:\n                rv.add('.'.join(component[si:]))\n            else:\n                rv.add('.'.join(component[si:ei]))\n        except:\n            pass\n    return ', '.join(sorted(rv, key=sort_key))\n", 
     "sublist": "def evaluate(self, formatter, kwargs, mi, locals, val, start_index, end_index, sep):\n    if not val:\n        return ''\n    si = int(start_index)\n    ei = int(end_index)\n    val = val.split(sep)\n    try:\n        if ei == 0:\n            return sep.join(val[si:])\n        else:\n            return sep.join(val[si:ei])\n    except:\n        return ''\n", 
@@ -32,9 +33,10 @@
     "count": "def evaluate(self, formatter, kwargs, mi, locals, val, sep):\n    return unicode(len(val.split(sep)))\n", 
     "lowercase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return val.lower()\n", 
     "substr": "def evaluate(self, formatter, kwargs, mi, locals, str_, start_, end_):\n    return str_[int(start_): len(str_) if int(end_) == 0 else int(end_)]\n", 
-    "assign": "def evaluate(self, formatter, kwargs, mi, locals, target, value):\n    locals[target] = value\n    return value\n", 
-    "switch": "def evaluate(self, formatter, kwargs, mi, locals, val, *args):\n    if (len(args) % 2) != 1:\n        raise ValueError(_('switch requires an odd number of arguments'))\n    i = 0\n    while i < len(args):\n        if i + 1 >= len(args):\n            return args[i]\n        if re.search(args[i], val):\n            return args[i+1]\n        i += 2\n", 
     "or": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    while i < len(args):\n        if args[i]:\n            return '1'\n        i += 1\n    return ''\n", 
+    "switch": "def evaluate(self, formatter, kwargs, mi, locals, val, *args):\n    if (len(args) % 2) != 1:\n        raise ValueError(_('switch requires an odd number of arguments'))\n    i = 0\n    while i < len(args):\n        if i + 1 >= len(args):\n            return args[i]\n        if re.search(args[i], val, flags=re.I):\n            return args[i+1]\n        i += 2\n", 
+    "ondevice": "def evaluate(self, formatter, kwargs, mi, locals):\n    if mi.ondevice_col:\n        return _('Yes')\n    return ''\n", 
+    "assign": "def evaluate(self, formatter, kwargs, mi, locals, target, value):\n    locals[target] = value\n    return value\n", 
     "raw_field": "def evaluate(self, formatter, kwargs, mi, locals, name):\n    return unicode(getattr(mi, name, None))\n", 
     "cmp": "def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    if x < y:\n        return lt\n    if x == y:\n        return eq\n    return gt\n"
 }

setup/installer/windows/wix-template.xml

@@ -17,6 +17,7 @@
                        IncludeMaximum="yes"
                        OnlyDetect="no"
                        Language="1033"
+                       MigrateFeatures="yes"
                        Property="OLDPRODUCTFOUND"/>
                     <UpgradeVersion Minimum="{version}"
                        IncludeMinimum="no"
@@ -69,7 +70,7 @@
                              WorkingDirectory="APPLICATIONROOTDIRECTORY" />
                         <util:InternetShortcut Id="OnlineDocumentationShortcut"
                             Name="User Manual" Type="url"
-                               Target="http://calibre-ebook.com/user_manual"/>
+                               Target="http://manual.calibre-ebook.com"/>
                         <util:InternetShortcut Id="GetInvolvedS"
                             Name="Get Involved" Type="url"
                             Target="http://calibre-ebook.com/get-involved"/>

setup/upload.py

@@ -16,7 +16,7 @@ from setup import Command, __version__, installer_name, __appname__
 PREFIX = "/var/www/calibre-ebook.com"
 DOWNLOADS = PREFIX+"/htdocs/downloads"
 BETAS = DOWNLOADS +'/betas'
-USER_MANUAL = PREFIX+'/htdocs/user_manual'
+USER_MANUAL = '/var/www/localhost/htdocs/'
 HTML2LRF = "calibre/ebooks/lrf/html/demo"
 TXT2LRF  = "src/calibre/ebooks/lrf/txt/demo"
 MOBILEREAD = 'ftp://dev.mobileread.com/calibre/'
@@ -365,7 +365,7 @@ class UploadUserManual(Command): # {{{
             self.build_plugin_example(x)
 
         check_call(' '.join(['scp', '-r', 'src/calibre/manual/.build/html/*',
-                    'divok:%s'%USER_MANUAL]), shell=True)
+                    'bugs:%s'%USER_MANUAL]), shell=True)
 # }}}
 
 class UploadDemo(Command): # {{{

src/calibre/customize/__init__.py

@@ -623,6 +623,8 @@ class StoreBase(Plugin): # {{{
     headquarters = ''
     # All formats the store distributes ebooks in.
     formats = []
+    # Is this store on an affiliate program?
+    affiliate = False
 
     def load_actual_plugin(self, gui):
         '''

src/calibre/customize/builtins.py

@@ -1108,9 +1108,9 @@ class StoreAmazonKindleStore(StoreBase):
     description = u'Kindle books from Amazon.'
     actual_plugin = 'calibre.gui2.store.amazon_plugin:AmazonKindleStore'
 
-    drm_free_only = False
     headquarters = 'US'
     formats = ['KINDLE']
+    affiliate = True
 
 class StoreAmazonDEKindleStore(StoreBase):
     name = 'Amazon DE Kindle'
@@ -1118,9 +1118,9 @@ class StoreAmazonDEKindleStore(StoreBase):
     description = u'Kindle Bücher von Amazon.'
     actual_plugin = 'calibre.gui2.store.amazon_de_plugin:AmazonDEKindleStore'
 
-    drm_free_only = False
     headquarters = 'DE'
     formats = ['KINDLE']
+    affiliate = True
 
 class StoreAmazonUKKindleStore(StoreBase):
     name = 'Amazon UK Kindle'
@@ -1128,9 +1128,9 @@ class StoreAmazonUKKindleStore(StoreBase):
     description = u'Kindle books from Amazon\'s UK web site. Also, includes French language ebooks.'
     actual_plugin = 'calibre.gui2.store.amazon_uk_plugin:AmazonUKKindleStore'
 
-    drm_free_only = False
     headquarters = 'UK'
     formats = ['KINDLE']
+    affiliate = True
 
 class StoreArchiveOrgStore(StoreBase):
     name = 'Archive.org'
@@ -1155,9 +1155,9 @@ class StoreBNStore(StoreBase):
     description = u'The world\'s largest book seller. As the ultimate destination for book lovers, Barnes & Noble.com offers an incredible array of content.'
     actual_plugin = 'calibre.gui2.store.bn_plugin:BNStore'
 
-    drm_free_only = False
     headquarters = 'US'
     formats = ['NOOK']
+    affiliate = True
 
 class StoreBeamEBooksDEStore(StoreBase):
     name = 'Beam EBooks DE'
@@ -1168,6 +1168,7 @@ class StoreBeamEBooksDEStore(StoreBase):
     drm_free_only = True
     headquarters = 'DE'
     formats = ['EPUB', 'MOBI', 'PDF']
+    affiliate = True
 
 class StoreBeWriteStore(StoreBase):
     name = 'BeWrite Books'
@@ -1183,18 +1184,18 @@ class StoreDieselEbooksStore(StoreBase):
     description = u'Instant access to over 2.4 million titles from hundreds of publishers including Harlequin, HarperCollins, John Wiley & Sons, McGraw-Hill, Simon & Schuster and Random House.'
     actual_plugin = 'calibre.gui2.store.diesel_ebooks_plugin:DieselEbooksStore'
 
-    drm_free_only = False
     headquarters = 'US'
     formats = ['EPUB', 'PDF']
+    affiliate = True
 
 class StoreEbookscomStore(StoreBase):
     name = 'eBooks.com'
     description = u'Sells books in multiple electronic formats in all categories. Technical infrastructure is cutting edge, robust and scalable, with servers in the US and Europe.'
     actual_plugin = 'calibre.gui2.store.ebooks_com_plugin:EbookscomStore'
 
-    drm_free_only = False
     headquarters = 'US'
     formats = ['EPUB', 'LIT', 'MOBI', 'PDF']
+    affiliate = True
 
 class StoreEPubBuyDEStore(StoreBase):
     name = 'EPUBBuy DE'
@@ -1205,22 +1206,41 @@ class StoreEPubBuyDEStore(StoreBase):
     drm_free_only = True
     headquarters = 'DE'
     formats = ['EPUB']
+    affiliate = True
+
+class StoreEBookShoppeUKStore(StoreBase):
+    name = 'ebookShoppe UK'
+    author = u'Charles Haley'
+    description = u'We made this website in an attempt to offer the widest range of UK eBooks possible across and as many formats as we could manage.'
+    actual_plugin = 'calibre.gui2.store.ebookshoppe_uk_plugin:EBookShoppeUKStore'
+
+    headquarters = 'UK'
+    formats = ['EPUB', 'PDF']
+    affiliate = True
 
 class StoreEHarlequinStore(StoreBase):
     name = 'eHarlequin'
     description = u'A global leader in series romance and one of the world\'s leading publishers of books for women. Offers women a broad range of reading from romance to bestseller fiction, from young adult novels to erotic literature, from nonfiction to fantasy, from African-American novels to inspirational romance, and more.'
     actual_plugin = 'calibre.gui2.store.eharlequin_plugin:EHarlequinStore'
 
-    drm_free_only = False
     headquarters = 'CA'
     formats = ['EPUB', 'PDF']
+    affiliate = True
+
+class StoreEpubBudStore(StoreBase):
+    name = 'ePub Bud'
+    description = 'Well, it\'s pretty much just "YouTube for Children\'s eBooks. A not-for-profit organization devoted to brining self published childrens books to the world.'
+    actual_plugin = 'calibre.gui2.store.epubbud_plugin:EpubBudStore'
+    
+    drm_free_only = True
+    headquarters = 'US'
+    formats = ['EPUB']
 
 class StoreFeedbooksStore(StoreBase):
     name = 'Feedbooks'
     description = u'Feedbooks is a cloud publishing and distribution service, connected to a large ecosystem of reading systems and social networks. Provides a variety of genres from independent and classic books.'
     actual_plugin = 'calibre.gui2.store.feedbooks_plugin:FeedbooksStore'
 
-    drm_free_only = False
     headquarters = 'FR'
     formats = ['EPUB', 'MOBI', 'PDF']
 
@@ -1230,9 +1250,9 @@ class StoreFoylesUKStore(StoreBase):
     description = u'Foyles of London\'s ebook store. Provides extensive range covering all subjects.'
     actual_plugin = 'calibre.gui2.store.foyles_uk_plugin:FoylesUKStore'
 
-    drm_free_only = False
     headquarters = 'UK'
     formats = ['EPUB', 'PDF']
+    affiliate = True
 
 class StoreGandalfStore(StoreBase):
     name = 'Gandalf'
@@ -1240,7 +1260,6 @@ class StoreGandalfStore(StoreBase):
     description = u'Księgarnia internetowa Gandalf.'
     actual_plugin = 'calibre.gui2.store.gandalf_plugin:GandalfStore'
 
-    drm_free_only = False
     headquarters = 'PL'
     formats = ['EPUB', 'PDF']
 
@@ -1249,7 +1268,6 @@ class StoreGoogleBooksStore(StoreBase):
     description = u'Google Books'
     actual_plugin = 'calibre.gui2.store.google_books_plugin:GoogleBooksStore'
 
-    drm_free_only = False
     headquarters = 'US'
     formats = ['EPUB', 'PDF', 'TXT']
 
@@ -1267,9 +1285,9 @@ class StoreKoboStore(StoreBase):
     description = u'With over 2.3 million eBooks to browse we have engaged readers in over 200 countries in Kobo eReading. Our eBook listings include New York Times Bestsellers, award winners, classics and more!'
     actual_plugin = 'calibre.gui2.store.kobo_plugin:KoboStore'
 
-    drm_free_only = False
     headquarters = 'CA'
     formats = ['EPUB']
+    affiliate = True
 
 class StoreLegimiStore(StoreBase):
     name = 'Legimi'
@@ -1277,7 +1295,6 @@ class StoreLegimiStore(StoreBase):
     description = u'Tanie oraz darmowe ebooki, egazety i blogi w formacie EPUB, wprost na Twój e-czytnik, iPhone, iPad, Android i komputer'
     actual_plugin = 'calibre.gui2.store.legimi_plugin:LegimiStore'
 
-    drm_free_only = False
     headquarters = 'PL'
     formats = ['EPUB']
 
@@ -1305,9 +1322,9 @@ class StoreNextoStore(StoreBase):
     description = u'Największy w Polsce sklep internetowy z audiobookami mp3, ebookami pdf oraz prasą do pobrania on-line.'
     actual_plugin = 'calibre.gui2.store.nexto_plugin:NextoStore'
 
-    drm_free_only = False
     headquarters = 'PL'
     formats = ['EPUB', 'PDF']
+    affiliate = True
 
 class StoreOpenLibraryStore(StoreBase):
     name = 'Open Library'
@@ -1344,6 +1361,7 @@ class StoreSmashwordsStore(StoreBase):
     drm_free_only = True
     headquarters = 'US'
     formats = ['EPUB', 'HTML', 'LRF', 'MOBI', 'PDB', 'RTF', 'TXT']
+    affiliate = True
 
 class StoreVirtualoStore(StoreBase):
     name = 'Virtualo'
@@ -1351,7 +1369,6 @@ class StoreVirtualoStore(StoreBase):
     description = u'Księgarnia internetowa, która oferuje bezpieczny i szeroki dostęp do książek w formie cyfrowej.'
     actual_plugin = 'calibre.gui2.store.virtualo_plugin:VirtualoStore'
 
-    drm_free_only = False
     headquarters = 'PL'
     formats = ['EPUB', 'PDF']
 
@@ -1361,7 +1378,6 @@ class StoreWaterstonesUKStore(StoreBase):
     description = u'Waterstone\'s mission is to be the leading Bookseller on the High Street and online providing customers the widest choice, great value and expert advice from a team passionate about Bookselling.'
     actual_plugin = 'calibre.gui2.store.waterstones_uk_plugin:WaterstonesUKStore'
 
-    drm_free_only = False
     headquarters = 'UK'
     formats = ['EPUB', 'PDF']
 
@@ -1374,6 +1390,15 @@ class StoreWeightlessBooksStore(StoreBase):
     headquarters = 'US'
     formats = ['EPUB', 'HTML', 'LIT', 'MOBI', 'PDF']
 
+class StoreWHSmithUKStore(StoreBase):
+    name = 'WH Smith UK'
+    author = 'Charles Haley'
+    description = u"Shop for savings on Books, discounted Magazine subscriptions and great prices on Stationery, Toys & Games"
+    actual_plugin = 'calibre.gui2.store.whsmith_uk_plugin:WHSmithUKStore'
+
+    headquarters = 'UK'
+    formats = ['EPUB', 'PDF']
+
 class StoreWizardsTowerBooksStore(StoreBase):
     name = 'Wizards Tower Books'
     description = u'A science fiction and fantasy publisher. Concentrates mainly on making out-of-print works available once more as e-books, and helping other small presses exploit the e-book market. Also publishes a small number of limited-print-run anthologies with a view to encouraging diversity in the science fiction and fantasy field.'
@@ -1389,7 +1414,6 @@ class StoreWoblinkStore(StoreBase):
     description = u'Czytanie zdarza się wszędzie!'
     actual_plugin = 'calibre.gui2.store.woblink_plugin:WoblinkStore'
 
-    drm_free_only = False
     headquarters = 'PL'
     formats = ['EPUB']
 
@@ -1404,8 +1428,10 @@ plugins += [
     StoreBeWriteStore,
     StoreDieselEbooksStore,
     StoreEbookscomStore,
+    StoreEBookShoppeUKStore,
     StoreEPubBuyDEStore,
     StoreEHarlequinStore,
+    StoreEpubBudStore,
     StoreFeedbooksStore,
     StoreFoylesUKStore,
     StoreGandalfStore,
@@ -1421,8 +1447,9 @@ plugins += [
     StorePragmaticBookshelfStore,
     StoreSmashwordsStore,
     StoreVirtualoStore,
-    #StoreWaterstonesUKStore,
+    StoreWaterstonesUKStore,
     StoreWeightlessBooksStore,
+    StoreWHSmithUKStore,
     StoreWizardsTowerBooksStore,
     StoreWoblinkStore
 ]

src/calibre/devices/android/driver.py

@@ -52,6 +52,7 @@ class ANDROID(USBMS):
             0x04e8 : { 0x681d : [0x0222, 0x0223, 0x0224, 0x0400],
                        0x681c : [0x0222, 0x0224, 0x0400],
                        0x6640 : [0x0100],
+                       0x685e : [0x0400],
                        0x6877 : [0x0400],
                      },
 
@@ -109,10 +110,12 @@ class ANDROID(USBMS):
             'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H',
             'IDEOS_TABLET', 'MYTOUCH_4G', 'UMS_COMPOSITE', 'SCH-I800_CARD',
             '7', 'A956', 'A955', 'A43', 'ANDROID_PLATFORM', 'TEGRA_2',
-            'MB860', 'MULTI-CARD', 'MID7015A', 'INCREDIBLE', 'A7EB', 'STREAK']
+            'MB860', 'MULTI-CARD', 'MID7015A', 'INCREDIBLE', 'A7EB', 'STREAK',
+            'MB525']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
-            'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD']
+            'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD',
+            '__UMS_COMPOSITE']
 
     OSX_MAIN_MEM = 'Android Device Main Memory'
 

src/calibre/devices/apple/driver.py

@@ -2743,7 +2743,6 @@ class ITUNES(DriverBase):
         # Update metadata from plugboard
         # If self.plugboard is None (no transforms), original metadata is returned intact
         metadata_x = self._xform_metadata_via_plugboard(metadata, this_book.format)
-        self.log("metadata.title_sort: %s  metadata_x.title_sort: %s" % (metadata.title_sort, metadata_x.title_sort))
         if isosx:
             if lb_added:
                 lb_added.name.set(metadata_x.title)
@@ -3024,6 +3023,8 @@ class ITUNES_ASYNC(ITUNES):
                 pythoncom.CoInitialize()
                 self._launch_iTunes()
             except:
+                import traceback
+                traceback.print_exc()
                 raise UserFeedback('unable to launch iTunes', details=None, level=UserFeedback.WARN)
             finally:
                 pythoncom.CoUninitialize()

src/calibre/ebooks/conversion/cli.py

@@ -40,7 +40,7 @@ To get help on them specify the input and output file and then use the -h \
 option.
 
 For full documentation of the conversion system see
-''') + 'http://calibre-ebook.com/user_manual/conversion.html'
+''') + 'http://manual.calibre-ebook.com/conversion.html'
 
 HEURISTIC_OPTIONS = ['markup_chapter_headings',
                       'italicize_common_cases', 'fix_indents',

src/calibre/ebooks/epub/fix/container.py

@@ -101,7 +101,7 @@ class Container(object):
             return None
         return existing[0]
 
-    def add_name_to_manifest(self, name):
+    def add_name_to_manifest(self, name, mt=None):
         item = self.manifest_item_for_name(name)
         if item is not None:
             return
@@ -109,11 +109,27 @@ class Container(object):
         item = manifest.makeelement('{%s}item'%OPF_NS, nsmap={'opf':OPF_NS},
                 href=self.name_to_href(name, posixpath.dirname(self.opf_name)),
                 id=self.generate_manifest_id())
-        mt = guess_type(posixpath.basename(name))[0]
+        if not mt:
+            mt = guess_type(posixpath.basename(name))[0]
         if not mt:
             mt = 'application/octest-stream'
         item.set('media-type', mt)
         manifest.append(item)
+        self.fix_tail(item)
+
+    def fix_tail(self, item):
+        '''
+        Designed only to work with self closing elements after item has
+        just been inserted/appended
+        '''
+        parent = item.getparent()
+        idx = parent.index(item)
+        if idx == 0:
+            item.tail = parent.text
+        else:
+            item.tail = parent[idx-1].tail
+            if idx == len(parent)-1:
+                parent[idx-1].tail = parent.text
 
     def generate_manifest_id(self):
         items = self.opf.xpath('//opf:manifest/opf:item[@id]',

src/calibre/ebooks/metadata/book/base.py

@@ -41,27 +41,34 @@ field_metadata = FieldMetadata()
 
 class SafeFormat(TemplateFormatter):
 
-    def get_value(self, key, args, kwargs):
+    def get_value(self, orig_key, args, kwargs):
+        if not orig_key:
+            return ''
+        orig_key = orig_key.lower()
+        key = orig_key
+        if key != 'title_sort' and key not in TOP_LEVEL_IDENTIFIERS:
+            key = field_metadata.search_term_to_field_key(key)
+            if key is None or (self.book and
+                                key not in self.book.all_field_keys()):
+                if hasattr(self.book, orig_key):
+                    key = orig_key
+                else:
+                    raise ValueError(_('Value: unknown field ') + orig_key)
         try:
-            key = key.lower()
-            if key != 'title_sort' and key not in TOP_LEVEL_IDENTIFIERS:
-                key = field_metadata.search_term_to_field_key(key)
             b = self.book.get_user_metadata(key, False)
-            if b and b['datatype'] == 'int' and self.book.get(key, 0) == 0:
-                v = ''
-            elif b and b['datatype'] == 'float' and self.book.get(key, 0.0) == 0.0:
-                v = ''
-            else:
-                v = self.book.format_field(key, series_with_index=False)[1]
-            if v is None:
-                return ''
-            if v == '':
-                return ''
-            return v
         except:
-            if DEBUG:
+            b = None
-                traceback.print_exc()
+        if b and b['datatype'] == 'int' and self.book.get(key, 0) == 0:
-            return key
+            v = ''
+        elif b and b['datatype'] == 'float' and self.book.get(key, 0.0) == 0.0:
+            v = ''
+        else:
+            v = self.book.format_field(key, series_with_index=False)[1]
+        if v is None:
+            return ''
+        if v == '':
+            return ''
+        return v
 
 composite_formatter = SafeFormat()
 
@@ -75,7 +82,7 @@ class Metadata(object):
     Metadata from custom columns should be accessed via the get() method,
     passing in the lookup name for the column, for example: "#mytags".
 
-    Use the :meth:`is_null` method to test if a filed is null.
+    Use the :meth:`is_null` method to test if a field is null.
 
     This object also has functions to format fields into strings.
 
@@ -106,7 +113,7 @@ class Metadata(object):
 
     def is_null(self, field):
         '''
-        Return True if the value of filed is null in this object.
+        Return True if the value of field is null in this object.
         'null' means it is unknown or evaluates to False. So a title of
         _('Unknown') is null or a language of 'und' is null.
 
@@ -628,6 +635,12 @@ class Metadata(object):
                 res = _('Yes') if res else _('No')
             elif datatype == 'rating':
                 res = res/2.0
+            elif datatype in ['int', 'float']:
+                try:
+                    fmt = cmeta['display'].get('number_format', None)
+                    res = fmt.format(res)
+                except:
+                    pass
             return (name, unicode(res), orig_res, cmeta)
 
         # convert top-level ids into their value

src/calibre/ebooks/metadata/sources/douban.py

@@ -46,6 +46,8 @@ cover_url      = XPath("descendant::atom:link[@rel='image']/attribute::href")
 
 def get_details(browser, url, timeout): # {{{
     try:
+        if Douban.DOUBAN_API_KEY and Douban.DOUBAN_API_KEY != '':
+            url = url + "?apikey=" + Douban.DOUBAN_API_KEY
         raw = browser.open_novisit(url, timeout=timeout).read()
     except Exception as e:
         gc = getattr(e, 'getcode', lambda : -1)

src/calibre/ebooks/metadata/sources/identify.py

@@ -408,6 +408,8 @@ def identify(log, abort, # {{{
             for f in plugin.prefs['ignore_fields']:
                 if ':' not in f:
                     setattr(result, f, getattr(dummy, f))
+                if f == 'series':
+                    result.series_index = dummy.series_index
             result.relevance_in_source = i
             result.has_cached_cover_url = (plugin.cached_cover_url_is_reliable
                     and plugin.get_cached_cover_url(result.identifiers) is not

src/calibre/ebooks/oeb/transforms/split.py

@@ -120,7 +120,19 @@ class Split(object):
         for i, x in enumerate(page_breaks):
             x.set('id', x.get('id', 'calibre_pb_%d'%i))
             id = x.get('id')
-            page_breaks_.append((XPath('//*[@id=%r]'%id),
+            try:
+                xp = XPath('//*[@id="%s"]'%id)
+            except:
+                try:
+                    xp = XPath("//*[@id='%s']"%id)
+                except:
+                    # The id has both a quote and an apostrophe or some other
+                    # Just replace it since I doubt its going to work anywhere else
+                    # either
+                    id = 'calibre_pb_%d'%i
+                    x.set('id', id)
+                    xp = XPath('//*[@id=%r]'%id)
+            page_breaks_.append((xp,
                 x.get('pb_before', False)))
             page_break_ids.append(id)
 

src/calibre/gui2/__init__.py

@@ -48,8 +48,9 @@ else:
     gprefs.defaults['action-layout-menubar-device'] = ()
     gprefs.defaults['action-layout-toolbar'] = (
         'Add Books', 'Edit Metadata', None, 'Convert Books', 'View', None,
-        'Choose Library', 'Donate', None, 'Fetch News', 'Store', 'Save To Disk',
+        'Store', 'Donate', 'Fetch News', 'Help', None,
-        'Connect Share', None, 'Remove Books', None, 'Help', 'Preferences',
+        'Remove Books', 'Choose Library', 'Save To Disk',
+        'Connect Share', 'Preferences',
         )
     gprefs.defaults['action-layout-toolbar-device'] = (
         'Add Books', 'Edit Metadata', None, 'Convert Books', 'View',
@@ -75,7 +76,7 @@ gprefs.defaults['action-layout-context-menu-device'] = (
 gprefs.defaults['show_splash_screen'] = True
 gprefs.defaults['toolbar_icon_size'] = 'medium'
 gprefs.defaults['automerge'] = 'ignore'
-gprefs.defaults['toolbar_text'] = 'auto'
+gprefs.defaults['toolbar_text'] = 'always'
 gprefs.defaults['font'] = None
 gprefs.defaults['tags_browser_partition_method'] = 'first letter'
 gprefs.defaults['tags_browser_collapse_at'] = 100

src/calibre/gui2/actions/convert.py

@@ -171,10 +171,9 @@ class ConvertAction(InterfaceAction):
                 raise Exception(_('Empty output file, '
                     'probably the conversion process crashed'))
 
-            data = open(temp_files[-1].name, 'rb')
+            with open(temp_files[-1].name, 'rb') as data:
-            self.gui.library_view.model().db.add_format(book_id, \
+                self.gui.library_view.model().db.add_format(book_id, \
                     fmt, data, index_is_id=True)
-            data.close()
             self.gui.status_bar.show_message(job.description + \
                     (' completed'), 2000)
         finally:

src/calibre/gui2/actions/delete.py

@@ -152,7 +152,8 @@ class DeleteAction(InterfaceAction):
         if not ids:
             return
         fmts = self._get_selected_formats(
-            '<p>'+_('Choose formats <b>not</b> to be deleted'), ids)
+            '<p>'+_('Choose formats <b>not</b> to be deleted.<p>Note that '
+                'this will never remove all formats from a book.'), ids)
         if fmts is None:
             return
         for id in ids:
@@ -161,9 +162,12 @@ class DeleteAction(InterfaceAction):
                 continue
             bfmts = set([x.lower() for x in bfmts.split(',')])
             rfmts = bfmts - set(fmts)
-            for fmt in rfmts:
+            if bfmts - rfmts:
-                self.gui.library_view.model().db.remove_format(id, fmt,
+                # Do not delete if it will leave the book with no
-                        index_is_id=True, notify=False)
+                # formats
+                for fmt in rfmts:
+                    self.gui.library_view.model().db.remove_format(id, fmt,
+                            index_is_id=True, notify=False)
         self.gui.library_view.model().refresh_ids(ids)
         self.gui.library_view.model().current_changed(self.gui.library_view.currentIndex(),
                 self.gui.library_view.currentIndex())

src/calibre/gui2/actions/edit_metadata.py

@@ -482,6 +482,8 @@ class EditMetadataAction(InterfaceAction):
             if mi.identifiers:
                 idents.update(mi.identifiers)
             mi.identifiers = idents
+            if mi.is_null('series'):
+                mi.series_index = None
             db.set_metadata(i, mi, commit=False, set_title=set_title,
                     set_authors=set_authors, notify=False)
             self.applied_ids.append(i)

src/calibre/gui2/actions/help.py

@@ -19,7 +19,7 @@ class HelpAction(InterfaceAction):
         self.qaction.triggered.connect(self.show_help)
 
     def show_help(self, *args):
-        open_url(QUrl('http://calibre-ebook.com/user_manual'))
+        open_url(QUrl('http://manual.calibre-ebook.com'))
 
 
 

src/calibre/gui2/actions/store.py

@@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
 
 from functools import partial
 
-from PyQt4.Qt import QMenu
+from PyQt4.Qt import QMenu, QIcon, QSize
 
 from calibre.gui2 import error_dialog
 from calibre.gui2.actions import InterfaceAction
@@ -32,8 +32,13 @@ class StoreAction(InterfaceAction):
         self.store_menu.addAction(_('Search for this book'), self.search_author_title)
         self.store_menu.addSeparator()
         self.store_list_menu = self.store_menu.addMenu(_('Stores'))
+        icon = QIcon()
+        icon.addFile(I('donate.png'), QSize(16, 16))
         for n, p in sorted(self.gui.istores.items(), key=lambda x: x[0].lower()):
-            self.store_list_menu.addAction(n, partial(self.open_store, p))
+            if p.base_plugin.affiliate:
+                self.store_list_menu.addAction(icon, n, partial(self.open_store, p))
+            else:
+                self.store_list_menu.addAction(n, partial(self.open_store, p))
         self.store_menu.addSeparator()
         self.store_menu.addAction(_('Choose stores'), self.choose)
         self.qaction.setMenu(self.store_menu)

src/calibre/gui2/convert/font_key.ui

@@ -33,7 +33,7 @@
      <property name="text">
       <string>&lt;p&gt;This wizard will help you choose an appropriate font size key for your needs. Just enter the base font size of the input document and then enter an input font size. The wizard will display what font size it will be mapped to, by the font rescaling algorithm. You can adjust the algorithm by adjusting the output base font size and font key below. When you find values suitable for you, click OK.&lt;/p&gt;
 &lt;p&gt;By default, if the output base font size is zero and/or no font size key is specified, calibre will use the values from the current Output Profile. &lt;/p&gt;
-&lt;p&gt;See the &lt;a href=&quot;http://calibre-ebook.com/user_manual/conversion.html#font-size-rescaling&quot;&gt;User Manual&lt;/a&gt; for a discussion of how font size rescaling works.&lt;/p&gt;</string>
+&lt;p&gt;See the &lt;a href=&quot;http://manual.calibre-ebook.com/conversion.html#font-size-rescaling&quot;&gt;User Manual&lt;/a&gt; for a discussion of how font size rescaling works.&lt;/p&gt;</string>
      </property>
      <property name="wordWrap">
       <bool>true</bool>

src/calibre/gui2/convert/heuristics.ui

@@ -17,7 +17,7 @@
    <item>
     <widget class="QLabel" name="label">
      <property name="text">
-      <string>&lt;b&gt;Heuristic processing&lt;/b&gt; means that calibre will scan your book for common patterns and fix them. As the name implies, this involves guesswork, which means that it could end up worsening the result of a conversion, if calibre guesses wrong. Therefore, it is disabled by default. Often, if a conversion does not turn out as you expect, turning on heuristics can improve matters. Read more about the various heuristic processing options in the &lt;a href=&quot;http://calibre-ebook.com/user_manual/conversion.html#heuristic-processing&quot;&gt;User Manual&lt;/a&gt;.</string>
+      <string>&lt;b&gt;Heuristic processing&lt;/b&gt; means that calibre will scan your book for common patterns and fix them. As the name implies, this involves guesswork, which means that it could end up worsening the result of a conversion, if calibre guesses wrong. Therefore, it is disabled by default. Often, if a conversion does not turn out as you expect, turning on heuristics can improve matters. Read more about the various heuristic processing options in the &lt;a href=&quot;http://manual.calibre-ebook.com/conversion.html#heuristic-processing&quot;&gt;User Manual&lt;/a&gt;.</string>
      </property>
      <property name="wordWrap">
       <bool>true</bool>

src/calibre/gui2/convert/search_and_replace.ui

@@ -188,7 +188,7 @@
    <item row="0" column="0">
     <widget class="QLabel" name="label">
      <property name="text">
-      <string>&lt;p&gt;Search and replace uses &lt;i&gt;regular expressions&lt;/i&gt;. See the &lt;a href=&quot;http://calibre-ebook.com/user_manual/regexp.html&quot;&gt;regular expressions tutorial&lt;/a&gt; to get started with regular expressions. Also clicking the wizard buttons below will allow you to test your regular expression against the current input document.</string>
+      <string>&lt;p&gt;Search and replace uses &lt;i&gt;regular expressions&lt;/i&gt;. See the &lt;a href=&quot;http://manual.calibre-ebook.com/regexp.html&quot;&gt;regular expressions tutorial&lt;/a&gt; to get started with regular expressions. Also clicking the wizard buttons below will allow you to test your regular expression against the current input document.</string>
      </property>
      <property name="wordWrap">
       <bool>true</bool>

src/calibre/gui2/convert/xpath_wizard.ui

@@ -127,7 +127,7 @@
    <item>
     <widget class="QLabel" name="label_5">
      <property name="text">
-      <string>&lt;p&gt;For example, to match all h2 tags that have class=&quot;chapter&quot;, set tag to &lt;i&gt;h2&lt;/i&gt;, attribute to &lt;i&gt;class&lt;/i&gt; and value to &lt;i&gt;chapter&lt;/i&gt;.&lt;/p&gt;&lt;p&gt;Leaving attribute blank will match any attribute and leaving value blank will match any value. Setting tag to * will match any tag.&lt;/p&gt;&lt;p&gt;To learn more advanced usage of XPath see the &lt;a href=&quot;http://calibre-ebook.com/user_manual/xpath.html&quot;&gt;XPath Tutorial&lt;/a&gt;.</string>
+      <string>&lt;p&gt;For example, to match all h2 tags that have class=&quot;chapter&quot;, set tag to &lt;i&gt;h2&lt;/i&gt;, attribute to &lt;i&gt;class&lt;/i&gt; and value to &lt;i&gt;chapter&lt;/i&gt;.&lt;/p&gt;&lt;p&gt;Leaving attribute blank will match any attribute and leaving value blank will match any value. Setting tag to * will match any tag.&lt;/p&gt;&lt;p&gt;To learn more advanced usage of XPath see the &lt;a href=&quot;http://manual.calibre-ebook.com/xpath.html&quot;&gt;XPath Tutorial&lt;/a&gt;.</string>
      </property>
      <property name="wordWrap">
       <bool>true</bool>

src/calibre/gui2/device.py

@@ -1294,7 +1294,8 @@ class DeviceMixin(object): # {{{
             self.book_db_uuid_path_map = None
             return
 
-        if not hasattr(self, 'db_book_uuid_cache'):
+        if not self.device_manager.is_device_connected or \
+                        not hasattr(self, 'db_book_uuid_cache'):
             return loc
 
         if self.book_db_id_cache is None:

src/calibre/gui2/dialogs/search.ui

@@ -177,7 +177,7 @@
              </size>
             </property>
             <property name="text">
-             <string>See the &lt;a href=&quot;http://calibre-ebook.com/user_manual/gui.html#the-search-interface&quot;&gt;User Manual&lt;/a&gt; for more help</string>
+             <string>See the &lt;a href=&quot;http://manual.calibre-ebook.com/gui.html#the-search-interface&quot;&gt;User Manual&lt;/a&gt; for more help</string>
             </property>
             <property name="openExternalLinks">
              <bool>true</bool>

src/calibre/gui2/dialogs/select_formats.py

@@ -44,7 +44,7 @@ class SelectFormats(QDialog):
         self.setLayout(self._l)
         self.setWindowTitle(_('Choose formats'))
         self._m = QLabel(msg)
-        self._m.setWordWrap = True
+        self._m.setWordWrap(True)
         self._l.addWidget(self._m)
         self.formats = Formats(fmt_list)
         self.fview = QListView(self)

src/calibre/gui2/dialogs/template_dialog.py

@@ -5,12 +5,15 @@ __license__   = 'GPL v3'
 
 import json
 
-from PyQt4.Qt import (Qt, QDialog, QDialogButtonBox, QSyntaxHighlighter,
+from PyQt4.Qt import (Qt, QDialog, QDialogButtonBox, QSyntaxHighlighter, QFont,
-                      QRegExp, QApplication,
+                      QRegExp, QApplication, QTextCharFormat, QColor, QCursor)
-                      QTextCharFormat, QFont, QColor, QCursor)
 
+from calibre.gui2 import error_dialog
 from calibre.gui2.dialogs.template_dialog_ui import Ui_TemplateDialog
 from calibre.utils.formatter_functions import formatter_functions
+from calibre.ebooks.metadata.book.base import composite_formatter, Metadata
+from calibre.library.coloring import (displayable_columns)
+
 
 class ParenPosition:
 
@@ -194,10 +197,25 @@ class TemplateHighlighter(QSyntaxHighlighter):
 
 class TemplateDialog(QDialog, Ui_TemplateDialog):
 
-    def __init__(self, parent, text):
+    def __init__(self, parent, text, mi=None, fm=None, color_field=None):
         QDialog.__init__(self, parent)
         Ui_TemplateDialog.__init__(self)
         self.setupUi(self)
+
+        self.coloring = color_field is not None
+        if self.coloring:
+            cols = sorted([k for k in displayable_columns(fm)])
+            self.colored_field.addItems(cols)
+            self.colored_field.setCurrentIndex(self.colored_field.findText(color_field))
+        else:
+            self.colored_field.setVisible(False)
+            self.colored_field_label.setVisible(False)
+
+        if mi:
+            self.mi = mi
+        else:
+            self.mi = Metadata(None, None)
+
         # Remove help icon on title bar
         icon = self.windowIcon()
         self.setWindowFlags(self.windowFlags()&(~Qt.WindowContextHelpButtonHint))
@@ -233,21 +251,27 @@ class TemplateDialog(QDialog, Ui_TemplateDialog):
         self.function.addItems(func_names)
         self.function.setCurrentIndex(0)
         self.function.currentIndexChanged[str].connect(self.function_changed)
+        self.textbox_changed()
+        self.rule = (None, '')
 
     def textbox_changed(self):
         cur_text = unicode(self.textbox.toPlainText())
         if self.last_text != cur_text:
             self.last_text = cur_text
             self.highlighter.regenerate_paren_positions()
+            self.text_cursor_changed()
+            self.template_value.setText(
+                composite_formatter.safe_format(cur_text, self.mi,
+                                                _('EXCEPTION: '), self.mi))
 
     def text_cursor_changed(self):
         cursor = self.textbox.textCursor()
-        block_number = cursor.blockNumber()
-        pos_in_block = cursor.positionInBlock()
         position = cursor.position()
         t = unicode(self.textbox.toPlainText())
-        if position < len(t):
+        if position > 0 and position <= len(t):
-            self.highlighter.check_cursor_pos(t[position], block_number,
+            block_number = cursor.blockNumber()
+            pos_in_block = cursor.positionInBlock() - 1
+            self.highlighter.check_cursor_pos(t[position-1], block_number,
                                               pos_in_block)
 
     def function_changed(self, toWhat):
@@ -262,3 +286,17 @@ class TemplateDialog(QDialog, Ui_TemplateDialog):
             else:
                 self.source_code.setPlainText(self.funcs[name].program_text)
 
+    def accept(self):
+        txt = unicode(self.textbox.toPlainText()).rstrip()
+        if self.coloring:
+            if self.colored_field.currentIndex() == -1:
+                error_dialog(self, _('No column chosen'),
+                    _('You must specify a column to be colored'), show=True)
+                return
+            if not txt:
+                error_dialog(self, _('No template provided'),
+                    _('The template box cannot be empty'), show=True)
+                return
+
+        self.rule = (unicode(self.colored_field.currentText()), txt)
+        QDialog.accept(self)

src/calibre/gui2/dialogs/template_dialog.ui

@@ -20,22 +20,60 @@
    <string>Edit Comments</string>
   </property>
   <layout class="QVBoxLayout" name="verticalLayout">
+   <item>
+    <layout class="QHBoxLayout">
+     <item>
+      <widget class="QLabel" name="colored_field_label">
+       <property name="text">
+        <string>Set the color of the column:</string>
+       </property>
+       <property name="buddy">
+        <cstring>colored_field</cstring>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QComboBox" name="colored_field">
+      </widget>
+     </item>
+    </layout>
+   </item>
    <item>
     <widget class="QPlainTextEdit" name="textbox"/>
    </item>
-   <item>
-    <widget class="QDialogButtonBox" name="buttonBox">
-     <property name="orientation">
-      <enum>Qt::Horizontal</enum>
-     </property>
-     <property name="standardButtons">
-      <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
-     </property>
-    </widget>
-   </item>
    <item>
     <layout class="QGridLayout" name="gridLayout">
-     <item row="0" column="0">
+     <item row="5" column="0">
+      <widget class="QLabel">
+       <property name="text">
+        <string>Template value:</string>
+       </property>
+       <property name="buddy">
+        <cstring>template_value</cstring>
+       </property>
+       <property name="toolTip">
+        <string>The value the of the template using the current book in the library view</string>
+       </property>
+      </widget>
+     </item>
+     <item row="5" column="1">
+      <widget class="QLineEdit" name="template_value">
+       <property name="readOnly">
+        <bool>true</bool>
+       </property>
+      </widget>
+     </item>
+     <item row="6" column="1">
+      <widget class="QDialogButtonBox" name="buttonBox">
+       <property name="orientation">
+        <enum>Qt::Horizontal</enum>
+       </property>
+       <property name="standardButtons">
+        <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
+       </property>
+      </widget>
+     </item>
+     <item row="6" column="0">
       <widget class="QLabel" name="label">
        <property name="text">
         <string>Function &amp;name:</string>
@@ -45,10 +83,10 @@
        </property>
       </widget>
      </item>
-     <item row="0" column="1">
+     <item row="7" column="1">
       <widget class="QComboBox" name="function"/>
      </item>
-     <item row="1" column="0">
+     <item row="8" column="0">
       <widget class="QLabel" name="label_2">
        <property name="text">
         <string>&amp;Documentation:</string>
@@ -61,7 +99,7 @@
        </property>
       </widget>
      </item>
-     <item row="2" column="0">
+     <item row="9" column="0">
       <widget class="QLabel" name="label_3">
        <property name="text">
         <string>Python &amp;code:</string>
@@ -74,7 +112,7 @@
        </property>
       </widget>
      </item>
-     <item row="1" column="1">
+     <item row="8" column="1">
       <widget class="QPlainTextEdit" name="documentation">
        <property name="maximumSize">
         <size>
@@ -84,7 +122,7 @@
        </property>
       </widget>
      </item>
-     <item row="2" column="1">
+     <item row="9" column="1">
       <widget class="QPlainTextEdit" name="source_code"/>
      </item>
     </layout>

src/calibre/gui2/dialogs/template_line_editor.py

@@ -5,12 +5,10 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-from PyQt4.Qt import (QLineEdit, QDialog, QGridLayout, QLabel,
+
-                      QDialogButtonBox, QColor, QComboBox, QIcon)
+from PyQt4.Qt import QLineEdit
 
 from calibre.gui2.dialogs.template_dialog import TemplateDialog
-from calibre.gui2.complete import MultiCompleteLineEdit
-from calibre.gui2 import error_dialog
 
 class TemplateLineEditor(QLineEdit):
 
@@ -20,113 +18,26 @@ class TemplateLineEditor(QLineEdit):
 
     def __init__(self, parent):
         QLineEdit.__init__(self, parent)
-        self.tags = None
+        self.mi   = None
 
-    def set_tags(self, tags):
+    def set_mi(self, mi):
-        self.tags = tags
+        self.mi = mi
 
     def contextMenuEvent(self, event):
         menu = self.createStandardContextMenu()
         menu.addSeparator()
 
+        action_clear_field = menu.addAction(_('Remove any template from the box'))
+        action_clear_field.triggered.connect(self.clear_field)
         action_open_editor = menu.addAction(_('Open Template Editor'))
         action_open_editor.triggered.connect(self.open_editor)
-        if self.tags:
-            action_tag_wizard = menu.addAction(_('Open Tag Wizard'))
-            action_tag_wizard.triggered.connect(self.tag_wizard)
         menu.exec_(event.globalPos())
 
+    def clear_field(self):
+        self.setText('')
+
     def open_editor(self):
-        t = TemplateDialog(self, self.text())
+        t = TemplateDialog(self, self.text(), mi=self.mi)
         t.setWindowTitle(_('Edit template'))
         if t.exec_():
-            self.setText(t.textbox.toPlainText())
+            self.setText(t.rule[1])
-
-    def tag_wizard(self):
-        txt = unicode(self.text())
-        if txt and not txt.startswith('program:\n#tag wizard'):
-            error_dialog(self, _('Invalid text'),
-                 _('The text in the box was not generated by this wizard'),
-                 show=True, show_copy_button=False)
-            return
-        d = TagWizard(self, self.tags, unicode(self.text()))
-        if d.exec_():
-            self.setText(d.template)
-
-class TagWizard(QDialog):
-
-    def __init__(self, parent, tags, txt):
-        QDialog.__init__(self, parent)
-        self.setWindowTitle(_('Tag Wizard'))
-        self.setWindowIcon(QIcon(I('wizard.png')))
-
-        self.tags = tags
-        l = QGridLayout()
-        self.setLayout(l)
-        l.setColumnStretch(0, 1)
-        l.setColumnMinimumWidth(0, 300)
-        l.addWidget(QLabel(_('Tags (more than one per box permitted)')), 0, 0, 1, 1)
-        l.addWidget(QLabel(_('Color')), 0, 1, 1, 1)
-        self.tagboxes = []
-        self.colorboxes = []
-        self.colors = [unicode(s) for s in list(QColor.colorNames())]
-        self.colors.insert(0, '')
-        for i in range(0, 10):
-            tb = MultiCompleteLineEdit(self)
-            tb.set_separator(', ')
-            tb.update_items_cache(self.tags)
-            self.tagboxes.append(tb)
-            l.addWidget(tb, i+1, 0, 1, 1)
-            cb = QComboBox(self)
-            cb.addItems(self.colors)
-            self.colorboxes.append(cb)
-            l.addWidget(cb, i+1, 1, 1, 1)
-
-        if txt:
-            lines = txt.split('\n')[3:]
-            i = 0
-            for line in lines:
-                if line.startswith('#'):
-                    t,c = line[1:].split(':|:')
-                    try:
-                        self.colorboxes[i].setCurrentIndex(self.colorboxes[i].findText(c))
-                        self.tagboxes[i].setText(t)
-                    except:
-                        pass
-                    i += 1
-
-        bb = QDialogButtonBox(QDialogButtonBox.Ok|QDialogButtonBox.Cancel, parent=self)
-        l.addWidget(bb, 100, 1, 1, 1)
-        bb.accepted.connect(self.accepted)
-        bb.rejected.connect(self.reject)
-        self.template = ''
-
-    def accepted(self):
-        res = ("program:\n#tag wizard -- do not directly edit\n"
-               "  t = field('tags');\n  first_non_empty(\n")
-        lines = []
-        for tb, cb in zip(self.tagboxes, self.colorboxes):
-            tags = [t.strip() for t in unicode(tb.text()).split(',') if t.strip()]
-            tags = '$|^'.join(tags)
-            c = unicode(cb.currentText()).strip()
-            if not tags or not c:
-                continue
-            if c not in self.colors:
-                error_dialog(self, _('Invalid color'),
-                             _('The color {0} is not valid').format(c),
-                             show=True, show_copy_button=False)
-                return False
-            lines.append("    in_list(t, ',', '^{0}$', '{1}', '')".format(tags, c))
-        res += ',\n'.join(lines)
-        res += ')\n'
-        self.template = res
-        res = ''
-        for tb, cb in zip(self.tagboxes, self.colorboxes):
-            t = unicode(tb.text()).strip()
-            if t.endswith(','):
-                t = t[:-1]
-            c = unicode(cb.currentText()).strip()
-            if t and c:
-                res += '#' + t + ':|:' + c  + '\n'
-        self.template += res
-        self.accept()

src/calibre/gui2/dialogs/user_profiles.ui

@@ -386,7 +386,7 @@ p, li { white-space: pre-wrap; }
                  <item>
                   <widget class="QLabel" name="label_8">
                    <property name="text">
-                    <string>For help with writing advanced news recipes, please visit &lt;a href=&quot;http://__appname__-ebook.com/user_manual/news.html&quot;&gt;User Recipes&lt;/a&gt;</string>
+                    <string>For help with writing advanced news recipes, please visit &lt;a href=&quot;http://manual.__appname__-ebook.com/news.html&quot;&gt;User Recipes&lt;/a&gt;</string>
                    </property>
                    <property name="wordWrap">
                     <bool>true</bool>

src/calibre/gui2/filename_pattern.ui

@@ -19,7 +19,7 @@
      <property name="text">
       <string>&lt;div style=&quot;font-size:10pt;&quot;&gt;
 &lt;p&gt;Set a regular expression pattern to use when trying to guess ebook metadata from filenames. &lt;/p&gt;
-&lt;p&gt;A &lt;a href=&quot;http://calibre-ebook.com/user_manual/regexp.html&quot;&gt;tutorial&lt;/a&gt; on using regular expressions is available.&lt;/p&gt;
+&lt;p&gt;A &lt;a href=&quot;http://manual.calibre-ebook.com/regexp.html&quot;&gt;tutorial&lt;/a&gt; on using regular expressions is available.&lt;/p&gt;
 &lt;p&gt;Use the &lt;b&gt;Test&lt;/b&gt; functionality below to test your regular expression on a few sample filenames (remember to include the file extension). The group names for the various metadata entries are documented in tooltips.&lt;/p&gt;&lt;/div&gt;</string>
      </property>
      <property name="textFormat">

src/calibre/gui2/layout.py

@@ -238,7 +238,6 @@ class Spacer(QWidget): # {{{
         self.l.addStretch(10)
 # }}}
 
-
 class MainWindowMixin(object): # {{{
 
     def __init__(self, db):

src/calibre/gui2/library/delegates.py

@@ -249,6 +249,23 @@ class CcDateDelegate(QStyledItemDelegate): # {{{
 # }}}
 
 class CcTextDelegate(QStyledItemDelegate): # {{{
+    '''
+    Delegate for text data.
+    '''
+
+    def createEditor(self, parent, option, index):
+        m = index.model()
+        col = m.column_map[index.column()]
+        editor = MultiCompleteLineEdit(parent)
+        editor.set_separator(None)
+        complete_items = sorted(list(m.db.all_custom(label=m.db.field_metadata.key_to_label(col))),
+                                key=sort_key)
+        editor.update_items_cache(complete_items)
+        return editor
+
+# }}}
+
+class CcNumberDelegate(QStyledItemDelegate): # {{{
     '''
     Delegate for text/int/float data.
     '''
@@ -256,25 +273,23 @@ class CcTextDelegate(QStyledItemDelegate): # {{{
     def createEditor(self, parent, option, index):
         m = index.model()
         col = m.column_map[index.column()]
-        typ = m.custom_columns[col]['datatype']
+        if m.custom_columns[col]['datatype'] == 'int':
-        if typ == 'int':
             editor = QSpinBox(parent)
             editor.setRange(-100, 100000000)
             editor.setSpecialValueText(_('Undefined'))
             editor.setSingleStep(1)
-        elif typ == 'float':
+        else:
             editor = QDoubleSpinBox(parent)
             editor.setSpecialValueText(_('Undefined'))
             editor.setRange(-100., 100000000)
             editor.setDecimals(2)
-        else:
-            editor = MultiCompleteLineEdit(parent)
-            editor.set_separator(None)
-            complete_items = sorted(list(m.db.all_custom(label=m.db.field_metadata.key_to_label(col))),
-                                    key=sort_key)
-            editor.update_items_cache(complete_items)
         return editor
 
+    def setEditorData(self, editor, index):
+        m = index.model()
+        val = m.db.data[index.row()][m.custom_columns[m.column_map[index.column()]]['rec_index']]
+        editor.setValue(val)
+
 # }}}
 
 class CcEnumDelegate(QStyledItemDelegate): # {{{
@@ -403,14 +418,15 @@ class CcTemplateDelegate(QStyledItemDelegate): # {{{
 
     def createEditor(self, parent, option, index):
         m = index.model()
+        mi = m.db.get_metadata(index.row(), index_is_id=False)
         text = m.custom_columns[m.column_map[index.column()]]['display']['composite_template']
-        editor = TemplateDialog(parent, text)
+        editor = TemplateDialog(parent, text, mi)
         editor.setWindowTitle(_("Edit template"))
         editor.textbox.setTabChangesFocus(False)
         editor.textbox.setTabStopWidth(20)
         d = editor.exec_()
         if d:
-            m.setData(index, QVariant(editor.textbox.toPlainText()), Qt.EditRole)
+            m.setData(index, QVariant(editor.rule[1]), Qt.EditRole)
         return None
 
     def setModelData(self, editor, model, index):

src/calibre/gui2/library/models.py

@@ -7,6 +7,7 @@ __docformat__ = 'restructuredtext en'
 
 import shutil, functools, re, os, traceback
 from contextlib import closing
+from collections import defaultdict
 
 from PyQt4.Qt import (QAbstractTableModel, Qt, pyqtSignal, QIcon, QImage,
         QModelIndex, QVariant, QDate, QColor)
@@ -87,6 +88,7 @@ class BooksModel(QAbstractTableModel): # {{{
         self.column_map = []
         self.headers = {}
         self.alignment_map = {}
+        self.color_cache = defaultdict(dict)
         self.buffer_size = buffer
         self.metadata_backup = None
         self.bool_yes_icon = QIcon(I('ok.png'))
@@ -97,8 +99,7 @@ class BooksModel(QAbstractTableModel): # {{{
         self.ids_to_highlight_set = set()
         self.current_highlighted_idx = None
         self.highlight_only = False
-        self.column_color_map = {}
+        self.colors = frozenset([unicode(c) for c in QColor.colorNames()])
-        self.colors = [unicode(c) for c in QColor.colorNames()]
         self.read_config()
 
     def change_alignment(self, colname, alignment):
@@ -154,7 +155,6 @@ class BooksModel(QAbstractTableModel): # {{{
                 self.headers[col] = self.custom_columns[col]['name']
 
         self.build_data_convertors()
-        self.set_color_templates(reset=False)
         self.reset()
         self.database_changed.emit(db)
         self.stop_metadata_backup()
@@ -172,11 +172,13 @@ class BooksModel(QAbstractTableModel): # {{{
 
 
     def refresh_ids(self, ids, current_row=-1):
+        self.color_cache = defaultdict(dict)
         rows = self.db.refresh_ids(ids)
         if rows:
             self.refresh_rows(rows, current_row=current_row)
 
     def refresh_rows(self, rows, current_row=-1):
+        self.color_cache = defaultdict(dict)
         for row in rows:
             if row == current_row:
                 self.new_bookdisplay_data.emit(
@@ -206,6 +208,7 @@ class BooksModel(QAbstractTableModel): # {{{
         return ret
 
     def count_changed(self, *args):
+        self.color_cache = defaultdict(dict)
         self.count_changed_signal.emit(self.db.count())
 
     def row_indices(self, index):
@@ -336,6 +339,10 @@ class BooksModel(QAbstractTableModel): # {{{
         self.db.refresh(field=None)
         self.resort(reset=reset)
 
+    def reset(self):
+        self.color_cache = defaultdict(dict)
+        QAbstractTableModel.reset(self)
+
     def resort(self, reset=True):
         if not self.db:
             return
@@ -536,16 +543,6 @@ class BooksModel(QAbstractTableModel): # {{{
             img = self.default_image
         return img
 
-    def set_color_templates(self, reset=True):
-        self.column_color_map = {}
-        for i in range(1,self.db.column_color_count+1):
-            name = self.db.prefs.get('column_color_name_'+str(i))
-            if name:
-                self.column_color_map[name] = \
-                        self.db.prefs.get('column_color_template_'+str(i))
-        if reset:
-            self.reset()
-
     def build_data_convertors(self):
         def authors(r, idx=-1):
             au = self.db.data[r][idx]
@@ -623,7 +620,12 @@ class BooksModel(QAbstractTableModel): # {{{
                 return None
             return QVariant(text)
 
-        def number_type(r, idx=-1):
+        def number_type(r, idx=-1, fmt=None):
+            if fmt is not None:
+                try:
+                    return QVariant(fmt.format(self.db.data[r][idx]))
+                except:
+                    pass
             return QVariant(self.db.data[r][idx])
 
         self.dc = {
@@ -674,7 +676,8 @@ class BooksModel(QAbstractTableModel): # {{{
                             bool_cols_are_tristate=
                                 self.db.prefs.get('bools_are_tristate'))
             elif datatype in ('int', 'float'):
-                self.dc[col] = functools.partial(number_type, idx=idx)
+                fmt = self.custom_columns[col]['display'].get('number_format', None)
+                self.dc[col] = functools.partial(number_type, idx=idx, fmt=fmt)
             elif datatype == 'datetime':
                 self.dc[col] = functools.partial(datetime_type, idx=idx)
             elif datatype == 'bool':
@@ -711,18 +714,27 @@ class BooksModel(QAbstractTableModel): # {{{
                 return QVariant(QColor('lightgreen'))
         elif role == Qt.ForegroundRole:
             key = self.column_map[col]
-            if key in self.column_color_map:
+            mi = None
-                mi = self.db.get_metadata(self.id(index), index_is_id=True)
+            for k, fmt in self.db.prefs['column_color_rules']:
-                fmt = self.column_color_map[key]
+                if k != key:
+                    continue
+                id_ = self.id(index)
+                if id_ in self.color_cache:
+                    if key in self.color_cache[id_]:
+                        return self.color_cache[id_][key]
+                if mi is None:
+                    mi = self.db.get_metadata(id_, index_is_id=True)
                 try:
                     color = composite_formatter.safe_format(fmt, mi, '', mi)
                     if color in self.colors:
                         color = QColor(color)
                         if color.isValid():
-                            return QVariant(color)
+                            color = QVariant(color)
+                            self.color_cache[id_][key] = color
+                            return color
                 except:
-                    return NONE
+                    continue
-            elif self.is_custom_column(key) and \
+            if self.is_custom_column(key) and \
                         self.custom_columns[key]['datatype'] == 'enumeration':
                 cc = self.custom_columns[self.column_map[col]]['display']
                 colors = cc.get('enum_colors', [])

src/calibre/gui2/library/views.py

@@ -15,7 +15,7 @@ from PyQt4.Qt import QTableView, Qt, QAbstractItemView, QMenu, pyqtSignal, \
 from calibre.gui2.library.delegates import RatingDelegate, PubDateDelegate, \
     TextDelegate, DateDelegate, CompleteDelegate, CcTextDelegate, \
     CcBoolDelegate, CcCommentsDelegate, CcDateDelegate, CcTemplateDelegate, \
-    CcEnumDelegate
+    CcEnumDelegate, CcNumberDelegate
 from calibre.gui2.library.models import BooksModel, DeviceBooksModel
 from calibre.utils.config import tweaks, prefs
 from calibre.gui2 import error_dialog, gprefs
@@ -89,6 +89,7 @@ class BooksView(QTableView): # {{{
         self.cc_bool_delegate = CcBoolDelegate(self)
         self.cc_comments_delegate = CcCommentsDelegate(self)
         self.cc_template_delegate = CcTemplateDelegate(self)
+        self.cc_number_delegate = CcNumberDelegate(self)
         self.display_parent = parent
         self._model = modelcls(self)
         self.setModel(self._model)
@@ -501,8 +502,10 @@ class BooksView(QTableView): # {{{
                                                           self.tags_delegate)
                     else:
                         self.setItemDelegateForColumn(cm.index(colhead), self.cc_text_delegate)
-                elif cc['datatype'] in ('series', 'int', 'float'):
+                elif cc['datatype'] == 'series':
                     self.setItemDelegateForColumn(cm.index(colhead), self.cc_text_delegate)
+                elif cc['datatype'] in ('int', 'float'):
+                    self.setItemDelegateForColumn(cm.index(colhead), self.cc_number_delegate)
                 elif cc['datatype'] == 'bool':
                     self.setItemDelegateForColumn(cm.index(colhead), self.cc_bool_delegate)
                 elif cc['datatype'] == 'rating':

src/calibre/gui2/metadata/single_download.py

@@ -35,8 +35,9 @@ from calibre import force_unicode
 
 class RichTextDelegate(QStyledItemDelegate): # {{{
 
-    def __init__(self, parent=None):
+    def __init__(self, parent=None, max_width=160):
         QStyledItemDelegate.__init__(self, parent)
+        self.max_width = max_width
 
     def to_doc(self, index):
         doc = QTextDocument()
@@ -46,8 +47,8 @@ class RichTextDelegate(QStyledItemDelegate): # {{{
     def sizeHint(self, option, index):
         doc = self.to_doc(index)
         ans = doc.size().toSize()
-        if ans.width() > 150:
+        if ans.width() > self.max_width - 10:
-            ans.setWidth(160)
+            ans.setWidth(self.max_width)
         ans.setHeight(ans.height()+10)
         return ans
 

src/calibre/gui2/preferences/coloring.py

@@ -0,0 +1,614 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from PyQt4.Qt import (QWidget, QDialog, QLabel, QGridLayout, QComboBox, QSize,
+        QLineEdit, QIntValidator, QDoubleValidator, QFrame, QColor, Qt, QIcon,
+        QScrollArea, QPushButton, QVBoxLayout, QDialogButtonBox, QToolButton,
+        QListView, QAbstractListModel, pyqtSignal, QSizePolicy, QSpacerItem)
+
+from calibre.utils.icu import sort_key
+from calibre.gui2 import error_dialog
+from calibre.gui2.dialogs.template_dialog import TemplateDialog
+from calibre.gui2.metadata.single_download import RichTextDelegate
+from calibre.library.coloring import (Rule, conditionable_columns,
+    displayable_columns, rule_from_template)
+
+class ConditionEditor(QWidget): # {{{
+
+    def __init__(self, fm, parent=None):
+        QWidget.__init__(self, parent)
+        self.fm = fm
+
+        self.action_map = {
+            'bool' : (
+                    (_('is true'), 'is true',),
+                    (_('is false'), 'is false'),
+                    (_('is undefined'), 'is undefined')
+            ),
+            'ondevice' : (
+                    (_('is true'), 'is set',),
+                    (_('is false'), 'is not set'),
+            ),
+            'identifiers' : (
+                (_('has id'), 'has id'),
+                (_('does not have id'), 'does not have id'),
+            ),
+            'int' : (
+                (_('is equal to'), 'eq'),
+                (_('is less than'), 'lt'),
+                (_('is greater than'), 'gt')
+            ),
+            'multiple' : (
+                (_('has'), 'has'),
+                (_('does not have'), 'does not have'),
+                (_('has pattern'), 'has pattern'),
+                (_('does not have pattern'), 'does not have pattern'),
+                (_('is set'), 'is set'),
+                (_('is not set'), 'is not set'),
+            ),
+            'single'   : (
+                (_('is'), 'is'),
+                (_('is not'), 'is not'),
+                (_('matches pattern'), 'matches pattern'),
+                (_('does not match pattern'), 'does not match pattern'),
+                (_('is set'), 'is set'),
+                (_('is not set'), 'is not set'),
+            ),
+        }
+
+        for x in ('float', 'rating', 'datetime'):
+            self.action_map[x] = self.action_map['int']
+
+        self.l = l = QGridLayout(self)
+        self.setLayout(l)
+
+        self.l1 = l1 = QLabel(_('If the '))
+        l.addWidget(l1, 0, 0)
+
+        self.column_box = QComboBox(self)
+        l.addWidget(self.column_box, 0, 1)
+
+        self.l2 = l2 = QLabel(_(' column '))
+        l.addWidget(l2, 0, 2)
+
+        self.action_box = QComboBox(self)
+        l.addWidget(self.action_box, 0, 3)
+
+        self.l3 = l3 = QLabel(_(' value '))
+        l.addWidget(l3, 0, 4)
+
+        self.value_box = QLineEdit(self)
+        l.addWidget(self.value_box, 0, 5)
+
+        self.column_box.addItem('', '')
+        for key in sorted(
+                conditionable_columns(fm),
+                key=sort_key):
+            self.column_box.addItem(key, key)
+        self.column_box.setCurrentIndex(0)
+
+        self.column_box.currentIndexChanged.connect(self.init_action_box)
+        self.action_box.currentIndexChanged.connect(self.init_value_box)
+
+        for b in (self.column_box, self.action_box):
+            b.setSizeAdjustPolicy(b.AdjustToMinimumContentsLengthWithIcon)
+            b.setMinimumContentsLength(15)
+
+    @dynamic_property
+    def current_col(self):
+        def fget(self):
+            idx = self.column_box.currentIndex()
+            return unicode(self.column_box.itemData(idx).toString())
+        def fset(self, val):
+            for idx in range(self.column_box.count()):
+                c = unicode(self.column_box.itemData(idx).toString())
+                if c == val:
+                    self.column_box.setCurrentIndex(idx)
+                    return
+            raise ValueError('Column %r not found'%val)
+        return property(fget=fget, fset=fset)
+
+    @dynamic_property
+    def current_action(self):
+        def fget(self):
+            idx = self.action_box.currentIndex()
+            return unicode(self.action_box.itemData(idx).toString())
+        def fset(self, val):
+            for idx in range(self.action_box.count()):
+                c = unicode(self.action_box.itemData(idx).toString())
+                if c == val:
+                    self.action_box.setCurrentIndex(idx)
+                    return
+            raise ValueError('Action %r not valid for current column'%val)
+        return property(fget=fget, fset=fset)
+
+    @property
+    def current_val(self):
+        return unicode(self.value_box.text()).strip()
+
+    @dynamic_property
+    def condition(self):
+
+        def fget(self):
+            c, a, v = (self.current_col, self.current_action,
+                    self.current_val)
+            if not c or not a:
+                return None
+            return (c, a, v)
+
+        def fset(self, condition):
+            c, a, v = condition
+            if not v:
+                v = ''
+            v = v.strip()
+            self.current_col = c
+            self.current_action = a
+            self.value_box.setText(v)
+
+        return property(fget=fget, fset=fset)
+
+    def init_action_box(self):
+        self.action_box.blockSignals(True)
+        self.action_box.clear()
+        self.action_box.addItem('', '')
+        col = self.current_col
+        m = self.fm[col]
+        dt = m['datatype']
+        if dt in self.action_map:
+            actions = self.action_map[dt]
+        else:
+            if col == 'ondevice':
+                k = 'ondevice'
+            elif col == 'identifiers':
+                k = 'identifiers'
+            else:
+                k = 'multiple' if m['is_multiple'] else 'single'
+            actions = self.action_map[k]
+
+        for text, key in actions:
+            self.action_box.addItem(text, key)
+        self.action_box.setCurrentIndex(0)
+        self.action_box.blockSignals(False)
+        self.init_value_box()
+
+    def init_value_box(self):
+        self.value_box.setEnabled(True)
+        self.value_box.setText('')
+        self.value_box.setInputMask('')
+        self.value_box.setValidator(None)
+        col = self.current_col
+        m = self.fm[col]
+        dt = m['datatype']
+        action = self.current_action
+        if not col or not action:
+            return
+        tt = ''
+        if col == 'identifiers':
+            tt = _('Enter either an identifier type or an '
+                    'identifier type and value of the form identifier:value')
+        elif dt in ('int', 'float', 'rating'):
+            tt = _('Enter a number')
+            v = QIntValidator if dt == 'int' else QDoubleValidator
+            self.value_box.setValidator(v(self.value_box))
+        elif dt == 'datetime':
+            self.value_box.setInputMask('9999-99-99')
+            tt = _('Enter a date in the format YYYY-MM-DD')
+        else:
+            tt = _('Enter a string.')
+            if 'pattern' in action:
+                tt = _('Enter a regular expression')
+            elif m.get('is_multiple', False):
+                tt += '\n' + _('You can match multiple values by separating'
+                        ' them with %s')%m['is_multiple']
+        self.value_box.setToolTip(tt)
+        if action in ('is set', 'is not set', 'is true', 'is false',
+                'is undefined'):
+            self.value_box.setEnabled(False)
+# }}}
+
+class RuleEditor(QDialog): # {{{
+
+    def __init__(self, fm, parent=None):
+        QDialog.__init__(self, parent)
+        self.fm = fm
+
+        self.setWindowIcon(QIcon(I('format-fill-color.png')))
+        self.setWindowTitle(_('Create/edit a column coloring rule'))
+
+        self.l = l = QGridLayout(self)
+        self.setLayout(l)
+
+        self.l1 = l1 = QLabel(_('Create a coloring rule by'
+            ' filling in the boxes below'))
+        l.addWidget(l1, 0, 0, 1, 5)
+
+        self.f1 = QFrame(self)
+        self.f1.setFrameShape(QFrame.HLine)
+        l.addWidget(self.f1, 1, 0, 1, 5)
+
+        self.l2 = l2 = QLabel(_('Set the color of the column:'))
+        l.addWidget(l2, 2, 0)
+
+        self.column_box = QComboBox(self)
+        l.addWidget(self.column_box, 2, 1)
+
+        self.l3 = l3 = QLabel(_('to'))
+        l.addWidget(l3, 2, 2)
+
+        self.color_box = QComboBox(self)
+        l.addWidget(self.color_box, 2, 3)
+        l.addItem(QSpacerItem(10, 10, QSizePolicy.Expanding), 2, 4)
+
+        self.l4 = l4 = QLabel(
+            _('Only if the following conditions are all satisfied:'))
+        l.addWidget(l4, 3, 0, 1, 5)
+
+        self.scroll_area = sa = QScrollArea(self)
+        sa.setMinimumHeight(300)
+        sa.setMinimumWidth(950)
+        sa.setWidgetResizable(True)
+        l.addWidget(sa, 4, 0, 1, 5)
+
+        self.add_button = b = QPushButton(QIcon(I('plus.png')),
+                _('Add another condition'))
+        l.addWidget(b, 5, 0, 1, 5)
+        b.clicked.connect(self.add_blank_condition)
+
+        self.l5 = l5 = QLabel(_('You can disable a condition by'
+            ' blanking all of its boxes'))
+        l.addWidget(l5, 6, 0, 1, 5)
+
+        self.bb = bb = QDialogButtonBox(
+                QDialogButtonBox.Ok|QDialogButtonBox.Cancel)
+        bb.accepted.connect(self.accept)
+        bb.rejected.connect(self.reject)
+        l.addWidget(bb, 7, 0, 1, 5)
+
+        self.conditions_widget = QWidget(self)
+        sa.setWidget(self.conditions_widget)
+        self.conditions_widget.setLayout(QVBoxLayout())
+        self.conditions_widget.layout().setAlignment(Qt.AlignTop)
+        self.conditions = []
+
+        for b in (self.column_box, self.color_box):
+            b.setSizeAdjustPolicy(b.AdjustToMinimumContentsLengthWithIcon)
+            b.setMinimumContentsLength(15)
+
+        for key in sorted(
+                displayable_columns(fm),
+                key=sort_key):
+            name = fm[key]['name']
+            if name:
+                self.column_box.addItem(key, key)
+        self.column_box.setCurrentIndex(0)
+
+        self.color_box.addItems(QColor.colorNames())
+        self.color_box.setCurrentIndex(0)
+
+        self.resize(self.sizeHint())
+
+    def add_blank_condition(self):
+        c = ConditionEditor(self.fm, parent=self.conditions_widget)
+        self.conditions.append(c)
+        self.conditions_widget.layout().addWidget(c)
+
+    def apply_rule(self, col, rule):
+        for i in range(self.column_box.count()):
+            c = unicode(self.column_box.itemData(i).toString())
+            if col == c:
+                self.column_box.setCurrentIndex(i)
+                break
+        if rule.color:
+            idx = self.color_box.findText(rule.color)
+            if idx >= 0:
+                self.color_box.setCurrentIndex(idx)
+        for c in rule.conditions:
+            ce = ConditionEditor(self.fm, parent=self.conditions_widget)
+            self.conditions.append(ce)
+            self.conditions_widget.layout().addWidget(ce)
+            try:
+                ce.condition = c
+            except:
+                import traceback
+                traceback.print_exc()
+
+
+    def accept(self):
+        if self.validate():
+            QDialog.accept(self)
+
+    def validate(self):
+        r = Rule(self.fm)
+        for c in self.conditions:
+            condition = c.condition
+            if condition is not None:
+                try:
+                    r.add_condition(*condition)
+                except Exception as e:
+                    import traceback
+                    error_dialog(self, _('Invalid condition'),
+                            _('One of the conditions for this rule is'
+                                ' invalid: <b>%s</b>')%e,
+                            det_msg=traceback.format_exc(), show=True)
+                    return False
+        if len(r.conditions) < 1:
+            error_dialog(self, _('No conditions'),
+                    _('You must specify at least one non-empty condition'
+                        ' for this rule'), show=True)
+            return False
+        return True
+
+    @property
+    def rule(self):
+        r = Rule(self.fm)
+        r.color = unicode(self.color_box.currentText())
+        idx = self.column_box.currentIndex()
+        col = unicode(self.column_box.itemData(idx).toString())
+        for c in self.conditions:
+            condition = c.condition
+            if condition is not None:
+                r.add_condition(*condition)
+
+        return col, r
+# }}}
+
+class RulesModel(QAbstractListModel): # {{{
+
+    def __init__(self, prefs, fm, parent=None):
+        QAbstractListModel.__init__(self, parent)
+
+        self.fm = fm
+        rules = list(prefs['column_color_rules'])
+        self.rules = []
+        for col, template in rules:
+            try:
+                rule = rule_from_template(self.fm, template)
+            except:
+                rule = template
+            self.rules.append((col, rule))
+
+    def rowCount(self, *args):
+        return len(self.rules)
+
+    def data(self, index, role):
+        row = index.row()
+        try:
+            col, rule = self.rules[row]
+        except:
+            return None
+
+        if role == Qt.DisplayRole:
+            return self.rule_to_html(col, rule)
+        if role == Qt.UserRole:
+            return (col, rule)
+
+    def add_rule(self, col, rule):
+        self.rules.append((col, rule))
+        self.reset()
+        return self.index(len(self.rules)-1)
+
+    def replace_rule(self, index, col, r):
+        self.rules[index.row()] = (col, r)
+        self.dataChanged.emit(index, index)
+
+    def remove_rule(self, index):
+        self.rules.remove(self.rules[index.row()])
+        self.reset()
+
+    def commit(self, prefs):
+        rules = []
+        for col, r in self.rules:
+            if isinstance(r, Rule):
+                r = r.template
+            if r is not None:
+                rules.append((col, r))
+        prefs['column_color_rules'] = rules
+
+    def move(self, idx, delta):
+        row = idx.row() + delta
+        if row >= 0 and row < len(self.rules):
+            t = self.rules[row]
+            self.rules[row] = self.rules[row-delta]
+            self.rules[row-delta] = t
+            self.dataChanged.emit(idx, idx)
+            idx = self.index(row)
+            self.dataChanged.emit(idx, idx)
+            return idx
+
+    def clear(self):
+        self.rules = []
+        self.reset()
+
+    def rule_to_html(self, col, rule):
+        if not isinstance(rule, Rule):
+            return _('''
+            <p>Advanced Rule for column <b>%s</b>:
+            <pre>%s</pre>
+            ''')%(col, rule)
+        conditions = [self.condition_to_html(c) for c in rule.conditions]
+        return _('''\
+            <p>Set the color of <b>%s</b> to <b>%s</b> if the following
+            conditions are met:</p>
+            <ul>%s</ul>
+            ''') % (col, rule.color, ''.join(conditions))
+
+    def condition_to_html(self, condition):
+        return (
+            _('<li>If the <b>%s</b> column <b>%s</b> value: <b>%s</b>') %
+                tuple(condition))
+
+# }}}
+
+class EditRules(QWidget): # {{{
+
+    changed = pyqtSignal()
+
+    def __init__(self, parent=None):
+        QWidget.__init__(self, parent)
+
+        self.l = l = QGridLayout(self)
+        self.setLayout(l)
+
+        self.l1 = l1 = QLabel(_(
+            'You can control the color of columns in the'
+            ' book list by creating "rules" that tell calibre'
+            ' what color to use. Click the Add Rule button below'
+            ' to get started. You can change an existing rule by double'
+            ' clicking it.'))
+        l1.setWordWrap(True)
+        l.addWidget(l1, 0, 0, 1, 2)
+
+        self.add_button = QPushButton(QIcon(I('plus.png')), _('Add Rule'),
+                self)
+        self.remove_button = QPushButton(QIcon(I('minus.png')),
+                _('Remove Rule'), self)
+        self.add_button.clicked.connect(self.add_rule)
+        self.remove_button.clicked.connect(self.remove_rule)
+        l.addWidget(self.add_button, 1, 0)
+        l.addWidget(self.remove_button, 1, 1)
+
+        self.g = g = QGridLayout()
+        self.rules_view = QListView(self)
+        self.rules_view.doubleClicked.connect(self.edit_rule)
+        self.rules_view.setSelectionMode(self.rules_view.SingleSelection)
+        self.rules_view.setAlternatingRowColors(True)
+        self.rtfd = RichTextDelegate(parent=self.rules_view, max_width=400)
+        self.rules_view.setItemDelegate(self.rtfd)
+        g.addWidget(self.rules_view, 0, 0, 2, 1)
+
+        self.up_button = b = QToolButton(self)
+        b.setIcon(QIcon(I('arrow-up.png')))
+        b.setToolTip(_('Move the selected rule up'))
+        b.clicked.connect(self.move_up)
+        g.addWidget(b, 0, 1, 1, 1, Qt.AlignTop)
+        self.down_button = b = QToolButton(self)
+        b.setIcon(QIcon(I('arrow-down.png')))
+        b.setToolTip(_('Move the selected rule down'))
+        b.clicked.connect(self.move_down)
+        g.addWidget(b, 1, 1, 1, 1, Qt.AlignBottom)
+
+        l.addLayout(g, 2, 0, 1, 2)
+        l.setRowStretch(2, 10)
+
+        self.add_advanced_button = b = QPushButton(QIcon(I('plus.png')),
+                _('Add Advanced Rule'), self)
+        b.clicked.connect(self.add_advanced)
+        l.addWidget(b, 3, 0, 1, 2)
+
+    def initialize(self, fm, prefs, mi):
+        self.model = RulesModel(prefs, fm)
+        self.rules_view.setModel(self.model)
+        self.fm = fm
+        self.mi = mi
+
+    def _add_rule(self, dlg):
+        if dlg.exec_() == dlg.Accepted:
+            col, r = dlg.rule
+            if r and col:
+                idx = self.model.add_rule(col, r)
+                self.rules_view.scrollTo(idx)
+                self.changed.emit()
+
+    def add_rule(self):
+            d = RuleEditor(self.model.fm)
+            d.add_blank_condition()
+            self._add_rule(d)
+
+    def add_advanced(self):
+        td = TemplateDialog(self, '', mi=self.mi, fm=self.fm, color_field='')
+        self._add_rule(td)
+
+    def edit_rule(self, index):
+        try:
+            col, rule = self.model.data(index, Qt.UserRole)
+        except:
+            return
+        if isinstance(rule, Rule):
+            d = RuleEditor(self.model.fm)
+            d.apply_rule(col, rule)
+        else:
+            d = TemplateDialog(self, rule, mi=self.mi, fm=self.fm, color_field=col)
+        if d.exec_() == d.Accepted:
+            col, r = d.rule
+            if r is not None and col:
+                self.model.replace_rule(index, col, r)
+                self.rules_view.scrollTo(index)
+                self.changed.emit()
+
+    def get_selected_row(self, txt):
+        sm = self.rules_view.selectionModel()
+        rows = list(sm.selectedRows())
+        if not rows:
+            error_dialog(self, _('No rule selected'),
+                    _('No rule selected for %s.')%txt, show=True)
+            return None
+        return rows[0]
+
+    def remove_rule(self):
+        row = self.get_selected_row(_('removal'))
+        if row is not None:
+            self.model.remove_rule(row)
+            self.changed.emit()
+
+    def move_up(self):
+        idx = self.rules_view.currentIndex()
+        if idx.isValid():
+            idx = self.model.move(idx, -1)
+            if idx is not None:
+                sm = self.rules_view.selectionModel()
+                sm.select(idx, sm.ClearAndSelect)
+                self.rules_view.setCurrentIndex(idx)
+                self.changed.emit()
+
+    def move_down(self):
+        idx = self.rules_view.currentIndex()
+        if idx.isValid():
+            idx = self.model.move(idx, 1)
+            if idx is not None:
+                sm = self.rules_view.selectionModel()
+                sm.select(idx, sm.ClearAndSelect)
+                self.rules_view.setCurrentIndex(idx)
+                self.changed.emit()
+
+    def clear(self):
+        self.model.clear()
+        self.changed.emit()
+
+    def commit(self, prefs):
+        self.model.commit(prefs)
+
+# }}}
+
+if __name__ == '__main__':
+    from PyQt4.Qt import QApplication
+    app = QApplication([])
+
+    from calibre.library import db
+
+    db = db()
+
+    if True:
+        d = RuleEditor(db.field_metadata)
+        d.add_blank_condition()
+        d.exec_()
+
+        col, r = d.rule
+
+        print ('Column to be colored:', col)
+        print ('Template:')
+        print (r.template)
+    else:
+        d = EditRules()
+        d.resize(QSize(800, 600))
+        d.initialize(db.field_metadata, db.prefs, None)
+        d.show()
+        app.exec_()
+        d.commit(db.prefs)
+
+

src/calibre/gui2/preferences/create_custom_column.py

@@ -127,6 +127,9 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
         elif ct == 'enumeration':
             self.enum_box.setText(','.join(c['display'].get('enum_values', [])))
             self.enum_colors.setText(','.join(c['display'].get('enum_colors', [])))
+        elif ct in ['int', 'float']:
+            if c['display'].get('number_format', None):
+                self.number_format_box.setText(c['display'].get('number_format', ''))
         self.datatype_changed()
         if ct in ['text', 'composite', 'enumeration']:
             self.use_decorations.setChecked(c['display'].get('use_decorations', False))
@@ -171,6 +174,7 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             col_type = None
         for x in ('box', 'default_label', 'label'):
             getattr(self, 'date_format_'+x).setVisible(col_type == 'datetime')
+            getattr(self, 'number_format_'+x).setVisible(col_type in ['int', 'float'])
         for x in ('box', 'default_label', 'label', 'sort_by', 'sort_by_label',
                   'make_category'):
             getattr(self, 'composite_'+x).setVisible(col_type in ['composite', '*composite'])
@@ -178,6 +182,18 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             getattr(self, 'enum_'+x).setVisible(col_type == 'enumeration')
         self.use_decorations.setVisible(col_type in ['text', 'composite', 'enumeration'])
         self.is_names.setVisible(col_type == '*text')
+        if col_type == 'int':
+            self.number_format_box.setToolTip('<p>' +
+                _('Examples: The format <code>{0:0>4d}</code> '
+                  'gives a 4-digit number with leading zeros. The format '
+                  '<code>{0:d}&nbsp;days</code> prints the number then the word "days"')+ '</p>')
+        elif col_type == 'float':
+            self.number_format_box.setToolTip('<p>' +
+                _('Examples: The format <code>{0:.1f}</code> gives a floating '
+                  'point number with 1 digit after the decimal point. The format '
+                  '<code>Price:&nbsp;$&nbsp;{0:,.2f}</code> prints '
+                  '"Price&nbsp;$&nbsp;" then displays the number with 2 digits '
+                  'after the decimal point and thousands separated by commas.') + '</p>')
 
     def accept(self):
         col = unicode(self.column_name_box.text()).strip()
@@ -267,6 +283,11 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             display_dict = {'enum_values': l, 'enum_colors': c}
         elif col_type == 'text' and is_multiple:
             display_dict = {'is_names': self.is_names.isChecked()}
+        elif col_type in ['int', 'float']:
+            if unicode(self.number_format_box.text()).strip():
+                display_dict = {'number_format':unicode(self.number_format_box.text()).strip()}
+            else:
+                display_dict = {'number_format': None}
 
         if col_type in ['text', 'composite', 'enumeration'] and not is_multiple:
             display_dict['use_decorations'] = self.use_decorations.checkState()

src/calibre/gui2/preferences/create_custom_column.ui

@@ -171,6 +171,16 @@ Everything else will show nothing.</string>
            </property>
           </widget>
          </item>
+         <item>
+          <widget class="QLineEdit" name="number_format_box">
+           <property name="sizePolicy">
+            <sizepolicy hsizetype="Maximum" vsizetype="Fixed">
+             <horstretch>0</horstretch>
+             <verstretch>0</verstretch>
+            </sizepolicy>
+           </property>
+          </widget>
+         </item>
          <item>
           <widget class="QLabel" name="date_format_default_label">
            <property name="toolTip">
@@ -181,6 +191,21 @@ Everything else will show nothing.</string>
            </property>
           </widget>
          </item>
+         <item>
+          <widget class="QLabel" name="number_format_default_label">
+           <property name="toolTip">
+            <string>&lt;p&gt;The format specifier must begin with &lt;code&gt;{0:&lt;/code&gt;
+and end with &lt;code&gt;}&lt;/code&gt; You can have text before and after the format specifier.
+            </string>
+           </property>
+           <property name="text">
+            <string>&lt;p&gt;Default: Not formatted. For format language details see &lt;a href="http://docs.python.org/library/string.html#format-string-syntax"&gt;the python documentation&lt;/a&gt;</string>
+           </property>
+           <property name="openExternalLinks">
+            <bool>true</bool>
+           </property>
+          </widget>
+         </item>
         </layout>
        </item>
        <item row="4" column="0">
@@ -193,6 +218,16 @@ Everything else will show nothing.</string>
          </property>
         </widget>
        </item>
+       <item row="4" column="0">
+        <widget class="QLabel" name="number_format_label">
+         <property name="text">
+          <string>Format for &amp;numbers</string>
+         </property>
+         <property name="buddy">
+          <cstring>number_format_box</cstring>
+         </property>
+        </widget>
+       </item>
        <item row="5" column="0">
         <widget class="QLabel" name="composite_label">
          <property name="text">

src/calibre/gui2/preferences/look_feel.py

@@ -6,7 +6,7 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 from PyQt4.Qt import (QApplication, QFont, QFontInfo, QFontDialog,
-        QAbstractListModel, Qt, QColor)
+        QAbstractListModel, Qt, QIcon)
 
 from calibre.gui2.preferences import ConfigWidgetBase, test_widget, CommaSeparatedList
 from calibre.gui2.preferences.look_feel_ui import Ui_Form
@@ -17,6 +17,7 @@ from calibre.utils.config import prefs
 from calibre.utils.icu import sort_key
 from calibre.gui2 import NONE
 from calibre.gui2.book_details import get_field_list
+from calibre.gui2.preferences.coloring import EditRules
 
 class DisplayedFields(QAbstractListModel): # {{{
 
@@ -129,7 +130,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
             (_('Medium'), 'medium'), (_('Large'), 'large')]
         r('toolbar_icon_size', gprefs, choices=choices)
 
-        choices = [(_('Automatic'), 'auto'), (_('Always'), 'always'),
+        choices = [(_('If there is enough room'), 'auto'), (_('Always'), 'always'),
             (_('Never'), 'never')]
         r('toolbar_text', gprefs, choices=choices)
 
@@ -159,61 +160,11 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.df_up_button.clicked.connect(self.move_df_up)
         self.df_down_button.clicked.connect(self.move_df_down)
 
-        self.color_help_text.setText('<p>' +
+        self.edit_rules = EditRules(self.tabWidget)
-                _('Here you can specify coloring rules for columns shown in the '
+        self.edit_rules.changed.connect(self.changed_signal)
-                  'library view. Choose the column you wish to color, then '
+        self.tabWidget.addTab(self.edit_rules,
-                  'supply a template that specifies the color to use based on '
+                QIcon(I('format-fill-color.png')), _('Column coloring'))
-                  'the values in the column. There is a '
+        self.tabWidget.setCurrentIndex(0)
-                  '<a href="http://calibre-ebook.com/user_manual/template_lang.html">'
-                  'tutorial</a> on using templates.') +
-                 '</p><p>' +
-                _('If you want to color a field based on tags, then click the '
-                  'button next to an empty line to open the tags wizard. '
-                  'It will build a template for you. You can later edit that '
-                  'template with the same wizard. If you edit it by hand, the '
-                  'wizard might not work or might restore old values.') +
-                 '</p><p>' +
-                _('The template must evaluate to one of the color names shown '
-                  'below. You can use any legal template expression. '
-                  'For example, you can set the title to always display in '
-                  'green using the template "green" (without the quotes). '
-                  'To show the title in the color named in the custom column '
-                  '#column, use "{#column}". To show the title in blue if the '
-                  'custom column #column contains the value "foo", in red if the '
-                  'column contains the value "bar", otherwise in black, use '
-                  '<pre>{#column:switch(foo,blue,bar,red,black)}</pre>'
-                  'To show the title in blue if the book has the exact tag '
-                  '"Science Fiction", red if the book has the exact tag '
-                  '"Mystery", or black if the book has neither tag, use'
-                  "<pre>program: \n"
-                  "    t = field('tags'); \n"
-                  "    first_non_empty(\n"
-                  "        in_list(t, ',', '^Science Fiction$', 'blue', ''), \n"
-                  "        in_list(t, ',', '^Mystery$', 'red', 'black'))</pre>"
-                  'To show the title in green if it has one format, blue if it '
-                  'two formats, and red if more, use'
-                  "<pre>program:cmp(count(field('formats'),','), 2, 'green', 'blue', 'red')</pre>") +
-                               '</p><p>' +
-                _('You can access a multi-line template editor from the '
-                  'context menu (right-click).') + '</p><p>' +
-                _('<b>Note:</b> if you want to color a "custom column with a fixed set '
-                  'of values", it is often easier to specify the '
-                  'colors in the column definition dialog. There you can '
-                  'provide a color for each value without using a template.')+ '</p>')
-        choices = db.field_metadata.displayable_field_keys()
-        choices.sort(key=sort_key)
-        choices.insert(0, '')
-        self.column_color_count = db.column_color_count+1
-        tags = db.all_tags()
-        for i in range(1, self.column_color_count):
-            r('column_color_name_'+str(i), db.prefs, choices=choices)
-            r('column_color_template_'+str(i), db.prefs)
-            tpl = getattr(self, 'opt_column_color_template_'+str(i))
-            tpl.set_tags(tags)
-            toolbutton = getattr(self, 'opt_column_color_wizard_'+str(i))
-            toolbutton.clicked.connect(tpl.tag_wizard)
-        all_colors = [unicode(s) for s in list(QColor.colorNames())]
-        self.colors_box.setText(', '.join(all_colors))
 
     def initialize(self):
         ConfigWidgetBase.initialize(self)
@@ -224,6 +175,13 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.current_font = self.initial_font = font
         self.update_font_display()
         self.display_model.initialize()
+        db = self.gui.current_db
+        try:
+            idx = self.gui.library_view.currentIndex().row()
+            mi = db.get_metadata(idx, index_is_id=False)
+        except:
+            mi=None
+        self.edit_rules.initialize(db.field_metadata, db.prefs, mi)
 
     def restore_defaults(self):
         ConfigWidgetBase.restore_defaults(self)
@@ -233,6 +191,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
             self.changed_signal.emit()
             self.update_font_display()
         self.display_model.restore_defaults()
+        self.edit_rules.clear()
         self.changed_signal.emit()
 
     def build_font_obj(self):
@@ -282,12 +241,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
             self.changed_signal.emit()
 
     def commit(self, *args):
-        for i in range(1, self.column_color_count):
-            col = getattr(self, 'opt_column_color_name_'+str(i))
-            tpl = getattr(self, 'opt_column_color_template_'+str(i))
-            if not col.currentIndex() or not unicode(tpl.text()).strip():
-                col.setCurrentIndex(0)
-                tpl.setText('')
         rr = ConfigWidgetBase.commit(self, *args)
         if self.current_font != self.initial_font:
             gprefs['font'] = (self.current_font[:4] if self.current_font else
@@ -297,10 +250,11 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
             QApplication.setFont(self.font_display.font())
             rr = True
         self.display_model.commit()
+        self.edit_rules.commit(self.gui.current_db.prefs)
         return rr
 
     def refresh_gui(self, gui):
-        gui.library_view.model().set_color_templates()
+        gui.library_view.model().reset()
         self.update_font_display()
         gui.tags_view.reread_collapse_parameters()
         gui.library_view.refresh_book_details()

src/calibre/gui2/preferences/look_feel.ui

@@ -6,7 +6,7 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>717</width>
+    <width>820</width>
     <height>519</height>
    </rect>
   </property>
@@ -407,193 +407,6 @@ then the tags will be displayed each on their own line.</string>
        </item>
       </layout>
      </widget>
-     <widget class="QWidget" name="tab_5">
-      <attribute name="icon">
-       <iconset resource="../../../../resources/images.qrc">
-        <normaloff>:/images/format-fill-color.png</normaloff>:/images/format-fill-color.png</iconset>
-      </attribute>
-      <attribute name="title">
-       <string>Column Coloring</string>
-      </attribute>
-      <layout class="QGridLayout" name="column_color_layout">
-       <item row="1" column="0">
-        <widget class="QLabel" name="label">
-         <property name="text">
-          <string>Column to color</string>
-         </property>
-        </widget>
-       </item>
-       <item row="1" column="1">
-        <widget class="QLabel" name="label">
-         <property name="text">
-          <string>Color selection template</string>
-         </property>
-        </widget>
-       </item>
-       <item row="2" column="0">
-        <widget class="QComboBox" name="opt_column_color_name_1"/>
-       </item>
-       <item row="2" column="1">
-        <widget class="TemplateLineEditor" name="opt_column_color_template_1"/>
-       </item>
-       <item row="2" column="2">
-        <widget class="QToolButton" name="opt_column_color_wizard_1">
-         <property name="icon">
-          <iconset resource="../../../../resources/images.qrc">
-           <normaloff>:/images/wizard.png</normaloff>:/images/wizard.png</iconset>
-         </property>
-         <property name="toolTip">
-          <string>Open the tags wizard.</string>
-         </property>
-        </widget>
-       </item>
-       <item row="3" column="0">
-        <widget class="QComboBox" name="opt_column_color_name_2"/>
-       </item>
-       <item row="3" column="1">
-        <widget class="TemplateLineEditor" name="opt_column_color_template_2"/>
-       </item>
-       <item row="3" column="2">
-        <widget class="QToolButton" name="opt_column_color_wizard_2">
-         <property name="icon">
-          <iconset resource="../../../../resources/images.qrc">
-           <normaloff>:/images/wizard.png</normaloff>:/images/wizard.png</iconset>
-         </property>
-         <property name="toolTip">
-          <string>Open the tags wizard.</string>
-         </property>
-        </widget>
-       </item>
-       <item row="4" column="0">
-        <widget class="QComboBox" name="opt_column_color_name_3"/>
-       </item>
-       <item row="4" column="1">
-        <widget class="TemplateLineEditor" name="opt_column_color_template_3"/>
-       </item>
-       <item row="4" column="2">
-        <widget class="QToolButton" name="opt_column_color_wizard_3">
-         <property name="icon">
-          <iconset resource="../../../../resources/images.qrc">
-           <normaloff>:/images/wizard.png</normaloff>:/images/wizard.png</iconset>
-         </property>
-         <property name="toolTip">
-          <string>Open the tags wizard.</string>
-         </property>
-        </widget>
-       </item>
-       <item row="5" column="0">
-        <widget class="QComboBox" name="opt_column_color_name_4"/>
-       </item>
-       <item row="5" column="1">
-        <widget class="TemplateLineEditor" name="opt_column_color_template_4"/>
-       </item>
-       <item row="5" column="2">
-        <widget class="QToolButton" name="opt_column_color_wizard_4">
-         <property name="icon">
-          <iconset resource="../../../../resources/images.qrc">
-           <normaloff>:/images/wizard.png</normaloff>:/images/wizard.png</iconset>
-         </property>
-         <property name="toolTip">
-          <string>Open the tags wizard.</string>
-         </property>
-        </widget>
-       </item>
-       <item row="6" column="0">
-        <widget class="QComboBox" name="opt_column_color_name_5"/>
-       </item>
-       <item row="6" column="1">
-        <widget class="TemplateLineEditor" name="opt_column_color_template_5"/>
-       </item>
-       <item row="6" column="2">
-        <widget class="QToolButton" name="opt_column_color_wizard_5">
-         <property name="icon">
-          <iconset resource="../../../../resources/images.qrc">
-           <normaloff>:/images/wizard.png</normaloff>:/images/wizard.png</iconset>
-         </property>
-         <property name="toolTip">
-          <string>Open the tags wizard.</string>
-         </property>
-        </widget>
-       </item>
-       <item row="20" column="0">
-        <widget class="QLabel" name="label">
-         <property name="text">
-          <string>Color names</string>
-         </property>
-        </widget>
-       </item>
-       <item row="0" column="0" colspan="3">
-        <widget class="QScrollArea" name="scrollArea">
-         <property name="minimumSize">
-          <size>
-           <width>0</width>
-           <height>200</height>
-          </size>
-         </property>
-         <property name="widgetResizable">
-          <bool>true</bool>
-         </property>
-         <property name="alignment">
-          <set>Qt::AlignCenter</set>
-         </property>
-         <widget class="QWidget" name="scrollAreaWidgetContents">
-          <property name="geometry">
-           <rect>
-            <x>0</x>
-            <y>0</y>
-            <width>687</width>
-            <height>194</height>
-           </rect>
-          </property>
-          <layout class="QVBoxLayout" name="verticalLayout">
-           <item>
-            <widget class="QLabel" name="color_help_text">
-             <property name="wordWrap">
-              <bool>true</bool>
-             </property>
-             <property name="openExternalLinks">
-              <bool>true</bool>
-             </property>
-            </widget>
-           </item>
-          </layout>
-         </widget>
-        </widget>
-       </item>
-       <item row="21" column="0" colspan="3">
-        <widget class="QScrollArea" name="scrollArea_2">
-         <property name="maximumSize">
-          <size>
-           <width>16777215</width>
-           <height>120</height>
-          </size>
-         </property>
-         <property name="widgetResizable">
-          <bool>true</bool>
-         </property>
-         <widget class="QWidget" name="scrollAreaWidgetContents_2">
-          <property name="geometry">
-           <rect>
-            <x>0</x>
-            <y>0</y>
-            <width>687</width>
-            <height>61</height>
-           </rect>
-          </property>
-          <layout class="QVBoxLayout" name="verticalLayout_2">
-           <item>
-            <widget class="QLabel" name="colors_box">
-             <property name="wordWrap">
-              <bool>true</bool>
-             </property>
-            </widget>
-           </item>
-          </layout>
-         </widget>
-        </widget>
-       </item>
-      </layout>
-     </widget>
     </widget>
    </item>
   </layout>
@@ -604,11 +417,6 @@ then the tags will be displayed each on their own line.</string>
    <extends>QLineEdit</extends>
    <header>calibre/gui2/complete.h</header>
   </customwidget>
-  <customwidget>
-   <class>TemplateLineEditor</class>
-   <extends>QLineEdit</extends>
-   <header>calibre/gui2/dialogs/template_line_editor.h</header>
-  </customwidget>
  </customwidgets>
  <resources>
   <include location="../../../../resources/images.qrc"/>

src/calibre/gui2/preferences/template_functions.py

@@ -7,7 +7,9 @@ __docformat__ = 'restructuredtext en'
 
 import json, traceback
 
-from calibre.gui2 import error_dialog
+from PyQt4.Qt import QDialogButtonBox
+
+from calibre.gui2 import error_dialog, warning_dialog
 from calibre.gui2.preferences import ConfigWidgetBase, test_widget
 from calibre.gui2.preferences.template_functions_ui import Ui_Form
 from calibre.gui2.widgets import PythonHighlighter
@@ -152,10 +154,15 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                          _('Name already used'), show=True)
             return
         if self.argument_count.value() == 0:
-            error_dialog(self.gui, _('Template functions'),
+            box = warning_dialog(self.gui, _('Template functions'),
-                         _('Argument count must be -1 or greater than zero'),
+                         _('Argument count should be -1 or greater than zero.'
-                         show=True)
+                           'Setting it to zero means that this function cannot '
-            return
+                           'be used in single function mode.'), det_msg = '',
+                         show=False)
+            box.bb.setStandardButtons(box.bb.standardButtons() | QDialogButtonBox.Cancel)
+            box.det_msg_toggle.setVisible(False)
+            if not box.exec_():
+                return
         try:
             prog = unicode(self.program.toPlainText())
             cls = compile_user_function(name, unicode(self.documentation.toPlainText()),

src/calibre/gui2/store/amazon_uk_plugin.py

@@ -6,11 +6,17 @@ __license__ = 'GPL 3'
 __copyright__ = '2011, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
+import urllib
+from contextlib import closing
+
+from lxml import html
 
 from PyQt4.Qt import QUrl
 
+from calibre import browser
 from calibre.gui2 import open_url
 from calibre.gui2.store.amazon_plugin import AmazonKindleStore
+from calibre.gui2.store.search_result import SearchResult
 
 class AmazonUKKindleStore(AmazonKindleStore):
     '''
@@ -28,3 +34,81 @@ class AmazonUKKindleStore(AmazonKindleStore):
             aff_id['asin'] = detail_item
             store_link = 'http://www.amazon.co.uk/gp/redirect.html?ie=UTF8&location=http://www.amazon.co.uk/dp/%(asin)s&tag=%(tag)s&linkCode=ur2&camp=1634&creative=6738' % aff_id
         open_url(QUrl(store_link))
+
+    def search(self, query, max_results=10, timeout=60):
+        url =  self.search_url + urllib.quote_plus(query)
+        br = browser()
+
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+
+            # Amazon has two results pages.
+            is_shot = doc.xpath('boolean(//div[@id="shotgunMainResults"])')
+            # Horizontal grid of books.
+            if is_shot:
+                data_xpath = '//div[contains(@class, "result")]'
+                cover_xpath = './/div[@class="productTitle"]//img/@src'
+            # Vertical list of books.
+            else:
+                data_xpath = '//div[contains(@class, "product")]'
+                cover_xpath = './div[@class="productImage"]/a/img/@src'
+
+            for data in doc.xpath(data_xpath):
+                if counter <= 0:
+                    break
+
+                # We must have an asin otherwise we can't easily reference the
+                # book later.
+                asin = ''.join(data.xpath('./@name'))
+                if not asin:
+                    continue
+                cover_url = ''.join(data.xpath(cover_xpath))
+
+                title = ''.join(data.xpath('.//div[@class="productTitle"]/a/text()'))
+                price = ''.join(data.xpath('.//div[@class="newPrice"]/span/text()'))
+
+                counter -= 1
+
+                s = SearchResult()
+                s.cover_url = cover_url.strip()
+                s.title = title.strip()
+                s.price = price.strip()
+                s.detail_item = asin.strip()
+                s.formats = 'Kindle'
+
+                if is_shot:
+                    # Amazon UK does not include the author on the grid layout
+                    s.author = ''
+                    self.get_details(s, timeout)
+                else:
+                    author = ''.join(data.xpath('.//div[@class="productTitle"]/span[@class="ptBrand"]/text()'))
+                    s.author = author.split(' by ')[-1].strip()
+
+                yield s
+
+    def get_details(self, search_result, timeout):
+        # We might already have been called.
+        if search_result.drm:
+            return
+
+        url = self.details_url
+
+        br = browser()
+        with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf:
+            idata = html.fromstring(nf.read())
+            if not search_result.author:
+                search_result.author = ''.join(idata.xpath('//div[@class="buying" and contains(., "Author")]/a/text()'))
+            if idata.xpath('boolean(//div[@class="content"]//li/b[contains(text(), "' +
+                           self.drm_search_text + '")])'):
+                if idata.xpath('boolean(//div[@class="content"]//li[contains(., "' +
+                               self.drm_free_text + '") and contains(b, "' +
+                               self.drm_search_text + '")])'):
+                    search_result.drm = SearchResult.DRM_UNLOCKED
+                else:
+                    search_result.drm = SearchResult.DRM_UNKNOWN
+            else:
+                search_result.drm = SearchResult.DRM_LOCKED
+        return True
+
+

src/calibre/gui2/store/config/chooser/adv_search_builder.py

@@ -45,8 +45,9 @@ class AdvSearchBuilderDialog(QDialog, Ui_Dialog):
         self.description_box.setText('')
         self.headquarters_box.setText('')
         self.format_box.setText('')
-        self.enabled_combo.setIndex(0)
+        self.enabled_combo.setCurrentIndex(0)
-        self.drm_combo.setIndex(0)
+        self.drm_combo.setCurrentIndex(0)
+        self.affiliate_combo.setCurrentIndex(0)
 
     def tokens(self, raw):
         phrases = re.findall(r'\s*".*?"\s*', raw)
@@ -126,6 +127,9 @@ class AdvSearchBuilderDialog(QDialog, Ui_Dialog):
         drm = unicode(self.drm_combo.currentText()).strip()
         if drm:
             ans.append('drm:' + drm)
+        affiliate = unicode(self.affiliate_combo.currentText()).strip()
+        if affiliate:
+            ans.append('affiliate:' + affiliate)
         if ans:
             return ' and '.join(ans)
         return ''

src/calibre/gui2/store/config/chooser/adv_search_builder.ui

@@ -149,7 +149,7 @@
              </size>
             </property>
             <property name="text">
-             <string>See the &lt;a href=&quot;http://calibre-ebook.com/user_manual/gui.html#the-search-interface&quot;&gt;User Manual&lt;/a&gt; for more help</string>
+             <string>See the &lt;a href=&quot;http://manual.calibre-ebook.com/gui.html#the-search-interface&quot;&gt;User Manual&lt;/a&gt; for more help</string>
             </property>
             <property name="openExternalLinks">
              <bool>true</bool>
@@ -226,7 +226,7 @@
          </property>
         </widget>
        </item>
-       <item row="8" column="0" colspan="2">
+       <item row="9" column="0" colspan="2">
         <layout class="QHBoxLayout" name="horizontalLayout_6">
          <item>
           <widget class="QPushButton" name="clear_button">
@@ -244,7 +244,7 @@
          </item>
         </layout>
        </item>
-       <item row="7" column="1">
+       <item row="8" column="1">
         <spacer name="verticalSpacer">
          <property name="orientation">
           <enum>Qt::Vertical</enum>
@@ -335,6 +335,32 @@
          </item>
         </widget>
        </item>
+       <item row="7" column="0">
+        <widget class="QLabel" name="label_13">
+         <property name="text">
+          <string>Affiliate:</string>
+         </property>
+        </widget>
+       </item>
+       <item row="7" column="1">
+        <widget class="QComboBox" name="affiliate_combo">
+         <item>
+          <property name="text">
+           <string/>
+          </property>
+         </item>
+         <item>
+          <property name="text">
+           <string>true</string>
+          </property>
+         </item>
+         <item>
+          <property name="text">
+           <string>false</string>
+          </property>
+         </item>
+        </widget>
+       </item>
       </layout>
      </widget>
     </widget>

src/calibre/gui2/store/config/chooser/chooser_widget.py

@@ -17,18 +17,20 @@ class StoreChooserWidget(QWidget, Ui_Form):
         QWidget.__init__(self)
         self.setupUi(self)
         
+        self.query.initialize('store_config_chooser_query')
+        
         self.adv_search_builder.setIcon(QIcon(I('search.png')))
         
         self.search.clicked.connect(self.do_search)
         self.adv_search_builder.clicked.connect(self.build_adv_search)
-        self.results_view.activated.connect(self.toggle_plugin)
+        self.enable_all.clicked.connect(self.results_view.model().enable_all)
+        self.enable_none.clicked.connect(self.results_view.model().enable_none)
+        self.enable_invert.clicked.connect(self.results_view.model().enable_invert)
+        self.results_view.activated.connect(self.results_view.model().toggle_plugin)
 
     def do_search(self):
         self.results_view.model().search(unicode(self.query.text()))
 
-    def toggle_plugin(self, index):
-        self.results_view.model().toggle_plugin(index)
-
     def build_adv_search(self):
         adv = AdvSearchBuilderDialog(self)
         if adv.exec_() == QDialog.Accepted:

src/calibre/gui2/store/config/chooser/chooser_widget.ui

@@ -31,7 +31,14 @@
       </widget>
      </item>
      <item>
-      <widget class="QLineEdit" name="query"/>
+      <widget class="HistoryLineEdit" name="query">
+       <property name="sizePolicy">
+        <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
+         <horstretch>0</horstretch>
+         <verstretch>0</verstretch>
+        </sizepolicy>
+       </property>
+      </widget>
      </item>
      <item>
       <widget class="QPushButton" name="search">
@@ -73,6 +80,51 @@
      </attribute>
     </widget>
    </item>
+   <item>
+    <layout class="QHBoxLayout" name="horizontalLayout_2">
+     <item>
+      <widget class="QLabel" name="label_2">
+       <property name="text">
+        <string>Enable</string>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QPushButton" name="enable_all">
+       <property name="text">
+        <string>All</string>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QPushButton" name="enable_none">
+       <property name="text">
+        <string>None</string>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QPushButton" name="enable_invert">
+       <property name="text">
+        <string>Invert</string>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <spacer name="horizontalSpacer">
+       <property name="orientation">
+        <enum>Qt::Horizontal</enum>
+       </property>
+       <property name="sizeHint" stdset="0">
+        <size>
+         <width>40</width>
+         <height>20</height>
+        </size>
+       </property>
+      </spacer>
+     </item>
+    </layout>
+   </item>
   </layout>
  </widget>
  <customwidgets>
@@ -81,6 +133,11 @@
    <extends>QTreeView</extends>
    <header>results_view.h</header>
   </customwidget>
+  <customwidget>
+   <class>HistoryLineEdit</class>
+   <extends>QLineEdit</extends>
+   <header>widgets.h</header>
+  </customwidget>
  </customwidgets>
  <resources/>
  <connections/>

src/calibre/gui2/store/config/chooser/models.py

@@ -6,7 +6,7 @@ __license__ = 'GPL 3'
 __copyright__ = '2011, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
-from PyQt4.Qt import (Qt, QAbstractItemModel, QIcon, QVariant, QModelIndex)
+from PyQt4.Qt import (Qt, QAbstractItemModel, QIcon, QVariant, QModelIndex, QSize)
 
 from calibre.gui2 import NONE
 from calibre.customize.ui import is_disabled, disable_plugin, enable_plugin
@@ -18,13 +18,15 @@ from calibre.utils.search_query_parser import SearchQueryParser
 
 class Matches(QAbstractItemModel):
 
-    HEADERS = [_('Enabled'), _('Name'), _('No DRM'), _('Headquarters'), _('Formats')]
+    HEADERS = [_('Enabled'), _('Name'), _('No DRM'), _('Headquarters'), _('Affiliate'), _('Formats')]
     HTML_COLS = [1]
 
     def __init__(self, plugins):
         QAbstractItemModel.__init__(self)
 
         self.NO_DRM_ICON = QIcon(I('ok.png'))
+        self.DONATE_ICON = QIcon()
+        self.DONATE_ICON.addFile(I('donate.png'), QSize(16, 16))
 
         self.all_matches = plugins
         self.matches = plugins
@@ -53,6 +55,22 @@ class Matches(QAbstractItemModel):
         self.layoutChanged.emit()
         self.sort(self.sort_col, self.sort_order)
 
+    def enable_all(self):
+        for i in xrange(len(self.matches)):
+            index = self.createIndex(i, 0)
+            data = QVariant(True)
+            self.setData(index, data, Qt.CheckStateRole)
+    
+    def enable_none(self):
+        for i in xrange(len(self.matches)):
+            index = self.createIndex(i, 0)
+            data = QVariant(False)
+            self.setData(index, data, Qt.CheckStateRole)
+    
+    def enable_invert(self):
+        for i in xrange(len(self.matches)):
+            self.toggle_plugin(self.createIndex(i, 0))
+
     def toggle_plugin(self, index):
         new_index = self.createIndex(index.row(), 0)
         data = QVariant(is_disabled(self.get_plugin(index)))
@@ -91,12 +109,15 @@ class Matches(QAbstractItemModel):
                 return QVariant('<b>%s</b><br><i>%s</i>' % (result.name, result.description))
             elif col == 3:
                 return QVariant(result.headquarters)
-            elif col == 4:
+            elif col == 5:
                 return QVariant(', '.join(result.formats).upper())
         elif role == Qt.DecorationRole:
             if col == 2:
                 if result.drm_free_only:
                     return QVariant(self.NO_DRM_ICON)
+            if col == 4:
+                if result.affiliate:
+                    return QVariant(self.DONATE_ICON)
         elif role == Qt.CheckStateRole:
             if col == 0:
                 if is_disabled(result):
@@ -105,20 +126,23 @@ class Matches(QAbstractItemModel):
         elif role == Qt.ToolTipRole:
             if col == 0:
                 if is_disabled(result):
-                    return QVariant(_('<p>This store is currently diabled and cannot be used in other parts of calibre.</p>'))
+                    return QVariant('<p>' + _('This store is currently diabled and cannot be used in other parts of calibre.') + '</p>')
                 else:
-                    return QVariant(_('<p>This store is currently enabled and can be used in other parts of calibre.</p>'))
+                    return QVariant('<p>' + _('This store is currently enabled and can be used in other parts of calibre.') + '</p>')
             elif col == 1:
                 return QVariant('<p>%s</p>' % result.description)
             elif col == 2:
                 if result.drm_free_only:
-                    return QVariant(_('<p>This store only distributes ebooks with DRM.</p>'))
+                    return QVariant('<p>' + _('This store only distributes ebooks with DRM.') + '</p>')
                 else:
-                    return QVariant(_('<p>This store distributes ebooks with DRM. It may have some titles without DRM, but you will need to check on a per title basis.</p>'))
+                    return QVariant('<p>' + _('This store distributes ebooks with DRM. It may have some titles without DRM, but you will need to check on a per title basis.') + '</p>')
             elif col == 3:
-                return QVariant(_('<p>This store is headquartered in %s. This is a good indication of what market the store caters to. However, this does not necessarily mean that the store is limited to that market only.</p>') % result.headquarters)
+                return QVariant('<p>' + _('This store is headquartered in %s. This is a good indication of what market the store caters to. However, this does not necessarily mean that the store is limited to that market only.') % result.headquarters + '</p>')
             elif col == 4:
-                return QVariant(_('<p>This store distributes ebooks in the following formats: %s</p>') % ', '.join(result.formats))
+                if result.affiliate:
+                    return QVariant('<p>' + _('Buying from this store supports the calibre developer: %s.') % result.author + '</p>')
+            elif col == 5:
+                return QVariant('<p>' + _('This store distributes ebooks in the following formats: %s') % ', '.join(result.formats) + '</p>')
         return NONE
 
     def setData(self, index, data, role):
@@ -148,6 +172,8 @@ class Matches(QAbstractItemModel):
             text = 'a' if getattr(match, 'drm_free_only', True) else 'b'
         elif col == 3:
             text = getattr(match, 'headquarters', '')
+        elif col == 4:
+            text = 'a' if getattr(match, 'affiliate', False) else 'b'
         return text
 
     def sort(self, col, order, reset=True):
@@ -167,6 +193,7 @@ class SearchFilter(SearchQueryParser):
 
     USABLE_LOCATIONS = [
         'all',
+        'affiliate',
         'description',
         'drm',
         'enabled',
@@ -207,6 +234,7 @@ class SearchFilter(SearchQueryParser):
         all_locs = set(self.USABLE_LOCATIONS) - set(['all'])
         locations = all_locs if location == 'all' else [location]
         q = {
+             'affiliate': lambda x: x.affiliate,
              'description': lambda x: x.description.lower(),
              'drm': lambda x: not x.drm_free_only,
              'enabled': lambda x: not is_disabled(x),
@@ -219,21 +247,21 @@ class SearchFilter(SearchQueryParser):
             for locvalue in locations:
                 accessor = q[locvalue]
                 if query == 'true':
-                    if locvalue in ('drm', 'enabled'):
+                    if locvalue in ('affiliate', 'drm', 'enabled'):
                         if accessor(sr) == True:
                             matches.add(sr)
                     elif accessor(sr) is not None:
                         matches.add(sr)
                     continue
                 if query == 'false':
-                    if locvalue in ('drm', 'enabled'):
+                    if locvalue in ('affiliate', 'drm', 'enabled'):
                         if accessor(sr) == False:
                             matches.add(sr)
                     elif accessor(sr) is None:
                         matches.add(sr)
                     continue
                 # this is bool, so can't match below
-                if locvalue in ('drm', 'enabled'):
+                if locvalue in ('affiliate', 'drm', 'enabled'):
                     continue
                 try:
                     ### Can't separate authors because comma is used for name sep and author sep

src/calibre/gui2/store/config/chooser/results_view.py

@@ -6,7 +6,9 @@ __license__ = 'GPL 3'
 __copyright__ = '2011, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
-from PyQt4.Qt import (Qt, QTreeView, QSize)
+from functools import partial
+
+from PyQt4.Qt import (Qt, QTreeView, QSize, QMenu)
 
 from calibre.customize.ui import store_plugins
 from calibre.gui2.metadata.single_download import RichTextDelegate
@@ -32,3 +34,20 @@ class ResultsView(QTreeView):
             
         self.model().sort(1, Qt.AscendingOrder)
         self.header().setSortIndicator(self.model().sort_col, self.model().sort_order)
+
+    def contextMenuEvent(self, event):
+        index = self.indexAt(event.pos())
+        
+        if not index.isValid():
+            return
+        
+        plugin = self.model().get_plugin(index)
+        
+        menu = QMenu()
+        ca = menu.addAction(_('Configure...'), partial(self.configure_plugin, plugin))
+        if not plugin.is_customizable():
+            ca.setEnabled(False)
+        menu.exec_(event.globalPos())
+
+    def configure_plugin(self, plugin):
+        plugin.do_user_config(self)

src/calibre/gui2/store/ebookshoppe_uk_plugin.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import urllib2
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class EBookShoppeUKStore(BasicStoreConfig, StorePlugin):
+
+    def open(self, parent=None, detail_item=None, external=False):
+        url_details = 'http://www.awin1.com/cread.php?awinmid=1414&awinaffid=120917&clickref=&p={0}'
+        url = 'http://www.awin1.com/awclick.php?mid=2666&id=120917'
+
+        if external or self.config.get('open_external', False):
+            if detail_item:
+                url = url_details.format(detail_item)
+            open_url(QUrl(url))
+        else:
+            detail_url = None
+            if detail_item:
+                detail_url = url_details.format(detail_item)
+            d = WebStoreDialog(self.gui, url, parent, detail_url)
+            d.setWindowTitle(self.name)
+            d.set_tags(self.config.get('tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        url = 'http://www.ebookshoppe.com/search.php?search_query=' + urllib2.quote(query)
+        br = browser()
+
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//ul[@class="ProductList"]/li'):
+                if counter <= 0:
+                    break
+
+                id = ''.join(data.xpath('./div[@class="ProductDetails"]/'
+                                        'strong/a/@href')).strip()
+                if not id:
+                    continue
+                cover_url = ''.join(data.xpath('./div[@class="ProductImage"]/a/img/@src'))
+                title = ''.join(data.xpath('./div[@class="ProductDetails"]/strong/a/text()'))
+                price = ''.join(data.xpath('./div[@class="ProductPriceRating"]/em/text()'))
+                counter -= 1
+
+                s = SearchResult()
+                s.cover_url = cover_url
+                s.title = title.strip()
+                s.price = price
+                s.drm = SearchResult.DRM_UNLOCKED
+                s.detail_item = id
+
+                self.get_author_and_formats(s, timeout)
+                if not s.author:
+                    continue
+
+                yield s
+
+    def get_author_and_formats(self, search_result, timeout):
+        br = browser()
+        with closing(br.open(search_result.detail_item, timeout=timeout)) as nf:
+            idata = html.fromstring(nf.read())
+            author = ''.join(idata.xpath('//div[@id="ProductOtherDetails"]/dl/dd[1]/text()'))
+            if author:
+                search_result.author = author
+            formats = idata.xpath('//dl[@class="ProductAddToCart"]/dd/'
+                                  'ul[@class="ProductOptionList"]/li/label/text()')
+            if formats:
+                search_result.formats = ', '.join(formats)
+            search_result.drm = SearchResult.DRM_UNKNOWN
+        return True

src/calibre/gui2/store/epubbud_plugin.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import urllib
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class EpubBudStore(BasicStoreConfig, StorePlugin):
+
+    def open(self, parent=None, detail_item=None, external=False):
+        url = 'http://epubbud.com/'
+
+        if external or self.config.get('open_external', False):
+            open_url(QUrl(url_slash_cleaner(detail_item if detail_item else url)))
+        else:
+            d = WebStoreDialog(self.gui, url, parent, detail_item)
+            d.setWindowTitle(self.name)
+            d.set_tags(self.config.get('tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        '''
+        OPDS based search.
+        
+        We really should get the catelog from http://pragprog.com/catalog.opds
+        and look for the application/opensearchdescription+xml entry.
+        Then get the opensearch description to get the search url and
+        format. However, we are going to be lazy and hard code it.
+        '''
+        url = 'http://www.epubbud.com/search.php?format=atom&q=' + urllib.quote_plus(query)
+        
+        br = browser()
+        
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            # Use html instead of etree as html allows us
+            # to ignore the namespace easily.
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//entry'):
+                if counter <= 0:
+                    break
+
+                id = ''.join(data.xpath('.//id/text()'))
+                if not id:
+                    continue
+
+                cover_url = ''.join(data.xpath('.//link[@rel="http://opds-spec.org/thumbnail"]/@href'))
+                
+                title = u''.join(data.xpath('.//title/text()'))
+                author = u''.join(data.xpath('.//author/name/text()'))
+
+                counter -= 1
+                
+                s = SearchResult()
+                s.cover_url = cover_url
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = '$0.00'
+                s.detail_item = id.strip()
+                s.drm = SearchResult.DRM_UNLOCKED
+                s.formats = 'EPUB'
+                
+                yield s

src/calibre/gui2/store/foyles_uk_plugin.py

@@ -23,12 +23,13 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog
 class FoylesUKStore(BasicStoreConfig, StorePlugin):
 
     def open(self, parent=None, detail_item=None, external=False):
-        url = 'http://www.awin1.com/cread.php?awinmid=1414&awinaffid=120917&clickref=&p='
+        url = 'http://www.awin1.com/awclick.php?mid=1414&id=120917'
+        detail_url = 'http://www.awin1.com/cread.php?awinmid=1414&awinaffid=120917&clickref=&p='
         url_redirect = 'http://www.foyles.co.uk'
 
         if external or self.config.get('open_external', False):
             if detail_item:
-                url = url + url_redirect + detail_item
+                url = detail_url + url_redirect + detail_item
             open_url(QUrl(url_slash_cleaner(url)))
         else:
             detail_url = None
@@ -54,6 +55,10 @@ class FoylesUKStore(BasicStoreConfig, StorePlugin):
                 if not id:
                     continue
 
+                # filter out the audio books
+                if not data.xpath('boolean(.//div[@class="Relative"]/ul/li[contains(text(), "ePub")])'):
+                    continue
+
                 cover_url = ''.join(data.xpath('.//a[@class="Jacket"]/img/@src'))
                 if cover_url:
                     cover_url = 'http://www.foyles.co.uk' + cover_url

src/calibre/gui2/store/manybooks_plugin.py

@@ -78,7 +78,8 @@ class ManyBooksStore(BasicStoreConfig, StorePlugin):
                     cover_name = mo.group()
                     cover_name = cover_name.replace('etext', '')
                     cover_id = id.split('.')[0]
-                    cover_url = 'http://manybooks_images.s3.amazonaws.com/original_covers/' + id[0] + '/' + cover_name + '/' + cover_id + '-thumb.jpg' 
+                    cover_url = 'http://www.manybooks.net/images/' + id[0] + '/' + cover_name + '/' + cover_id + '-thumb.jpg'
+                print(cover_url)
 
                 counter -= 1
                 

src/calibre/gui2/store/mobileread/adv_search_builder.py

@@ -0,0 +1,119 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import re
+
+from PyQt4.Qt import (QDialog, QDialogButtonBox)
+
+from calibre.gui2.store.mobileread.adv_search_builder_ui import Ui_Dialog
+from calibre.library.caches import CONTAINS_MATCH, EQUALS_MATCH
+
+class AdvSearchBuilderDialog(QDialog, Ui_Dialog):
+
+    def __init__(self, parent):
+        QDialog.__init__(self, parent)
+        self.setupUi(self)
+        
+        self.buttonBox.accepted.connect(self.advanced_search_button_pushed)
+        self.tab_2_button_box.accepted.connect(self.accept)
+        self.tab_2_button_box.rejected.connect(self.reject)
+        self.clear_button.clicked.connect(self.clear_button_pushed)
+        self.adv_search_used = False
+        self.mc = ''
+        
+        self.tabWidget.setCurrentIndex(0)
+        self.tabWidget.currentChanged[int].connect(self.tab_changed)
+        self.tab_changed(0)
+        
+    def tab_changed(self, idx):
+        if idx == 1:
+            self.tab_2_button_box.button(QDialogButtonBox.Ok).setDefault(True)
+        else:
+            self.buttonBox.button(QDialogButtonBox.Ok).setDefault(True)
+
+    def advanced_search_button_pushed(self):
+        self.adv_search_used = True
+        self.accept()
+
+    def clear_button_pushed(self):
+        self.title_box.setText('')
+        self.author_box.setText('')
+        self.format_box.setText('')
+
+    def tokens(self, raw):
+        phrases = re.findall(r'\s*".*?"\s*', raw)
+        for f in phrases:
+            raw = raw.replace(f, ' ')
+        phrases = [t.strip('" ') for t in phrases]
+        return ['"' + self.mc + t + '"' for t in phrases + [r.strip() for r in raw.split()]]
+
+    def search_string(self):
+        if self.adv_search_used:
+            return self.adv_search_string()
+        else:
+            return self.box_search_string()
+
+    def adv_search_string(self):
+        mk = self.matchkind.currentIndex()
+        if mk == CONTAINS_MATCH:
+            self.mc = ''
+        elif mk == EQUALS_MATCH:
+            self.mc = '='
+        else:
+            self.mc = '~'
+        all, any, phrase, none = map(lambda x: unicode(x.text()),
+                (self.all, self.any, self.phrase, self.none))
+        all, any, none = map(self.tokens, (all, any, none))
+        phrase = phrase.strip()
+        all = ' and '.join(all)
+        any = ' or '.join(any)
+        none = ' and not '.join(none)
+        ans = ''
+        if phrase:
+            ans += '"%s"'%phrase
+        if all:
+            ans += (' and ' if ans else '') + all
+        if none:
+            ans += (' and not ' if ans else 'not ') + none
+        if any:
+            ans += (' or ' if ans else '') + any
+        return ans
+
+    def token(self):
+        txt = unicode(self.text.text()).strip()
+        if txt:
+            if self.negate.isChecked():
+                txt = '!'+txt
+            tok = self.FIELDS[unicode(self.field.currentText())]+txt
+            if re.search(r'\s', tok):
+                tok = '"%s"'%tok
+            return tok
+
+    def box_search_string(self):
+        mk = self.matchkind.currentIndex()
+        if mk == CONTAINS_MATCH:
+            self.mc = ''
+        elif mk == EQUALS_MATCH:
+            self.mc = '='
+        else:
+            self.mc = '~'
+
+        ans = []
+        self.box_last_values = {}
+        title = unicode(self.title_box.text()).strip()
+        if title:
+            ans.append('title:"' + self.mc + title + '"')
+        author = unicode(self.author_box.text()).strip()
+        if author:
+            ans.append('author:"' + self.mc + author + '"')
+        format = unicode(self.format_box.text()).strip()
+        if format:
+            ans.append('format:"' + self.mc + format + '"')            
+        if ans:
+            return ' and '.join(ans)
+        return ''

src/calibre/gui2/store/mobileread/adv_search_builder.ui

@@ -0,0 +1,350 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Dialog</class>
+ <widget class="QDialog" name="Dialog">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>752</width>
+    <height>472</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Advanced Search</string>
+  </property>
+  <property name="windowIcon">
+   <iconset>
+    <normaloff>:/images/search.png</normaloff>:/images/search.png</iconset>
+  </property>
+  <layout class="QGridLayout" name="gridLayout_2">
+   <item row="0" column="0">
+    <widget class="QLabel" name="label_5">
+     <property name="text">
+      <string>&amp;What kind of match to use:</string>
+     </property>
+     <property name="buddy">
+      <cstring>matchkind</cstring>
+     </property>
+    </widget>
+   </item>
+   <item row="0" column="1">
+    <widget class="QComboBox" name="matchkind">
+     <item>
+      <property name="text">
+       <string>Contains: the word or phrase matches anywhere in the metadata field</string>
+      </property>
+     </item>
+     <item>
+      <property name="text">
+       <string>Equals: the word or phrase must match the entire metadata field</string>
+      </property>
+     </item>
+     <item>
+      <property name="text">
+       <string>Regular expression: the expression must match anywhere in the metadata field</string>
+      </property>
+     </item>
+    </widget>
+   </item>
+   <item row="2" column="0" colspan="2">
+    <widget class="QTabWidget" name="tabWidget">
+     <property name="currentIndex">
+      <number>0</number>
+     </property>
+     <widget class="QWidget" name="tab">
+      <attribute name="title">
+       <string>A&amp;dvanced Search</string>
+      </attribute>
+      <layout class="QGridLayout" name="gridLayout_3">
+       <item row="0" column="0">
+        <widget class="QGroupBox" name="groupBox">
+         <property name="title">
+          <string>Find entries that have...</string>
+         </property>
+         <layout class="QVBoxLayout" name="verticalLayout">
+          <item>
+           <layout class="QHBoxLayout" name="horizontalLayout">
+            <item>
+             <widget class="QLabel" name="label">
+              <property name="text">
+               <string>&amp;All these words:</string>
+              </property>
+              <property name="buddy">
+               <cstring>all</cstring>
+              </property>
+             </widget>
+            </item>
+            <item>
+             <widget class="QLineEdit" name="all"/>
+            </item>
+           </layout>
+          </item>
+          <item>
+           <layout class="QHBoxLayout" name="horizontalLayout_2">
+            <item>
+             <widget class="QLabel" name="label_2">
+              <property name="text">
+               <string>This exact &amp;phrase:</string>
+              </property>
+              <property name="buddy">
+               <cstring>all</cstring>
+              </property>
+             </widget>
+            </item>
+            <item>
+             <widget class="QLineEdit" name="phrase"/>
+            </item>
+           </layout>
+          </item>
+          <item>
+           <layout class="QHBoxLayout" name="horizontalLayout_3">
+            <item>
+             <widget class="QLabel" name="label_3">
+              <property name="text">
+               <string>&amp;One or more of these words:</string>
+              </property>
+              <property name="buddy">
+               <cstring>all</cstring>
+              </property>
+             </widget>
+            </item>
+            <item>
+             <widget class="QLineEdit" name="any"/>
+            </item>
+           </layout>
+          </item>
+         </layout>
+        </widget>
+       </item>
+       <item row="1" column="0">
+        <widget class="QGroupBox" name="groupBox_2">
+         <property name="title">
+          <string>But dont show entries that have...</string>
+         </property>
+         <layout class="QVBoxLayout" name="verticalLayout_2">
+          <item>
+           <layout class="QHBoxLayout" name="horizontalLayout_4">
+            <item>
+             <widget class="QLabel" name="label_4">
+              <property name="text">
+               <string>Any of these &amp;unwanted words:</string>
+              </property>
+              <property name="buddy">
+               <cstring>all</cstring>
+              </property>
+             </widget>
+            </item>
+            <item>
+             <widget class="QLineEdit" name="none"/>
+            </item>
+           </layout>
+          </item>
+          <item>
+           <widget class="QLabel" name="label_6">
+            <property name="maximumSize">
+             <size>
+              <width>16777215</width>
+              <height>30</height>
+             </size>
+            </property>
+            <property name="text">
+             <string>See the &lt;a href=&quot;http://calibre-ebook.com/user_manual/gui.html#the-search-interface&quot;&gt;User Manual&lt;/a&gt; for more help</string>
+            </property>
+            <property name="openExternalLinks">
+             <bool>true</bool>
+            </property>
+           </widget>
+          </item>
+         </layout>
+        </widget>
+       </item>
+       <item row="2" column="0">
+        <spacer name="verticalSpacer_2">
+         <property name="orientation">
+          <enum>Qt::Vertical</enum>
+         </property>
+         <property name="sizeHint" stdset="0">
+          <size>
+           <width>20</width>
+           <height>40</height>
+          </size>
+         </property>
+        </spacer>
+       </item>
+       <item row="3" column="0">
+        <widget class="QDialogButtonBox" name="buttonBox">
+         <property name="orientation">
+          <enum>Qt::Horizontal</enum>
+         </property>
+         <property name="standardButtons">
+          <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
+         </property>
+        </widget>
+       </item>
+      </layout>
+     </widget>
+     <widget class="QWidget" name="tab_2">
+      <attribute name="title">
+       <string>Titl&amp;e/Author/Price ...</string>
+      </attribute>
+      <layout class="QGridLayout" name="gridLayout">
+       <item row="1" column="0">
+        <widget class="QLabel" name="label_7">
+         <property name="text">
+          <string>&amp;Title:</string>
+         </property>
+         <property name="buddy">
+          <cstring>title_box</cstring>
+         </property>
+        </widget>
+       </item>
+       <item row="1" column="1">
+        <widget class="EnLineEdit" name="title_box">
+         <property name="toolTip">
+          <string>Enter the title.</string>
+         </property>
+        </widget>
+       </item>
+       <item row="2" column="0">
+        <widget class="QLabel" name="label_8">
+         <property name="text">
+          <string>&amp;Author:</string>
+         </property>
+         <property name="buddy">
+          <cstring>author_box</cstring>
+         </property>
+        </widget>
+       </item>
+       <item row="5" column="0" colspan="2">
+        <layout class="QHBoxLayout" name="horizontalLayout_6">
+         <item>
+          <widget class="QPushButton" name="clear_button">
+           <property name="text">
+            <string>&amp;Clear</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QDialogButtonBox" name="tab_2_button_box">
+           <property name="standardButtons">
+            <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
+           </property>
+          </widget>
+         </item>
+        </layout>
+       </item>
+       <item row="4" column="1">
+        <spacer name="verticalSpacer">
+         <property name="orientation">
+          <enum>Qt::Vertical</enum>
+         </property>
+         <property name="sizeHint" stdset="0">
+          <size>
+           <width>20</width>
+           <height>40</height>
+          </size>
+         </property>
+        </spacer>
+       </item>
+       <item row="0" column="0" colspan="2">
+        <widget class="QLabel" name="label_11">
+         <property name="text">
+          <string>Search only in specific fields:</string>
+         </property>
+        </widget>
+       </item>
+       <item row="2" column="1">
+        <widget class="EnLineEdit" name="author_box"/>
+       </item>
+       <item row="3" column="1">
+        <widget class="QLineEdit" name="format_box"/>
+       </item>
+       <item row="3" column="0">
+        <widget class="QLabel" name="label_10">
+         <property name="text">
+          <string>&amp;Format:</string>
+         </property>
+         <property name="buddy">
+          <cstring>format_box</cstring>
+         </property>
+        </widget>
+       </item>
+      </layout>
+     </widget>
+    </widget>
+   </item>
+   <item row="1" column="1">
+    <spacer name="verticalSpacer_3">
+     <property name="orientation">
+      <enum>Qt::Vertical</enum>
+     </property>
+     <property name="sizeHint" stdset="0">
+      <size>
+       <width>20</width>
+       <height>40</height>
+      </size>
+     </property>
+    </spacer>
+   </item>
+  </layout>
+ </widget>
+ <customwidgets>
+  <customwidget>
+   <class>EnLineEdit</class>
+   <extends>QLineEdit</extends>
+   <header>widgets.h</header>
+  </customwidget>
+ </customwidgets>
+ <tabstops>
+  <tabstop>all</tabstop>
+  <tabstop>phrase</tabstop>
+  <tabstop>any</tabstop>
+  <tabstop>none</tabstop>
+  <tabstop>buttonBox</tabstop>
+  <tabstop>title_box</tabstop>
+  <tabstop>author_box</tabstop>
+  <tabstop>format_box</tabstop>
+  <tabstop>clear_button</tabstop>
+  <tabstop>tab_2_button_box</tabstop>
+  <tabstop>tabWidget</tabstop>
+  <tabstop>matchkind</tabstop>
+ </tabstops>
+ <resources>
+  <include location="../../../../resources/images.qrc"/>
+ </resources>
+ <connections>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>accepted()</signal>
+   <receiver>Dialog</receiver>
+   <slot>accept()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>248</x>
+     <y>254</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>157</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>rejected()</signal>
+   <receiver>Dialog</receiver>
+   <slot>reject()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>316</x>
+     <y>260</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>286</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
+</ui>

src/calibre/gui2/store/mobileread/mobileread_plugin.py

@@ -18,7 +18,7 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog
 from calibre.gui2.store.mobileread.models import SearchFilter
 from calibre.gui2.store.mobileread.cache_progress_dialog import CacheProgressDialog
 from calibre.gui2.store.mobileread.cache_update_thread import CacheUpdateThread
-from calibre.gui2.store.mobileread.store_dialog import MobeReadStoreDialog
+from calibre.gui2.store.mobileread.store_dialog import MobileReadStoreDialog
 
 class MobileReadStore(BasicStoreConfig, StorePlugin):
     
@@ -38,7 +38,7 @@ class MobileReadStore(BasicStoreConfig, StorePlugin):
                 d.exec_()
             else:
                 self.update_cache(parent, 30)
-                d = MobeReadStoreDialog(self, parent)
+                d = MobileReadStoreDialog(self, parent)
                 d.setWindowTitle(self.name)
                 d.exec_()
 

src/calibre/gui2/store/mobileread/store_dialog.py

@@ -9,11 +9,11 @@ __docformat__ = 'restructuredtext en'
 
 from PyQt4.Qt import (Qt, QDialog, QIcon)
 
-from calibre.gui2.store.search.adv_search_builder import AdvSearchBuilderDialog
+from calibre.gui2.store.mobileread.adv_search_builder import AdvSearchBuilderDialog
 from calibre.gui2.store.mobileread.models import BooksModel
 from calibre.gui2.store.mobileread.store_dialog_ui import Ui_Dialog
 
-class MobeReadStoreDialog(QDialog, Ui_Dialog):
+class MobileReadStoreDialog(QDialog, Ui_Dialog):
     
     def __init__(self, plugin, *args):
         QDialog.__init__(self, *args)
@@ -49,8 +49,6 @@ class MobeReadStoreDialog(QDialog, Ui_Dialog):
     
     def build_adv_search(self):
         adv = AdvSearchBuilderDialog(self)
-        adv.price_label.hide()
-        adv.price_box.hide()
         if adv.exec_() == QDialog.Accepted:
             self.search_query.setText(adv.search_string())
     

src/calibre/gui2/store/search/adv_search_builder.py

@@ -45,6 +45,7 @@ class AdvSearchBuilderDialog(QDialog, Ui_Dialog):
         self.author_box.setText('')
         self.price_box.setText('')
         self.format_box.setText('')
+        self.affiliate_combo.setCurrentIndex(0)
 
     def tokens(self, raw):
         phrases = re.findall(r'\s*".*?"\s*', raw)
@@ -118,6 +119,9 @@ class AdvSearchBuilderDialog(QDialog, Ui_Dialog):
         format = unicode(self.format_box.text()).strip()
         if format:
             ans.append('format:"' + self.mc + format + '"')
+        affiliate = unicode(self.affiliate_combo.currentText()).strip()
+        if affiliate:
+            ans.append('affiliate:' + affiliate)  
         if ans:
             return ' and '.join(ans)
         return ''

src/calibre/gui2/store/search/adv_search_builder.ui

@@ -149,7 +149,7 @@
              </size>
             </property>
             <property name="text">
-             <string>See the &lt;a href=&quot;http://calibre-ebook.com/user_manual/gui.html#the-search-interface&quot;&gt;User Manual&lt;/a&gt; for more help</string>
+             <string>See the &lt;a href=&quot;http://manual.calibre-ebook.com/gui.html#the-search-interface&quot;&gt;User Manual&lt;/a&gt; for more help</string>
             </property>
             <property name="openExternalLinks">
              <bool>true</bool>
@@ -226,7 +226,7 @@
          </property>
         </widget>
        </item>
-       <item row="6" column="0" colspan="2">
+       <item row="7" column="0" colspan="2">
         <layout class="QHBoxLayout" name="horizontalLayout_6">
          <item>
           <widget class="QPushButton" name="clear_button">
@@ -244,7 +244,7 @@
          </item>
         </layout>
        </item>
-       <item row="5" column="1">
+       <item row="6" column="1">
         <spacer name="verticalSpacer">
          <property name="orientation">
           <enum>Qt::Vertical</enum>
@@ -283,6 +283,32 @@
        <item row="3" column="1">
         <widget class="EnLineEdit" name="price_box"/>
        </item>
+       <item row="5" column="0">
+        <widget class="QLabel" name="label_9">
+         <property name="text">
+          <string>Affiliate:</string>
+         </property>
+        </widget>
+       </item>
+       <item row="5" column="1">
+        <widget class="QComboBox" name="affiliate_combo">
+         <item>
+          <property name="text">
+           <string/>
+          </property>
+         </item>
+         <item>
+          <property name="text">
+           <string>true</string>
+          </property>
+         </item>
+         <item>
+          <property name="text">
+           <string>false</string>
+          </property>
+         </item>
+        </widget>
+       </item>
       </layout>
      </widget>
     </widget>

src/calibre/gui2/store/search/download_thread.py

@@ -120,6 +120,8 @@ class SearchThread(Thread):
                     if not self._run:
                         return
                     res.store_name = store_name
+                    res.affiliate = store_plugin.base_plugin.affiliate
+                    res.plugin_author = store_plugin.base_plugin.author
                     self.results.put((res, store_plugin))
                 self.tasks.task_done()
             except:

src/calibre/gui2/store/search/models.py

@@ -12,7 +12,7 @@ from operator import attrgetter
 from PyQt4.Qt import (Qt, QAbstractItemModel, QVariant, QPixmap, QModelIndex, QSize,
                       pyqtSignal)
 
-from calibre.gui2 import NONE
+from calibre.gui2 import NONE, FunctionDispatcher
 from calibre.gui2.store.search_result import SearchResult
 from calibre.gui2.store.search.download_thread import DetailsThreadPool, \
     CoverThreadPool
@@ -33,7 +33,7 @@ class Matches(QAbstractItemModel):
 
     total_changed = pyqtSignal(int)
 
-    HEADERS = [_('Cover'), _('Title'), _('Price'), _('DRM'), _('Store')]
+    HEADERS = [_('Cover'), _('Title'), _('Price'), _('DRM'), _('Store'), _('')]
     HTML_COLS = (1, 4)
 
     def __init__(self, cover_thread_count=2, detail_thread_count=4):
@@ -45,6 +45,8 @@ class Matches(QAbstractItemModel):
                 Qt.SmoothTransformation)
         self.DRM_UNKNOWN_ICON = QPixmap(I('dialog_question.png')).scaledToHeight(64,
                 Qt.SmoothTransformation)
+        self.DONATE_ICON = QPixmap(I('donate.png')).scaledToHeight(16,
+                Qt.SmoothTransformation)
 
         # All matches. Used to determine the order to display
         # self.matches because the SearchFilter returns
@@ -57,6 +59,9 @@ class Matches(QAbstractItemModel):
         self.cover_pool = CoverThreadPool(cover_thread_count)
         self.details_pool = DetailsThreadPool(detail_thread_count)
         
+        self.filter_results_dispatcher = FunctionDispatcher(self.filter_results)
+        self.got_result_details_dispatcher = FunctionDispatcher(self.got_result_details)
+
         self.sort_col = 2
         self.sort_order = Qt.AscendingOrder
 
@@ -82,10 +87,10 @@ class Matches(QAbstractItemModel):
             self.search_filter.add_search_result(result)
             if result.cover_url:
                 result.cover_queued = True
-                self.cover_pool.add_task(result, self.filter_results)
+                self.cover_pool.add_task(result, self.filter_results_dispatcher)
             else:
                 result.cover_queued = False
-            self.details_pool.add_task(result, store_plugin, self.got_result_details)
+            self.details_pool.add_task(result, store_plugin, self.got_result_details_dispatcher)
             self.filter_results()
             self.layoutChanged.emit()
 
@@ -112,7 +117,7 @@ class Matches(QAbstractItemModel):
     def got_result_details(self, result):
         if not result.cover_queued and result.cover_url:
             result.cover_queued = True
-            self.cover_pool.add_task(result, self.filter_results)
+            self.cover_pool.add_task(result, self.filter_results_dispatcher)
         if result in self.matches:
             row = self.matches.index(result)
             self.dataChanged.emit(self.index(row, 0), self.index(row, self.columnCount() - 1))
@@ -150,6 +155,8 @@ class Matches(QAbstractItemModel):
 
     def data(self, index, role):
         row, col = index.row(), index.column()
+        if row >= len(self.matches):
+            return NONE
         result = self.matches[row]
         if role == Qt.DisplayRole:
             if col == 1:
@@ -173,6 +180,10 @@ class Matches(QAbstractItemModel):
                     return QVariant(self.DRM_UNLOCKED_ICON)
                 elif result.drm == SearchResult.DRM_UNKNOWN:
                     return QVariant(self.DRM_UNKNOWN_ICON)
+            if col == 5:
+                if result.affiliate:
+                    return QVariant(self.DONATE_ICON)
+                return NONE
         elif role == Qt.ToolTipRole:
             if col == 1:
                 return QVariant('<p>%s</p>' % result.title)
@@ -187,6 +198,9 @@ class Matches(QAbstractItemModel):
                     return QVariant('<p>' + _('The DRM status of this book could not be determined. There is a very high likelihood that this book is actually DRM restricted.') + '</p>')
             elif col == 4:
                 return QVariant('<p>%s</p>' % result.formats)
+            elif col == 5:
+                if result.affiliate:
+                    return QVariant('<p>' + _('Buying from this store supports the calibre developer: %s.') % result.plugin_author + '</p>')
         elif role == Qt.SizeHintRole:
             return QSize(64, 64)
         return NONE
@@ -206,6 +220,11 @@ class Matches(QAbstractItemModel):
                 text = 'c'
         elif col == 4:
             text = result.store_name
+        elif col == 5:
+            if result.affiliate:
+                text = 'a'
+            else:
+                text = 'b'
         return text
 
     def sort(self, col, order, reset=True):
@@ -234,6 +253,7 @@ class SearchFilter(SearchQueryParser):
 
     USABLE_LOCATIONS = [
         'all',
+        'affiliate',
         'author',
         'authors',
         'cover',
@@ -284,6 +304,7 @@ class SearchFilter(SearchQueryParser):
         all_locs = set(self.USABLE_LOCATIONS) - set(['all'])
         locations = all_locs if location == 'all' else [location]
         q = {
+             'affiliate': attrgetter('affiliate'),
              'author': lambda x: x.author.lower(),
              'cover': attrgetter('cover_url'),
              'drm': attrgetter('drm'),
@@ -298,23 +319,35 @@ class SearchFilter(SearchQueryParser):
             for locvalue in locations:
                 accessor = q[locvalue]
                 if query == 'true':
-                    if locvalue == 'drm':
+                    # True/False.
+                    if locvalue == 'affiliate':
+                        if accessor(sr):
+                            matches.add(sr)
+                    # Special that are treated as True/False.
+                    elif locvalue == 'drm':
                         if accessor(sr) == SearchResult.DRM_LOCKED:
                             matches.add(sr)
+                    # Testing for something or nothing.
                     else:
                         if accessor(sr) is not None:
                             matches.add(sr)
                     continue
                 if query == 'false':
-                    if locvalue == 'drm':
+                    # True/False.
+                    if locvalue == 'affiliate':
+                        if not accessor(sr):
+                            matches.add(sr)
+                    # Special that are treated as True/False.
+                    elif locvalue == 'drm':
                         if accessor(sr) == SearchResult.DRM_UNLOCKED:
                             matches.add(sr)
+                    # Testing for something or nothing.
                     else:
                         if accessor(sr) is None:
                             matches.add(sr)
                     continue
-                # this is bool, so can't match below
+                # this is bool or treated as bool, so can't match below.
-                if locvalue == 'drm':
+                if locvalue in ('affiliate', 'drm'):
                     continue
                 try:
                     ### Can't separate authors because comma is used for name sep and author sep

src/calibre/gui2/store/search/search.py

@@ -9,8 +9,8 @@ __docformat__ = 'restructuredtext en'
 import re
 from random import shuffle
 
-from PyQt4.Qt import (Qt, QDialog, QDialogButtonBox, QTimer, QCheckBox,
+from PyQt4.Qt import (Qt, QDialog, QDialogButtonBox, QTimer, QCheckBox, QLabel,
-                      QVBoxLayout, QIcon, QWidget, QTabWidget)
+                      QVBoxLayout, QIcon, QWidget, QTabWidget, QGridLayout)
 
 from calibre.gui2 import JSONConfig, info_dialog
 from calibre.gui2.progress_indicator import ProgressIndicator
@@ -92,14 +92,21 @@ class SearchDialog(QDialog, Ui_Dialog):
         self.store_checks = {}
 
         stores_check_widget = QWidget()
-        store_list_layout = QVBoxLayout()
+        store_list_layout = QGridLayout()
         stores_check_widget.setLayout(store_list_layout)
-        for x in sorted(self.gui.istores.keys(), key=lambda x: x.lower()):
+
+        icon = QIcon(I('donate.png'))
+        for i, x in enumerate(sorted(self.gui.istores.keys(), key=lambda x: x.lower())):
             cbox = QCheckBox(x)
             cbox.setChecked(existing.get(x, False))
-            store_list_layout.addWidget(cbox)
+            store_list_layout.addWidget(cbox, i, 0, 1, 1)
+            if self.gui.istores[x].base_plugin.affiliate:
+                iw = QLabel(self)
+                iw.setToolTip('<p>' + _('Buying from this store supports the calibre developer: %s</p>') % self.gui.istores[x].base_plugin.author + '</p>')
+                iw.setPixmap(icon.pixmap(16, 16))
+                store_list_layout.addWidget(iw, i, 1, 1, 1)
             self.store_checks[x] = cbox
-        store_list_layout.addStretch()
+        store_list_layout.setRowStretch(store_list_layout.rowCount(), 10)
         self.store_list.setWidget(stores_check_widget)
 
     def build_adv_search(self):
@@ -244,6 +251,8 @@ class SearchDialog(QDialog, Ui_Dialog):
         # search widget.
         self.config['open_external'] = self.open_external.isChecked()
 
+        # Create the config dialog. It's going to put two config widgets
+        # into a QTabWidget for displaying all of the settings.
         d = QDialog(self)
         button_box = QDialogButtonBox(QDialogButtonBox.Close)
         v = QVBoxLayout(d)
@@ -261,7 +270,22 @@ class SearchDialog(QDialog, Ui_Dialog):
         tab_widget.addTab(chooser_config_widget, _('Choose stores'))
         tab_widget.addTab(search_config_widget, _('Configure search'))
 
+        # Restore dialog state.
+        geometry = self.config.get('config_dialog_geometry', None)
+        if geometry:
+            d.restoreGeometry(geometry)
+        else:
+            d.resize(800, 600)
+        tab_index = self.config.get('config_dialog_tab_index', 0)
+        tab_index = min(tab_index, tab_widget.count() - 1)
+        tab_widget.setCurrentIndex(tab_index)
+
         d.exec_()
+        
+        # Save dialog state.
+        self.config['config_dialog_geometry'] = bytearray(d.saveGeometry())
+        self.config['config_dialog_tab_index'] = tab_widget.currentIndex()
+        
         search_config_widget.save_settings()
         self.config_changed()
         self.gui.load_store_plugins()

src/calibre/gui2/store/search_result.py

@@ -22,6 +22,8 @@ class SearchResult(object):
         self.detail_item = ''
         self.drm = None
         self.formats = ''
+        self.affiliate = False
+        self.plugin_author = ''
 
     def __eq__(self, other):
         return self.title == other.title and self.author == other.author and self.store_name == other.store_name

src/calibre/gui2/store/whsmith_uk_plugin.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import urllib2
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class WHSmithUKStore(BasicStoreConfig, StorePlugin):
+
+    def open(self, parent=None, detail_item=None, external=False):
+        url = 'http://www.whsmith.co.uk/'
+        url_details = ''
+
+        if external or self.config.get('open_external', False):
+            if detail_item:
+                url = url_details + detail_item
+            open_url(QUrl(url))
+        else:
+            detail_url = None
+            if detail_item:
+                detail_url = url_details + detail_item
+            d = WebStoreDialog(self.gui, url, parent, detail_url)
+            d.setWindowTitle(self.name)
+            d.set_tags(self.config.get('tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        url = ('http://www.whsmith.co.uk/CatalogAndSearch/SearchWithinCategory.aspx'
+               '?cat=\Books\eb_eBooks&gq=' + urllib2.quote(query))
+
+        br = browser()
+
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//div[@class="product-search"]/'
+                                    'div[contains(@id, "whsSearchResultItem")]'):
+                if counter <= 0:
+                    break
+
+                id = ''.join(data.xpath('.//a[contains(@id, "labelProductTitle")]/@href'))
+                if not id:
+                    continue
+                cover_url = ''.join(data.xpath('.//a[contains(@id, "hlinkProductImage")]/img/@src'))
+                title = ''.join(data.xpath('.//a[contains(@id, "labelProductTitle")]/text()'))
+                author = ', '.join(data.xpath('.//div[@class="author"]/h3/span/text()'))
+                price = ''.join(data.xpath('.//span[contains(@id, "labelProductPrice")]/text()'))
+                pdf = data.xpath('boolean(.//span[contains(@id, "labelFormatText") and '
+                                                 'contains(., "PDF")])')
+                epub = data.xpath('boolean(.//span[contains(@id, "labelFormatText") and '
+                                                  'contains(., "ePub")])')
+                counter -= 1
+
+                s = SearchResult()
+                s.cover_url = cover_url
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price
+                s.drm = SearchResult.DRM_LOCKED
+                s.detail_item = id
+                formats = []
+                if epub:
+                    formats.append('ePub')
+                if pdf:
+                    formats.append('PDF')
+                s.formats = ', '.join(formats)
+
+                yield s

src/calibre/gui2/update.py

@@ -49,10 +49,12 @@ class UpdateNotification(QDialog):
         self.logo.setMaximumWidth(110)
         self.logo.setPixmap(QPixmap(I('lt.png')).scaled(100, 100,
             Qt.IgnoreAspectRatio, Qt.SmoothTransformation))
-        self.label = QLabel('<p>'+
+        self.label = QLabel(('<p>'+
             _('%s has been updated to version <b>%s</b>. '
             'See the <a href="http://calibre-ebook.com/whats-new'
-            '">new features</a>.')%(__appname__, version))
+            '">new features</a>.') + '<p>'+_('Update <b>only</b> if one of the '
+            'new features or bug fixes is important to you. '
+            'If the current version works well for you, do not update.'))%(__appname__, version))
         self.label.setOpenExternalLinks(True)
         self.label.setWordWrap(True)
         self.setWindowTitle(_('Update available!'))

src/calibre/gui2/wizard/finish.ui

@@ -62,7 +62,7 @@
    <item>
     <widget class="QLabel" name="label_3">
      <property name="text">
-      <string>&lt;h2&gt;User Manual&lt;/h2&gt;A User Manual is also available &lt;a href=&quot;http://calibre-ebook.com/user_manual&quot;&gt;online&lt;/a&gt;.</string>
+      <string>&lt;h2&gt;User Manual&lt;/h2&gt;A User Manual is also available &lt;a href=&quot;http://manual.calibre-ebook.com&quot;&gt;online&lt;/a&gt;.</string>
      </property>
      <property name="wordWrap">
       <bool>true</bool>

src/calibre/library/caches.py

@@ -200,6 +200,11 @@ class CacheRow(list): # {{{
     def __getslice__(self, i, j):
         return self.__getitem__(slice(i, j))
 
+    def refresh_composites(self):
+        for c in self._composites:
+            self[c] =  None
+        self._must_do = True
+
 # }}}
 
 class ResultCache(SearchQueryParser): # {{{
@@ -918,6 +923,7 @@ class ResultCache(SearchQueryParser): # {{{
         for item in self._data:
             if item is not None:
                 item[ondevice_col] = db.book_on_device_string(item[0])
+                item.refresh_composites()
 
     def refresh(self, db, field=None, ascending=True):
         temp = db.conn.get('SELECT * FROM meta2')

src/calibre/library/coloring.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+from future_builtins import map
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import binascii, re, json
+from textwrap import dedent
+
+class Rule(object): # {{{
+
+    SIGNATURE = '# BasicColorRule():'
+
+    def __init__(self, fm, color=None):
+        self.color = color
+        self.fm = fm
+        self.conditions = []
+
+    def add_condition(self, col, action, val):
+        if col not in self.fm:
+            raise ValueError('%r is not a valid column name'%col)
+        v = self.validate_condition(col, action, val)
+        if v:
+            raise ValueError(v)
+        self.conditions.append((col, action, val))
+
+    def validate_condition(self, col, action, val):
+        m = self.fm[col]
+        dt = m['datatype']
+        if (dt in ('int', 'float', 'rating') and action in ('lt', 'eq', 'gt')):
+            try:
+                int(val) if dt == 'int' else float(val)
+            except:
+                return '%r is not a valid numerical value'%val
+
+        if (dt in ('comments', 'series', 'text', 'enumeration') and 'pattern'
+                in action):
+            try:
+                re.compile(val)
+            except:
+                return '%r is not a valid regular expression'%val
+
+    @property
+    def signature(self):
+        args = (self.color, self.conditions)
+        sig = json.dumps(args, ensure_ascii=False)
+        return self.SIGNATURE + binascii.hexlify(sig.encode('utf-8'))
+
+    @property
+    def template(self):
+        if not self.color or not self.conditions:
+            return None
+        conditions = map(self.apply_condition, self.conditions)
+        conditions = (',\n' + ' '*9).join(conditions)
+        return dedent('''\
+                program:
+                {sig}
+                test(and(
+                         {conditions}
+                    ), '{color}', '');
+                ''').format(sig=self.signature, conditions=conditions,
+                        color=self.color)
+
+    def apply_condition(self, condition):
+        col, action, val = condition
+        m = self.fm[col]
+        dt = m['datatype']
+
+        if col == 'ondevice':
+            return self.ondevice_condition(col, action, val)
+
+        if col == 'identifiers':
+            return self.identifiers_condition(col, action, val)
+
+        if dt == 'bool':
+            return self.bool_condition(col, action, val)
+
+        if dt in ('int', 'float', 'rating'):
+            return self.number_condition(col, action, val)
+
+        if dt == 'datetime':
+            return self.date_condition(col, action, val)
+
+        if dt in ('comments', 'series', 'text', 'enumeration'):
+            ism = m.get('is_multiple', False)
+            if ism:
+                return self.multiple_condition(col, action, val, ism)
+            return self.text_condition(col, action, val)
+
+    def identifiers_condition(self, col, action, val):
+        if action == 'has id':
+            return "identifier_in_list(field('identifiers'), '%s', '1', '')"
+        return "identifier_in_list(field('identifiers'), '%s', '', '1')"
+
+    def ondevice_condition(self, col, action, val):
+        if action == 'is set':
+            return "test('%s', '1', '')"%col
+        if action == 'is not set':
+            return "test('%s', '', '1')"%col
+
+    def bool_condition(self, col, action, val):
+        test = {'is true': 'True',
+                'is false': 'False',
+                'is undefined': 'None'}[action]
+        return "strcmp('%s', raw_field('%s'), '', '1', '')"%(test, col)
+
+    def number_condition(self, col, action, val):
+        lt, eq, gt = {
+                'eq': ('', '1', ''),
+                'lt': ('1', '', ''),
+                'gt': ('', '', '1')
+        }[action]
+        lt, eq, gt = '', '1', ''
+        return "cmp(raw_field('%s'), %s, '%s', '%s', '%s')" % (col, val, lt, eq, gt)
+
+    def date_condition(self, col, action, val):
+        lt, eq, gt = {
+                'eq': ('', '1', ''),
+                'lt': ('1', '', ''),
+                'gt': ('', '', '1')
+        }[action]
+        return "cmp(format_date(raw_field('%s'), 'yyyy-MM-dd'), %s, '%s', '%s', '%s')" % (col,
+                val, lt, eq, gt)
+
+    def multiple_condition(self, col, action, val, sep):
+        if action == 'is set':
+            return "test('%s', '1', '')"%col
+        if action == 'is not set':
+            return "test('%s', '', '1')"%col
+        if action == 'has':
+            return "str_in_list(field('%s'), '%s', \"%s\", '1', '')"%(col, sep, val)
+        if action == 'does not have':
+            return "str_in_list(field('%s'), '%s', \"%s\", '', '1')"%(col, sep, val)
+        if action == 'has pattern':
+            return "in_list(field('%s'), '%s', \"%s\", '1', '')"%(col, sep, val)
+        if action == 'does not have pattern':
+            return "in_list(field('%s'), '%s', \"%s\", '', '1')"%(col, sep, val)
+
+    def text_condition(self, col, action, val):
+        if action == 'is set':
+            return "test('%s', '1', '')"%col
+        if action == 'is not set':
+            return "test('%s', '', '1')"%col
+        if action == 'is':
+            return "strcmp(field('%s'), \"%s\", '', '1', '')"%(col, val)
+        if action == 'is not':
+            return "strcmp(field('%s'), \"%s\", '1', '', '1')"%(col, val)
+        if action == 'matches pattern':
+            return "contains(field('%s'), \"%s\", '1', '')"%(col, val)
+        if action == 'does not match pattern':
+            return "contains(field('%s'), \"%s\", '', '1')"%(col, val)
+
+# }}}
+
+def rule_from_template(fm, template):
+    ok_lines = []
+    for line in template.splitlines():
+        if line.startswith(Rule.SIGNATURE):
+            raw = line[len(Rule.SIGNATURE):].strip()
+            try:
+                color, conditions = json.loads(binascii.unhexlify(raw).decode('utf-8'))
+            except:
+                continue
+            r = Rule(fm)
+            r.color = color
+            for c in conditions:
+                try:
+                    r.add_condition(*c)
+                except:
+                    continue
+            if r.color and r.conditions:
+                return r
+        else:
+            ok_lines.append(line)
+    return '\n'.join(ok_lines)
+
+def conditionable_columns(fm):
+    for key in fm:
+        m = fm[key]
+        dt = m['datatype']
+        if m.get('name', False) and dt in ('bool', 'int', 'float', 'rating', 'series',
+                'comments', 'text', 'enumeration', 'datetime'):
+            if key == 'sort':
+                yield 'title_sort'
+            else:
+                yield key
+
+def displayable_columns(fm):
+    for key in fm.displayable_field_keys():
+        if key not in ('sort', 'author_sort', 'comments', 'formats',
+                'identifiers', 'path'):
+            yield key
+
+def migrate_old_rule(fm, template):
+    if template.startswith('program:\n#tag wizard'):
+        rules = []
+        for line in template.splitlines():
+            if line.startswith('#') and ':|:' in line:
+                value, color = line[1:].split(':|:')
+                r = Rule(fm, color=color)
+                r.add_condition('tags', 'has', value)
+                rules.append(r.template)
+        return rules
+    return [template]
+

src/calibre/library/database2.py

@@ -211,10 +211,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         defs = self.prefs.defaults
         defs['gui_restriction'] = defs['cs_restriction'] = ''
         defs['categories_using_hierarchy'] = []
-        self.column_color_count = 5
+        defs['column_color_rules'] = []
-        for i in range(1,self.column_color_count+1):
-            defs['column_color_name_'+str(i)] = ''
-            defs['column_color_template_'+str(i)] = ''
 
         # Migrate the bool tristate tweak
         defs['bools_are_tristate'] = \
@@ -222,6 +219,24 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if self.prefs.get('bools_are_tristate') is None:
             self.prefs.set('bools_are_tristate', defs['bools_are_tristate'])
 
+        # Migrate column coloring rules
+        if self.prefs.get('column_color_name_1', None) is not None:
+            from calibre.library.coloring import migrate_old_rule
+            old_rules = []
+            for i in range(1, 6):
+                col = self.prefs.get('column_color_name_'+str(i), None)
+                templ = self.prefs.get('column_color_template_'+str(i), None)
+                if col and templ:
+                    try:
+                        del self.prefs['column_color_name_'+str(i)]
+                        rules = migrate_old_rule(self.field_metadata, templ)
+                        for templ in rules:
+                            old_rules.append((col, templ))
+                    except:
+                        pass
+            if old_rules:
+                self.prefs['column_color_rules'] += old_rules
+
         # Migrate saved search and user categories to db preference scheme
         def migrate_preference(key, default):
             oldval = prefs[key]
@@ -860,6 +875,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         mi.uuid        = row[fm['uuid']]
         mi.title_sort  = row[fm['sort']]
         mi.book_size   = row[fm['size']]
+        mi.ondevice_col= row[fm['ondevice']]
         mi.last_modified = row[fm['last_modified']]
         formats = row[fm['formats']]
         if not formats:
@@ -1555,13 +1571,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             if ids is not None:
                 count = self.conn.get('''SELECT COUNT(id)
                                        FROM data
-                                       WHERE format="%s" AND
+                                       WHERE format=? AND
-                                       books_list_filter(book)'''%fmt,
+                                       books_list_filter(book)''', (fmt,),
                                        all=False)
             else:
                 count = self.conn.get('''SELECT COUNT(id)
                                        FROM data
-                                       WHERE format="%s"'''%fmt,
+                                       WHERE format=?''', (fmt,),
                                        all=False)
             if count > 0:
                 categories['formats'].append(Tag(fmt, count=count, icon=icon,
@@ -1583,13 +1599,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             if ids is not None:
                 count = self.conn.get('''SELECT COUNT(book)
                                        FROM identifiers
-                                       WHERE type="%s" AND
+                                       WHERE type=? AND
-                                       books_list_filter(book)'''%ident,
+                                       books_list_filter(book)''', (ident,),
                                        all=False)
             else:
                 count = self.conn.get('''SELECT COUNT(id)
                                        FROM identifiers
-                                       WHERE type="%s"'''%ident,
+                                       WHERE type=?''', (ident,),
                                        all=False)
             if count > 0:
                 categories['identifiers'].append(Tag(ident, count=count, icon=icon,

src/calibre/library/field_metadata.py

@@ -374,6 +374,8 @@ class FieldMetadata(dict):
         self.get = self._tb_cats.get
 
     def __getitem__(self, key):
+        if key == 'title_sort':
+            return self._tb_cats['sort']
         return self._tb_cats[key]
 
     def __setitem__(self, key, val):
@@ -390,6 +392,8 @@ class FieldMetadata(dict):
         return self.has_key(key)
 
     def has_key(self, key):
+        if key == 'title_sort':
+            return True
         return key in self._tb_cats
 
     def keys(self):

src/calibre/library/save_to_disk.py

@@ -134,6 +134,8 @@ class SafeFormat(TemplateFormatter):
     '''
 
     def get_value(self, key, args, kwargs):
+        if key == '':
+            return ''
         try:
             key = key.lower()
             try:

src/calibre/manual/conf.py

@@ -43,7 +43,7 @@ language = 'en'
 
 # General substitutions.
 project = __appname__
-copyright = '2008, Kovid Goyal'
+copyright = 'Kovid Goyal'
 
 # The default replacements for |version| and |release|, also used in various
 # other places throughout the built documents.
@@ -86,11 +86,19 @@ pygments_style = 'sphinx'
 # given in html_static_path.
 html_theme = 'default'
 html_theme_options = {'stickysidebar':'true', 'relbarbgcolor':'black'}
+# Put the quick search box on top
+html_sidebars = {
+        '**' : ['searchbox.html', 'localtoc.html', 'relations.html',
+            'sourcelink.html'],
+}
+
+# The favicon
+html_favicon = 'favicon.ico'
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['resources']
+html_static_path = ['resources', '../../../icons/favicon.ico']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
@@ -108,7 +116,7 @@ html_logo        = 'resources/logo.png'
 epub_author      = 'Kovid Goyal'
 epub_cover       = 'epub_cover.jpg'
 epub_publisher   = 'Kovid Goyal'
-epub_identifier  = 'http://calibre-ebook.com/user_manual'
+epub_identifier  = 'http://manual.calibre-ebook.com'
 epub_scheme      = 'url'
 epub_uid         = 'S54a88f8e9d42455e9c6db000e989225f'
 epub_tocdepth    = 4
@@ -131,7 +139,7 @@ html_copy_source = True
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'calibredoc'
 
-html_use_opensearch = 'http://calibre-ebook.com/user_manual'
+html_use_opensearch = 'http://manual.calibre-ebook.com'
 
 html_show_sphinx = False
 

src/calibre/manual/custom.py

@@ -240,11 +240,21 @@ def cli_docs(app):
             raw += '\n'+'\n'.join(lines)
             update_cli_doc(os.path.join('cli', cmd+'.rst'), raw, info)
 
+def generate_docs(app):
+    cli_docs(app)
+    template_docs(app)
+
+def template_docs(app):
+    from template_ref_generate import generate_template_language_help
+    info = app.builder.info
+    raw = generate_template_language_help()
+    update_cli_doc('template_ref.rst', raw, info)
+
 def setup(app):
     app.add_config_value('epub_cover', None, False)
     app.add_builder(EPUBHelpBuilder)
     app.connect('doctree-read', substitute)
-    app.connect('builder-inited', cli_docs)
+    app.connect('builder-inited', generate_docs)
     app.connect('build-finished', finished)
 
 def finished(app, exception):

src/calibre/manual/develop.rst

@@ -65,7 +65,7 @@ this, make your changes, then run::
     bzr send -o my-changes
 
 This will create a :file:`my-changes` file in the current directory,
-simply attach that to a ticket on the |app| `bug tracker <http://bugs.calibre-ebook.com/newticket>`_.
+simply attach that to a ticket on the |app| `bug tracker <http://calibre-ebook.com/bugs>`_.
 
 If you plan to do a lot of development on |app|, then the best method is to create a 
 `Launchpad <http://launchpad.net>`_ account. Once you have the account, you can use it to register

src/calibre/manual/faq.rst

@@ -560,7 +560,7 @@ I want some feature added to |app|. What can I do?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 You have two choices:
  1. Create a patch by hacking on |app| and send it to me for review and inclusion. See `Development <http://calibre-ebook.com/get-involved>`_.
- 2. `Open a ticket <http://bugs.calibre-ebook.com/newticket>`_ (you have to register and login first). Remember that |app| development is done by volunteers, so if you get no response to your feature request, it means no one feels like implementing it.
+ 2. `Open a ticket <http://calibre-ebook.com/bugs>`_ (you have to register and login first). Remember that |app| development is done by volunteers, so if you get no response to your feature request, it means no one feels like implementing it.
 
 How is |app| licensed?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

src/calibre/manual/gui.rst

@@ -352,7 +352,7 @@ The syntax for searching for dates is::
 If the date is ambiguous, the current locale is used for date comparison. For example, in an mm/dd/yyyy
 locale, 2/1/2009 is interpreted as 1 Feb 2009. In a dd/mm/yyyy locale, it is interpreted as 2 Jan 2009.  Some
 special date strings are available. The string ``today`` translates to today's date, whatever it is. The
-strings `yesterday`` and ``thismonth`` also work. In addition, the string ``daysago`` can be used to compare
+strings ``yesterday`` and ``thismonth`` also work. In addition, the string ``daysago`` can be used to compare
 to a date some number of days ago, for example: date:>10daysago, date:<=45daysago.
 
 You can search for books that have a format of a certain size like this::

src/calibre/manual/template_lang.rst

@@ -114,13 +114,15 @@ The syntax for using functions is ``{field:function(arguments)}``, or ``{field:f
 
 If you have programming experience, please note that the syntax in this mode (single function) is not what you might expect. Strings are not quoted. Spaces are significant. All arguments must be constants; there is no sub-evaluation. Use :ref:`template program mode <template_mode>` and :ref:`general program mode <general_mode>` to avoid these differences.
 
+Many functions use regular expressions. In all cases, regular expression matching is case-insensitive.
+
 The functions available are:
 
     * ``lowercase()``	-- return value of the field in lower case.
     * ``uppercase()``	-- return the value of the field in upper case.
     * ``titlecase()``	-- return the value of the field in title case.
     * ``capitalize()``	-- return the value with the first letter upper case and the rest lower case.
-    * ``contains(pattern, text if match, text if not match`` -- checks if field contains matches for the regular expression `pattern`. Returns `text if match` if matches are found, otherwise it returns `text if no match`.
+    * ``contains(pattern, text if match, text if not match)`` -- checks if field contains matches for the regular expression `pattern`. Returns `text if match` if matches are found, otherwise it returns `text if no match`.
     * ``count(separator)`` -- interprets the value as a list of items separated by `separator`, returning the number of items in the list. Most lists use a comma as the separator, but authors uses an ampersand. Examples: `{tags:count(,)}`, `{authors:count(&)}`
     * ``ifempty(text)``	-- if the field is not empty, return the value of the field. Otherwise return `text`.
     * ``in_list(separator, pattern, found_val, not_found_val)`` -- interpret the field as a list of items separated by `separator`, comparing the `pattern` against each value in the list. If the pattern matches a value, return `found_val`, otherwise return `not_found_val`.
@@ -130,6 +132,7 @@ The functions available are:
     * ``switch(pattern, value, pattern, value, ..., else_value)`` -- for each ``pattern, value`` pair, checks if the field matches the regular expression ``pattern`` and if so, returns that ``value``. If no ``pattern`` matches, then ``else_value`` is returned. You can have as many ``pattern, value`` pairs as you want.
     * ``lookup(pattern, field, pattern, field, ..., else_field)`` -- like switch, except the arguments are field (metadata) names, not text. The value of the appropriate field will be fetched and used. Note that because composite columns are fields, you can use this function in one composite field to use the value of some other composite field. This is extremely useful when constructing variable save paths (more later).
     * ``select(key)`` -- interpret the field as a comma-separated list of items, with the items being of the form "id:value". Find the pair with the id equal to key, and return the corresponding value. This function is particularly useful for extracting a value such as an isbn from the set of identifiers for a book.
+    * ``str_in_list(val, separator, string, found_val, not_found_val)`` -- treat val as a list of items separated by separator, comparing the string against each value in the list. If the string matches a value, return found_val, otherwise return not_found_val. If the string contains separators, then it is also treated as a list and each value is checked.
     * ``subitems(val, start_index, end_index)`` -- This function is used to break apart lists of tag-like hierarchical items such as genres. It interprets the value as a comma-separated list of tag-like items, where each item is a period-separated list. Returns a new list made by first finding all the period-separated tag-like items, then for each such item extracting the components from `start_index` to `end_index`, then combining the results back together. The first component in a period-separated list has an index of zero. If an index is negative, then it counts from the end of the list. As a special case, an end_index of zero is assumed to be the length of the list. Examples::
 
         Assuming a #genre column containing "A.B.C":    
@@ -255,6 +258,7 @@ The following functions are available in addition to those described in single-f
     * ``not(value)`` -- returns the string "1" if the value is empty, otherwise returns the empty string. This function works well with test or first_non_empty. You can have as many values as you want.
     * ``merge_lists(list1, list2, separator)`` -- return a list made by merging the items in list1 and list2, removing duplicate items using a case-insensitive compare. If items differ in case, the one in list1 is used. The items in list1 and list2 are separated by separator, as are the items in the returned list.
     * ``multiply(x, y)`` -- returns x * y. Throws an exception if either x or y are not numbers.
+    * ``ondevice()`` -- return the string "Yes" if ondevice is set, otherwise return the empty string
     * ``or(value, value, ...)`` -- returns the string "1" if any value is not empty, otherwise returns the empty string. This function works well with test or first_non_empty. You can have as many values as you want.
     * ``print(a, b, ...)`` -- prints the arguments to standard output. Unless you start calibre from the command line (``calibre-debug -g``), the output will go to a black hole.
     * ``raw_field(name)`` -- returns the metadata field named by name without applying any formatting.
@@ -264,20 +268,14 @@ The following functions are available in addition to those described in single-f
     * ``subtract(x, y)`` -- returns x - y. Throws an exception if either x or y are not numbers.
     * ``template(x)`` -- evaluates x as a template. The evaluation is done in its own context, meaning that variables are not shared between the caller and the template evaluation. Because the `{` and `}` characters are special, you must use `[[` for the `{` character and `]]` for the '}' character; they are converted automatically. For example, ``template('[[title_sort]]') will evaluate the template ``{title_sort}`` and return its value.
     
-Function classification summary:
+Function classification
+---------------------------
+
+.. toctree::
+    :maxdepth: 3
+
+    template_ref
 
-    * Get values from metadata: ``field``. ``raw_field``. In some situations, ``lookup`` can be used in place of ``field``.
-    * Arithmetic: ``add``, ``subtract``, ``multiply``, ``divide``
-    * Boolean: ``and``, ``or``, ``not``. The function ``if_empty`` is similar to ``and`` called with one argument.
-    * If-then-else: ``contains``, ``test``
-    * Iterating over values: ``first_non_empty``, ``lookup``, ``switch``
-    * List lookup: ``in_list``, ``list_item``, ``select``, 
-    * List manipulation: ``count``, ``merge_lists``, ``sublist``, ``subitems``
-    * Recursion: ``eval``, ``template``
-    * Relational: ``cmp`` , ``strcmp`` for strings
-    * String case changes: ``lowercase``, ``uppercase``, ``titlecase``, ``capitalize``
-    * String manipulation: ``re``, ``shorten``, ``substr``
-    * Other: ``assign``, ``booksize``, ``print``, ``format_date``, 
         
 .. _general_mode:
 
@@ -422,19 +420,8 @@ You might find the following tips useful.
     * In a plugboard, you can set a field to empty (or whatever is equivalent to empty) by using the special template ``{null}``. This template will always evaluate to an empty string.
     * The technique described above to show numbers even if they have a zero value works with the standard field series_index.
 
-API of the Metadata objects
+.. toctree::
-----------------------------
+  :hidden:
 
-.. module:: calibre.ebooks.metadata.book.base
+  template_ref
-
-.. autoclass:: Metadata
-   :members:
-   :member-order: bysource
-
-.. data:: STANDARD_METADATA_FIELDS
-
-    The set of standard metadata fields.
-
-.. literalinclude:: ../ebooks/metadata/book/__init__.py
-   :lines: 7-
 

src/calibre/manual/template_ref_generate.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
+
+from collections import defaultdict
+
+PREAMBLE = '''\
+.. include:: global.rst
+
+.. _templaterefcalibre:
+
+Reference for all builtin template language functions
+========================================================
+
+Here, we document all the builtin functions available in the |app| template language. Every function is implemented as a class in python and you can click the source links to see the source code, in case the documentation is insufficient. The functions are arranged in logical groups by type.
+
+.. contents::
+    :depth: 2
+    :local:
+
+.. module:: calibre.utils.formatter_functions
+
+'''
+
+CATEGORY_TEMPLATE = '''\
+{category}
+{dashes}
+
+'''
+
+FUNCTION_TEMPLATE = '''\
+{fs}
+{hats}
+
+.. autoclass:: {cn}
+
+'''
+
+POSTAMBLE = '''\
+
+API of the Metadata objects
+----------------------------
+
+The python implementation of the template functions is passed in a Metadata object. Knowing it's API is useful if you want to define your own template functions.
+
+.. module:: calibre.ebooks.metadata.book.base
+
+.. autoclass:: Metadata
+   :members:
+   :member-order: bysource
+
+.. data:: STANDARD_METADATA_FIELDS
+
+    The set of standard metadata fields.
+
+.. literalinclude:: ../ebooks/metadata/book/__init__.py
+   :lines: 7-
+'''
+
+
+def generate_template_language_help():
+    from calibre.utils.formatter_functions import all_builtin_functions
+
+    funcs = defaultdict(dict)
+
+    for func in all_builtin_functions:
+        class_name = func.__class__.__name__
+        func_sig = getattr(func, 'doc')
+        x = func_sig.find(' -- ')
+        if x < 0:
+            print 'No sig for ', class_name
+            continue
+        func_sig = func_sig[:x]
+        func_cat = getattr(func, 'category')
+        funcs[func_cat][func_sig] = class_name
+
+    output = PREAMBLE
+    cats = sorted(funcs.keys())
+    for cat in cats:
+        output += CATEGORY_TEMPLATE.format(category=cat, dashes='-'*len(cat))
+        entries = [k for k in sorted(funcs[cat].keys())]
+        for entry in entries:
+            output += FUNCTION_TEMPLATE.format(fs = entry, cn=funcs[cat][entry],
+                                               hats='^'*len(entry))
+
+    output += POSTAMBLE
+    return output
+
+if __name__ == '__main__':
+    generate_template_language_help()

src/calibre/utils/formatter.py

@@ -98,7 +98,10 @@ class _Parser(object):
                     cls = funcs['assign']
                     return cls.eval_(self.parent, self.parent.kwargs,
                                     self.parent.book, self.parent.locals, id, self.expr())
-                return self.parent.locals.get(id, _('unknown id ') + id)
+                val = self.parent.locals.get(id, None)
+                if val is None:
+                    self.error(_('Unknown identifier ') + id)
+                return val
             # We have a function.
             # Check if it is a known one. We do this here so error reporting is
             # better, as it can identify the tokens near the problem.
@@ -317,8 +320,8 @@ class TemplateFormatter(string.Formatter):
         try:
             ans = self.vformat(fmt, [], kwargs).strip()
         except Exception as e:
-            if DEBUG:
+#            if DEBUG:
-                traceback.print_exc()
+#                traceback.print_exc()
             ans = error_value + ' ' + e.message
         return ans
 
@@ -339,6 +342,8 @@ class EvalFormatter(TemplateFormatter):
     A template formatter that uses a simple dict instead of an mi instance
     '''
     def get_value(self, key, args, kwargs):
+        if key == '':
+            return ''
         key = key.lower()
         return kwargs.get(key, _('No such variable ') + key)
 

src/calibre/utils/formatter_functions.py

@@ -8,7 +8,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import inspect, re, traceback, sys
+import inspect, re, traceback
 
 from calibre.utils.titlecase import titlecase
 from calibre.utils.icu import capitalize, strcmp, sort_key
@@ -57,26 +57,20 @@ class FormatterFunction(object):
 
     doc = _('No documentation provided')
     name = 'no name provided'
+    category = 'Unknown'
     arg_count = 0
 
     def evaluate(self, formatter, kwargs, mi, locals, *args):
         raise NotImplementedError()
 
     def eval_(self, formatter, kwargs, mi, locals, *args):
-        try:
+        ret = self.evaluate(formatter, kwargs, mi, locals, *args)
-            ret = self.evaluate(formatter, kwargs, mi, locals, *args)
+        if isinstance(ret, (str, unicode)):
-            if isinstance(ret, (str, unicode)):
+            return ret
-                return ret
+        if isinstance(ret, (int, float, bool)):
-            if isinstance(ret, (int, float, bool)):
+            return unicode(ret)
-                return unicode(ret)
+        if isinstance(ret, list):
-            if isinstance(ret, list):
+            return ','.join(list)
-                return ','.join(list)
-        except:
-            traceback.print_exc()
-            exc_type, exc_value, exc_traceback = sys.exc_info()
-            info = ': '.join(traceback.format_exception(exc_type, exc_value,
-                                        exc_traceback)[-2:]).replace('\n', '')
-            return _('Exception ') + info
 
 all_builtin_functions = []
 class BuiltinFormatterFunction(FormatterFunction):
@@ -94,7 +88,8 @@ class BuiltinFormatterFunction(FormatterFunction):
 class BuiltinStrcmp(BuiltinFormatterFunction):
     name = 'strcmp'
     arg_count = 5
-    doc = _('strcmp(x, y, lt, eq, gt) -- does a case-insensitive comparison of x '
+    category = 'Relational'
+    __doc__ = doc = _('strcmp(x, y, lt, eq, gt) -- does a case-insensitive comparison of x '
             'and y as strings. Returns lt if x < y. Returns eq if x == y. '
             'Otherwise returns gt.')
 
@@ -108,13 +103,14 @@ class BuiltinStrcmp(BuiltinFormatterFunction):
 
 class BuiltinCmp(BuiltinFormatterFunction):
     name = 'cmp'
+    category = 'Relational'
     arg_count = 5
-    doc =   _('cmp(x, y, lt, eq, gt) -- compares x and y after converting both to '
+    __doc__ = doc =   _('cmp(x, y, lt, eq, gt) -- compares x and y after converting both to '
             'numbers. Returns lt if x < y. Returns eq if x == y. Otherwise returns gt.')
 
     def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):
-        x = float(x if x else 0)
+        x = float(x if x and x != 'None' else 0)
-        y = float(y if y else 0)
+        y = float(y if y and y != 'None' else 0)
         if x < y:
             return lt
         if x == y:
@@ -124,7 +120,8 @@ class BuiltinCmp(BuiltinFormatterFunction):
 class BuiltinStrcat(BuiltinFormatterFunction):
     name = 'strcat'
     arg_count = -1
-    doc = _('strcat(a, b, ...) -- can take any number of arguments. Returns a '
+    category = 'String Manipulation'
+    __doc__ = doc = _('strcat(a, b, ...) -- can take any number of arguments. Returns a '
             'string formed by concatenating all the arguments')
 
     def evaluate(self, formatter, kwargs, mi, locals, *args):
@@ -137,7 +134,8 @@ class BuiltinStrcat(BuiltinFormatterFunction):
 class BuiltinAdd(BuiltinFormatterFunction):
     name = 'add'
     arg_count = 2
-    doc = _('add(x, y) -- returns x + y. Throws an exception if either x or y are not numbers.')
+    category = 'Arithmetic'
+    __doc__ = doc = _('add(x, y) -- returns x + y. Throws an exception if either x or y are not numbers.')
 
     def evaluate(self, formatter, kwargs, mi, locals, x, y):
         x = float(x if x else 0)
@@ -147,7 +145,8 @@ class BuiltinAdd(BuiltinFormatterFunction):
 class BuiltinSubtract(BuiltinFormatterFunction):
     name = 'subtract'
     arg_count = 2
-    doc = _('subtract(x, y) -- returns x - y. Throws an exception if either x or y are not numbers.')
+    category = 'Arithmetic'
+    __doc__ = doc = _('subtract(x, y) -- returns x - y. Throws an exception if either x or y are not numbers.')
 
     def evaluate(self, formatter, kwargs, mi, locals, x, y):
         x = float(x if x else 0)
@@ -157,7 +156,8 @@ class BuiltinSubtract(BuiltinFormatterFunction):
 class BuiltinMultiply(BuiltinFormatterFunction):
     name = 'multiply'
     arg_count = 2
-    doc = _('multiply(x, y) -- returns x * y. Throws an exception if either x or y are not numbers.')
+    category = 'Arithmetic'
+    __doc__ = doc = _('multiply(x, y) -- returns x * y. Throws an exception if either x or y are not numbers.')
 
     def evaluate(self, formatter, kwargs, mi, locals, x, y):
         x = float(x if x else 0)
@@ -167,7 +167,8 @@ class BuiltinMultiply(BuiltinFormatterFunction):
 class BuiltinDivide(BuiltinFormatterFunction):
     name = 'divide'
     arg_count = 2
-    doc = _('divide(x, y) -- returns x / y. Throws an exception if either x or y are not numbers.')
+    category = 'Arithmetic'
+    __doc__ = doc = _('divide(x, y) -- returns x / y. Throws an exception if either x or y are not numbers.')
 
     def evaluate(self, formatter, kwargs, mi, locals, x, y):
         x = float(x if x else 0)
@@ -177,7 +178,9 @@ class BuiltinDivide(BuiltinFormatterFunction):
 class BuiltinTemplate(BuiltinFormatterFunction):
     name = 'template'
     arg_count = 1
-    doc = _('template(x) -- evaluates x as a template. The evaluation is done '
+    category = 'Recursion'
+
+    __doc__ = doc = _('template(x) -- evaluates x as a template. The evaluation is done '
             'in its own context, meaning that variables are not shared between '
             'the caller and the template evaluation. Because the { and } '
             'characters are special, you must use [[ for the { character and '
@@ -192,7 +195,8 @@ class BuiltinTemplate(BuiltinFormatterFunction):
 class BuiltinEval(BuiltinFormatterFunction):
     name = 'eval'
     arg_count = 1
-    doc = _('eval(template) -- evaluates the template, passing the local '
+    category = 'Recursion'
+    __doc__ = doc = _('eval(template) -- evaluates the template, passing the local '
             'variables (those \'assign\'ed to) instead of the book metadata. '
             ' This permits using the template processor to construct complex '
             'results from local variables.')
@@ -205,7 +209,8 @@ class BuiltinEval(BuiltinFormatterFunction):
 class BuiltinAssign(BuiltinFormatterFunction):
     name = 'assign'
     arg_count = 2
-    doc = _('assign(id, val) -- assigns val to id, then returns val. '
+    category = 'Other'
+    __doc__ = doc = _('assign(id, val) -- assigns val to id, then returns val. '
             'id must be an identifier, not an expression')
 
     def evaluate(self, formatter, kwargs, mi, locals, target, value):
@@ -215,7 +220,8 @@ class BuiltinAssign(BuiltinFormatterFunction):
 class BuiltinPrint(BuiltinFormatterFunction):
     name = 'print'
     arg_count = -1
-    doc = _('print(a, b, ...) -- prints the arguments to standard output. '
+    category = 'Other'
+    __doc__ = doc = _('print(a, b, ...) -- prints the arguments to standard output. '
             'Unless you start calibre from the command line (calibre-debug -g), '
             'the output will go to a black hole.')
 
@@ -226,15 +232,17 @@ class BuiltinPrint(BuiltinFormatterFunction):
 class BuiltinField(BuiltinFormatterFunction):
     name = 'field'
     arg_count = 1
-    doc = _('field(name) -- returns the metadata field named by name')
+    category = 'Get values from metadata'
+    __doc__ = doc = _('field(name) -- returns the metadata field named by name')
 
     def evaluate(self, formatter, kwargs, mi, locals, name):
         return formatter.get_value(name, [], kwargs)
 
-class BuiltinRaw_field(BuiltinFormatterFunction):
+class BuiltinRawField(BuiltinFormatterFunction):
     name = 'raw_field'
     arg_count = 1
-    doc = _('raw_field(name) -- returns the metadata field named by name '
+    category = 'Get values from metadata'
+    __doc__ = doc = _('raw_field(name) -- returns the metadata field named by name '
             'without applying any formatting.')
 
     def evaluate(self, formatter, kwargs, mi, locals, name):
@@ -243,7 +251,8 @@ class BuiltinRaw_field(BuiltinFormatterFunction):
 class BuiltinSubstr(BuiltinFormatterFunction):
     name = 'substr'
     arg_count = 3
-    doc = _('substr(str, start, end) -- returns the start\'th through the end\'th '
+    category = 'String Manipulation'
+    __doc__ = doc = _('substr(str, start, end) -- returns the start\'th through the end\'th '
             'characters of str. The first character in str is the zero\'th '
             'character. If end is negative, then it indicates that many '
             'characters counting from the right. If end is zero, then it '
@@ -256,7 +265,8 @@ class BuiltinSubstr(BuiltinFormatterFunction):
 class BuiltinLookup(BuiltinFormatterFunction):
     name = 'lookup'
     arg_count = -1
-    doc = _('lookup(val, pattern, field, pattern, field, ..., else_field) -- '
+    category = 'Iterating over values'
+    __doc__ = doc = _('lookup(val, pattern, field, pattern, field, ..., else_field) -- '
             'like switch, except the arguments are field (metadata) names, not '
             'text. The value of the appropriate field will be fetched and used. '
             'Note that because composite columns are fields, you can use this '
@@ -276,14 +286,15 @@ class BuiltinLookup(BuiltinFormatterFunction):
         while i < len(args):
             if i + 1 >= len(args):
                 return formatter.vformat('{' + args[i].strip() + '}', [], kwargs)
-            if re.search(args[i], val):
+            if re.search(args[i], val, flags=re.I):
                 return formatter.vformat('{'+args[i+1].strip() + '}', [], kwargs)
             i += 2
 
 class BuiltinTest(BuiltinFormatterFunction):
     name = 'test'
     arg_count = 3
-    doc = _('test(val, text if not empty, text if empty) -- return `text if not '
+    category = 'If-then-else'
+    __doc__ = doc = _('test(val, text if not empty, text if empty) -- return `text if not '
             'empty` if the field is not empty, otherwise return `text if empty`')
 
     def evaluate(self, formatter, kwargs, mi, locals, val, value_if_set, value_not_set):
@@ -295,14 +306,15 @@ class BuiltinTest(BuiltinFormatterFunction):
 class BuiltinContains(BuiltinFormatterFunction):
     name = 'contains'
     arg_count = 4
-    doc = _('contains(val, pattern, text if match, text if not match) -- checks '
+    category = 'If-then-else'
+    __doc__ = doc = _('contains(val, pattern, text if match, text if not match) -- checks '
             'if field contains matches for the regular expression `pattern`. '
             'Returns `text if match` if matches are found, otherwise it returns '
             '`text if no match`')
 
     def evaluate(self, formatter, kwargs, mi, locals,
                  val, test, value_if_present, value_if_not):
-        if re.search(test, val):
+        if re.search(test, val, flags=re.I):
             return value_if_present
         else:
             return value_if_not
@@ -310,7 +322,8 @@ class BuiltinContains(BuiltinFormatterFunction):
 class BuiltinSwitch(BuiltinFormatterFunction):
     name = 'switch'
     arg_count = -1
-    doc = _('switch(val, pattern, value, pattern, value, ..., else_value) -- '
+    category = 'Iterating over values'
+    __doc__ = doc = _('switch(val, pattern, value, pattern, value, ..., else_value) -- '
             'for each `pattern, value` pair, checks if the field matches '
             'the regular expression `pattern` and if so, returns that '
             '`value`. If no pattern matches, then else_value is returned. '
@@ -323,14 +336,15 @@ class BuiltinSwitch(BuiltinFormatterFunction):
         while i < len(args):
             if i + 1 >= len(args):
                 return args[i]
-            if re.search(args[i], val):
+            if re.search(args[i], val, flags=re.I):
                 return args[i+1]
             i += 2
 
 class BuiltinInList(BuiltinFormatterFunction):
     name = 'in_list'
     arg_count = 5
-    doc = _('in_list(val, separator, pattern, found_val, not_found_val) -- '
+    category = 'List Lookup'
+    __doc__ = doc = _('in_list(val, separator, pattern, found_val, not_found_val) -- '
             'treat val as a list of items separated by separator, '
             'comparing the pattern against each value in the list. If the '
             'pattern matches a value, return found_val, otherwise return '
@@ -338,26 +352,76 @@ class BuiltinInList(BuiltinFormatterFunction):
 
     def evaluate(self, formatter, kwargs, mi, locals, val, sep, pat, fv, nfv):
         l = [v.strip() for v in val.split(sep) if v.strip()]
-        for v in l:
+        if l:
-            if re.search(pat, v):
+            for v in l:
-                return fv
+                if re.search(pat, v, flags=re.I):
+                    return fv
+        return nfv
+
+class BuiltinStrInList(BuiltinFormatterFunction):
+    name = 'str_in_list'
+    arg_count = 5
+    category = 'List lookup'
+    __doc__ = doc = _('str_in_list(val, separator, string, found_val, not_found_val) -- '
+            'treat val as a list of items separated by separator, '
+            'comparing the string against each value in the list. If the '
+            'string matches a value, return found_val, otherwise return '
+            'not_found_val. If the string contains separators, then it is '
+            'also treated as a list and each value is checked.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val, sep, str, fv, nfv):
+        l = [v.strip() for v in val.split(sep) if v.strip()]
+        c = [v.strip() for v in str.split(sep) if v.strip()]
+        if l:
+            for v in l:
+                for t in c:
+                    if strcmp(t, v) == 0:
+                        return fv
+        return nfv
+
+class BuiltinIdentifierInList(BuiltinFormatterFunction):
+    name = 'identifier_in_list'
+    arg_count = 4
+    category = 'List lookup'
+    __doc__ = doc = _('identifier_in_list(val, id, found_val, not_found_val) -- '
+            'treat val as a list of identifiers separated by commas, '
+            'comparing the string against each value in the list. An identifier '
+            'has the format "identifier:value". The id parameter should be '
+            'either "id" or "id:regexp". The first case matches if there is any '
+            'identifier with that id. The second case matches if the regexp '
+            'matches the identifier\'s value. If there is a match, '
+            'return found_val, otherwise return not_found_val.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val, ident, fv, nfv):
+        l = [v.strip() for v in val.split(',') if v.strip()]
+        (id, _, regexp) = ident.partition(':')
+        if not id:
+            return nfv
+        id += ':'
+        if l:
+            for v in l:
+                if v.startswith(id):
+                    if not regexp or re.search(regexp, v[len(id):], flags=re.I):
+                        return fv
         return nfv
 
 class BuiltinRe(BuiltinFormatterFunction):
     name = 're'
     arg_count = 3
-    doc = _('re(val, pattern, replacement) -- return the field after applying '
+    category = 'String Manipulation'
+    __doc__ = doc = _('re(val, pattern, replacement) -- return the field after applying '
             'the regular expression. All instances of `pattern` are replaced '
             'with `replacement`. As in all of calibre, these are '
             'python-compatible regular expressions')
 
     def evaluate(self, formatter, kwargs, mi, locals, val, pattern, replacement):
-        return re.sub(pattern, replacement, val)
+        return re.sub(pattern, replacement, val, flags=re.I)
 
 class BuiltinIfempty(BuiltinFormatterFunction):
     name = 'ifempty'
     arg_count = 2
-    doc = _('ifempty(val, text if empty) -- return val if val is not empty, '
+    category = 'If-then-else'
+    __doc__ = doc = _('ifempty(val, text if empty) -- return val if val is not empty, '
             'otherwise return `text if empty`')
 
     def evaluate(self, formatter, kwargs, mi, locals, val, value_if_empty):
@@ -369,7 +433,8 @@ class BuiltinIfempty(BuiltinFormatterFunction):
 class BuiltinShorten(BuiltinFormatterFunction):
     name = 'shorten'
     arg_count = 4
-    doc = _('shorten(val, left chars, middle text, right chars) -- Return a '
+    category = 'String Manipulation'
+    __doc__ = doc = _('shorten(val, left chars, middle text, right chars) -- Return a '
             'shortened version of the field, consisting of `left chars` '
             'characters from the beginning of the field, followed by '
             '`middle text`, followed by `right chars` characters from '
@@ -394,7 +459,8 @@ class BuiltinShorten(BuiltinFormatterFunction):
 class BuiltinCount(BuiltinFormatterFunction):
     name = 'count'
     arg_count = 2
-    doc = _('count(val, separator) -- interprets the value as a list of items '
+    category = 'List Manipulation'
+    __doc__ = doc = _('count(val, separator) -- interprets the value as a list of items '
             'separated by `separator`, returning the number of items in the '
             'list. Most lists use a comma as the separator, but authors '
             'uses an ampersand. Examples: {tags:count(,)}, {authors:count(&)}')
@@ -405,7 +471,8 @@ class BuiltinCount(BuiltinFormatterFunction):
 class BuiltinListitem(BuiltinFormatterFunction):
     name = 'list_item'
     arg_count = 3
-    doc = _('list_item(val, index, separator) -- interpret the value as a list of '
+    category = 'List Lookup'
+    __doc__ = doc = _('list_item(val, index, separator) -- interpret the value as a list of '
             'items separated by `separator`, returning the `index`th item. '
             'The first item is number zero. The last item can be returned '
             'using `list_item(-1,separator)`. If the item is not in the list, '
@@ -425,7 +492,8 @@ class BuiltinListitem(BuiltinFormatterFunction):
 class BuiltinSelect(BuiltinFormatterFunction):
     name = 'select'
     arg_count = 2
-    doc = _('select(val, key) -- interpret the value as a comma-separated list '
+    category = 'List Lookup'
+    __doc__ = doc = _('select(val, key) -- interpret the value as a comma-separated list '
             'of items, with the items being "id:value". Find the pair with the'
             'id equal to key, and return the corresponding value.'
             )
@@ -442,9 +510,10 @@ class BuiltinSelect(BuiltinFormatterFunction):
 class BuiltinSublist(BuiltinFormatterFunction):
     name = 'sublist'
     arg_count = 4
-    doc = _('sublist(val, start_index, end_index, separator) -- interpret the '
+    category = 'List Manipulation'
+    __doc__ = doc = _('sublist(val, start_index, end_index, separator) -- interpret the '
             'value as a list of items separated by `separator`, returning a '
-            'new list made from the `start_index`th to the `end_index`th item. '
+            'new list made from the `start_index` to the `end_index` item. '
             'The first item is number zero. If an index is negative, then it '
             'counts from the end of the list. As a special case, an end_index '
             'of zero is assumed to be the length of the list. Examples using '
@@ -452,7 +521,8 @@ class BuiltinSublist(BuiltinFormatterFunction):
             'comma-separated) contains "A, B, C": '
             '{tags:sublist(0,1,\,)} returns "A". '
             '{tags:sublist(-1,0,\,)} returns "C". '
-            '{tags:sublist(0,-1,\,)} returns "A, B".')
+            '{tags:sublist(0,-1,\,)} returns "A, B".'
+            )
 
     def evaluate(self, formatter, kwargs, mi, locals, val, start_index, end_index, sep):
         if not val:
@@ -471,12 +541,13 @@ class BuiltinSublist(BuiltinFormatterFunction):
 class BuiltinSubitems(BuiltinFormatterFunction):
     name = 'subitems'
     arg_count = 3
-    doc = _('subitems(val, start_index, end_index) -- This function is used to '
+    category = 'List Manipulation'
+    __doc__ = doc = _('subitems(val, start_index, end_index) -- This function is used to '
             'break apart lists of items such as genres. It interprets the value '
             'as a comma-separated list of items, where each item is a period-'
             'separated list. Returns a new list made by first finding all the '
             'period-separated items, then for each such item extracting the '
-            'start_index`th to the `end_index`th components, then combining '
+            'start_index` to the `end_index` components, then combining '
             'the results back together. The first component in a period-'
             'separated list has an index of zero. If an index is negative, '
             'then it counts from the end of the list. As a special case, an '
@@ -505,11 +576,12 @@ class BuiltinSubitems(BuiltinFormatterFunction):
                 pass
         return ', '.join(sorted(rv, key=sort_key))
 
-class BuiltinFormat_date(BuiltinFormatterFunction):
+class BuiltinFormatDate(BuiltinFormatterFunction):
     name = 'format_date'
     arg_count = 2
-    doc = _('format_date(val, format_string) -- format the value, which must '
+    category = 'Get values from metadata'
-            'be a date field, using the format_string, returning a string. '
+    __doc__ = doc = _('format_date(val, format_string) -- format the value, '
+            'which must be a date, using the format_string, returning a string. '
             'The formatting codes are: '
             'd    : the day as number without a leading zero (1 to 31) '
             'dd   : the day as number with a leading zero (01 to 31) '
@@ -524,7 +596,7 @@ class BuiltinFormat_date(BuiltinFormatterFunction):
             'iso  : the date with time and timezone. Must be the only format present')
 
     def evaluate(self, formatter, kwargs, mi, locals, val, format_string):
-        if not val:
+        if not val or val == 'None':
             return ''
         try:
             dt = parse_date(val)
@@ -536,7 +608,8 @@ class BuiltinFormat_date(BuiltinFormatterFunction):
 class BuiltinUppercase(BuiltinFormatterFunction):
     name = 'uppercase'
     arg_count = 1
-    doc = _('uppercase(val) -- return value of the field in upper case')
+    category = 'String case changes'
+    __doc__ = doc = _('uppercase(val) -- return value of the field in upper case')
 
     def evaluate(self, formatter, kwargs, mi, locals, val):
         return val.upper()
@@ -544,7 +617,8 @@ class BuiltinUppercase(BuiltinFormatterFunction):
 class BuiltinLowercase(BuiltinFormatterFunction):
     name = 'lowercase'
     arg_count = 1
-    doc = _('lowercase(val) -- return value of the field in lower case')
+    category = 'String case changes'
+    __doc__ = doc = _('lowercase(val) -- return value of the field in lower case')
 
     def evaluate(self, formatter, kwargs, mi, locals, val):
         return val.lower()
@@ -552,7 +626,8 @@ class BuiltinLowercase(BuiltinFormatterFunction):
 class BuiltinTitlecase(BuiltinFormatterFunction):
     name = 'titlecase'
     arg_count = 1
-    doc = _('titlecase(val) -- return value of the field in title case')
+    category = 'String case changes'
+    __doc__ = doc = _('titlecase(val) -- return value of the field in title case')
 
     def evaluate(self, formatter, kwargs, mi, locals, val):
         return titlecase(val)
@@ -560,7 +635,8 @@ class BuiltinTitlecase(BuiltinFormatterFunction):
 class BuiltinCapitalize(BuiltinFormatterFunction):
     name = 'capitalize'
     arg_count = 1
-    doc = _('capitalize(val) -- return value of the field capitalized')
+    category = 'String case changes'
+    __doc__ = doc = _('capitalize(val) -- return value of the field capitalized')
 
     def evaluate(self, formatter, kwargs, mi, locals, val):
         return capitalize(val)
@@ -568,7 +644,8 @@ class BuiltinCapitalize(BuiltinFormatterFunction):
 class BuiltinBooksize(BuiltinFormatterFunction):
     name = 'booksize'
     arg_count = 0
-    doc = _('booksize() -- return value of the field capitalized')
+    category = 'Get values from metadata'
+    __doc__ = doc = _('booksize() -- return value of the size field')
 
     def evaluate(self, formatter, kwargs, mi, locals):
         if mi.book_size is not None:
@@ -578,10 +655,23 @@ class BuiltinBooksize(BuiltinFormatterFunction):
                 pass
         return ''
 
+class BuiltinOndevice(BuiltinFormatterFunction):
+    name = 'ondevice'
+    arg_count = 0
+    category = 'Get values from metadata'
+    __doc__ = doc = _('ondevice() -- return Yes if ondevice is set, otherwise return '
+            'the empty string')
+
+    def evaluate(self, formatter, kwargs, mi, locals):
+        if mi.ondevice_col:
+            return _('Yes')
+        return ''
+
 class BuiltinFirstNonEmpty(BuiltinFormatterFunction):
     name = 'first_non_empty'
     arg_count = -1
-    doc = _('first_non_empty(value, value, ...) -- '
+    category = 'Iterating over values'
+    __doc__ = doc = _('first_non_empty(value, value, ...) -- '
             'returns the first value that is not empty. If all values are '
             'empty, then the empty value is returned.'
             'You can have as many values as you want.')
@@ -597,7 +687,8 @@ class BuiltinFirstNonEmpty(BuiltinFormatterFunction):
 class BuiltinAnd(BuiltinFormatterFunction):
     name = 'and'
     arg_count = -1
-    doc = _('and(value, value, ...) -- '
+    category = 'Boolean'
+    __doc__ = doc = _('and(value, value, ...) -- '
             'returns the string "1" if all values are not empty, otherwise '
             'returns the empty string. This function works well with test or '
             'first_non_empty. You can have as many values as you want.')
@@ -613,7 +704,8 @@ class BuiltinAnd(BuiltinFormatterFunction):
 class BuiltinOr(BuiltinFormatterFunction):
     name = 'or'
     arg_count = -1
-    doc = _('or(value, value, ...) -- '
+    category = 'Boolean'
+    __doc__ = doc = _('or(value, value, ...) -- '
             'returns the string "1" if any value is not empty, otherwise '
             'returns the empty string. This function works well with test or '
             'first_non_empty. You can have as many values as you want.')
@@ -629,7 +721,8 @@ class BuiltinOr(BuiltinFormatterFunction):
 class BuiltinNot(BuiltinFormatterFunction):
     name = 'not'
     arg_count = 1
-    doc = _('not(value) -- '
+    category = 'Boolean'
+    __doc__ = doc = _('not(value) -- '
             'returns the string "1" if the value is empty, otherwise '
             'returns the empty string. This function works well with test or '
             'first_non_empty. You can have as many values as you want.')
@@ -645,7 +738,8 @@ class BuiltinNot(BuiltinFormatterFunction):
 class BuiltinMergeLists(BuiltinFormatterFunction):
     name = 'merge_lists'
     arg_count = 3
-    doc = _('merge_lists(list1, list2, separator) -- '
+    category = 'List Manipulation'
+    __doc__ = doc = _('merge_lists(list1, list2, separator) -- '
             'return a list made by merging the items in list1 and list2, '
             'removing duplicate items using a case-insensitive compare. If '
             'items differ in case, the one in list1 is used. '
@@ -678,7 +772,8 @@ builtin_divide      = BuiltinDivide()
 builtin_eval        = BuiltinEval()
 builtin_first_non_empty = BuiltinFirstNonEmpty()
 builtin_field       = BuiltinField()
-builtin_format_date = BuiltinFormat_date()
+builtin_format_date = BuiltinFormatDate()
+builtin_identifier_in_list = BuiltinIdentifierInList()
 builtin_ifempty     = BuiltinIfempty()
 builtin_in_list     = BuiltinInList()
 builtin_list_item   = BuiltinListitem()
@@ -687,14 +782,16 @@ builtin_lowercase   = BuiltinLowercase()
 builtin_merge_lists = BuiltinMergeLists()
 builtin_multiply    = BuiltinMultiply()
 builtin_not         = BuiltinNot()
+builtin_ondevice    = BuiltinOndevice()
 builtin_or          = BuiltinOr()
 builtin_print       = BuiltinPrint()
-builtin_raw_field   = BuiltinRaw_field()
+builtin_raw_field   = BuiltinRawField()
 builtin_re          = BuiltinRe()
 builtin_select      = BuiltinSelect()
 builtin_shorten     = BuiltinShorten()
 builtin_strcat      = BuiltinStrcat()
 builtin_strcmp      = BuiltinStrcmp()
+builtin_str_in_list = BuiltinStrInList()
 builtin_subitems    = BuiltinSubitems()
 builtin_sublist     = BuiltinSublist()
 builtin_substr      = BuiltinSubstr()

src/calibre/utils/help2man.py

@@ -52,7 +52,7 @@ def create_man_page(prog, parser):
 
     lines += ['.SH SEE ALSO',
               'The User Manual is available at '
-              'http://calibre-ebook.com/user_manual',
+              'http://manual.calibre-ebook.com',
               '.PP', '.B Created by '+__author__]
 
     lines = [x if isinstance(x, unicode) else unicode(x, 'utf-8', 'replace') for

src/calibre/utils/zipfile.py

@@ -1123,6 +1123,13 @@ class ZipFile:
                     targetpath = os.sep.join(components)
                     with open(targetpath, 'wb') as target:
                         shutil.copyfileobj(source, target)
+        # Kovid: Try to preserve the timestamps in the ZIP file
+        try:
+            mtime = time.localtime()
+            mtime = time.mktime(member.date_time + (0, 0) + (mtime.tm_isdst,))
+            os.utime(targetpath, (mtime, mtime))
+        except:
+            pass
         self.extract_mapping[member.filename] = targetpath
         return targetpath