merge from trunk

.bzrignore

@@ -4,6 +4,7 @@ src/calibre/plugins
 resources/images.qrc
 src/calibre/manual/.build/
 src/calibre/manual/cli/
+src/calibre/manual/template_ref.rst
 build
 dist
 docs
@@ -13,6 +14,7 @@ resources/scripts.pickle
 resources/ebook-convert-complete.pickle
 resources/builtin_recipes.xml
 resources/builtin_recipes.zip
+resources/template-functions.json
 setup/installer/windows/calibre/build.log
 src/calibre/translations/.errors
 src/cssutils/.svn/

Changelog.yaml

@@ -19,6 +19,74 @@
 #  new recipes:
 #    - title: 
 
+- version: 0.8.4
+  date: 2011-06-03
+
+  new features:
+    - title: "New and much simpler interface for specifying column coloring via Preferences->Look & Feel->Column Coloring"
+
+    - title: "Driver for Trekstor eBook Player 5M, Samsung Galaxy SII I9100, Motorola Defy and miBuk GAMMA 6.2"
+      tickets: [792091, 791216]
+
+    - title: "Get Books: Add EpubBud, WH Smits and E-book Shoppe stores"
+
+    - title: "When deleting 'all formats except ...', do not delete if it leaves a book with no formats"
+
+    - title: "Change default toolbar to make it a little more new user friendly. The icons have been re-arranged and now the text is always visiblke by default. You can change that in Preferences->Look & Feeel and Preferences->Toolbar"
+
+    - title: "Windows installer: Remember and use previous settings for installing desktop icons, adding to path, etc. This makes the installer a little slower, complaints should go to Microsoft."
+
+    - title: "Template language: Add str_in_list and on_device formatter functions. Make debugging templates a little easier"
+
+    - title: "Allow the user to specify formatting for number type custom columns"
+  
+  bug fixes:
+    - title: "Fix typo in NOOK TSR driver that prevented it from working on windows"
+
+    - title: "Fix quotes in identifiers causing Tag Browser to be blank."
+      tickets: [791044]
+
+    - title: "Speedup auto complete when there are lots of items (>2500) the downside being that non ASCII characters are not sorted correctly. The threshold can be controlled via Preferences->Tweaks"
+      tickets: [792191]
+
+    - title: "RTF Output: Fix handling of curly brackets"
+      tickets: [791805]
+
+    - title: "Fix searching in Get Books not working with non ASCII characters"
+      tickets: [791788]
+
+    - title: "Fix excessive memory consumption when moving very large files during a metadata change"
+      tickets: [791806]
+
+    - title: "Fix series index being overwritten even when series is turned off in bulk metadata download"
+      tickets: [789990]
+
+    - title: "Fix regression in templates where id and other non standard fields no longer worked."
+
+    - title: "EPUB Output: Fix crash caused by ids with non-ascii characters in them"
+
+    - title: "Try to preserve the timestamps of files in a ZIP container"
+
+    - title: "After adding books always select the most recently added book."
+      tickets: [789343]
+
+  improved recipes:
+    - bild.de
+    - CNN
+    - BBC News (fast)
+    - Dilema Veche
+
+  new recipes:
+    - title: Metro UK
+      author: Dave Asbury
+
+    - title: Alt om Herning and Version2.dk
+      author: Rasmus Lauritsen
+
+    - title: Observatorul cultural
+      author: song2
+
+
 - version: 0.8.3
   date: 2011-05-27
 

recipes/bild_de.recipe

@@ -1,10 +1,11 @@
 # -*- coding: utf-8 -*-
 from calibre.web.feeds.recipes import BasicNewsRecipe
 class AdvancedUserRecipe1303841067(BasicNewsRecipe):
+
     title          = u'Bild.de'
     __author__  = 'schuster'
     oldest_article = 1
-    max_articles_per_feed = 50
+    max_articles_per_feed = 100
     no_stylesheets         = True
     use_embedded_content   = False
     language               = 'de'
@@ -12,11 +13,25 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
 
 # get cover from myspace
     cover_url = 'http://a3.l3-images.myspacecdn.com/images02/56/0232f842170b4d349779f8379c27e073/l.jpg'
+    masthead_url = 'http://a3.l3-images.myspacecdn.com/images02/56/0232f842170b4d349779f8379c27e073/l.jpg'
 
 # set what to fetch on the site
     remove_tags_before =  dict(name = 'h2', attrs={'id':'cover'})
     remove_tags_after = dict(name ='div', attrs={'class':'back'})
 
+
+# remove things on the site that we don't want
+    remove_tags = [dict(name='div', attrs={'class':'credit'}),
+        dict(name='div', attrs={'class':'index'}),
+        dict(name='div', attrs={'id':'zstart31'}),
+        dict(name='div', attrs={'class':'hentry'}),
+        dict(name='div', attrs={'class':'back'}),
+        dict(name='div', attrs={'class':'pagination'}),
+        dict(name='div', attrs={'class':'header'}),
+        dict(name='div', attrs={'class':'element floatL'}),
+        dict(name='div', attrs={'class':'stWrap'})
+]
+
 #  thanx to kiklop74 for code (see sticky thread -> Recipes - Re-usable code)
 # this one removes a lot of direct-link's
     def preprocess_html(self, soup):
@@ -42,5 +57,18 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
                           (u'Unterhaltung', u'http://rss.bild.de/bild-unterhaltung.xml'),
                           (u'Sport', u'http://rss.bild.de/bild-sport.xml'),
                           (u'Lifestyle', u'http://rss.bild.de/bild-lifestyle.xml'),
-                          (u'Ratgeber', u'http://rss.bild.de/bild-ratgeber.xml')
+                          (u'Ratgeber', u'http://rss.bild.de/bild-ratgeber.xml'),
+                          (u'Reg. - Berlin', u'http://rss.bild.de/bild-berlin.xml'),
+                          (u'Reg. - Bremen', u'http://rss.bild.de/bild-bremen.xml'),
+                          (u'Reg. - Dresden', u'http://rss.bild.de/bild-dresden.xml'),
+                          (u'Reg. - Düsseldorf', u'http://rss.bild.de/bild-duesseldorf.xml'),
+                          (u'Reg. - Frankfurt-Main', u'http://rss.bild.de/bild-frankfurt-main.xml'),
+                          (u'Reg. - Hamburg', u'http://rss.bild.de/bild-hamburg.xml'),
+                          (u'Reg. - Hannover', u'http://rss.bild.de/bild-hannover.xml'),
+                          (u'Reg. - Köln', u'http://rss.bild.de/bild-koeln.xml'),
+                          (u'Reg. - Leipzig', u'http://rss.bild.de/bild-leipzig.xml'),
+                          (u'Reg. - München', u'http://rss.bild.de/bild-muenchen.xml'),
+                          (u'Reg. - Ruhrgebiet', u'http://rss.bild.de/bild-ruhrgebiet.xml'),
+                          (u'Reg. - Stuttgart', u'http://rss.bild.de/bild-stuttgart.xml')
 ]
+

recipes/brigitte_de.recipe

@@ -0,0 +1,36 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe(BasicNewsRecipe):
+
+    title          = u'Brigitte.de'
+    __author__  = 'schuster'
+    oldest_article = 14
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    language               = 'de'
+    remove_javascript      = True
+    remove_empty_feeds = True
+    timeout = 10
+    cover_url = 'http://www.medienmilch.de/typo3temp/pics/Brigitte-Logo_d5feb4a6e4.jpg'
+    masthead_url = 'http://www.medienmilch.de/typo3temp/pics/Brigitte-Logo_d5feb4a6e4.jpg'
+
+
+    remove_tags = [dict(attrs={'class':['linklist', 'head', 'indent right relatedContent', 'artikel-meta segment', 'segment', 'comment commentFormWrapper segment borderBG', 'segment borderBG comments', 'segment borderBG box', 'center', 'segment nextPageLink', 'inCar']}),
+                dict(id=['header', 'artTools', 'context', 'interact', 'footer-navigation', 'bwNet', 'copy', 'keyboardNavigationHint']),
+                dict(name=['hjtrs', 'kud'])]
+
+    feeds          = [(u'Mode', u'http://www.brigitte.de/mode/feed.rss'),
+                          (u'Beauty', u'http://www.brigitte.de/beauty/feed.rss'),
+                          (u'Luxus', u'http://www.brigitte.de/luxus/feed.rss'),
+                          (u'Figur', u'http://www.brigitte.de/figur/feed.rss'),
+                          (u'Gesundheit', u'http://www.brigitte.de/gesundheit/feed.rss'),
+                          (u'Liebe&Sex', u'http://www.brigitte.de/liebe-sex/feed.rss'),
+                          (u'Gesellschaft', u'http://www.brigitte.de/gesellschaft/feed.rss'),
+                          (u'Kultur', u'http://www.brigitte.de/kultur/feed.rss'),
+                          (u'Reise', u'http://www.brigitte.de/reise/feed.rss'),
+                          (u'Kochen', u'http://www.brigitte.de/kochen/feed.rss'),
+                          (u'Wohnen', u'http://www.brigitte.de/wohnen/feed.rss'),
+                          (u'Job', u'http://www.brigitte.de/job/feed.rss'),
+                          (u'Erfahrungen', u'http://www.brigitte.de/erfahrungen/feed.rss'),
+]

recipes/cnn.recipe

@@ -3,71 +3,39 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 '''
 Profile to download CNN
 '''
+
+import re
 from calibre.web.feeds.news import BasicNewsRecipe
-from calibre.ebooks.BeautifulSoup import BeautifulSoup
 
 class CNN(BasicNewsRecipe):
 
     title = 'CNN'
     description = 'Global news'
     timefmt  = ' [%d %b %Y]'
-    __author__ = 'Krittika Goyal and Sujata Raman'
+    __author__ = 'Kovid Goyal'
     language = 'en'
 
     no_stylesheets = True
     use_embedded_content   = False
     oldest_article        = 15
-    recursions = 1
+    #recursions = 1
-    match_regexps = [r'http://sportsillustrated.cnn.com/.*/[1-9].html']
+    #match_regexps = [r'http://sportsillustrated.cnn.com/.*/[1-9].html']
     max_articles_per_feed = 25
 
-    extra_css = '''
+    preprocess_regexps = [
-                .cnn_strycntntlft{font-family :Arial,Helvetica,sans-serif;}
+        (re.compile(r'<!--\[if.*if\]-->', re.DOTALL), lambda m: ''),
-                h2{font-family :Arial,Helvetica,sans-serif; font-size:x-small}
+        (re.compile(r'<script.*?</script>', re.DOTALL), lambda m: ''),
-                .cnnTxtCmpnt{font-family :Arial,Helvetica,sans-serif; font-size:x-small}
+        (re.compile(r'<style.*?</style>', re.DOTALL), lambda m: ''),
-                .cnnTMcontent{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#575757}
-                .storytext{font-family :Arial,Helvetica,sans-serif; font-size:small}
-                .storybyline{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#575757}
-                .credit{font-family :Arial,Helvetica,sans-serif; font-size:xx-small; color:#575757}
-                .storyBrandingBanner{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#575757}
-                .storytimestamp{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#575757}
-                .timestamp{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#575757}
-                .cnn_strytmstmp{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#666666;}
-                .cnn_stryimg640caption{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#666666;}
-                .cnn_strylccimg300cntr{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#666666;}
-                .cnn_stryichgfcpt{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#666666;}
-                .cnnByline{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#666666;}
-                .cnn_bulletbin cnnStryHghLght{ font-size:xx-small;}
-                .subhead p{font-family :Arial,Helvetica,sans-serif; font-size:x-small;}
-                .cnnStoryContent{font-family :Arial,Helvetica,sans-serif; font-size:x-small}
-                .cnnContentContainer{font-family :Arial,Helvetica,sans-serif; font-size:x-small}
-                .col1{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#666666;}
-                .col3{color:#333333; font-family :Arial,Helvetica,sans-serif; font-size:x-small;font-weight:bold;}
-                .cnnInlineT1Caption{font-family :Arial,Helvetica,sans-serif; font-size:x-small;font-weight:bold;}
-                .cnnInlineT1Credit{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#333333;}
-                .col10{color:#5A637E;}
-                .cnnInlineRailBulletList{color:black;}
-                .cnnLine0{font-family :Arial,Helvetica,sans-serif; color:#666666;font-weight:bold;}
-                .cnnTimeStamp{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#333333;}
-                .galleryhedDek{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#575757;}
-                .galleryWidgetHeader{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#004276;}
-                .article-content{font-family :Arial,Helvetica,sans-serif; font-size:x-small}
-                .cnnRecapStory{font-family :Arial,Helvetica,sans-serif; font-size:x-small}
-                h1{font-family :Arial,Helvetica,sans-serif; font-size:x-large}
-                .captionname{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#575757;}
-                inStoryIE{{font-family :Arial,Helvetica,sans-serif; font-size:x-small;}
-                '''
-
-    #remove_tags_before = dict(name='h1', attrs={'class':'heading'})
-    #remove_tags_after  = dict(name='td', attrs={'class':'newptool1'})
-    remove_tags = [
-       dict(name='iframe'),
-       dict(name='div', attrs={'class':['cnnEndOfStory', 'cnnShareThisItem', 'cnn_strylctcntr cnn_strylctcqrelt', 'cnnShareBoxContent', 'cnn_strybtmcntnt', 'cnn_strycntntrgt']}),
-       dict(name='div', attrs={'id':['IEContainer', 'clickIncludeBox']}),
-       #dict(name='ul', attrs={'class':'article-tools'}),
-       #dict(name='ul', attrs={'class':'articleTools'}),
     ]
 
+    keep_only_tags = [dict(id='cnnContentContainer')]
+    remove_tags = [
+            {'class':['cnn_strybtntools', 'cnn_strylftcntnt',
+                'cnn_strybtntools', 'cnn_strybtntoolsbttm', 'cnn_strybtmcntnt',
+                'cnn_strycntntrgt']},
+    ]
+
+
     feeds =  [
              ('Top News', 'http://rss.cnn.com/rss/cnn_topstories.rss'),
              ('World', 'http://rss.cnn.com/rss/cnn_world.rss'),
@@ -84,15 +52,8 @@ class CNN(BasicNewsRecipe):
              ('Offbeat', 'http://rss.cnn.com/rss/cnn_offbeat.rss'),
              ('Most Popular', 'http://rss.cnn.com/rss/cnn_mostpopular.rss')
              ]
-    def preprocess_html(self, soup):
+
-        story = soup.find(name='div', attrs={'class':'cnnBody_Left'})
+    def get_article_url(self, article):
-        if story is None:
+        ans = BasicNewsRecipe.get_article_url(self, article)
-           story = soup.find(name='div', attrs={'id':'cnnContentContainer'})
+        return ans.partition('?')[0]
-           soup = BeautifulSoup('<html><head><title>t</title></head><body></body></html>')
+
-           body = soup.find(name='body')
-           body.insert(0, story)
-        else:
-           soup = BeautifulSoup('<html><head><title>t</title></head><body></body></html>')
-           body = soup.find(name='body')
-           body.insert(0, story)
-        return soup

recipes/express_de.recipe

@@ -1,5 +1,4 @@
 from calibre.web.feeds.news import BasicNewsRecipe
-
 class AdvancedUserRecipe1303841067(BasicNewsRecipe):
 
     title          = u'Express.de'
@@ -12,7 +11,6 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
     extra_css = '''
                     h2{font-family:Arial,Helvetica,sans-serif; font-size: x-small;}
                     h1{ font-family:Arial,Helvetica,sans-serif;  font-size:x-large; font-weight:bold;}
-
                 '''
     remove_javascript      = True
     remove_tags_befor = [dict(name='div', attrs={'class':'Datum'})]
@@ -25,6 +23,7 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
            dict(id='Logo'),
            dict(id='MainLinkSpacer'),
            dict(id='MainLinks'),
+           dict(id='ContainerPfad'), #neu
            dict(title='Diese Seite Bookmarken'),
 
                                            dict(name='span'),
@@ -44,7 +43,8 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
            dict(name='div', attrs={'class':'HeaderSearch'}),
            dict(name='div', attrs={'class':'sbutton'}),
            dict(name='div', attrs={'class':'active'}),
-
+           dict(name='div', attrs={'class':'MoreNews'}), #neu
+           dict(name='div', attrs={'class':'ContentBoxSubline'}) #neu
 ]
 
 
@@ -68,7 +68,5 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
                           (u'Fortuna D~Dorf', u'http://www.express.de/sport/fussball/fortuna/-/3292/3292/-/view/asFeed/-/index.xml'),
                           (u'Basketball News', u'http://www.express.de/sport/basketball/-/3190/3190/-/view/asFeed/-/index.xml'),
                           (u'Big Brother', u'http://www.express.de/news/promi-show/big-brother/-/2402/2402/-/view/asFeed/-/index.xml'),
-
-
-
                     ]
+

recipes/guardian.recipe

@@ -87,7 +87,13 @@ class Guardian(BasicNewsRecipe):
         idx = soup.find('div', id='book-index')
         for s in idx.findAll('strong', attrs={'class':'book'}):
             a = s.find('a', href=True)
-            yield (self.tag_to_string(a), a['href'])
+            section_title = self.tag_to_string(a)
+            if not section_title in self.ignore_sections:
+                prefix = ''
+                if section_title != 'Main section':
+                    prefix = section_title + ': '
+                for subsection in s.parent.findAll('a', attrs={'class':'book-section'}):
+                    yield (prefix + self.tag_to_string(subsection), subsection['href'])
     
     def find_articles(self, url):
         soup = self.index_to_soup(url)
@@ -114,10 +120,7 @@ class Guardian(BasicNewsRecipe):
         try:
             feeds = []
             for title, href in self.find_sections():
-                if not title in self.ignore_sections:
                 feeds.append((title, list(self.find_articles(href))))
             return feeds
         except:
             raise NotImplementedError
-
-

recipes/heise_online.recipe

@@ -0,0 +1,52 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+class AdvancedUserRecipe(BasicNewsRecipe):
+
+    title = 'Heise-online'
+    description = 'News vom Heise-Verlag'
+    __author__ = 'schuster'
+    use_embedded_content   = False
+    language = 'de'
+    oldest_article = 2
+    max_articles_per_feed = 35
+    rescale_images = True
+    remove_empty_feeds = True
+    timeout = 5
+    no_stylesheets = True
+
+
+    remove_tags_after = dict(name ='p', attrs={'class':'editor'})
+    remove_tags = [dict(id='navi_top_container'),
+                            dict(id='navi_bottom'),
+                            dict(id='mitte_rechts'),
+                            dict(id='navigation'),
+                            dict(id='subnavi'),
+                            dict(id='social_bookmarks'),
+                            dict(id='permalink'),
+                            dict(id='content_foren'),
+                            dict(id='seiten_navi'),
+                            dict(id='adbottom'),
+                            dict(id='sitemap')]
+
+    feeds =  [
+                   ('Newsticker', 'http://www.heise.de/newsticker/heise.rdf'),
+                   ('Auto', 'http://www.heise.de/autos/rss/news.rdf'),
+                   ('Foto ', 'http://www.heise.de/foto/rss/news-atom.xml'),
+                   ('Mac&i', 'http://www.heise.de/mac-and-i/news.rdf'),
+                   ('Mobile ', 'http://www.heise.de/mobil/newsticker/heise-atom.xml'),
+                   ('Netz ', 'http://www.heise.de/netze/rss/netze-atom.xml'),
+                   ('Open ', 'http://www.heise.de/open/news/news-atom.xml'),
+                   ('Resale ', 'http://www.heise.de/resale/rss/resale.rdf'),
+                   ('Security ', 'http://www.heise.de/security/news/news-atom.xml'),
+                   ('C`t', 'http://www.heise.de/ct/rss/artikel-atom.xml'),
+                   ('iX', 'http://www.heise.de/ix/news/news.rdf'),
+                   ('Mach-flott', 'http://www.heise.de/mach-flott/rss/mach-flott-atom.xml'),
+                   ('Blog: Babel-Bulletin', 'http://www.heise.de/developer/rss/babel-bulletin/blog.rdf'),
+                   ('Blog: Der Dotnet-Doktor', 'http://www.heise.de/developer/rss/dotnet-doktor/blog.rdf'),
+                   ('Blog: Bernds Management-Welt', 'http://www.heise.de/developer/rss/bernds-management-welt/blog.rdf'),
+                   ('Blog: IT conversation', 'http://www.heise.de/developer/rss/world-of-it/blog.rdf'),
+                   ('Blog: Kais bewegtes Web', 'http://www.heise.de/developer/rss/kais-bewegtes-web/blog.rdf')
+]
+
+    def print_version(self, url):
+        return url + '?view=print'
+

recipes/lemonde_dip.recipe

@@ -1,5 +1,5 @@
 __license__   = 'GPL v3'
-__copyright__ = '2008-2010, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2011, Darko Miletic <darko.miletic at gmail.com>'
 '''
 mondediplo.com
 '''
@@ -11,7 +11,7 @@ from calibre.web.feeds.news import BasicNewsRecipe
 class LeMondeDiplomatiqueEn(BasicNewsRecipe):
     title                  = 'Le Monde diplomatique - English edition'
     __author__             = 'Darko Miletic'
-    description            = 'Real journalism making sense of the world around us'
+    description            = "Le Monde diplomatique is the place you go when you want to know what's really happening. This is a major international paper that is truly independent, that sees the world in fresh ways, that focuses on places no other publications reach. We offer a clear, considered view of the conflicting interests and complexities of a modern global world. LMD in English is a concise version of the Paris-based parent edition, publishing all the major stories each month, expertly translated, and with some London-based commissions too. We offer a taster of LMD quality on our website where a selection of articles are available each month."
     publisher              = 'Le Monde diplomatique'
     category               = 'news, politics, world'
     no_stylesheets         = True
@@ -26,7 +26,13 @@ class LeMondeDiplomatiqueEn(BasicNewsRecipe):
     INDEX                  = PREFIX + strftime('%Y/%m/')
     use_embedded_content   = False
     language               = 'en'
-    extra_css              = ' body{font-family: "Luxi sans","Lucida sans","Lucida Grande",Lucida,"Lucida Sans Unicode",sans-serif} .surtitre{font-size: 1.2em; font-variant: small-caps; margin-bottom: 0.5em} .chapo{font-size: 1.2em; font-weight: bold; margin: 1em 0 0.5em} .texte{font-family: Georgia,"Times New Roman",serif} h1{color: #990000} .notes{border-top: 1px solid #CCCCCC; font-size: 0.9em; line-height: 1.4em} '
+    extra_css              = """
+                                body{font-family: "Luxi sans","Lucida sans","Lucida Grande",Lucida,"Lucida Sans Unicode",sans-serif}
+                                .surtitre{font-size: 1.2em; font-variant: small-caps; margin-bottom: 0.5em}
+                                .chapo{font-size: 1.2em; font-weight: bold; margin: 1em 0 0.5em}
+                                .texte{font-family: Georgia,"Times New Roman",serif} h1{color: #990000}
+                                .notes{border-top: 1px solid #CCCCCC; font-size: 0.9em; line-height: 1.4em}
+                            """
 
     conversion_options = {
                           'comment'   : description
@@ -51,7 +57,7 @@ class LeMondeDiplomatiqueEn(BasicNewsRecipe):
                         , dict(name='div',attrs={'class':'notes surlignable'})
                         ]
     remove_tags = [dict(name=['object','link','script','iframe','base'])]
-    remove_attributes = ['height','width']
+    remove_attributes = ['height','width','name','lang']
 
     def parse_index(self):
         articles = []
@@ -75,3 +81,24 @@ class LeMondeDiplomatiqueEn(BasicNewsRecipe):
                                 })
         return [(self.title, articles)]
 
+    def get_cover_url(self):
+        cover_url = None
+        soup = self.index_to_soup(self.INDEX)
+        cover_item = soup.find('div',attrs={'class':'current'})
+        if cover_item:
+           ap = cover_item.find('img',attrs={'class':'spip_logos'})
+           if ap:
+              cover_url = self.INDEX + ap['src']
+        return cover_url
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll('a'):
+            if item.string is not None:
+               str = item.string
+               item.replaceWith(str)
+            else:
+               str = self.tag_to_string(item)
+               item.replaceWith(str)
+        return soup

recipes/max_planck.recipe

@@ -3,9 +3,6 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
 
     title          = u'Max-Planck-Inst.'
     __author__  = 'schuster'
-    remove_tags = [dict(attrs={'class':['clearfix', 'lens', 'col2_box_list', 'col2_box_teaser group_ext no_print', 'dotted_line', 'col2_box_teaser', 'box_image small', 'bold', 'col2_box_teaser no_print', 'print_kontakt']}),
-                dict(id=['ie_clearing', 'col2', 'col2_content']),
-                dict(name=['script', 'noscript', 'style'])]
     oldest_article = 30
     max_articles_per_feed = 100
     no_stylesheets         = True
@@ -13,6 +10,11 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
     language               = 'de'
     remove_javascript      = True
 
+    remove_tags = [dict(attrs={'class':['box_url', 'print_kontakt']}),
+                dict(id=['skiplinks'])]
+
+
+
     def print_version(self, url):
         split_url = url.split("/")
         print_url = 'http://www.mpg.de/print/' +  split_url[3]

recipes/newsweek.recipe

@@ -69,7 +69,11 @@ class Newsweek(BasicNewsRecipe):
         for section, shref in self.newsweek_sections():
             self.log('Processing section', section, shref)
             articles = []
+            try:
                 soups = [self.index_to_soup(shref)]
+            except:
+                self.log.warn('Section %s not found, skipping'%section)
+                continue
             na = soups[0].find('a', rel='next')
             if na:
                 soups.append(self.index_to_soup(self.BASE_URL+na['href']))

recipes/polizeipress_de.recipe

@@ -0,0 +1,35 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe(BasicNewsRecipe):
+
+    title          = u'Polizeipresse - Deutschland'
+    __author__  = 'schuster'
+    description            =    'Tagesaktuelle "Polizeiberichte" aus ganz Deutschland (bis auf Ortsebene).' 'Um deinen Ort/Stadt/Kreis usw. einzubinden, gehe auf "http://www.presseportal.de/polizeipresse/" und suche im oberen "Suchfeld" nach dem Namen.'   'Oberhalb der Suchergebnisse (Folgen:) auf den üblichen link zu den RSS-Feeds klicken und den RSS-link im Rezept unter "feeds" eintragen wie üblich.'   'Die Auswahl von Orten kann vereinfacht werden wenn man den Suchbegriff wie folgt eingibt:' '"Stadt-Ort".'
+    oldest_article = 21
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    language               = 'de'
+    remove_javascript      = True
+    masthead_url = 'http://www.alt-heliservice.de/images/34_BPOL_Logo_4C_g_schutzbereich.jpg'
+    cover_url = 'http://berlinstadtservice.de/buerger/Bundespolizei-Logo.png'
+
+    remove_tags = [
+        dict(name='div', attrs={'id':'logo'}),
+        dict(name='div', attrs={'id':'origin'}),
+        dict(name='pre', attrs={'class':'xml_contact'})]
+
+    def print_version(self,url):
+        segments = url.split('/')
+        printURL = 'http://www.presseportal.de/print.htx?nr=' + '/'.join(segments[5:6]) + '&type=polizei'
+        return printURL
+
+    feeds          = [(u'Frimmerdorf', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Grevenbroich-frimmersdorf&w=public_service'),
+          (u'Neurath', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Grevenbroich-neurath&w=public_service'),
+          (u'Gustorf', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Grevenbroich-gustorf&w=public_service'),
+          (u'Neuenhausen', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Grevenbroich-neuenhausen&w=public_service'),
+          (u'Wevelinghoven', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Grevenbroich-Wevelinghoven&w=public_service'),
+          (u'Grevenbroich ges.', u'http://www.presseportal.de/rss/rss2_vts.htx?q=grevenbroich&w=public_service'),
+          (u'Kreis Neuss ges.', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Rhein-Kreis+Neuss&w=public_service'),
+          ]
+

recipes/runa.recipe

@@ -1,52 +0,0 @@
-#!/usr/bin/env  python
-
-__license__ = 'GPL v3'
-__author__ = 'Mori'
-__version__ = 'v. 0.1'
-'''
-www.runa.pl/blog
-'''
-
-from calibre.web.feeds.news import BasicNewsRecipe
-import re
-
-class FantazmatyRecipe(BasicNewsRecipe):
-	__author__ = 'Mori'
-	language = 'pl'
-
-	title = u'Fantazmaty'
-	publisher = u'Agencja Wydawnicza Runa'
-	description = u'Blog Agencji Wydawniczej Runa'
-	
-	no_stylesheets = True
-	remove_javascript = True
-	encoding = 'utf-8'
-	
-	oldest_article = 100
-	max_articles_per_feed = 100
-	
-	extra_css = '''
-		img{float: left; padding-right: 10px; padding-bottom: 5px;}
-	'''
-	
-	feeds = [
-		(u'Fantazmaty', u'http://www.runa.pl/blog/rss.xml')
-	]
-	
-	remove_tags = [
-		dict(name = 'div', attrs = {'class' : 'path'}),
-		dict(name = 'div', attrs = {'class' : 'drdot'}),
-		dict(name = 'div', attrs = {'class' : 'picture'})
-	]
-	
-	remove_tags_after = [
-		dict(name = 'div', attrs = {'class' : 'content'})
-	]
-	
-	preprocess_regexps = [
-		(re.compile(i[0], re.IGNORECASE | re.DOTALL), i[1]) for i in 
-		[
-			(r'<body>.*?<div id="primary"', lambda match: '<body><div id="primary"'),
-			(r'<!--.*?-->', lambda match: '')
-		]
-	]

recipes/telegraph_uk.recipe

@@ -49,6 +49,7 @@ class TelegraphUK(BasicNewsRecipe):
                          (u'UK News'        , u'http://www.telegraph.co.uk/news/uknews/rss'                                      )
                         ,(u'World News'     , u'http://www.telegraph.co.uk/news/worldnews/rss'                                   )
                         ,(u'Politics'       , u'http://www.telegraph.co.uk/news/newstopics/politics/rss'                         )
+                        ,(u'Finance'        , u'http://www.telegraph.co.uk/finance/rss'                                          )
                         ,(u'Technology News', u'http://www.telegraph.co.uk/scienceandtechnology/technology/technologynews/rss'   )
                         ,(u'UK News'        , u'http://www.telegraph.co.uk/scienceandtechnology/technology/technologyreviews/rss')
                         ,(u'Science News'   , u'http://www.telegraph.co.uk/scienceandtechnology/science/sciencenews/rss'         )

recipes/time_magazine.recipe

@@ -10,8 +10,8 @@ import re
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Time(BasicNewsRecipe):
-    recipe_disabled = ('This recipe has been disabled as TIME no longer'
+    #recipe_disabled = ('This recipe has been disabled as TIME no longer'
-            ' publish complete articles on the web.')
+    #        ' publish complete articles on the web.')
     title                 = u'Time'
     __author__            = 'Kovid Goyal and Sujata Raman'
     description           = 'Weekly magazine'

recipes/zaobao.recipe

@@ -82,7 +82,7 @@ class ZAOBAO(BasicNewsRecipe):
         return soup
 
     def parse_feeds(self):
-        self.log_debug(_('ZAOBAO overrided parse_feeds()'))
+        self.log(_('ZAOBAO overrided parse_feeds()'))
         parsed_feeds = BasicNewsRecipe.parse_feeds(self)
 
         for id, obj in enumerate(self.INDEXES):
@@ -99,7 +99,7 @@ class ZAOBAO(BasicNewsRecipe):
                     a_title = self.tag_to_string(a)
                     date = ''
                     description = ''
-                    self.log_debug(_('adding %s at %s')%(a_title,a_url))
+                    self.log(_('adding %s at %s')%(a_title,a_url))
                     articles.append({
                                     'title':a_title,
                                     'date':date,
@@ -110,23 +110,23 @@ class ZAOBAO(BasicNewsRecipe):
             pfeeds = feeds_from_index([(title, articles)], oldest_article=self.oldest_article,
                                      max_articles_per_feed=self.max_articles_per_feed)
 
-            self.log_debug(_('adding %s to feed')%(title))
+            self.log(_('adding %s to feed')%(title))
             for feed in pfeeds:
-                self.log_debug(_('adding feed: %s')%(feed.title))
+                self.log(_('adding feed: %s')%(feed.title))
                 feed.description = self.DESC_SENSE
                 parsed_feeds.append(feed)
                 for a, article in enumerate(feed):
-                    self.log_debug(_('added article %s from %s')%(article.title, article.url))
+                    self.log(_('added article %s from %s')%(article.title, article.url))
-                self.log_debug(_('added feed %s')%(feed.title))
+                self.log(_('added feed %s')%(feed.title))
 
         for i, feed in enumerate(parsed_feeds):
             # workaorund a strange problem: Somethimes the xml encoding is not apllied correctly by parse()
             weired_encoding_detected = False
             if not isinstance(feed.description, unicode) and self.encoding and feed.description:
-                self.log_debug(_('Feed %s is not encoded correctly, manually replace it')%(feed.title))
+                self.log(_('Feed %s is not encoded correctly, manually replace it')%(feed.title))
                 feed.description = feed.description.decode(self.encoding, 'replace')
             elif feed.description.find(self.DESC_SENSE) == -1 and self.encoding and feed.description:
-                self.log_debug(_('Feed %s is weired encoded, manually redo all')%(feed.title))
+                self.log(_('Feed %s is weired encoded, manually redo all')%(feed.title))
                 feed.description = feed.description.encode('cp1252', 'replace').decode(self.encoding, 'replace')
                 weired_encoding_detected = True
 
@@ -148,7 +148,7 @@ class ZAOBAO(BasicNewsRecipe):
                         article.text_summary = article.text_summary.encode('cp1252', 'replace').decode(self.encoding, 'replace')
 
                 if article.title == "Untitled article":
-                    self.log_debug(_('Removing empty article %s from %s')%(article.title, article.url))
+                    self.log(_('Removing empty article %s from %s')%(article.title, article.url))
                     # remove the article
                     feed.articles[a:a+1] = []
         return parsed_feeds

resources/content_server/browse/browse.html

@@ -20,7 +20,7 @@
     <script type="text/javascript"
         src="{prefix}/static/jquery.multiselect.min.js"></script>
 
-
+    <script type="text/javascript" src="{prefix}/static/stacktrace.js"></script>
     <script type="text/javascript" src="{prefix}/static/browse/browse.js"></script>
 
     <script type="text/javascript">

resources/content_server/browse/browse.js

@@ -129,7 +129,13 @@ function toplevel() {
 // }}}
 
 function render_error(msg) {
-    return '<div class="ui-widget"><div class="ui-state-error ui-corner-all" style="padding: 0pt 0.7em"><p><span class="ui-icon ui-icon-alert" style="float: left; margin-right: 0.3em">&nbsp;</span><strong>Error: </strong>'+msg+"</p></div></div>"
+    var st = "";
+    try {
+        var st = printStackTrace();
+        st = st.join('\n\n');
+    } catch(e) {
+    }
+    return '<div class="ui-widget"><div class="ui-state-error ui-corner-all" style="padding: 0pt 0.7em"><p><span class="ui-icon ui-icon-alert" style="float: left; margin-right: 0.3em">&nbsp;</span><strong>Error: </strong>'+msg+"<pre>"+st+"</pre></p></div></div>"
 }
 
 // Category feed {{{

resources/content_server/stacktrace.js

@@ -0,0 +1,371 @@
+// Domain Public by Eric Wendelin http://eriwen.com/ (2008)
+//                  Luke Smith http://lucassmith.name/ (2008)
+//                  Loic Dachary <loic@dachary.org> (2008)
+//                  Johan Euphrosine <proppy@aminche.com> (2008)
+//                  Oyvind Sean Kinsey http://kinsey.no/blog (2010)
+//                  Victor Homyakov <victor-homyakov@users.sourceforge.net> (2010)
+//
+// Information and discussions
+// http://jspoker.pokersource.info/skin/test-printstacktrace.html
+// http://eriwen.com/javascript/js-stack-trace/
+// http://eriwen.com/javascript/stacktrace-update/
+// http://pastie.org/253058
+//
+// guessFunctionNameFromLines comes from firebug
+//
+// Software License Agreement (BSD License)
+//
+// Copyright (c) 2007, Parakey Inc.
+// All rights reserved.
+//
+// Redistribution and use of this software in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above
+//   copyright notice, this list of conditions and the
+//   following disclaimer.
+//
+// * Redistributions in binary form must reproduce the above
+//   copyright notice, this list of conditions and the
+//   following disclaimer in the documentation and/or other
+//   materials provided with the distribution.
+//
+// * Neither the name of Parakey Inc. nor the names of its
+//   contributors may be used to endorse or promote products
+//   derived from this software without specific prior
+//   written permission of Parakey Inc.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+// FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+// OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/**
+ * Main function giving a function stack trace with a forced or passed in Error
+ *
+ * @cfg {Error} e The error to create a stacktrace from (optional)
+ * @cfg {Boolean} guess If we should try to resolve the names of anonymous functions
+ * @return {Array} of Strings with functions, lines, files, and arguments where possible
+ */
+function printStackTrace(options) {
+    options = options || {guess: true};
+    var ex = options.e || null, guess = !!options.guess;
+    var p = new printStackTrace.implementation(), result = p.run(ex);
+    return (guess) ? p.guessAnonymousFunctions(result) : result;
+}
+
+printStackTrace.implementation = function() {
+};
+
+printStackTrace.implementation.prototype = {
+    run: function(ex) {
+        ex = ex || this.createException();
+        // Do not use the stored mode: different exceptions in Chrome
+        // may or may not have arguments or stack
+        var mode = this.mode(ex);
+        // Use either the stored mode, or resolve it
+        //var mode = this._mode || this.mode(ex);
+        if (mode === 'other') {
+            return this.other(arguments.callee);
+        } else {
+            return this[mode](ex);
+        }
+    },
+
+    createException: function() {
+        try {
+            this.undef();
+            return null;
+        } catch (e) {
+            return e;
+        }
+    },
+
+    /**
+     * @return {String} mode of operation for the environment in question.
+     */
+    mode: function(e) {
+        if (e['arguments'] && e.stack) {
+            return (this._mode = 'chrome');
+        } else if (e.message && typeof window !== 'undefined' && window.opera) {
+            return (this._mode = e.stacktrace ? 'opera10' : 'opera');
+        } else if (e.stack) {
+            return (this._mode = 'firefox');
+        }
+        return (this._mode = 'other');
+    },
+
+    /**
+     * Given a context, function name, and callback function, overwrite it so that it calls
+     * printStackTrace() first with a callback and then runs the rest of the body.
+     *
+     * @param {Object} context of execution (e.g. window)
+     * @param {String} functionName to instrument
+     * @param {Function} function to call with a stack trace on invocation
+     */
+    instrumentFunction: function(context, functionName, callback) {
+        context = context || window;
+        var original = context[functionName];
+        context[functionName] = function instrumented() {
+            callback.call(this, printStackTrace().slice(4));
+            return context[functionName]._instrumented.apply(this, arguments);
+        };
+        context[functionName]._instrumented = original;
+    },
+
+    /**
+     * Given a context and function name of a function that has been
+     * instrumented, revert the function to it's original (non-instrumented)
+     * state.
+     *
+     * @param {Object} context of execution (e.g. window)
+     * @param {String} functionName to de-instrument
+     */
+    deinstrumentFunction: function(context, functionName) {
+        if (context[functionName].constructor === Function &&
+                context[functionName]._instrumented &&
+                context[functionName]._instrumented.constructor === Function) {
+            context[functionName] = context[functionName]._instrumented;
+        }
+    },
+
+    /**
+     * Given an Error object, return a formatted Array based on Chrome's stack string.
+     *
+     * @param e - Error object to inspect
+     * @return Array<String> of function calls, files and line numbers
+     */
+    chrome: function(e) {
+        //return e.stack.replace(/^[^\(]+?[\n$]/gm, '').replace(/^\s+at\s+/gm, '').replace(/^Object.<anonymous>\s*\(/gm, '{anonymous}()@').split('\n');
+        return e.stack.replace(/^\S[^\(]+?[\n$]/gm, '').
+          replace(/^\s+at\s+/gm, '').
+          replace(/^([^\(]+?)([\n$])/gm, '{anonymous}()@$1$2').
+          replace(/^Object.<anonymous>\s*\(([^\)]+)\)/gm, '{anonymous}()@$1').split('\n');
+    },
+
+    /**
+     * Given an Error object, return a formatted Array based on Firefox's stack string.
+     *
+     * @param e - Error object to inspect
+     * @return Array<String> of function calls, files and line numbers
+     */
+    firefox: function(e) {
+        return e.stack.replace(/(?:\n@:0)?\s+$/m, '').replace(/^\(/gm, '{anonymous}(').split('\n');
+    },
+
+    /**
+     * Given an Error object, return a formatted Array based on Opera 10's stacktrace string.
+     *
+     * @param e - Error object to inspect
+     * @return Array<String> of function calls, files and line numbers
+     */
+    opera10: function(e) {
+        var stack = e.stacktrace;
+        var lines = stack.split('\n'), ANON = '{anonymous}', lineRE = /.*line (\d+), column (\d+) in ((<anonymous function\:?\s*(\S+))|([^\(]+)\([^\)]*\))(?: in )?(.*)\s*$/i, i, j, len;
+        for (i = 2, j = 0, len = lines.length; i < len - 2; i++) {
+            if (lineRE.test(lines[i])) {
+                var location = RegExp.$6 + ':' + RegExp.$1 + ':' + RegExp.$2;
+                var fnName = RegExp.$3;
+                fnName = fnName.replace(/<anonymous function\:?\s?(\S+)?>/g, ANON);
+                lines[j++] = fnName + '@' + location;
+            }
+        }
+
+        lines.splice(j, lines.length - j);
+        return lines;
+    },
+
+    // Opera 7.x-9.x only!
+    opera: function(e) {
+        var lines = e.message.split('\n'), ANON = '{anonymous}', lineRE = /Line\s+(\d+).*script\s+(http\S+)(?:.*in\s+function\s+(\S+))?/i, i, j, len;
+
+        for (i = 4, j = 0, len = lines.length; i < len; i += 2) {
+            //TODO: RegExp.exec() would probably be cleaner here
+            if (lineRE.test(lines[i])) {
+                lines[j++] = (RegExp.$3 ? RegExp.$3 + '()@' + RegExp.$2 + RegExp.$1 : ANON + '()@' + RegExp.$2 + ':' + RegExp.$1) + ' -- ' + lines[i + 1].replace(/^\s+/, '');
+            }
+        }
+
+        lines.splice(j, lines.length - j);
+        return lines;
+    },
+
+    // Safari, IE, and others
+    other: function(curr) {
+        var ANON = '{anonymous}', fnRE = /function\s*([\w\-$]+)?\s*\(/i, stack = [], fn, args, maxStackSize = 10;
+        while (curr && stack.length < maxStackSize) {
+            fn = fnRE.test(curr.toString()) ? RegExp.$1 || ANON : ANON;
+            args = Array.prototype.slice.call(curr['arguments'] || []);
+            stack[stack.length] = fn + '(' + this.stringifyArguments(args) + ')';
+            curr = curr.caller;
+        }
+        return stack;
+    },
+
+    /**
+     * Given arguments array as a String, subsituting type names for non-string types.
+     *
+     * @param {Arguments} object
+     * @return {Array} of Strings with stringified arguments
+     */
+    stringifyArguments: function(args) {
+        var slice = Array.prototype.slice;
+        for (var i = 0; i < args.length; ++i) {
+            var arg = args[i];
+            if (arg === undefined) {
+                args[i] = 'undefined';
+            } else if (arg === null) {
+                args[i] = 'null';
+            } else if (arg.constructor) {
+                if (arg.constructor === Array) {
+                    if (arg.length < 3) {
+                        args[i] = '[' + this.stringifyArguments(arg) + ']';
+                    } else {
+                        args[i] = '[' + this.stringifyArguments(slice.call(arg, 0, 1)) + '...' + this.stringifyArguments(slice.call(arg, -1)) + ']';
+                    }
+                } else if (arg.constructor === Object) {
+                    args[i] = '#object';
+                } else if (arg.constructor === Function) {
+                    args[i] = '#function';
+                } else if (arg.constructor === String) {
+                    args[i] = '"' + arg + '"';
+                }
+            }
+        }
+        return args.join(',');
+    },
+
+    sourceCache: {},
+
+    /**
+     * @return the text from a given URL.
+     */
+    ajax: function(url) {
+        var req = this.createXMLHTTPObject();
+        if (!req) {
+            return;
+        }
+        req.open('GET', url, false);
+        req.setRequestHeader('User-Agent', 'XMLHTTP/1.0');
+        req.send('');
+        return req.responseText;
+    },
+
+    /**
+     * Try XHR methods in order and store XHR factory.
+     *
+     * @return <Function> XHR function or equivalent
+     */
+    createXMLHTTPObject: function() {
+        var xmlhttp, XMLHttpFactories = [
+            function() {
+                return new XMLHttpRequest();
+            }, function() {
+                return new ActiveXObject('Msxml2.XMLHTTP');
+            }, function() {
+                return new ActiveXObject('Msxml3.XMLHTTP');
+            }, function() {
+                return new ActiveXObject('Microsoft.XMLHTTP');
+            }
+        ];
+        for (var i = 0; i < XMLHttpFactories.length; i++) {
+            try {
+                xmlhttp = XMLHttpFactories[i]();
+                // Use memoization to cache the factory
+                this.createXMLHTTPObject = XMLHttpFactories[i];
+                return xmlhttp;
+            } catch (e) {
+            }
+        }
+    },
+
+    /**
+     * Given a URL, check if it is in the same domain (so we can get the source
+     * via Ajax).
+     *
+     * @param url <String> source url
+     * @return False if we need a cross-domain request
+     */
+    isSameDomain: function(url) {
+        return url.indexOf(location.hostname) !== -1;
+    },
+
+    /**
+     * Get source code from given URL if in the same domain.
+     *
+     * @param url <String> JS source URL
+     * @return <Array> Array of source code lines
+     */
+    getSource: function(url) {
+        if (!(url in this.sourceCache)) {
+            this.sourceCache[url] = this.ajax(url).split('\n');
+        }
+        return this.sourceCache[url];
+    },
+
+    guessAnonymousFunctions: function(stack) {
+        for (var i = 0; i < stack.length; ++i) {
+            var reStack = /\{anonymous\}\(.*\)@(\w+:\/\/([\-\w\.]+)+(:\d+)?[^:]+):(\d+):?(\d+)?/;
+            var frame = stack[i], m = reStack.exec(frame);
+            if (m) {
+                var file = m[1], lineno = m[4], charno = m[7] || 0; //m[7] is character position in Chrome
+                if (file && this.isSameDomain(file) && lineno) {
+                    var functionName = this.guessAnonymousFunction(file, lineno, charno);
+                    stack[i] = frame.replace('{anonymous}', functionName);
+                }
+            }
+        }
+        return stack;
+    },
+
+    guessAnonymousFunction: function(url, lineNo, charNo) {
+        var ret;
+        try {
+            ret = this.findFunctionName(this.getSource(url), lineNo);
+        } catch (e) {
+            ret = 'getSource failed with url: ' + url + ', exception: ' + e.toString();
+        }
+        return ret;
+    },
+
+    findFunctionName: function(source, lineNo) {
+        // FIXME findFunctionName fails for compressed source
+        // (more than one function on the same line)
+        // TODO use captured args
+        // function {name}({args}) m[1]=name m[2]=args
+        var reFunctionDeclaration = /function\s+([^(]*?)\s*\(([^)]*)\)/;
+        // {name} = function ({args}) TODO args capture
+        // /['"]?([0-9A-Za-z_]+)['"]?\s*[:=]\s*function(?:[^(]*)/
+        var reFunctionExpression = /['"]?([0-9A-Za-z_]+)['"]?\s*[:=]\s*function\b/;
+        // {name} = eval()
+        var reFunctionEvaluation = /['"]?([0-9A-Za-z_]+)['"]?\s*[:=]\s*(?:eval|new Function)\b/;
+        // Walk backwards in the source lines until we find
+        // the line which matches one of the patterns above
+        var code = "", line, maxLines = 10, m;
+        for (var i = 0; i < maxLines; ++i) {
+            // FIXME lineNo is 1-based, source[] is 0-based
+            line = source[lineNo - i];
+            if (line) {
+                code = line + code;
+                m = reFunctionExpression.exec(code);
+                if (m && m[1]) {
+                    return m[1];
+                }
+                m = reFunctionDeclaration.exec(code);
+                if (m && m[1]) {
+                    //return m[1] + "(" + (m[2] || "") + ")";
+                    return m[1];
+                }
+                m = reFunctionEvaluation.exec(code);
+                if (m && m[1]) {
+                    return m[1];
+                }
+            }
+        }
+        return '(?)';
+    }
+};

resources/default_tweaks.py

@@ -37,7 +37,6 @@ series_index_auto_increment = 'next'
 # Can be either True or False
 authors_completer_append_separator = False
 
-
 #: Author sort name algorithm
 # The algorithm used to copy author to author_sort
 # Possible values are:
@@ -71,6 +70,15 @@ author_name_suffixes = ('Jr', 'Sr', 'Inc', 'Ph.D', 'Phd',
 #   categories_use_field_for_author_name = 'author_sort'
 categories_use_field_for_author_name = 'author'
 
+#: Completion sort order: choose when to change from lexicographic to ASCII-like
+# Calibre normally uses locale-dependent lexicographic ordering when showing
+# completion values. This means that the sort order is correct for the user's
+# language. However, this can be slow. Performance is improved by switching to
+# ascii ordering. This tweak controls when that switch happens. Set it to zero
+# to always use ascii ordering. Set it to something larger than zero to switch
+# to ascii ordering for performance reasons.
+completion_change_to_ascii_sorting = 2500
+
 #: Control partitioning of Tag Browser
 # When partitioning the tags browser, the format of the subcategory label is
 # controlled by a template: categories_collapsed_name_template if sorting by
@@ -93,7 +101,6 @@ categories_collapsed_name_template = r'{first.sort:shorten(4,,0)} - {last.sort:s
 categories_collapsed_rating_template = r'{first.avg_rating:4.2f:ifempty(0)} - {last.avg_rating:4.2f:ifempty(0)}'
 categories_collapsed_popularity_template = r'{first.count:d} - {last.count:d}'
 
-
 #: Specify columns to sort the booklist by on startup
 # Provide a set of columns to be sorted on when calibre starts
 #  The argument is None if saved sort history is to be used
@@ -244,17 +251,14 @@ sony_collection_name_template='{value}{category:| (|)}'
 # Default: empty (no rules), so no collection attributes are named.
 sony_collection_sorting_rules = []
 
-
 #: Control how tags are applied when copying books to another library
 # Set this to True to ensure that tags in 'Tags to add when adding
 # a book' are added when copying books to another library
 add_new_book_tags_when_importing_books = False
 
-
 #: Set the maximum number of tags to show per book in the content server
 max_content_server_tags_shown=5
 
-
 #: Set custom metadata fields that the content server will or will not display.
 # content_server_will_display is a list of custom fields to be displayed.
 # content_server_wont_display is a list of custom fields not to be displayed.
@@ -296,7 +300,6 @@ generate_cover_foot_font = None
 # Example: doubleclick_on_library_view = 'do_nothing'
 doubleclick_on_library_view = 'open_viewer'
 
-
 #: Language to use when sorting.
 # Setting this tweak will force sorting to use the
 # collating order for the specified language. This might be useful if you run

resources/template-functions.json

@@ -1,42 +0,0 @@
-{
-    "and": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    while i < len(args):\n        if not args[i]:\n            return ''\n        i += 1\n    return '1'\n", 
-    "contains": "def evaluate(self, formatter, kwargs, mi, locals,\n             val, test, value_if_present, value_if_not):\n    if re.search(test, val, flags=re.I):\n        return value_if_present\n    else:\n        return value_if_not\n", 
-    "divide": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x / y)\n", 
-    "uppercase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return val.upper()\n", 
-    "strcat": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    res = ''\n    for i in range(0, len(args)):\n        res += args[i]\n    return res\n", 
-    "in_list": "def evaluate(self, formatter, kwargs, mi, locals, val, sep, pat, fv, nfv):\n    l = [v.strip() for v in val.split(sep) if v.strip()]\n    if l:\n        for v in l:\n            if re.search(pat, v, flags=re.I):\n                return fv\n    return nfv\n", 
-    "multiply": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x * y)\n", 
-    "ifempty": "def evaluate(self, formatter, kwargs, mi, locals, val, value_if_empty):\n    if val:\n        return val\n    else:\n        return value_if_empty\n", 
-    "booksize": "def evaluate(self, formatter, kwargs, mi, locals):\n    if mi.book_size is not None:\n        try:\n            return str(mi.book_size)\n        except:\n            pass\n    return ''\n", 
-    "select": "def evaluate(self, formatter, kwargs, mi, locals, val, key):\n    if not val:\n        return ''\n    vals = [v.strip() for v in val.split(',')]\n    for v in vals:\n        if v.startswith(key+':'):\n            return v[len(key)+1:]\n    return ''\n", 
-    "strcmp": "def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):\n    v = strcmp(x, y)\n    if v < 0:\n        return lt\n    if v == 0:\n        return eq\n    return gt\n", 
-    "first_non_empty": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    while i < len(args):\n        if args[i]:\n            return args[i]\n        i += 1\n    return ''\n", 
-    "re": "def evaluate(self, formatter, kwargs, mi, locals, val, pattern, replacement):\n    return re.sub(pattern, replacement, val, flags=re.I)\n", 
-    "subtract": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x - y)\n", 
-    "list_item": "def evaluate(self, formatter, kwargs, mi, locals, val, index, sep):\n    if not val:\n        return ''\n    index = int(index)\n    val = val.split(sep)\n    try:\n        return val[index]\n    except:\n        return ''\n", 
-    "shorten": "def evaluate(self, formatter, kwargs, mi, locals,\n             val, leading, center_string, trailing):\n    l = max(0, int(leading))\n    t = max(0, int(trailing))\n    if len(val) > l + len(center_string) + t:\n        return val[0:l] + center_string + ('' if t == 0 else val[-t:])\n    else:\n        return val\n", 
-    "field": "def evaluate(self, formatter, kwargs, mi, locals, name):\n    return formatter.get_value(name, [], kwargs)\n", 
-    "add": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x + y)\n", 
-    "lookup": "def evaluate(self, formatter, kwargs, mi, locals, val, *args):\n    if len(args) == 2: # here for backwards compatibility\n        if val:\n            return formatter.vformat('{'+args[0].strip()+'}', [], kwargs)\n        else:\n            return formatter.vformat('{'+args[1].strip()+'}', [], kwargs)\n    if (len(args) % 2) != 1:\n        raise ValueError(_('lookup requires either 2 or an odd number of arguments'))\n    i = 0\n    while i < len(args):\n        if i + 1 >= len(args):\n            return formatter.vformat('{' + args[i].strip() + '}', [], kwargs)\n        if re.search(args[i], val, flags=re.I):\n            return formatter.vformat('{'+args[i+1].strip() + '}', [], kwargs)\n        i += 2\n", 
-    "template": "def evaluate(self, formatter, kwargs, mi, locals, template):\n    template = template.replace('[[', '{').replace(']]', '}')\n    return formatter.__class__().safe_format(template, kwargs, 'TEMPLATE', mi)\n", 
-    "print": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    print args\n    return None\n", 
-    "merge_lists": "def evaluate(self, formatter, kwargs, mi, locals, list1, list2, separator):\n    l1 = [l.strip() for l in list1.split(separator) if l.strip()]\n    l2 = [l.strip() for l in list2.split(separator) if l.strip()]\n    lcl1 = set([icu_lower(l) for l in l1])\n    res = []\n    for i in l1:\n        res.append(i)\n    for i in l2:\n        if icu_lower(i) not in lcl1:\n            res.append(i)\n    return ', '.join(sorted(res, key=sort_key))\n", 
-    "str_in_list": "def evaluate(self, formatter, kwargs, mi, locals, val, sep, str, fv, nfv):\n    l = [v.strip() for v in val.split(sep) if v.strip()]\n    c = [v.strip() for v in str.split(sep) if v.strip()]\n    if l:\n        for v in l:\n            for t in c:\n                if strcmp(t, v) == 0:\n                    return fv\n    return nfv\n", 
-    "titlecase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return titlecase(val)\n", 
-    "subitems": "def evaluate(self, formatter, kwargs, mi, locals, val, start_index, end_index):\n    if not val:\n        return ''\n    si = int(start_index)\n    ei = int(end_index)\n    items = [v.strip() for v in val.split(',')]\n    rv = set()\n    for item in items:\n        component = item.split('.')\n        try:\n            if ei == 0:\n                rv.add('.'.join(component[si:]))\n            else:\n                rv.add('.'.join(component[si:ei]))\n        except:\n            pass\n    return ', '.join(sorted(rv, key=sort_key))\n", 
-    "sublist": "def evaluate(self, formatter, kwargs, mi, locals, val, start_index, end_index, sep):\n    if not val:\n        return ''\n    si = int(start_index)\n    ei = int(end_index)\n    val = val.split(sep)\n    try:\n        if ei == 0:\n            return sep.join(val[si:])\n        else:\n            return sep.join(val[si:ei])\n    except:\n        return ''\n", 
-    "test": "def evaluate(self, formatter, kwargs, mi, locals, val, value_if_set, value_not_set):\n    if val:\n        return value_if_set\n    else:\n        return value_not_set\n", 
-    "eval": "def evaluate(self, formatter, kwargs, mi, locals, template):\n    from formatter import eval_formatter\n    template = template.replace('[[', '{').replace(']]', '}')\n    return eval_formatter.safe_format(template, locals, 'EVAL', None)\n", 
-    "not": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    while i < len(args):\n        if args[i]:\n            return '1'\n        i += 1\n    return ''\n", 
-    "format_date": "def evaluate(self, formatter, kwargs, mi, locals, val, format_string):\n    if not val:\n        return ''\n    try:\n        dt = parse_date(val)\n        s = format_date(dt, format_string)\n    except:\n        s = 'BAD DATE'\n    return s\n", 
-    "capitalize": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return capitalize(val)\n", 
-    "count": "def evaluate(self, formatter, kwargs, mi, locals, val, sep):\n    return unicode(len(val.split(sep)))\n", 
-    "lowercase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return val.lower()\n", 
-    "substr": "def evaluate(self, formatter, kwargs, mi, locals, str_, start_, end_):\n    return str_[int(start_): len(str_) if int(end_) == 0 else int(end_)]\n", 
-    "or": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    while i < len(args):\n        if args[i]:\n            return '1'\n        i += 1\n    return ''\n", 
-    "switch": "def evaluate(self, formatter, kwargs, mi, locals, val, *args):\n    if (len(args) % 2) != 1:\n        raise ValueError(_('switch requires an odd number of arguments'))\n    i = 0\n    while i < len(args):\n        if i + 1 >= len(args):\n            return args[i]\n        if re.search(args[i], val, flags=re.I):\n            return args[i+1]\n        i += 2\n", 
-    "ondevice": "def evaluate(self, formatter, kwargs, mi, locals):\n    if mi.ondevice_col:\n        return _('Yes')\n    return ''\n", 
-    "assign": "def evaluate(self, formatter, kwargs, mi, locals, target, value):\n    locals[target] = value\n    return value\n", 
-    "raw_field": "def evaluate(self, formatter, kwargs, mi, locals, name):\n    return unicode(getattr(mi, name, None))\n", 
-    "cmp": "def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    if x < y:\n        return lt\n    if x == y:\n        return eq\n    return gt\n"
-}

setup/install.py

@@ -8,8 +8,8 @@ __docformat__ = 'restructuredtext en'
 
 import sys, os, textwrap, subprocess, shutil, tempfile, atexit, stat, shlex
 
-from setup import Command, islinux, isfreebsd, isbsd, basenames, modules, functions, \
+from setup import (Command, islinux, isbsd, basenames, modules, functions,
-        __appname__, __version__
+        __appname__, __version__)
 
 HEADER = '''\
 #!/usr/bin/env python2

setup/installer/windows/MemoryModule.c

@@ -0,0 +1,689 @@
+/*
+ * Memory DLL loading code
+ * Version 0.0.2 with additions from Thomas Heller
+ *
+ * Copyright (c) 2004-2005 by Joachim Bauch / mail@joachim-bauch.de
+ * http://www.joachim-bauch.de
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is MemoryModule.c
+ *
+ * The Initial Developer of the Original Code is Joachim Bauch.
+ *
+ * Portions created by Joachim Bauch are Copyright (C) 2004-2005
+ * Joachim Bauch. All Rights Reserved.
+ *
+ * Portions Copyright (C) 2005 Thomas Heller.
+ *
+ */
+
+// disable warnings about pointer <-> DWORD conversions
+#pragma warning( disable : 4311 4312 )
+
+#include <Windows.h>
+#include <winnt.h>
+#if DEBUG_OUTPUT
+#include <stdio.h>
+#endif
+
+#ifndef IMAGE_SIZEOF_BASE_RELOCATION
+// Vista SDKs no longer define IMAGE_SIZEOF_BASE_RELOCATION!?
+# define IMAGE_SIZEOF_BASE_RELOCATION (sizeof(IMAGE_BASE_RELOCATION))
+#endif
+#include "MemoryModule.h"
+
+/*
+  XXX We need to protect at least walking the 'loaded' linked list with a lock!
+*/
+
+/******************************************************************/
+FINDPROC findproc;
+void *findproc_data = NULL;
+
+struct NAME_TABLE {
+	char *name;
+	DWORD ordinal;
+};
+
+typedef struct tagMEMORYMODULE {
+	PIMAGE_NT_HEADERS headers;
+	unsigned char *codeBase;
+	HMODULE *modules;
+	int numModules;
+	int initialized;
+
+	struct NAME_TABLE *name_table;
+
+	char *name;
+	int refcount;
+	struct tagMEMORYMODULE *next, *prev;
+} MEMORYMODULE, *PMEMORYMODULE;
+
+typedef BOOL (WINAPI *DllEntryProc)(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpReserved);
+
+#define GET_HEADER_DICTIONARY(module, idx)	&(module)->headers->OptionalHeader.DataDirectory[idx]
+
+MEMORYMODULE *loaded; /* linked list of loaded memory modules */
+
+/* private - insert a loaded library in a linked list */
+static void _Register(char *name, MEMORYMODULE *module)
+{
+	module->next = loaded;
+	if (loaded)
+		loaded->prev = module;
+	module->prev = NULL;
+	loaded = module;
+}
+
+/* private - remove a loaded library from a linked list */
+static void _Unregister(MEMORYMODULE *module)
+{
+	free(module->name);
+	if (module->prev)
+		module->prev->next = module->next;
+	if (module->next)
+		module->next->prev = module->prev;
+	if (module == loaded)
+		loaded = module->next;
+}
+
+/* public - replacement for GetModuleHandle() */
+HMODULE MyGetModuleHandle(LPCTSTR lpModuleName)
+{
+	MEMORYMODULE *p = loaded;
+	while (p) {
+		// If already loaded, only increment the reference count
+		if (0 == stricmp(lpModuleName, p->name)) {
+			return (HMODULE)p;
+		}
+		p = p->next;
+	}
+	return GetModuleHandle(lpModuleName);
+}
+
+/* public - replacement for LoadLibrary, but searches FIRST for memory
+   libraries, then for normal libraries.  So, it will load libraries AS memory
+   module if they are found by findproc().
+*/
+HMODULE MyLoadLibrary(char *lpFileName)
+{
+	MEMORYMODULE *p = loaded;
+	HMODULE hMod;
+
+	while (p) {
+		// If already loaded, only increment the reference count
+		if (0 == stricmp(lpFileName, p->name)) {
+			p->refcount++;
+			return (HMODULE)p;
+		}
+		p = p->next;
+	}
+	if (findproc && findproc_data) {
+		void *pdata = findproc(lpFileName, findproc_data);
+		if (pdata) {
+			hMod = MemoryLoadLibrary(lpFileName, pdata);
+			free(p);
+			return hMod;
+		}
+	}
+	hMod = LoadLibrary(lpFileName);
+	return hMod;
+}
+
+/* public - replacement for GetProcAddress() */
+FARPROC MyGetProcAddress(HMODULE hModule, LPCSTR lpProcName)
+{
+	MEMORYMODULE *p = loaded;
+	while (p) {
+		if ((HMODULE)p == hModule)
+			return MemoryGetProcAddress(p, lpProcName);
+		p = p->next;
+	}
+	return GetProcAddress(hModule, lpProcName);
+}
+
+/* public - replacement for FreeLibrary() */
+BOOL MyFreeLibrary(HMODULE hModule)
+{
+	MEMORYMODULE *p = loaded;
+	while (p) {
+		if ((HMODULE)p == hModule) {
+			if (--p->refcount == 0) {
+				_Unregister(p);
+				MemoryFreeLibrary(p);
+			}
+			return TRUE;
+		}
+		p = p->next;
+	}
+	return FreeLibrary(hModule);
+}
+
+#if DEBUG_OUTPUT
+static void
+OutputLastError(const char *msg)
+{
+	LPVOID tmp;
+	char *tmpmsg;
+	FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
+		NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR)&tmp, 0, NULL);
+	tmpmsg = (char *)LocalAlloc(LPTR, strlen(msg) + strlen(tmp) + 3);
+	sprintf(tmpmsg, "%s: %s", msg, tmp);
+	OutputDebugString(tmpmsg);
+	LocalFree(tmpmsg);
+	LocalFree(tmp);
+}
+#endif
+
+/*
+static int dprintf(char *fmt, ...)
+{
+	char Buffer[4096];
+	va_list marker;
+	int result;
+	
+	va_start(marker, fmt);
+	result = vsprintf(Buffer, fmt, marker);
+	OutputDebugString(Buffer);
+	return result;
+}
+*/
+
+static void
+CopySections(const unsigned char *data, PIMAGE_NT_HEADERS old_headers, PMEMORYMODULE module)
+{
+	int i, size;
+	unsigned char *codeBase = module->codeBase;
+	unsigned char *dest;
+	PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers);
+	for (i=0; i<module->headers->FileHeader.NumberOfSections; i++, section++)
+	{
+		if (section->SizeOfRawData == 0)
+		{
+			// section doesn't contain data in the dll itself, but may define
+			// uninitialized data
+			size = old_headers->OptionalHeader.SectionAlignment;
+			if (size > 0)
+			{
+				dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress,
+					size,
+					MEM_COMMIT,
+					PAGE_READWRITE);
+
+				section->Misc.PhysicalAddress = (DWORD)dest;
+				memset(dest, 0, size);
+			}
+
+			// section is empty
+			continue;
+		}
+
+		// commit memory block and copy data from dll
+		dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress,
+							section->SizeOfRawData,
+							MEM_COMMIT,
+							PAGE_READWRITE);
+		memcpy(dest, data + section->PointerToRawData, section->SizeOfRawData);
+		section->Misc.PhysicalAddress = (DWORD)dest;
+	}
+}
+
+// Protection flags for memory pages (Executable, Readable, Writeable)
+static int ProtectionFlags[2][2][2] = {
+	{
+		// not executable
+		{PAGE_NOACCESS, PAGE_WRITECOPY},
+		{PAGE_READONLY, PAGE_READWRITE},
+	}, {
+		// executable
+		{PAGE_EXECUTE, PAGE_EXECUTE_WRITECOPY},
+		{PAGE_EXECUTE_READ, PAGE_EXECUTE_READWRITE},
+	},
+};
+
+static void
+FinalizeSections(PMEMORYMODULE module)
+{
+	int i;
+	PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers);
+	
+	// loop through all sections and change access flags
+	for (i=0; i<module->headers->FileHeader.NumberOfSections; i++, section++)
+	{
+		DWORD protect, oldProtect, size;
+		int executable = (section->Characteristics & IMAGE_SCN_MEM_EXECUTE) != 0;
+		int readable =   (section->Characteristics & IMAGE_SCN_MEM_READ) != 0;
+		int writeable =  (section->Characteristics & IMAGE_SCN_MEM_WRITE) != 0;
+
+		if (section->Characteristics & IMAGE_SCN_MEM_DISCARDABLE)
+		{
+			// section is not needed any more and can safely be freed
+			VirtualFree((LPVOID)section->Misc.PhysicalAddress, section->SizeOfRawData, MEM_DECOMMIT);
+			continue;
+		}
+
+		// determine protection flags based on characteristics
+		protect = ProtectionFlags[executable][readable][writeable];
+		if (section->Characteristics & IMAGE_SCN_MEM_NOT_CACHED)
+			protect |= PAGE_NOCACHE;
+
+		// determine size of region
+		size = section->SizeOfRawData;
+		if (size == 0)
+		{
+			if (section->Characteristics & IMAGE_SCN_CNT_INITIALIZED_DATA)
+				size = module->headers->OptionalHeader.SizeOfInitializedData;
+			else if (section->Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA)
+				size = module->headers->OptionalHeader.SizeOfUninitializedData;
+		}
+
+		if (size > 0)
+		{
+			// change memory access flags
+			if (VirtualProtect((LPVOID)section->Misc.PhysicalAddress, section->SizeOfRawData, protect, &oldProtect) == 0)
+#if DEBUG_OUTPUT
+				OutputLastError("Error protecting memory page")
+#endif
+			;
+		}
+	}
+}
+
+static void
+PerformBaseRelocation(PMEMORYMODULE module, DWORD delta)
+{
+	DWORD i;
+	unsigned char *codeBase = module->codeBase;
+
+	PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_BASERELOC);
+	if (directory->Size > 0)
+	{
+		PIMAGE_BASE_RELOCATION relocation = (PIMAGE_BASE_RELOCATION)(codeBase + directory->VirtualAddress);
+		for (; relocation->VirtualAddress > 0; )
+		{
+			unsigned char *dest = (unsigned char *)(codeBase + relocation->VirtualAddress);
+			unsigned short *relInfo = (unsigned short *)((unsigned char *)relocation + IMAGE_SIZEOF_BASE_RELOCATION);
+			for (i=0; i<((relocation->SizeOfBlock-IMAGE_SIZEOF_BASE_RELOCATION) / 2); i++, relInfo++)
+			{
+				DWORD *patchAddrHL;
+				int type, offset;
+
+				// the upper 4 bits define the type of relocation
+				type = *relInfo >> 12;
+				// the lower 12 bits define the offset
+				offset = *relInfo & 0xfff;
+				
+				switch (type)
+				{
+				case IMAGE_REL_BASED_ABSOLUTE:
+					// skip relocation
+					break;
+
+				case IMAGE_REL_BASED_HIGHLOW:
+					// change complete 32 bit address
+					patchAddrHL = (DWORD *)(dest + offset);
+					*patchAddrHL += delta;
+					break;
+
+				default:
+					//printf("Unknown relocation: %d\n", type);
+					break;
+				}
+			}
+
+			// advance to next relocation block
+			relocation = (PIMAGE_BASE_RELOCATION)(((DWORD)relocation) + relocation->SizeOfBlock);
+		}
+	}
+}
+
+static int
+BuildImportTable(PMEMORYMODULE module)
+{
+	int result=1;
+	unsigned char *codeBase = module->codeBase;
+
+	PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_IMPORT);
+	if (directory->Size > 0)
+	{
+		PIMAGE_IMPORT_DESCRIPTOR importDesc = (PIMAGE_IMPORT_DESCRIPTOR)(codeBase + directory->VirtualAddress);
+		for (; !IsBadReadPtr(importDesc, sizeof(IMAGE_IMPORT_DESCRIPTOR)) && importDesc->Name; importDesc++)
+		{
+			DWORD *thunkRef, *funcRef;
+			HMODULE handle;
+
+			handle = MyLoadLibrary(codeBase + importDesc->Name);
+			if (handle == INVALID_HANDLE_VALUE)
+			{
+				//LastError should already be set
+#if DEBUG_OUTPUT
+				OutputLastError("Can't load library");
+#endif
+				result = 0;
+				break;
+			}
+
+			module->modules = (HMODULE *)realloc(module->modules, (module->numModules+1)*(sizeof(HMODULE)));
+			if (module->modules == NULL)
+			{
+				SetLastError(ERROR_NOT_ENOUGH_MEMORY);
+				result = 0;
+				break;
+			}
+
+			module->modules[module->numModules++] = handle;
+			if (importDesc->OriginalFirstThunk)
+			{
+				thunkRef = (DWORD *)(codeBase + importDesc->OriginalFirstThunk);
+				funcRef = (DWORD *)(codeBase + importDesc->FirstThunk);
+			} else {
+				// no hint table
+				thunkRef = (DWORD *)(codeBase + importDesc->FirstThunk);
+				funcRef = (DWORD *)(codeBase + importDesc->FirstThunk);
+			}
+			for (; *thunkRef; thunkRef++, funcRef++)
+			{
+				if IMAGE_SNAP_BY_ORDINAL(*thunkRef) {
+					*funcRef = (DWORD)MyGetProcAddress(handle, (LPCSTR)IMAGE_ORDINAL(*thunkRef));
+				} else {
+					PIMAGE_IMPORT_BY_NAME thunkData = (PIMAGE_IMPORT_BY_NAME)(codeBase + *thunkRef);
+					*funcRef = (DWORD)MyGetProcAddress(handle, (LPCSTR)&thunkData->Name);
+				}
+				if (*funcRef == 0)
+				{
+					SetLastError(ERROR_PROC_NOT_FOUND);
+					result = 0;
+					break;
+				}
+			}
+
+			if (!result)
+				break;
+		}
+	}
+
+	return result;
+}
+
+/*
+  MemoryLoadLibrary - load a library AS MEMORY MODULE, or return
+  existing MEMORY MODULE with increased refcount.
+
+  This allows to load a library AGAIN as memory module which is
+  already loaded as HMODULE!
+
+*/
+HMEMORYMODULE MemoryLoadLibrary(char *name, const void *data)
+{
+	PMEMORYMODULE result;
+	PIMAGE_DOS_HEADER dos_header;
+	PIMAGE_NT_HEADERS old_header;
+	unsigned char *code, *headers;
+	DWORD locationDelta;
+	DllEntryProc DllEntry;
+	BOOL successfull;
+	MEMORYMODULE *p = loaded;
+
+	while (p) {
+		// If already loaded, only increment the reference count
+		if (0 == stricmp(name, p->name)) {
+			p->refcount++;
+			return (HMODULE)p;
+		}
+		p = p->next;
+	}
+
+	/* Do NOT check for GetModuleHandle here! */
+
+	dos_header = (PIMAGE_DOS_HEADER)data;
+	if (dos_header->e_magic != IMAGE_DOS_SIGNATURE)
+	{
+		SetLastError(ERROR_BAD_FORMAT);
+#if DEBUG_OUTPUT
+		OutputDebugString("Not a valid executable file.\n");
+#endif
+		return NULL;
+	}
+
+	old_header = (PIMAGE_NT_HEADERS)&((const unsigned char *)(data))[dos_header->e_lfanew];
+	if (old_header->Signature != IMAGE_NT_SIGNATURE)
+	{
+		SetLastError(ERROR_BAD_FORMAT);
+#if DEBUG_OUTPUT
+		OutputDebugString("No PE header found.\n");
+#endif
+		return NULL;
+	}
+
+	// reserve memory for image of library
+	code = (unsigned char *)VirtualAlloc((LPVOID)(old_header->OptionalHeader.ImageBase),
+		old_header->OptionalHeader.SizeOfImage,
+		MEM_RESERVE,
+		PAGE_READWRITE);
+
+    if (code == NULL)
+        // try to allocate memory at arbitrary position
+        code = (unsigned char *)VirtualAlloc(NULL,
+            old_header->OptionalHeader.SizeOfImage,
+            MEM_RESERVE,
+            PAGE_READWRITE);
+    
+	if (code == NULL)
+	{
+		SetLastError(ERROR_NOT_ENOUGH_MEMORY);
+#if DEBUG_OUTPUT
+		OutputLastError("Can't reserve memory");
+#endif
+		return NULL;
+	}
+
+	result = (PMEMORYMODULE)HeapAlloc(GetProcessHeap(), 0, sizeof(MEMORYMODULE));
+	result->codeBase = code;
+	result->numModules = 0;
+	result->modules = NULL;
+	result->initialized = 0;
+	result->next = result->prev = NULL;
+	result->refcount = 1;
+	result->name = strdup(name);
+	result->name_table = NULL;
+
+	// XXX: is it correct to commit the complete memory region at once?
+    //      calling DllEntry raises an exception if we don't...
+	VirtualAlloc(code,
+		old_header->OptionalHeader.SizeOfImage,
+		MEM_COMMIT,
+		PAGE_READWRITE);
+
+	// commit memory for headers
+	headers = (unsigned char *)VirtualAlloc(code,
+		old_header->OptionalHeader.SizeOfHeaders,
+		MEM_COMMIT,
+		PAGE_READWRITE);
+	
+	// copy PE header to code
+	memcpy(headers, dos_header, dos_header->e_lfanew + old_header->OptionalHeader.SizeOfHeaders);
+	result->headers = (PIMAGE_NT_HEADERS)&((const unsigned char *)(headers))[dos_header->e_lfanew];
+
+	// update position
+	result->headers->OptionalHeader.ImageBase = (DWORD)code;
+
+	// copy sections from DLL file block to new memory location
+	CopySections(data, old_header, result);
+
+	// adjust base address of imported data
+	locationDelta = (DWORD)(code - old_header->OptionalHeader.ImageBase);
+	if (locationDelta != 0)
+		PerformBaseRelocation(result, locationDelta);
+
+	// load required dlls and adjust function table of imports
+	if (!BuildImportTable(result))
+		goto error;
+
+	// mark memory pages depending on section headers and release
+	// sections that are marked as "discardable"
+	FinalizeSections(result);
+
+	// get entry point of loaded library
+	if (result->headers->OptionalHeader.AddressOfEntryPoint != 0)
+	{
+		DllEntry = (DllEntryProc)(code + result->headers->OptionalHeader.AddressOfEntryPoint);
+		if (DllEntry == 0)
+		{
+			SetLastError(ERROR_BAD_FORMAT); /* XXX ? */
+#if DEBUG_OUTPUT
+			OutputDebugString("Library has no entry point.\n");
+#endif
+			goto error;
+		}
+
+		// notify library about attaching to process
+		successfull = (*DllEntry)((HINSTANCE)code, DLL_PROCESS_ATTACH, 0);
+		if (!successfull)
+		{
+#if DEBUG_OUTPUT
+			OutputDebugString("Can't attach library.\n");
+#endif
+			goto error;
+		}
+		result->initialized = 1;
+	}
+
+	_Register(name, result);
+
+	return (HMEMORYMODULE)result;
+
+error:
+	// cleanup
+	free(result->name);
+	MemoryFreeLibrary(result);
+	return NULL;
+}
+
+int _compare(const struct NAME_TABLE *p1, const struct NAME_TABLE *p2)
+{
+	return stricmp(p1->name, p2->name);
+}
+
+int _find(const char **name, const struct NAME_TABLE *p)
+{
+	return stricmp(*name, p->name);
+}
+
+struct NAME_TABLE *GetNameTable(PMEMORYMODULE module)
+{
+	unsigned char *codeBase;
+	PIMAGE_EXPORT_DIRECTORY exports;
+	PIMAGE_DATA_DIRECTORY directory;
+	DWORD i, *nameRef;
+	WORD *ordinal;
+	struct NAME_TABLE *p, *ptab;
+
+	if (module->name_table)
+		return module->name_table;
+
+	codeBase = module->codeBase;
+	directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_EXPORT);
+	exports = (PIMAGE_EXPORT_DIRECTORY)(codeBase + directory->VirtualAddress);
+
+	nameRef = (DWORD *)(codeBase + exports->AddressOfNames);
+	ordinal = (WORD *)(codeBase + exports->AddressOfNameOrdinals);
+
+	p = ((PMEMORYMODULE)module)->name_table = (struct NAME_TABLE *)malloc(sizeof(struct NAME_TABLE)
+									      * exports->NumberOfNames);
+	if (p == NULL)
+		return NULL;
+	ptab = p;
+	for (i=0; i<exports->NumberOfNames; ++i) {
+		p->name = (char *)(codeBase + *nameRef++);
+		p->ordinal = *ordinal++;
+		++p;
+	}
+	qsort(ptab, exports->NumberOfNames, sizeof(struct NAME_TABLE), _compare);
+	return ptab;
+}
+
+FARPROC MemoryGetProcAddress(HMEMORYMODULE module, const char *name)
+{
+	unsigned char *codeBase = ((PMEMORYMODULE)module)->codeBase;
+	int idx=-1;
+	PIMAGE_EXPORT_DIRECTORY exports;
+	PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY((PMEMORYMODULE)module, IMAGE_DIRECTORY_ENTRY_EXPORT);
+
+	if (directory->Size == 0)
+		// no export table found
+		return NULL;
+
+	exports = (PIMAGE_EXPORT_DIRECTORY)(codeBase + directory->VirtualAddress);
+	if (exports->NumberOfNames == 0 || exports->NumberOfFunctions == 0)
+		// DLL doesn't export anything
+		return NULL;
+
+	if (HIWORD(name)) {
+		struct NAME_TABLE *ptab;
+		struct NAME_TABLE *found;
+		ptab = GetNameTable((PMEMORYMODULE)module);
+		if (ptab == NULL)
+			// some failure
+			return NULL;
+		found = bsearch(&name, ptab, exports->NumberOfNames, sizeof(struct NAME_TABLE), _find);
+		if (found == NULL)
+			// exported symbol not found
+			return NULL;
+	
+		idx = found->ordinal;
+	}
+	else
+		idx = LOWORD(name) - exports->Base;
+
+	if ((DWORD)idx > exports->NumberOfFunctions)
+		// name <-> ordinal number don't match
+		return NULL;
+	
+	// AddressOfFunctions contains the RVAs to the "real" functions
+	return (FARPROC)(codeBase + *(DWORD *)(codeBase + exports->AddressOfFunctions + (idx*4)));
+}
+
+void MemoryFreeLibrary(HMEMORYMODULE mod)
+{
+	int i;
+	PMEMORYMODULE module = (PMEMORYMODULE)mod;
+
+	if (module != NULL)
+	{
+		if (module->initialized != 0)
+		{
+			// notify library about detaching from process
+			DllEntryProc DllEntry = (DllEntryProc)(module->codeBase + module->headers->OptionalHeader.AddressOfEntryPoint);
+			(*DllEntry)((HINSTANCE)module->codeBase, DLL_PROCESS_DETACH, 0);
+			module->initialized = 0;
+		}
+
+		if (module->modules != NULL)
+		{
+			// free previously opened libraries
+			for (i=0; i<module->numModules; i++)
+				if (module->modules[i] != INVALID_HANDLE_VALUE)
+					MyFreeLibrary(module->modules[i]);
+
+			free(module->modules);
+		}
+
+		if (module->codeBase != NULL)
+			// release memory of library
+			VirtualFree(module->codeBase, 0, MEM_RELEASE);
+
+		if (module->name_table != NULL)
+			free(module->name_table);
+
+		HeapFree(GetProcessHeap(), 0, module);
+	}
+}

setup/installer/windows/MemoryModule.h

@@ -0,0 +1,58 @@
+/*
+ * Memory DLL loading code
+ * Version 0.0.2
+ *
+ * Copyright (c) 2004-2005 by Joachim Bauch / mail@joachim-bauch.de
+ * http://www.joachim-bauch.de
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is MemoryModule.h
+ *
+ * The Initial Developer of the Original Code is Joachim Bauch.
+ *
+ * Portions created by Joachim Bauch are Copyright (C) 2004-2005
+ * Joachim Bauch. All Rights Reserved.
+ *
+ */
+
+#ifndef __MEMORY_MODULE_HEADER
+#define __MEMORY_MODULE_HEADER
+
+#include <Windows.h>
+
+typedef void *HMEMORYMODULE;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef void *(*FINDPROC)();
+
+extern FINDPROC findproc;
+extern void *findproc_data;
+
+HMEMORYMODULE MemoryLoadLibrary(char *, const void *);
+
+FARPROC MemoryGetProcAddress(HMEMORYMODULE, const char *);
+
+void MemoryFreeLibrary(HMEMORYMODULE);
+
+BOOL MyFreeLibrary(HMODULE hModule);
+HMODULE MyLoadLibrary(char *lpFileName);
+FARPROC MyGetProcAddress(HMODULE hModule, LPCSTR lpProcName);
+HMODULE MyGetModuleHandle(LPCTSTR lpModuleName);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // __MEMORY_MODULE_HEADER

setup/installer/windows/__init__.py

@@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
 
 import os, shutil, subprocess
 
-from setup import Command, __appname__
+from setup import Command, __appname__, __version__
 from setup.installer import VMInstaller
 
 class Win(Command):
@@ -43,4 +43,11 @@ class Win32(VMInstaller):
             self.warn('Failed to freeze')
             raise SystemExit(1)
 
+        installer = 'dist/%s-portable-%s.zip'%(__appname__, __version__)
+        subprocess.check_call(('scp',
+            'xp_build:build/%s/%s'%(__appname__, installer), 'dist'))
+        if not os.path.exists(installer):
+            self.warn('Failed to get portable installer')
+            raise SystemExit(1)
+
 

setup/installer/windows/en-us.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <WixLocalization Culture="en-us" xmlns="http://schemas.microsoft.com/wix/2006/localization">
-    <String Id="AdvancedWelcomeEulaDlgDescriptionPerUser">If you are upgrading from a {app} version older than 0.6.17, please uninstall {app} first. Click Advanced to change installation settings.</String>
+    <String Id="AdvancedWelcomeEulaDlgDescriptionPerUser">Click Advanced to change installation settings.</String>
     <String Id="ProgressTextFileCost">Computing space requirements, this may take upto five minutes...</String>
     <String Id="ProgressTextCostInitialize">Computing space requirements, this may take upto five minutes...</String>
     <String Id="ProgressTextCostFinalize">Computing space requirements, this may take upto five minutes...</String>

setup/installer/windows/freeze.py

@@ -8,19 +8,19 @@ __docformat__ = 'restructuredtext en'
 
 import sys, os, shutil, glob, py_compile, subprocess, re, zipfile, time
 
-from setup import Command, modules, functions, basenames, __version__, \
+from setup import (Command, modules, functions, basenames, __version__,
-    __appname__
+    __appname__)
 from setup.build_environment import msvc, MT, RC
 from setup.installer.windows.wix import WixMixIn
 
 OPENSSL_DIR = r'Q:\openssl'
 QT_DIR = 'Q:\\Qt\\4.7.3'
 QT_DLLS = ['Core', 'Gui', 'Network', 'Svg', 'WebKit', 'Xml', 'XmlPatterns']
-LIBUSB_DIR       = 'C:\\libusb'
 LIBUNRAR         = 'C:\\Program Files\\UnrarDLL\\unrar.dll'
 SW               = r'C:\cygwin\home\kovid\sw'
 IMAGEMAGICK      = os.path.join(SW, 'build', 'ImageMagick-6.6.6',
         'VisualMagick', 'bin')
+CRT = r'C:\Microsoft.VC90.CRT'
 
 VERSION = re.sub('[a-z]\d+', '', __version__)
 WINVER = VERSION+'.0'
@@ -50,7 +50,7 @@ def walk(dir):
 
 class Win32Freeze(Command, WixMixIn):
 
-    description = 'Free windows calibre installation'
+    description = 'Freeze windows calibre installation'
 
     def add_options(self, parser):
         parser.add_option('--no-ice', default=False, action='store_true',
@@ -71,33 +71,69 @@ class Win32Freeze(Command, WixMixIn):
         self.rc_template = self.j(self.d(self.a(__file__)), 'template.rc')
         self.py_ver = ''.join(map(str, sys.version_info[:2]))
         self.lib_dir = self.j(self.base, 'Lib')
-        self.pydlib = self.j(self.base, 'pydlib')
         self.pylib = self.j(self.base, 'pylib.zip')
+        self.dll_dir = self.j(self.base, 'DLLs')
+        self.plugins_dir = os.path.join(self.base, 'plugins2')
+        self.portable_base = self.j(self.d(self.base), 'Calibre Portable')
+        self.obj_dir = self.j(self.src_root, 'build', 'launcher')
 
         self.initbase()
         self.build_launchers()
+        self.add_plugins()
         self.freeze()
         self.embed_manifests()
         self.install_site_py()
         self.archive_lib_dir()
+        self.remove_CRT_from_manifests()
         self.create_installer()
+        self.build_portable()
+
+    def remove_CRT_from_manifests(self):
+        '''
+        The dependency on the CRT is removed from the manifests of all DLLs.
+        This allows the CRT loaded by the .exe files to be used instead.
+        '''
+        search_pat = re.compile(r'(?is)<dependency>.*Microsoft\.VC\d+\.CRT')
+        repl_pat = re.compile(
+            r'(?is)<dependency>.*?Microsoft\.VC\d+\.CRT.*?</dependency>')
+
+        for dll in glob.glob(self.j(self.dll_dir, '*.dll')):
+            bn = self.b(dll)
+            with open(dll, 'rb') as f:
+                raw = f.read()
+            match = search_pat.search(raw)
+            if match is None:
+                continue
+            self.info('Removing CRT dependency from manifest of: %s'%bn)
+            # Blank out the bytes corresponding to the dependency specification
+            nraw = repl_pat.sub(lambda m: b' '*len(m.group()), raw)
+            if len(nraw) != len(raw):
+                raise Exception('Something went wrong with %s'%bn)
+            with open(dll, 'wb') as f:
+                f.write(nraw)
 
     def initbase(self):
         if self.e(self.base):
             shutil.rmtree(self.base)
         os.makedirs(self.base)
 
+    def add_plugins(self):
+        self.info('Adding plugins...')
+        tgt = self.plugins_dir
+        if os.path.exists(tgt):
+            shutil.rmtree(tgt)
+        os.mkdir(tgt)
+        base = self.j(self.SRC, 'calibre', 'plugins')
+        for f in glob.glob(self.j(base, '*.pyd')):
+            # We dont want the manifests as the manifest in the exe will be
+            # used instead
+            shutil.copy2(f, tgt)
+
     def freeze(self):
         shutil.copy2(self.j(self.src_root, 'LICENSE'), self.base)
 
-        self.info('Adding plugins...')
+        self.info('Adding CRT')
-        tgt = os.path.join(self.base, 'plugins')
+        shutil.copytree(CRT, self.j(self.base, os.path.basename(CRT)))
-        if not os.path.exists(tgt):
-            os.mkdir(tgt)
-        base = self.j(self.SRC, 'calibre', 'plugins')
-        for pat in ('*.pyd', '*.manifest'):
-            for f in glob.glob(self.j(base, pat)):
-                shutil.copy2(f, tgt)
 
         self.info('Adding resources...')
         tgt = self.j(self.base, 'resources')
@@ -106,7 +142,6 @@ class Win32Freeze(Command, WixMixIn):
         shutil.copytree(self.j(self.src_root, 'resources'), tgt)
 
         self.info('Adding Qt and python...')
-        self.dll_dir = self.j(self.base, 'DLLs')
         shutil.copytree(r'C:\Python%s\DLLs'%self.py_ver, self.dll_dir,
                 ignore=shutil.ignore_patterns('msvc*.dll', 'Microsoft.*'))
         for x in glob.glob(self.j(OPENSSL_DIR, 'bin', '*.dll')):
@@ -194,14 +229,13 @@ class Win32Freeze(Command, WixMixIn):
             if os.path.exists(tg):
                 shutil.rmtree(tg)
             shutil.copytree(imfd, tg)
+        for dirpath, dirnames, filenames in os.walk(tdir):
+            for x in filenames:
+                if not x.endswith('.dll'):
+                    os.remove(self.j(dirpath, x))
 
         print
         print 'Adding third party dependencies'
-        tdir = os.path.join(self.base, 'driver')
-        os.makedirs(tdir)
-        for pat in ('*.dll', '*.sys', '*.cat', '*.inf'):
-            for f in glob.glob(os.path.join(LIBUSB_DIR, pat)):
-                shutil.copyfile(f, os.path.join(tdir, os.path.basename(f)))
         print '\tAdding unrar'
         shutil.copyfile(LIBUNRAR,
                 os.path.join(self.dll_dir, os.path.basename(LIBUNRAR)))
@@ -256,7 +290,7 @@ class Win32Freeze(Command, WixMixIn):
     def embed_resources(self, module, desc=None):
         icon_base = self.j(self.src_root, 'icons')
         icon_map = {'calibre':'library', 'ebook-viewer':'viewer',
-                'lrfviewer':'viewer'}
+                'lrfviewer':'viewer', 'calibre-portable':'library'}
         file_type = 'DLL' if module.endswith('.dll') else 'APP'
         template = open(self.rc_template, 'rb').read()
         bname = self.b(module)
@@ -313,13 +347,67 @@ class Win32Freeze(Command, WixMixIn):
             self.info(p.stderr.read())
             sys.exit(1)
 
+    def build_portable(self):
+        base  = self.portable_base
+        if os.path.exists(base):
+            shutil.rmtree(base)
+        os.makedirs(base)
+        src = self.j(self.src_root, 'setup', 'installer', 'windows',
+                'portable.c')
+        obj = self.j(self.obj_dir, self.b(src)+'.obj')
+        cflags  = '/c /EHsc /MT /W3 /Ox /nologo /D_UNICODE'.split()
+
+        if self.newer(obj, [src]):
+            self.info('Compiling', obj)
+            cmd = [msvc.cc] + cflags + ['/Fo'+obj, '/Tc'+src]
+            self.run_builder(cmd)
+
+        exe = self.j(base, 'calibre-portable.exe')
+        if self.newer(exe, [obj]):
+            self.info('Linking', exe)
+            cmd = [msvc.linker] + ['/INCREMENTAL:NO', '/MACHINE:X86',
+                    '/LIBPATH:'+self.obj_dir, '/SUBSYSTEM:WINDOWS',
+                    '/RELEASE',
+                    '/OUT:'+exe, self.embed_resources(exe),
+                    obj, 'User32.lib']
+            self.run_builder(cmd)
+
+        self.info('Creating portable installer')
+        shutil.copytree(self.base, self.j(base, 'Calibre'))
+        os.mkdir(self.j(base, 'Calibre Library'))
+        os.mkdir(self.j(base, 'Calibre Settings'))
+
+        name = '%s-portable-%s.zip'%(__appname__, __version__)
+        with zipfile.ZipFile(self.j('dist', name), 'w', zipfile.ZIP_DEFLATED) as zf:
+            self.add_dir_to_zip(zf, base, 'Calibre Portable')
+
+    def add_dir_to_zip(self, zf, path, prefix=''):
+        '''
+        Add a directory recursively to the zip file with an optional prefix.
+        '''
+        if prefix:
+            zi = zipfile.ZipInfo(prefix+'/')
+            zi.external_attr = 16
+            zf.writestr(zi, '')
+        cwd = os.path.abspath(os.getcwd())
+        try:
+            os.chdir(path)
+            fp = (prefix + ('/' if prefix else '')).replace('//', '/')
+            for f in os.listdir('.'):
+                arcname = fp + f
+                if os.path.isdir(f):
+                    self.add_dir_to_zip(zf, f, prefix=arcname)
+                else:
+                    zf.write(f, arcname)
+        finally:
+            os.chdir(cwd)
+
     def build_launchers(self):
-        self.obj_dir = self.j(self.src_root, 'build', 'launcher')
         if not os.path.exists(self.obj_dir):
             os.makedirs(self.obj_dir)
         base = self.j(self.src_root, 'setup', 'installer', 'windows')
-        sources = [self.j(base, x) for x in ['util.c']]
+        sources = [self.j(base, x) for x in ['util.c', 'MemoryModule.c']]
-        headers = [self.j(base, x) for x in ['util.h']]
+        headers = [self.j(base, x) for x in ['util.h', 'MemoryModule.h']]
         objects = [self.j(self.obj_dir, self.b(x)+'.obj') for x in sources]
         cflags  = '/c /EHsc /MD /W3 /Ox /nologo /D_UNICODE'.split()
         cflags += ['/DPYDLL="python%s.dll"'%self.py_ver, '/IC:/Python%s/include'%self.py_ver]
@@ -371,43 +459,49 @@ class Win32Freeze(Command, WixMixIn):
 
     def archive_lib_dir(self):
         self.info('Putting all python code into a zip file for performance')
-        if os.path.exists(self.pydlib):
-            shutil.rmtree(self.pydlib)
-        os.makedirs(self.pydlib)
         self.zf_timestamp = time.localtime(time.time())[:6]
         self.zf_names = set()
         with zipfile.ZipFile(self.pylib, 'w', zipfile.ZIP_STORED) as zf:
+            # Add the .pyds from python and calibre to the zip file
+            for x in (self.plugins_dir, self.dll_dir):
+                for pyd in os.listdir(x):
+                    if pyd.endswith('.pyd') and pyd != 'sqlite_custom.pyd':
+                        # sqlite_custom has to be a file for
+                        # sqlite_load_extension to work
+                        self.add_to_zipfile(zf, pyd, x)
+                        os.remove(self.j(x, pyd))
+
+            # Add everything in Lib except site-packages to the zip file
             for x in os.listdir(self.lib_dir):
                 if x == 'site-packages':
                     continue
                 self.add_to_zipfile(zf, x, self.lib_dir)
 
             sp = self.j(self.lib_dir, 'site-packages')
-            handled = set(['site.pyo'])
+            # Special handling for PIL and pywin32
-            for pth in ('PIL.pth', 'pywin32.pth'):
+            handled = set(['PIL.pth', 'pywin32.pth', 'PIL', 'win32'])
-                handled.add(pth)
+            self.add_to_zipfile(zf, 'PIL', sp)
-                shutil.copyfile(self.j(sp, pth), self.j(self.pydlib, pth))
+            base = self.j(sp, 'win32', 'lib')
-                for d in self.get_pth_dirs(self.j(sp, pth)):
+            for x in os.listdir(base):
-                    shutil.copytree(d, self.j(self.pydlib, self.b(d)), True)
+                if os.path.splitext(x)[1] not in ('.exe',):
-                    handled.add(self.b(d))
+                    self.add_to_zipfile(zf, x, base)
+            base = self.d(base)
+            for x in os.listdir(base):
+                if not os.path.isdir(self.j(base, x)):
+                    if os.path.splitext(x)[1] not in ('.exe',):
+                        self.add_to_zipfile(zf, x, base)
 
             handled.add('easy-install.pth')
             for d in self.get_pth_dirs(self.j(sp, 'easy-install.pth')):
                 handled.add(self.b(d))
-                zip_safe = self.is_zip_safe(d)
                 for x in os.listdir(d):
                     if x == 'EGG-INFO':
                         continue
-                    if zip_safe:
                     self.add_to_zipfile(zf, x, d)
-                    else:
-                        absp = self.j(d, x)
-                        dest = self.j(self.pydlib, x)
-                        if os.path.isdir(absp):
-                            shutil.copytree(absp, dest, True)
-                        else:
-                            shutil.copy2(absp, dest)
 
+            # The rest of site-packages
+            # We dont want the site.py from site-packages
+            handled.add('site.pyo')
             for x in os.listdir(sp):
                 if x in handled or x.endswith('.egg-info'):
                     continue
@@ -415,33 +509,18 @@ class Win32Freeze(Command, WixMixIn):
                 if os.path.isdir(absp):
                     if not os.listdir(absp):
                         continue
-                    if self.is_zip_safe(absp):
                     self.add_to_zipfile(zf, x, sp)
-                    else:
-                        shutil.copytree(absp, self.j(self.pydlib, x), True)
-                else:
-                    if x.endswith('.pyd'):
-                        shutil.copy2(absp, self.j(self.pydlib, x))
                 else:
                     self.add_to_zipfile(zf, x, sp)
 
         shutil.rmtree(self.lib_dir)
 
-    def is_zip_safe(self, path):
-        for f in walk(path):
-            ext = os.path.splitext(f)[1].lower()
-            if ext in ('.pyd', '.dll', '.exe'):
-                return False
-        return True
-
     def get_pth_dirs(self, pth):
         base = os.path.dirname(pth)
         for line in open(pth).readlines():
             line = line.strip()
             if not line or line.startswith('#') or line.startswith('import'):
                 continue
-            if line == 'win32\\lib':
-                continue
             candidate = self.j(base, line)
             if os.path.exists(candidate):
                 yield candidate
@@ -463,10 +542,10 @@ class Win32Freeze(Command, WixMixIn):
                     self.add_to_zipfile(zf, name + os.sep + x, base)
         else:
             ext = os.path.splitext(name)[1].lower()
-            if ext in ('.pyd', '.dll', '.exe'):
+            if ext in ('.dll',):
                 raise ValueError('Cannot add %r to zipfile'%abspath)
             zinfo.external_attr = 0600 << 16
-            if ext in ('.py', '.pyc', '.pyo'):
+            if ext in ('.py', '.pyc', '.pyo', '.pyd'):
                 with open(abspath, 'rb') as f:
                     zf.writestr(zinfo, f.read())
 

setup/installer/windows/notes.rst

@@ -88,7 +88,9 @@ Qt uses its own routine to locate and load "system libraries" including the open
 
 Now, run configure and make::
 
-    configure -opensource -release -qt-zlib -qt-gif -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs -openssl -I Q:\openssl\include -L Q:\openssl\lib && nmake
+-no-plugin-manifests is needed so that loading the plugins does not fail looking for the CRT assembly
+
+    configure -opensource -release -qt-zlib -qt-gif -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs -no-plugin-manifests -openssl -I Q:\openssl\include -L Q:\openssl\lib && nmake
 
 SIP
 -----

setup/installer/windows/portable.c

@@ -0,0 +1,146 @@
+#ifndef UNICODE
+#define UNICODE
+#endif 
+
+#include <windows.h>
+#include <tchar.h>
+#include <stdio.h>
+
+#define BUFSIZE 4096
+
+void show_error(LPCTSTR msg) {
+    MessageBeep(MB_ICONERROR);
+    MessageBox(NULL, msg, TEXT("Error"), MB_OK|MB_ICONERROR);
+}
+
+void show_detailed_error(LPCTSTR preamble, LPCTSTR msg, int code) {
+    LPTSTR buf;
+    buf = (LPTSTR)LocalAlloc(LMEM_ZEROINIT, sizeof(TCHAR)*
+            (_tcslen(msg) + _tcslen(preamble) + 80));
+
+    _sntprintf_s(buf, 
+        LocalSize(buf) / sizeof(TCHAR), _TRUNCATE,
+        TEXT("%s\r\n  %s (Error Code: %d)\r\n"), 
+        preamble, msg, code);
+
+    show_error(buf);
+    LocalFree(buf);
+}
+
+void show_last_error_crt(LPCTSTR preamble) {
+    TCHAR buf[BUFSIZE];
+    int err = 0;
+
+    _get_errno(&err);
+    _wcserror_s(buf, BUFSIZE, err);
+    show_detailed_error(preamble, buf, err);
+}
+
+void show_last_error(LPCTSTR preamble) {
+    TCHAR *msg = NULL;
+    DWORD dw = GetLastError(); 
+
+    FormatMessage(
+        FORMAT_MESSAGE_ALLOCATE_BUFFER | 
+        FORMAT_MESSAGE_FROM_SYSTEM |
+        FORMAT_MESSAGE_IGNORE_INSERTS,
+        NULL,
+        dw,
+        MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+        &msg,
+        0, NULL );
+
+    show_detailed_error(preamble, msg, (int)dw);
+}
+
+
+LPTSTR get_app_dir() {
+    LPTSTR buf, buf2, buf3;
+    DWORD sz;
+    TCHAR drive[4] = TEXT("\0\0\0");
+    errno_t err;
+
+    buf = (LPTSTR)calloc(BUFSIZE, sizeof(TCHAR));
+    buf2 = (LPTSTR)calloc(BUFSIZE, sizeof(TCHAR));
+    buf3 = (LPTSTR)calloc(BUFSIZE, sizeof(TCHAR));
+
+    sz = GetModuleFileName(NULL, buf, BUFSIZE);
+
+    if (sz == 0 || sz > BUFSIZE-1) {
+        show_error(TEXT("Failed to get path to calibre-portable.exe"));
+        ExitProcess(1);
+    }
+
+    err = _tsplitpath_s(buf, drive, 4, buf2, BUFSIZE, NULL, 0, NULL, 0);
+
+    if (err != 0) {
+        show_last_error_crt(TEXT("Failed to split path to calibre-portable.exe"));
+        ExitProcess(1);
+    }
+
+    _sntprintf_s(buf3, BUFSIZE-1, _TRUNCATE, TEXT("%s%s"), drive, buf2);
+    free(buf); free(buf2);
+    return buf3;
+}
+
+void launch_calibre(LPCTSTR exe, LPCTSTR config_dir, LPCTSTR library_dir) {
+    DWORD dwFlags=0;
+    STARTUPINFO si;
+    PROCESS_INFORMATION pi;
+    BOOL fSuccess; 
+    TCHAR cmdline[BUFSIZE];
+
+    if (! SetEnvironmentVariable(TEXT("CALIBRE_CONFIG_DIRECTORY"), config_dir)) {
+        show_last_error(TEXT("Failed to set environment variables"));
+        ExitProcess(1);
+    }
+
+    dwFlags = CREATE_UNICODE_ENVIRONMENT | CREATE_NEW_PROCESS_GROUP;
+    _sntprintf_s(cmdline, BUFSIZE, _TRUNCATE, TEXT(" \"--with-library=%s\""), library_dir);
+
+    ZeroMemory( &si, sizeof(si) );
+    si.cb = sizeof(si);
+    ZeroMemory( &pi, sizeof(pi) );
+
+    fSuccess = CreateProcess(exe, cmdline,
+        NULL,           // Process handle not inheritable
+        NULL,           // Thread handle not inheritable
+        FALSE,          // Set handle inheritance to FALSE
+        dwFlags,        // Creation flags http://msdn.microsoft.com/en-us/library/ms684863(v=vs.85).aspx
+        NULL,           // Use parent's environment block
+        NULL,           // Use parent's starting directory 
+        &si,            // Pointer to STARTUPINFO structure
+        &pi             // Pointer to PROCESS_INFORMATION structure
+    );
+
+    if (fSuccess == 0) {
+        show_last_error(TEXT("Failed to launch the calibre program"));
+    }
+
+    // Close process and thread handles.
+    CloseHandle( pi.hProcess );
+    CloseHandle( pi.hThread );
+
+}
+
+int WINAPI wWinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, PWSTR pCmdLine, int nCmdShow)
+{
+    LPTSTR app_dir, config_dir, exe, library_dir;
+
+    app_dir = get_app_dir();
+    config_dir = (LPTSTR)calloc(BUFSIZE, sizeof(TCHAR));
+    library_dir = (LPTSTR)calloc(BUFSIZE, sizeof(TCHAR));
+    exe = (LPTSTR)calloc(BUFSIZE, sizeof(TCHAR));
+
+    _sntprintf_s(config_dir, BUFSIZE, _TRUNCATE, TEXT("%sCalibre Settings"), app_dir);
+    _sntprintf_s(exe, BUFSIZE, _TRUNCATE, TEXT("%sCalibre\\calibre.exe"), app_dir);
+    _sntprintf_s(library_dir, BUFSIZE, _TRUNCATE, TEXT("%sCalibre Library"), app_dir);
+
+    launch_calibre(exe, config_dir, library_dir);
+
+    free(app_dir); free(config_dir); free(exe); free(library_dir);
+
+    return 0;
+}
+
+

setup/installer/windows/site.py

@@ -1,12 +1,72 @@
 #!/usr/bin/env python
 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import with_statement
 
 __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import sys, os, linecache
+import sys
+import os
+import zipimport
+import _memimporter
+
+DEBUG_ZIPIMPORT = False
+
+class ZipExtensionImporter(zipimport.zipimporter):
+    '''
+    Taken, with thanks, from the py2exe source code
+    '''
+
+    def __init__(self, *args, **kwargs):
+        zipimport.zipimporter.__init__(self, *args, **kwargs)
+        # We know there are no dlls in the zip file, so dont set findproc
+        # (performance optimization)
+        #_memimporter.set_find_proc(self.locate_dll_image)
+
+    def find_module(self, fullname, path=None):
+        result = zipimport.zipimporter.find_module(self, fullname, path)
+        if result:
+            return result
+        fullname = fullname.replace(".", "\\")
+        if (fullname + '.pyd') in self._files:
+            return self
+        return None
+
+    def locate_dll_image(self, name):
+        # A callback function for_memimporter.import_module.  Tries to
+        # locate additional dlls.  Returns the image as Python string,
+        # or None if not found.
+        if name in self._files:
+            return self.get_data(name)
+        return None
+
+    def load_module(self, fullname):
+        if sys.modules.has_key(fullname):
+            mod = sys.modules[fullname]
+            if DEBUG_ZIPIMPORT:
+                sys.stderr.write("import %s # previously loaded from zipfile %s\n" % (fullname, self.archive))
+            return mod
+        try:
+            return zipimport.zipimporter.load_module(self, fullname)
+        except zipimport.ZipImportError:
+            pass
+        initname = "init" + fullname.split(".")[-1] # name of initfunction
+        filename = fullname.replace(".", "\\")
+        path = filename + '.pyd'
+        if path in self._files:
+            if DEBUG_ZIPIMPORT:
+                sys.stderr.write("# found %s in zipfile %s\n" % (path, self.archive))
+            code = self.get_data(path)
+            mod = _memimporter.import_module(code, initname, fullname, path)
+            mod.__file__ = "%s\\%s" % (self.archive, path)
+            mod.__loader__ = self
+            if DEBUG_ZIPIMPORT:
+                sys.stderr.write("import %s # loaded from zipfile %s\n" % (fullname, mod.__file__))
+            return mod
+        raise zipimport.ZipImportError, "can't find module %s" % fullname
+
+    def __repr__(self):
+        return "<%s object %r>" % (self.__class__.__name__, self.archive)
 
 
 def abs__file__():
@@ -32,7 +92,7 @@ def aliasmbcs():
 
 def add_calibre_vars():
     sys.resources_location = os.path.join(sys.app_dir, 'resources')
-    sys.extensions_location = os.path.join(sys.app_dir, 'plugins')
+    sys.extensions_location = os.path.join(sys.app_dir, 'plugins2')
 
     dv = os.environ.get('CALIBRE_DEVELOP_FROM', None)
     if dv and os.path.exists(dv):
@@ -42,42 +102,6 @@ def makepath(*paths):
     dir = os.path.abspath(os.path.join(*paths))
     return dir, os.path.normcase(dir)
 
-def addpackage(sitedir, name):
-    """Process a .pth file within the site-packages directory:
-       For each line in the file, either combine it with sitedir to a path,
-       or execute it if it starts with 'import '.
-    """
-    fullname = os.path.join(sitedir, name)
-    try:
-        f = open(fullname, "rU")
-    except IOError:
-        return
-    with f:
-        for line in f:
-            if line.startswith("#"):
-                continue
-            if line.startswith(("import ", "import\t")):
-                exec line
-                continue
-            line = line.rstrip()
-            dir, dircase = makepath(sitedir, line)
-            if os.path.exists(dir):
-                sys.path.append(dir)
-
-
-def addsitedir(sitedir):
-    """Add 'sitedir' argument to sys.path if missing and handle .pth files in
-    'sitedir'"""
-    sitedir, sitedircase = makepath(sitedir)
-    try:
-        names = os.listdir(sitedir)
-    except os.error:
-        return
-    dotpth = os.extsep + "pth"
-    names = [name for name in names if name.endswith(dotpth)]
-    for name in sorted(names):
-        addpackage(sitedir, name)
-
 def run_entry_point():
     bname, mod, func = sys.calibre_basename, sys.calibre_module, sys.calibre_function
     sys.argv[0] = bname+'.exe'
@@ -89,6 +113,10 @@ def main():
     sys.setdefaultencoding('utf-8')
     aliasmbcs()
 
+    sys.path_hooks.insert(0, ZipExtensionImporter)
+    sys.path_importer_cache.clear()
+
+    import linecache
     def fake_getline(filename, lineno, module_globals=None):
         return ''
     linecache.orig_getline = linecache.getline
@@ -96,10 +124,11 @@ def main():
 
     abs__file__()
 
-    addsitedir(os.path.join(sys.app_dir, 'pydlib'))
-
     add_calibre_vars()
 
+    # Needed for pywintypes to be able to load its DLL
+    sys.path.append(os.path.join(sys.app_dir, 'DLLs'))
+
     return run_entry_point()
 
 

setup/installer/windows/util.c

@@ -1,18 +1,130 @@
 /*
  * Copyright 2009 Kovid Goyal
+ * The memimporter code is taken from the py2exe project
  */
 
 #include "util.h"
+
 #include <delayimp.h>
 #include <io.h>
 #include <fcntl.h>
 
+
 static char GUI_APP = 0;
 static char python_dll[] = PYDLL;
 
 void set_gui_app(char yes) { GUI_APP = yes; }
 char is_gui_app() { return GUI_APP; }
 
+
+// memimporter {{{
+
+#include "MemoryModule.h"
+
+static char **DLL_Py_PackageContext = NULL;
+static PyObject **DLL_ImportError = NULL;
+static char module_doc[] =
+"Importer which can load extension modules from memory";
+
+
+static void *memdup(void *ptr, Py_ssize_t size)
+{
+	void *p = malloc(size);
+	if (p == NULL)
+		return NULL;
+	memcpy(p, ptr, size);
+	return p;
+}
+
+/*
+  Be sure to detect errors in FindLibrary - undetected errors lead to
+  very strange behaviour.
+*/
+static void* FindLibrary(char *name, PyObject *callback)
+{
+	PyObject *result;
+	char *p;
+	Py_ssize_t size;
+
+	if (callback == NULL)
+		return NULL;
+	result = PyObject_CallFunction(callback, "s", name);
+	if (result == NULL) {
+		PyErr_Clear();
+		return NULL;
+	}
+	if (-1 == PyString_AsStringAndSize(result, &p, &size)) {
+		PyErr_Clear();
+		Py_DECREF(result);
+		return NULL;
+	}
+	p = memdup(p, size);
+	Py_DECREF(result);
+	return p;
+}
+
+static PyObject *set_find_proc(PyObject *self, PyObject *args)
+{
+	PyObject *callback = NULL;
+	if (!PyArg_ParseTuple(args, "|O:set_find_proc", &callback))
+		return NULL;
+	Py_DECREF((PyObject *)findproc_data);
+	Py_INCREF(callback);
+	findproc_data = (void *)callback;
+    return Py_BuildValue("i", 1);
+}
+
+static PyObject *
+import_module(PyObject *self, PyObject *args)
+{
+	char *data;
+	int size;
+	char *initfuncname;
+	char *modname;
+	char *pathname;
+	HMEMORYMODULE hmem;
+	FARPROC do_init;
+
+	char *oldcontext;
+
+	/* code, initfuncname, fqmodulename, path */
+	if (!PyArg_ParseTuple(args, "s#sss:import_module",
+			      &data, &size,
+			      &initfuncname, &modname, &pathname))
+		return NULL;
+	hmem = MemoryLoadLibrary(pathname, data);
+	if (!hmem) {
+		PyErr_Format(*DLL_ImportError,
+			     "MemoryLoadLibrary() failed loading %s", pathname);
+		return NULL;
+	}
+	do_init = MemoryGetProcAddress(hmem, initfuncname);
+	if (!do_init) {
+		MemoryFreeLibrary(hmem);
+		PyErr_Format(*DLL_ImportError,
+			     "Could not find function %s in memory loaded pyd", initfuncname);
+		return NULL;
+	}
+
+    oldcontext = *DLL_Py_PackageContext;
+	*DLL_Py_PackageContext = modname;
+	do_init();
+	*DLL_Py_PackageContext = oldcontext;
+	if (PyErr_Occurred())
+		return NULL;
+	/* Retrieve from sys.modules */
+	return PyImport_ImportModule(modname);
+}
+
+static PyMethodDef methods[] = {
+	{ "import_module", import_module, METH_VARARGS,
+	  "import_module(code, initfunc, dllname[, finder]) -> module" },
+	{ "set_find_proc", set_find_proc, METH_VARARGS },
+	{ NULL, NULL },		/* Sentinel */
+};
+
+// }}}
+
 static int _show_error(const wchar_t *preamble, const wchar_t *msg, const int code) {
     wchar_t *buf, *cbuf;
     buf = (wchar_t*)LocalAlloc(LMEM_ZEROINIT, sizeof(wchar_t)*
@@ -61,7 +173,7 @@ int show_last_error(wchar_t *preamble) {
         NULL,
         dw,
         MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
-        msg,
+        &msg,
         0, NULL );
 
     return _show_error(preamble, msg, (int)dw);
@@ -185,7 +297,7 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr,
     char *dummy_argv[1] = {""};
 
     buf  = (char*)calloc(MAX_PATH, sizeof(char));
-    path = (char*)calloc(3*MAX_PATH, sizeof(char));
+    path = (char*)calloc(MAX_PATH, sizeof(char));
     if (!buf || !path) ExitProcess(_show_error(L"Out of memory", L"", 1));
 
     sz = GetModuleFileNameA(NULL, buf, MAX_PATH);
@@ -198,8 +310,7 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr,
     buf[strlen(buf)-1] = '\0';
 
     _snprintf_s(python_home, MAX_PATH, _TRUNCATE, "%s", buf);
-    _snprintf_s(path, 3*MAX_PATH, _TRUNCATE, "%s\\pylib.zip;%s\\pydlib;%s\\DLLs",
+    _snprintf_s(path, MAX_PATH, _TRUNCATE, "%s\\pylib.zip", buf);
-            buf, buf, buf);
     free(buf);
 
 
@@ -227,7 +338,10 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr,
     if (!flag) ExitProcess(_show_error(L"Failed to get debug flag", L"", 1));
     //*flag = 1;
 
-
+    DLL_Py_PackageContext = (char**)GetProcAddress(dll, "_Py_PackageContext");
+    if (!DLL_Py_PackageContext) ExitProcess(_show_error(L"Failed to load _Py_PackageContext from dll", L"", 1));
+    DLL_ImportError = (PyObject**)GetProcAddress(dll, "PyExc_ImportError");
+    if (!DLL_ImportError) ExitProcess(_show_error(L"Failed to load PyExc_ImportError from dll", L"", 1));
 
     Py_SetProgramName(program_name);
     Py_SetPythonHome(python_home);
@@ -263,6 +377,10 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr,
         PyList_SetItem(argv, i, v);
     }
     PySys_SetObject("argv", argv);
+
+    findproc = FindLibrary;
+	Py_InitModule3("_memimporter", methods, module_doc);
+
 }
 
 

setup/installer/windows/wix-template.xml

@@ -11,6 +11,10 @@
 						SummaryCodepage='1252' />
 
                 <Media Id="1" Cabinet="{app}.cab" CompressionLevel="{compression}" EmbedCab="yes" />
+                <!-- The following line ensures that DLLs are replaced even if their version is the same as before. This
+                     is necessary because of the manifest nuking that was part of making calibre isolated. But I think it
+                     is more rigorous anyway. -->
+                <Property Id='REINSTALLMODE' Value='emus'/>
 
                 <Upgrade Id="{upgrade_code}">
                     <UpgradeVersion Maximum="{version}"
@@ -33,7 +37,6 @@
                </Property>
 
                 <Directory Id='TARGETDIR' Name='SourceDir'>
-                        <Merge Id="VCRedist" SourceFile="{crt_msm}" DiskId="1" Language="0"/>
 						<Directory Id='ProgramFilesFolder' Name='PFiles'>
 								<Directory Id='APPLICATIONFOLDER' Name='{app}' />
 						</Directory>
@@ -100,10 +103,6 @@
                             <ComponentRef Id="RememberInstallDir"/>
                         </Feature>
 
-                        <Feature Id="VCRedist" Title="Visual C++ 8.0 Runtime" AllowAdvertise="no" Display="hidden" Level="1">
-                            <MergeRef Id="VCRedist"/>
-                        </Feature>
-
                         <Feature Id="FSMS" Title="Start menu shortcuts" Level="1"
                             Description="Program shortcuts installed in the Start Menu">
                             <ComponentRef Id="StartMenuShortcuts"/>
@@ -149,12 +148,13 @@
                 Set default folder name and allow only per machine installs.
                 For a per-machine installation, the default installation location
                 will be [ProgramFilesFolder][ApplicationFolderName] and the user
-                will be able to change it in the setup UI. This is because the installer
+                will be able to change it in the setup UI. This is no longer necessary
-                has to install the VC90 merge module into the system winsxs folder for python
+                (i.e. per user installs should work) but left this way as I
-                to work, so per user installs are impossible anyway.
+                dont want to deal with the complications 
                 -->
                 <Property Id="ApplicationFolderName" Value="Calibre2" />
                 <Property Id="WixAppFolder" Value="WixPerMachineFolder" />
+                <Property Id="ALLUSERS" Value="1" />
                 <WixVariable Id="WixUISupportPerUser" Value="0" />
 
                 <!-- Add option to launch calibre after install -->
@@ -164,10 +164,6 @@
                 <CustomAction Id="LaunchApplication" BinaryKey="WixCA"
                     DllEntry="WixShellExec" Impersonate="yes"/>
 
-                <InstallUISequence>
-                    <FileCost Suppress="yes" />
-                </InstallUISequence>
-
 		</Product>
 </Wix>
 

setup/installer/windows/wix.py

@@ -35,7 +35,6 @@ class WixMixIn:
                 exe_map           = self.smap,
                 main_icon         = self.j(self.src_root, 'icons', 'library.ico'),
                 web_icon          = self.j(self.src_root, 'icons', 'web.ico'),
-                crt_msm           = self.j(self.SW, 'Microsoft_VC90_CRT_x86.msm')
                 )
         template = open(self.j(self.d(__file__), 'en-us.xml'),
                 'rb').read()

setup/pygettext.py

@@ -187,7 +187,6 @@ msgstr ""
 
 '''%dict(appname=__appname__, version=version, year=time.strftime('%Y'))
 
-
 def usage(code, msg=''):
     print >> sys.stderr, __doc__ % globals()
     if msg:

setup/translations.py

@@ -85,7 +85,7 @@ class Translations(POT):
 
     def mo_file(self, po_file):
         locale = os.path.splitext(os.path.basename(po_file))[0]
-        return locale, os.path.join(self.DEST, locale, 'LC_MESSAGES', 'messages.mo')
+        return locale, os.path.join(self.DEST, locale, 'messages.mo')
 
 
     def run(self, opts):
@@ -94,7 +94,6 @@ class Translations(POT):
             base = os.path.dirname(dest)
             if not os.path.exists(base):
                 os.makedirs(base)
-            if self.newer(dest, f):
             self.info('\tCompiling translations for', locale)
             subprocess.check_call(['msgfmt', '-o', dest, f])
             if locale in ('en_GB', 'nds', 'te', 'yi'):
@@ -123,6 +122,16 @@ class Translations(POT):
                 shutil.copy2(f, dest)
 
         self.write_stats()
+        self.freeze_locales()
+
+    def freeze_locales(self):
+        zf = self.DEST + '.zip'
+        from calibre import CurrentDir
+        from calibre.utils.zipfile import ZipFile, ZIP_DEFLATED
+        with ZipFile(zf, 'w', ZIP_DEFLATED) as zf:
+            with CurrentDir(self.DEST):
+                zf.add_dir('.')
+        shutil.rmtree(self.DEST)
 
     @property
     def stats(self):

setup/upload.py

@@ -26,6 +26,7 @@ def installers():
     installers = list(map(installer_name, ('dmg', 'msi', 'tar.bz2')))
     installers.append(installer_name('tar.bz2', is64bit=True))
     installers.insert(0, 'dist/%s-%s.tar.gz'%(__appname__, __version__))
+    installers.append('dist/%s-portable-%s.zip'%(__appname__, __version__))
     return installers
 
 def installer_description(fname):
@@ -38,6 +39,8 @@ def installer_description(fname):
         return 'Windows installer'
     if fname.endswith('.dmg'):
         return 'OS X dmg'
+    if fname.endswith('.zip'):
+        return 'Calibre Portable'
     return 'Unknown file'
 
 class ReUpload(Command): # {{{

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 8, 3)
+numeric_version = (0, 8, 4)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 

src/calibre/customize/builtins.py

@@ -586,10 +586,10 @@ from calibre.devices.apple.driver import ITUNES
 from calibre.devices.hanlin.driver import HANLINV3, HANLINV5, BOOX, SPECTRA
 from calibre.devices.blackberry.driver import BLACKBERRY
 from calibre.devices.cybook.driver import CYBOOK, ORIZON
-from calibre.devices.eb600.driver import EB600, COOL_ER, SHINEBOOK, \
+from calibre.devices.eb600.driver import (EB600, COOL_ER, SHINEBOOK,
-                POCKETBOOK360, GER2, ITALICA, ECLICTO, DBOOK, INVESBOOK, \
+                POCKETBOOK360, GER2, ITALICA, ECLICTO, DBOOK, INVESBOOK,
-                BOOQ, ELONEX, POCKETBOOK301, MENTOR, POCKETBOOK602, \
+                BOOQ, ELONEX, POCKETBOOK301, MENTOR, POCKETBOOK602,
-                POCKETBOOK701
+                POCKETBOOK701, POCKETBOOK360P)
 from calibre.devices.iliad.driver import ILIAD
 from calibre.devices.irexdr.driver import IREXDR1000, IREXDR800
 from calibre.devices.jetbook.driver import JETBOOK, MIBUK, JETBOOK_MINI
@@ -608,9 +608,9 @@ from calibre.devices.edge.driver import EDGE
 from calibre.devices.teclast.driver import TECLAST_K3, NEWSMY, IPAPYRUS, \
         SOVOS, PICO, SUNSTECH_EB700, ARCHOS7O, STASH, WEXLER
 from calibre.devices.sne.driver import SNE
-from calibre.devices.misc import PALMPRE, AVANT, SWEEX, PDNOVEL, \
+from calibre.devices.misc import (PALMPRE, AVANT, SWEEX, PDNOVEL,
-        GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR, \
+        GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR,
-        TREKSTOR, EEEREADER, NEXTBOOK
+        TREKSTOR, EEEREADER, NEXTBOOK, ADAM)
 from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG
 from calibre.devices.kobo.driver import KOBO
 from calibre.devices.bambook.driver import BAMBOOK
@@ -689,7 +689,7 @@ plugins += [
     JETBOOK_MINI,
     MIBUK,
     SHINEBOOK,
-    POCKETBOOK360, POCKETBOOK301, POCKETBOOK602, POCKETBOOK701,
+    POCKETBOOK360, POCKETBOOK301, POCKETBOOK602, POCKETBOOK701, POCKETBOOK360P,
     KINDLE,
     KINDLE2,
     KINDLE_DX,
@@ -744,6 +744,7 @@ plugins += [
     TREKSTOR,
     EEEREADER,
     NEXTBOOK,
+    ADAM,
     ITUNES,
     BOEYE_BEX,
     BOEYE_BDX,
@@ -1227,6 +1228,15 @@ class StoreEHarlequinStore(StoreBase):
     formats = ['EPUB', 'PDF']
     affiliate = True
 
+class StoreEpubBudStore(StoreBase):
+    name = 'ePub Bud'
+    description = 'Well, it\'s pretty much just "YouTube for Children\'s eBooks. A not-for-profit organization devoted to brining self published childrens books to the world.'
+    actual_plugin = 'calibre.gui2.store.epubbud_plugin:EpubBudStore'
+
+    drm_free_only = True
+    headquarters = 'US'
+    formats = ['EPUB']
+
 class StoreFeedbooksStore(StoreBase):
     name = 'Feedbooks'
     description = u'Feedbooks is a cloud publishing and distribution service, connected to a large ecosystem of reading systems and social networks. Provides a variety of genres from independent and classic books.'
@@ -1408,6 +1418,15 @@ class StoreWoblinkStore(StoreBase):
     headquarters = 'PL'
     formats = ['EPUB']
 
+class StoreZixoStore(StoreBase):
+    name = 'Zixo'
+    author = u'Tomasz Długosz'
+    description = u'Księgarnia z ebookami oraz książkami audio. Aby otwierać książki w formacie Zixo należy zainstalować program dostępny na stronie księgarni. Umożliwia on m.in. dodawanie zakładek i dostosowywanie rozmiaru czcionki.'
+    actual_plugin = 'calibre.gui2.store.zixo_plugin:ZixoStore'
+
+    headquarters = 'PL'
+    formats = ['PDF, ZIXO']
+
 plugins += [
     StoreArchiveOrgStore,
     StoreAmazonKindleStore,
@@ -1422,6 +1441,7 @@ plugins += [
     StoreEBookShoppeUKStore,
     StoreEPubBuyDEStore,
     StoreEHarlequinStore,
+    StoreEpubBudStore,
     StoreFeedbooksStore,
     StoreFoylesUKStore,
     StoreGandalfStore,
@@ -1441,7 +1461,8 @@ plugins += [
     StoreWeightlessBooksStore,
     StoreWHSmithUKStore,
     StoreWizardsTowerBooksStore,
-    StoreWoblinkStore
+    StoreWoblinkStore,
+    StoreZixoStore
 ]
 
 # }}}

src/calibre/debug.py

@@ -53,6 +53,8 @@ Run an embedded python interpreter.
             default=False, action='store_true')
     parser.add_option('-m', '--inspect-mobi',
             help='Inspect the MOBI file at the specified path', default=None)
+    parser.add_option('--test-build', help='Test binary modules in build',
+            action='store_true', default=False)
 
     return parser
 
@@ -232,6 +234,9 @@ def main(args=sys.argv):
     elif opts.inspect_mobi is not None:
         from calibre.ebooks.mobi.debug import inspect_mobi
         inspect_mobi(opts.inspect_mobi)
+    elif opts.test_build:
+        from calibre.test_build import test
+        test()
     else:
         from calibre import ipython
         ipython()

src/calibre/devices/android/driver.py

@@ -52,6 +52,8 @@ class ANDROID(USBMS):
             0x04e8 : { 0x681d : [0x0222, 0x0223, 0x0224, 0x0400],
                        0x681c : [0x0222, 0x0224, 0x0400],
                        0x6640 : [0x0100],
+                       0x685e : [0x0400],
+                       0x6860 : [0x0400],
                        0x6877 : [0x0400],
                      },
 
@@ -113,7 +115,8 @@ class ANDROID(USBMS):
             'MB525']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
-            'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD']
+            'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD',
+            '__UMS_COMPOSITE']
 
     OSX_MAIN_MEM = 'Android Device Main Memory'
 

src/calibre/devices/apple/driver.py

@@ -2392,6 +2392,16 @@ class ITUNES(DriverBase):
                 self.iTunes.Windows[0].Minimized = True
             self.initial_status = 'launched'
 
+            try:
+                # Pre-emptive test to confirm functional iTunes automation interface
+                foo = self.iTunes.Version
+                foo
+            except:
+                self.iTunes = None
+                raise OpenFeedback('Unable to connect to iTunes.\n' +
+                             ' iTunes automation interface non-responsive, ' +
+                             'recommend reinstalling iTunes')
+
             # Read the current storage path for iTunes media from the XML file
             media_dir = ''
             string = None
@@ -2743,7 +2753,6 @@ class ITUNES(DriverBase):
         # Update metadata from plugboard
         # If self.plugboard is None (no transforms), original metadata is returned intact
         metadata_x = self._xform_metadata_via_plugboard(metadata, this_book.format)
-        self.log("metadata.title_sort: %s  metadata_x.title_sort: %s" % (metadata.title_sort, metadata_x.title_sort))
         if isosx:
             if lb_added:
                 lb_added.name.set(metadata_x.title)
@@ -2989,7 +2998,6 @@ class ITUNES(DriverBase):
             newmi = book
         return newmi
 
-
 class ITUNES_ASYNC(ITUNES):
     '''
     This subclass allows the user to interact directly with iTunes via a menu option
@@ -3024,6 +3032,8 @@ class ITUNES_ASYNC(ITUNES):
                 pythoncom.CoInitialize()
                 self._launch_iTunes()
             except:
+                import traceback
+                traceback.print_exc()
                 raise UserFeedback('unable to launch iTunes', details=None, level=UserFeedback.WARN)
             finally:
                 pythoncom.CoUninitialize()

src/calibre/devices/eb600/driver.py

@@ -35,8 +35,8 @@ class EB600(USBMS):
     PRODUCT_ID  = [0x1688]
     BCD         = [0x110]
 
-    VENDOR_NAME      = 'NETRONIX'
+    VENDOR_NAME      = ['NETRONIX', 'WOLDER']
-    WINDOWS_MAIN_MEM = 'EBOOK'
+    WINDOWS_MAIN_MEM = ['EBOOK', 'MIBUK_GAMMA_6.2']
     WINDOWS_CARD_A_MEM = 'EBOOK'
 
     OSX_MAIN_MEM = 'EB600 Internal Storage Media'
@@ -246,6 +246,16 @@ class POCKETBOOK602(USBMS):
     WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['PB602', 'PB603', 'PB902',
             'PB903', 'PB']
 
+class POCKETBOOK360P(POCKETBOOK602):
+
+    name = 'PocketBook 360+ Device Interface'
+    description    = _('Communicate with the PocketBook 360+ reader.')
+    BCD         = [0x0323]
+    EBOOK_DIR_MAIN = ''
+
+    VENDOR_NAME = '__POCKET'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'BOOK_USB_STORAGE'
+
 class POCKETBOOK701(USBMS):
 
     name = 'PocketBook 701 Device Interface'

src/calibre/devices/misc.py

@@ -224,13 +224,16 @@ class TREKSTOR(USBMS):
     FORMATS     = ['epub', 'txt', 'pdf']
 
     VENDOR_ID   = [0x1e68]
-    PRODUCT_ID  = [0x0041, 0x0042]
+    PRODUCT_ID  = [0x0041, 0x0042,
+            0x003e # This is for the EBOOK_PLAYER_5M https://bugs.launchpad.net/bugs/792091
+            ]
     BCD         = [0x0002]
 
     EBOOK_DIR_MAIN = 'Ebooks'
 
     VENDOR_NAME = 'TREKSTOR'
-    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'EBOOK_PLAYER_7'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['EBOOK_PLAYER_7',
+            'EBOOK_PLAYER_5M']
 
 class EEEREADER(USBMS):
 
@@ -252,6 +255,28 @@ class EEEREADER(USBMS):
     VENDOR_NAME = 'LINUX'
     WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'FILE-STOR_GADGET'
 
+class ADAM(USBMS):
+
+    name = 'Notion Ink Adam device interface'
+    gui_name = 'Adam'
+
+    description    = _('Communicate with the Adam tablet')
+    author         = 'Kovid Goyal'
+    supported_platforms = ['windows', 'osx', 'linux']
+
+    # Ordered list of supported formats
+    FORMATS     = ['epub', 'pdf', 'doc']
+
+    VENDOR_ID   = [0x0955]
+    PRODUCT_ID  = [0x7100]
+    BCD         = [0x9999]
+
+    EBOOK_DIR_MAIN = 'eBooks'
+
+    VENDOR_NAME = 'NI'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['ADAM']
+    SUPPORTS_SUB_DIRS = True
+
 class NEXTBOOK(USBMS):
 
     name           = 'Nextbook device interface'

src/calibre/devices/nook/driver.py

@@ -107,6 +107,9 @@ class NOOK_COLOR(NOOK):
 
         return filepath
 
+    def upload_cover(self, path, filename, metadata, filepath):
+        pass
+
 class NOOK_TSR(NOOK):
     gui_name       = _('Nook Simple')
     description    = _('Communicate with the Nook TSR eBook reader.')
@@ -115,5 +118,8 @@ class NOOK_TSR(NOOK):
     BCD         = [0x216]
 
     EBOOK_DIR_MAIN = EBOOK_DIR_CARD_A = 'My Files/Books'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'EBOOK_DISK'
 
+    def upload_cover(self, path, filename, metadata, filepath):
+        pass
 

src/calibre/devices/usbms/books.py

@@ -204,7 +204,8 @@ class CollectionsBookList(BookList):
                 elif fm['datatype'] == 'text' and fm['is_multiple']:
                     val = orig_val
                 elif fm['datatype'] == 'composite' and fm['is_multiple']:
-                    val = [v.strip() for v in val.split(fm['is_multiple'])]
+                    val = [v.strip() for v in
+                           val.split(fm['is_multiple']['ui_to_list'])]
                 else:
                     val = [val]
 

src/calibre/ebooks/comic/input.py

@@ -11,7 +11,7 @@ import os, shutil, traceback, textwrap, time, codecs
 from Queue import Empty
 
 from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation
-from calibre import extract, CurrentDir, prints
+from calibre import extract, CurrentDir, prints, walk
 from calibre.constants import filesystem_encoding
 from calibre.ptempfile import PersistentTemporaryDirectory
 from calibre.utils.ipc.server import Server
@@ -27,6 +27,11 @@ def extract_comic(path_to_comic_file):
         # names
         tdir = tdir.decode(filesystem_encoding)
     extract(path_to_comic_file, tdir)
+    for x in walk(tdir):
+        bn = os.path.basename(x)
+        nbn = bn.replace('#', '_')
+        if nbn != bn:
+            os.rename(x, os.path.join(os.path.dirname(x), nbn))
     return tdir
 
 def find_pages(dir, sort_on_mtime=False, verbose=False):
@@ -362,6 +367,7 @@ class ComicInput(InputFormatPlugin):
                 if not line:
                     continue
                 fname, title = line.partition(':')[0], line.partition(':')[-1]
+                fname = fname.replace('#', '_')
                 fname = os.path.join(tdir, *fname.split('/'))
                 if not title:
                     title = os.path.basename(fname).rpartition('.')[0]

src/calibre/ebooks/metadata/book/base.py

@@ -54,7 +54,10 @@ class SafeFormat(TemplateFormatter):
                     key = orig_key
                 else:
                     raise ValueError(_('Value: unknown field ') + orig_key)
+        try:
             b = self.book.get_user_metadata(key, False)
+        except:
+            b = None
         if b and b['datatype'] == 'int' and self.book.get(key, 0) == 0:
             v = ''
         elif b and b['datatype'] == 'float' and self.book.get(key, 0.0) == 0.0:
@@ -618,10 +621,7 @@ class Metadata(object):
             orig_res = res
             datatype = cmeta['datatype']
             if datatype == 'text' and cmeta['is_multiple']:
-                if cmeta['display'].get('is_names', False):
+                res = cmeta['is_multiple']['list_to_ui'].join(res)
-                    res = u' & '.join(res)
-                else:
-                    res = u', '.join(sorted(res, key=sort_key))
             elif datatype == 'series' and series_with_index:
                 if self.get_extra(key) is not None:
                     res = res + \
@@ -665,7 +665,7 @@ class Metadata(object):
             elif datatype == 'text' and fmeta['is_multiple']:
                 if isinstance(res, dict):
                     res = [k + ':' + v for k,v in res.items()]
-                res = u', '.join(sorted(res, key=sort_key))
+                res = fmeta['is_multiple']['list_to_ui'].join(sorted(res, key=sort_key))
             elif datatype == 'series' and series_with_index:
                 res = res + ' [%s]'%self.format_series_index()
             elif datatype == 'datetime':

src/calibre/ebooks/metadata/book/json_codec.py

@@ -5,8 +5,7 @@ Created on 4 Jun 2010
 '''
 
 from base64 import b64encode, b64decode
-import json
+import json, traceback
-import traceback
 
 from calibre.ebooks.metadata.book import SERIALIZABLE_FIELDS
 from calibre.constants import filesystem_encoding, preferred_encoding
@@ -69,6 +68,40 @@ def object_to_unicode(obj, enc=preferred_encoding):
         return ans
     return obj
 
+def encode_is_multiple(fm):
+    if fm.get('is_multiple', None):
+        # migrate is_multiple back to a character
+        fm['is_multiple2'] = fm.get('is_multiple', {})
+        dt = fm.get('datatype', None)
+        if dt == 'composite':
+            fm['is_multiple'] = ','
+        else:
+            fm['is_multiple'] =  '|'
+    else:
+        fm['is_multiple'] = None
+        fm['is_multiple2'] = {}
+
+def decode_is_multiple(fm):
+    im = fm.get('is_multiple2',  None)
+    if im:
+        fm['is_multiple'] = im
+        del fm['is_multiple2']
+    else:
+        # Must migrate the is_multiple from char to dict
+        im = fm.get('is_multiple',  {})
+        if im:
+            dt = fm.get('datatype', None)
+            if dt == 'composite':
+                im = {'cache_to_list': ',', 'ui_to_list': ',',
+                      'list_to_ui': ', '}
+            elif fm.get('display', {}).get('is_names', False):
+                im = {'cache_to_list': '|', 'ui_to_list': '&',
+                      'list_to_ui': ', '}
+            else:
+                im = {'cache_to_list': '|', 'ui_to_list': ',',
+                      'list_to_ui': ', '}
+        fm['is_multiple'] = im
+
 class JsonCodec(object):
 
     def __init__(self):
@@ -93,9 +126,10 @@ class JsonCodec(object):
     def encode_metadata_attr(self, book, key):
         if key == 'user_metadata':
             meta = book.get_all_user_metadata(make_copy=True)
-            for k in meta:
+            for fm in meta.itervalues():
-                if meta[k]['datatype'] == 'datetime':
+                if fm['datatype'] == 'datetime':
-                    meta[k]['#value#'] = datetime_to_string(meta[k]['#value#'])
+                    fm['#value#'] = datetime_to_string(fm['#value#'])
+                encode_is_multiple(fm)
             return meta
         if key in self.field_metadata:
             datatype = self.field_metadata[key]['datatype']
@@ -135,9 +169,10 @@ class JsonCodec(object):
         if key == 'classifiers':
             key = 'identifiers'
         if key == 'user_metadata':
-            for k in value:
+            for fm in value.itervalues():
-                if value[k]['datatype'] == 'datetime':
+                if fm['datatype'] == 'datetime':
-                    value[k]['#value#'] = string_to_datetime(value[k]['#value#'])
+                    fm['#value#'] = string_to_datetime(fm['#value#'])
+                decode_is_multiple(fm)
             return value
         elif key in self.field_metadata:
             if self.field_metadata[key]['datatype'] == 'datetime':

src/calibre/ebooks/metadata/opf2.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 lxml based OPF parser.
 '''
 
-import re, sys, unittest, functools, os, uuid, glob, cStringIO, json
+import re, sys, unittest, functools, os, uuid, glob, cStringIO, json, copy
 from urllib import unquote
 from urlparse import urlparse
 
@@ -453,10 +453,13 @@ class TitleSortField(MetadataField):
 
 def serialize_user_metadata(metadata_elem, all_user_metadata, tail='\n'+(' '*8)):
     from calibre.utils.config import to_json
-    from calibre.ebooks.metadata.book.json_codec import object_to_unicode
+    from calibre.ebooks.metadata.book.json_codec import (object_to_unicode,
+                                                         encode_is_multiple)
 
     for name, fm in all_user_metadata.items():
         try:
+            fm = copy.copy(fm)
+            encode_is_multiple(fm)
             fm = object_to_unicode(fm)
             fm = json.dumps(fm, default=to_json, ensure_ascii=False)
         except:
@@ -575,6 +578,7 @@ class OPF(object): # {{{
         self._user_metadata_ = {}
         temp = Metadata('x', ['x'])
         from calibre.utils.config import from_json
+        from calibre.ebooks.metadata.book.json_codec import decode_is_multiple
         elems = self.root.xpath('//*[name() = "meta" and starts-with(@name,'
                 '"calibre:user_metadata:") and @content]')
         for elem in elems:
@@ -585,6 +589,7 @@ class OPF(object): # {{{
             fm = elem.get('content')
             try:
                 fm = json.loads(fm, object_hook=from_json)
+                decode_is_multiple(fm)
                 temp.set_user_metadata(name, fm)
             except:
                 prints('Failed to read user metadata:', name)

src/calibre/ebooks/metadata/sources/amazon.py

@@ -42,6 +42,7 @@ class Worker(Thread): # Get details {{{
         months = {
                 'de': {
             1 : ['jän'],
+            2 : ['februar'],
             3 : ['märz'],
             5 : ['mai'],
             6 : ['juni'],

src/calibre/ebooks/oeb/iterator.py

@@ -18,7 +18,7 @@ from calibre.ebooks.chardet import xml_to_unicode
 from calibre.utils.zipfile import safe_replace
 from calibre.utils.config import DynamicConfig
 from calibre.utils.logging import Log
-from calibre import guess_type, prints
+from calibre import guess_type, prints, prepare_string_for_xml
 from calibre.ebooks.oeb.transforms.cover import CoverManager
 
 TITLEPAGE = CoverManager.SVG_TEMPLATE.decode('utf-8').replace(\
@@ -229,8 +229,8 @@ class EbookIterator(object):
         cover = self.opf.cover
         if self.ebook_ext in ('lit', 'mobi', 'prc', 'opf', 'fb2') and cover:
             cfile = os.path.join(self.base, 'calibre_iterator_cover.html')
-            chtml = (TITLEPAGE%os.path.relpath(cover, self.base).replace(os.sep,
+            rcpath = os.path.relpath(cover, self.base).replace(os.sep, '/')
-                '/')).encode('utf-8')
+            chtml = (TITLEPAGE%prepare_string_for_xml(rcpath, True)).encode('utf-8')
             open(cfile, 'wb').write(chtml)
             self.spine[0:0] = [SpineItem(cfile,
                 mime_type='application/xhtml+xml')]

src/calibre/ebooks/oeb/stylizer.py

@@ -13,7 +13,13 @@ from weakref import WeakKeyDictionary
 from xml.dom import SyntaxErr as CSSSyntaxError
 import cssutils
 from cssutils.css import (CSSStyleRule, CSSPageRule, CSSStyleDeclaration,
-    CSSValueList, CSSFontFaceRule, cssproperties)
+    CSSFontFaceRule, cssproperties)
+try:
+    from cssutils.css import CSSValueList
+    CSSValueList
+except ImportError:
+    # cssutils >= 0.9.8
+    from cssutils.css import PropertyValue as CSSValueList
 from cssutils import profile as cssprofiles
 from lxml import etree
 from lxml.cssselect import css_to_xpath, ExpressionError, SelectorSyntaxError

src/calibre/ebooks/rtf/rtfml.py

@@ -68,8 +68,13 @@ TODO:
 '''
 
 def txt2rtf(text):
+    # Escape { and } in the text.
+    text = text.replace('{', r'\'7b')
+    text = text.replace('}', r'\'7d')
+
     if not isinstance(text, unicode):
         return text
+
     buf = cStringIO.StringIO()
     for x in text:
         val = ord(x)

src/calibre/gui2/actions/convert.py

@@ -171,10 +171,9 @@ class ConvertAction(InterfaceAction):
                 raise Exception(_('Empty output file, '
                     'probably the conversion process crashed'))
 
-            data = open(temp_files[-1].name, 'rb')
+            with open(temp_files[-1].name, 'rb') as data:
                 self.gui.library_view.model().db.add_format(book_id, \
                     fmt, data, index_is_id=True)
-            data.close()
             self.gui.status_bar.show_message(job.description + \
                     (' completed'), 2000)
         finally:

src/calibre/gui2/actions/delete.py

@@ -94,6 +94,9 @@ class DeleteAction(InterfaceAction):
         self.delete_menu.addAction(
                 _('Remove all formats from selected books, except...'),
                 self.delete_all_but_selected_formats)
+        self.delete_menu.addAction(
+                _('Remove all formats from selected books'),
+                self.delete_all_formats)
         self.delete_menu.addAction(
                 _('Remove covers from selected books'), self.delete_covers)
         self.delete_menu.addSeparator()
@@ -174,6 +177,28 @@ class DeleteAction(InterfaceAction):
         if ids:
             self.gui.tags_view.recount()
 
+    def delete_all_formats(self, *args):
+        ids = self._get_selected_ids()
+        if not ids:
+            return
+        if not confirm('<p>'+_('<b>All formats</b> for the selected books will '
+                               'be <b>deleted</b> from your library.<br>'
+                               'The book metadata will be kept. Are you sure?')
+                            +'</p>', 'delete_all_formats', self.gui):
+            return
+        db = self.gui.library_view.model().db
+        for id in ids:
+            fmts = db.formats(id, index_is_id=True, verify_formats=False)
+            if fmts:
+                for fmt in fmts.split(','):
+                    self.gui.library_view.model().db.remove_format(id, fmt,
+                            index_is_id=True, notify=False)
+        self.gui.library_view.model().refresh_ids(ids)
+        self.gui.library_view.model().current_changed(self.gui.library_view.currentIndex(),
+                self.gui.library_view.currentIndex())
+        if ids:
+            self.gui.tags_view.recount()
+
     def remove_matching_books_from_device(self, *args):
         if not self.gui.device_manager.is_device_connected:
             d = error_dialog(self.gui, _('Cannot delete books'),

src/calibre/gui2/complete.py

@@ -12,16 +12,23 @@ from PyQt4.Qt import QLineEdit, QAbstractListModel, Qt, \
 from calibre.utils.icu import sort_key, lower
 from calibre.gui2 import NONE
 from calibre.gui2.widgets import EnComboBox, LineEditECM
+from calibre.utils.config_base import tweaks
 
 class CompleteModel(QAbstractListModel):
 
     def __init__(self, parent=None):
         QAbstractListModel.__init__(self, parent)
         self.items = []
+        self.sorting = QCompleter.UnsortedModel
 
     def set_items(self, items):
         items = [unicode(x.strip()) for x in items]
-        self.items = list(sorted(items, key=lambda x: sort_key(x)))
+        if len(items) < tweaks['completion_change_to_ascii_sorting']:
+            self.items = sorted(items, key=lambda x: sort_key(x))
+            self.sorting = QCompleter.UnsortedModel
+        else:
+            self.items = sorted(items, key=lambda x:x.lower())
+            self.sorting = QCompleter.CaseInsensitivelySortedModel
         self.lowered_items = [lower(x) for x in self.items]
         self.reset()
 
@@ -62,7 +69,7 @@ class MultiCompleteLineEdit(QLineEdit, LineEditECM):
         c.setWidget(self)
         c.setCompletionMode(QCompleter.PopupCompletion)
         c.setCaseSensitivity(Qt.CaseInsensitive)
-        c.setModelSorting(QCompleter.UnsortedModel)
+        c.setModelSorting(self._model.sorting)
         c.setCompletionRole(Qt.DisplayRole)
         p = c.popup()
         p.setMouseTracking(True)
@@ -146,6 +153,7 @@ class MultiCompleteLineEdit(QLineEdit, LineEditECM):
             return self._model.items
         def fset(self, items):
             self._model.set_items(items)
+            self._completer.setModelSorting(self._model.sorting)
         return property(fget=fget, fset=fset)
 
 class MultiCompleteComboBox(EnComboBox):

src/calibre/gui2/custom_column_widgets.py

@@ -226,16 +226,14 @@ class Comments(Base):
 class Text(Base):
 
     def setup_ui(self, parent):
-        if self.col_metadata['display'].get('is_names', False):
+        self.sep = self.col_metadata['multiple_seps']
-            self.sep = u' & '
-        else:
-            self.sep = u', '
         values = self.all_values = list(self.db.all_custom(num=self.col_id))
         values.sort(key=sort_key)
+
         if self.col_metadata['is_multiple']:
             w = MultiCompleteLineEdit(parent)
-            w.set_separator(self.sep.strip())
+            w.set_separator(self.sep['ui_to_list'])
-            if self.sep == u' & ':
+            if self.sep['ui_to_list'] == '&':
                 w.set_space_before_sep(True)
                 w.set_add_separator(tweaks['authors_completer_append_separator'])
             w.update_items_cache(values)
@@ -269,12 +267,12 @@ class Text(Base):
         if self.col_metadata['is_multiple']:
             if not val:
                 val = []
-            self.widgets[1].setText(self.sep.join(val))
+            self.widgets[1].setText(self.sep['list_to_ui'].join(val))
 
     def getter(self):
         if self.col_metadata['is_multiple']:
             val = unicode(self.widgets[1].text()).strip()
-            ans = [x.strip() for x in val.split(self.sep.strip()) if x.strip()]
+            ans = [x.strip() for x in val.split(self.sep['ui_to_list']) if x.strip()]
             if not ans:
                 ans = None
             return ans
@@ -899,9 +897,10 @@ class BulkText(BulkBase):
         if not self.a_c_checkbox.isChecked():
             return
         if self.col_metadata['is_multiple']:
+            ism = self.col_metadata['multiple_seps']
             if self.col_metadata['display'].get('is_names', False):
                 val = self.gui_val
-                add = [v.strip() for v in val.split('&') if v.strip()]
+                add = [v.strip() for v in val.split(ism['ui_to_list']) if v.strip()]
                 self.db.set_custom_bulk(book_ids, add, num=self.col_id)
             else:
                 remove_all, adding, rtext = self.gui_val
@@ -911,10 +910,10 @@ class BulkText(BulkBase):
                 else:
                     txt = rtext
                     if txt:
-                        remove = set([v.strip() for v in txt.split(',')])
+                        remove = set([v.strip() for v in txt.split(ism['ui_to_list'])])
                 txt = adding
                 if txt:
-                    add = set([v.strip() for v in txt.split(',')])
+                    add = set([v.strip() for v in txt.split(ism['ui_to_list'])])
                 else:
                     add = set()
                 self.db.set_custom_bulk_multiple(book_ids, add=add,

src/calibre/gui2/dialogs/message_box.py

@@ -154,13 +154,16 @@ _proceed_memory = []
 class ProceedNotification(MessageBox): # {{{
 
     def __init__(self, callback, payload, html_log, log_viewer_title, title, msg,
-            det_msg='', show_copy_button=False, parent=None):
+            det_msg='', show_copy_button=False, parent=None,
+            cancel_callback=None):
         '''
         A non modal popup that notifies the user that a background task has
         been completed.
 
         :param callback: A callable that is called with payload if the user
-        asks to proceed. Note that this is always called in the GUI thread
+        asks to proceed. Note that this is always called in the GUI thread.
+        :param cancel_callback: A callable that is called with the payload if
+        the users asks not to proceed.
         :param payload: Arbitrary object, passed to callback
         :param html_log: An HTML or plain text log
         :param log_viewer_title: The title for the log viewer window
@@ -181,7 +184,7 @@ class ProceedNotification(MessageBox): # {{{
         self.vlb.clicked.connect(self.show_log)
         self.det_msg_toggle.setVisible(bool(det_msg))
         self.setModal(False)
-        self.callback = callback
+        self.callback, self.cancel_callback = callback, cancel_callback
         _proceed_memory.append(self)
 
     def show_log(self):
@@ -192,9 +195,11 @@ class ProceedNotification(MessageBox): # {{{
         try:
             if result == self.Accepted:
                 self.callback(self.payload)
+            elif self.cancel_callback is not None:
+                self.cancel_callback(self.payload)
         finally:
             # Ensure this notification is garbage collected
-            self.callback = None
+            self.callback = self.cancel_callback = None
             self.setParent(None)
             self.finished.disconnect()
             self.vlb.clicked.disconnect()

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -520,7 +520,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
             elif not fm['is_multiple']:
                 val = [val]
             elif fm['datatype'] == 'composite':
-                val = [v.strip() for v in val.split(fm['is_multiple'])]
+                val = [v.strip() for v in val.split(fm['is_multiple']['ui_to_list'])]
             elif field == 'authors':
                 val = [v.replace('|', ',') for v in val]
         else:
@@ -655,19 +655,10 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
 
         if self.destination_field_fm['is_multiple']:
             if self.comma_separated.isChecked():
-                if dest == 'authors' or \
+                splitter = self.destination_field_fm['is_multiple']['ui_to_list']
-                        (self.destination_field_fm['is_custom'] and
-                         self.destination_field_fm['datatype'] == 'text' and
-                         self.destination_field_fm['display'].get('is_names', False)):
-                    splitter = ' & '
-                else:
-                    splitter = ','
-
                 res = []
                 for v in val:
-                    for x in v.split(splitter):
+                    res.extend([x.strip() for x in v.split(splitter) if x.strip()])
-                        if x.strip():
-                            res.append(x.strip())
                 val = res
             else:
                 val = [v.replace(',', '') for v in val]

src/calibre/gui2/dialogs/template_dialog.py

@@ -5,13 +5,15 @@ __license__   = 'GPL v3'
 
 import json
 
-from PyQt4.Qt import (Qt, QDialog, QDialogButtonBox, QSyntaxHighlighter,
+from PyQt4.Qt import (Qt, QDialog, QDialogButtonBox, QSyntaxHighlighter, QFont,
-                      QRegExp, QApplication,
+                      QRegExp, QApplication, QTextCharFormat, QColor, QCursor)
-                      QTextCharFormat, QFont, QColor, QCursor)
 
+from calibre.gui2 import error_dialog
 from calibre.gui2.dialogs.template_dialog_ui import Ui_TemplateDialog
 from calibre.utils.formatter_functions import formatter_functions
-from calibre.ebooks.metadata.book.base import composite_formatter
+from calibre.ebooks.metadata.book.base import composite_formatter, Metadata
+from calibre.library.coloring import (displayable_columns)
+
 
 class ParenPosition:
 
@@ -195,12 +197,24 @@ class TemplateHighlighter(QSyntaxHighlighter):
 
 class TemplateDialog(QDialog, Ui_TemplateDialog):
 
-    def __init__(self, parent, text, mi):
+    def __init__(self, parent, text, mi=None, fm=None, color_field=None):
         QDialog.__init__(self, parent)
         Ui_TemplateDialog.__init__(self)
         self.setupUi(self)
 
+        self.coloring = color_field is not None
+        if self.coloring:
+            cols = sorted([k for k in displayable_columns(fm)])
+            self.colored_field.addItems(cols)
+            self.colored_field.setCurrentIndex(self.colored_field.findText(color_field))
+        else:
+            self.colored_field.setVisible(False)
+            self.colored_field_label.setVisible(False)
+
+        if mi:
             self.mi = mi
+        else:
+            self.mi = Metadata(None, None)
 
         # Remove help icon on title bar
         icon = self.windowIcon()
@@ -238,24 +252,35 @@ class TemplateDialog(QDialog, Ui_TemplateDialog):
         self.function.setCurrentIndex(0)
         self.function.currentIndexChanged[str].connect(self.function_changed)
         self.textbox_changed()
+        self.rule = (None, '')
+
+        tt = _('Template language tutorial')
+        self.template_tutorial.setText(
+                '<a href="http://manual.calibre-ebook.com/template_lang.html">'
+                '%s</a>'%tt)
+        tt = _('Template function reference')
+        self.template_func_reference.setText(
+                '<a href="http://manual.calibre-ebook.com/template_ref.html">'
+                '%s</a>'%tt)
 
     def textbox_changed(self):
         cur_text = unicode(self.textbox.toPlainText())
         if self.last_text != cur_text:
             self.last_text = cur_text
             self.highlighter.regenerate_paren_positions()
+            self.text_cursor_changed()
             self.template_value.setText(
                 composite_formatter.safe_format(cur_text, self.mi,
                                                 _('EXCEPTION: '), self.mi))
 
     def text_cursor_changed(self):
         cursor = self.textbox.textCursor()
-        block_number = cursor.blockNumber()
-        pos_in_block = cursor.positionInBlock()
         position = cursor.position()
         t = unicode(self.textbox.toPlainText())
-        if position < len(t):
+        if position > 0 and position <= len(t):
-            self.highlighter.check_cursor_pos(t[position], block_number,
+            block_number = cursor.blockNumber()
+            pos_in_block = cursor.positionInBlock() - 1
+            self.highlighter.check_cursor_pos(t[position-1], block_number,
                                               pos_in_block)
 
     def function_changed(self, toWhat):
@@ -270,3 +295,17 @@ class TemplateDialog(QDialog, Ui_TemplateDialog):
             else:
                 self.source_code.setPlainText(self.funcs[name].program_text)
 
+    def accept(self):
+        txt = unicode(self.textbox.toPlainText()).rstrip()
+        if self.coloring:
+            if self.colored_field.currentIndex() == -1:
+                error_dialog(self, _('No column chosen'),
+                    _('You must specify a column to be colored'), show=True)
+                return
+            if not txt:
+                error_dialog(self, _('No template provided'),
+                    _('The template box cannot be empty'), show=True)
+                return
+
+        self.rule = (unicode(self.colored_field.currentText()), txt)
+        QDialog.accept(self)

src/calibre/gui2/dialogs/template_dialog.ui

@@ -20,12 +20,30 @@
    <string>Edit Comments</string>
   </property>
   <layout class="QVBoxLayout" name="verticalLayout">
+   <item>
+    <layout class="QHBoxLayout">
+     <item>
+      <widget class="QLabel" name="colored_field_label">
+       <property name="text">
+        <string>Set the color of the column:</string>
+       </property>
+       <property name="buddy">
+        <cstring>colored_field</cstring>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QComboBox" name="colored_field">
+      </widget>
+     </item>
+    </layout>
+   </item>
    <item>
     <widget class="QPlainTextEdit" name="textbox"/>
    </item>
    <item>
     <layout class="QGridLayout" name="gridLayout">
-     <item row="0" column="0">
+     <item row="5" column="0">
       <widget class="QLabel">
        <property name="text">
         <string>Template value:</string>
@@ -38,14 +56,14 @@
        </property>
       </widget>
      </item>
-     <item row="0" column="1">
+     <item row="5" column="1">
       <widget class="QLineEdit" name="template_value">
        <property name="readOnly">
         <bool>true</bool>
        </property>
       </widget>
      </item>
-     <item row="1" column="1">
+     <item row="6" column="1">
       <widget class="QDialogButtonBox" name="buttonBox">
        <property name="orientation">
         <enum>Qt::Horizontal</enum>
@@ -55,7 +73,7 @@
        </property>
       </widget>
      </item>
-     <item row="2" column="0">
+     <item row="6" column="0">
       <widget class="QLabel" name="label">
        <property name="text">
         <string>Function &amp;name:</string>
@@ -65,10 +83,10 @@
        </property>
       </widget>
      </item>
-     <item row="2" column="1">
+     <item row="7" column="1">
       <widget class="QComboBox" name="function"/>
      </item>
-     <item row="3" column="0">
+     <item row="8" column="0">
       <widget class="QLabel" name="label_2">
        <property name="text">
         <string>&amp;Documentation:</string>
@@ -81,7 +99,7 @@
        </property>
       </widget>
      </item>
-     <item row="4" column="0">
+     <item row="9" column="0">
       <widget class="QLabel" name="label_3">
        <property name="text">
         <string>Python &amp;code:</string>
@@ -94,7 +112,7 @@
        </property>
       </widget>
      </item>
-     <item row="3" column="1">
+     <item row="8" column="1">
       <widget class="QPlainTextEdit" name="documentation">
        <property name="maximumSize">
         <size>
@@ -104,9 +122,23 @@
        </property>
       </widget>
      </item>
-     <item row="4" column="1">
+     <item row="9" column="1">
       <widget class="QPlainTextEdit" name="source_code"/>
      </item>
+     <item row="10" column="1">
+      <widget class="QLabel" name="template_tutorial">
+       <property name="openExternalLinks">
+        <bool>true</bool>
+       </property>
+      </widget>
+     </item>
+     <item row="11" column="1">
+      <widget class="QLabel" name="template_func_reference">
+       <property name="openExternalLinks">
+        <bool>true</bool>
+       </property>
+      </widget>
+     </item>
     </layout>
    </item>
   </layout>

src/calibre/gui2/dialogs/template_line_editor.py

@@ -5,17 +5,10 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-from functools import partial
-from collections import defaultdict
 
-from PyQt4.Qt import (Qt, QLineEdit, QDialog, QGridLayout, QLabel, QCheckBox,
+from PyQt4.Qt import QLineEdit
-                      QIcon, QDialogButtonBox, QColor, QComboBox, QPushButton)
 
-from calibre.ebooks.metadata.book.base import composite_formatter
 from calibre.gui2.dialogs.template_dialog import TemplateDialog
-from calibre.gui2.complete import MultiCompleteLineEdit
-from calibre.gui2 import error_dialog
-from calibre.utils.icu import sort_key
 
 class TemplateLineEditor(QLineEdit):
 
@@ -25,16 +18,11 @@ class TemplateLineEditor(QLineEdit):
 
     def __init__(self, parent):
         QLineEdit.__init__(self, parent)
-        self.tags = None
         self.mi   = None
-        self.txt = None
 
     def set_mi(self, mi):
         self.mi = mi
 
-    def set_db(self, db):
-        self.db = db
-
     def contextMenuEvent(self, event):
         menu = self.createStandardContextMenu()
         menu.addSeparator()
@@ -46,494 +34,10 @@ class TemplateLineEditor(QLineEdit):
         menu.exec_(event.globalPos())
 
     def clear_field(self):
-        self.txt = None
         self.setText('')
-        self.setReadOnly(False)
-        self.setStyleSheet('TemplateLineEditor { color: black }')
 
     def open_editor(self):
-        if self.txt:
+        t = TemplateDialog(self, self.text(), mi=self.mi)
-            t = TemplateDialog(self, self.txt, self.mi)
-        else:
-            t = TemplateDialog(self, self.text(), self.mi)
         t.setWindowTitle(_('Edit template'))
         if t.exec_():
-            self.txt = None
+            self.setText(t.rule[1])
-            self.setText(t.textbox.toPlainText())
-
-    def enable_wizard_button(self, txt):
-        if not txt or txt.startswith('program:\n#tag wizard'):
-            return True
-        return False
-
-    def setText(self, txt):
-        txt = unicode(txt)
-        if txt and txt.startswith('program:\n#tag wizard'):
-            self.txt = txt
-            self.setReadOnly(True)
-            QLineEdit.setText(self, '')
-            QLineEdit.setText(self, _('Template generated by the wizard'))
-            self.setStyleSheet('TemplateLineEditor { color: gray }')
-        else:
-            QLineEdit.setText(self, txt)
-
-    def tag_wizard(self):
-        txt = unicode(self.text())
-        if txt and not self.txt:
-            error_dialog(self, _('Invalid text'),
-                 _('The text in the box was not generated by this wizard'),
-                 show=True, show_copy_button=False)
-            return
-        d = TagWizard(self, self.db, unicode(self.txt), self.mi)
-        if d.exec_():
-            self.setText(d.template)
-
-    def text(self):
-        if self.txt:
-            return self.txt
-        return QLineEdit.text(self)
-
-class TagWizard(QDialog):
-
-    text_template              = ("        strcmp(field('{f}'), '{v}', '{ltv}', '{eqv}', '{gtv}')", True)
-    text_empty_template        = ("        test(field('{f}'), '{fv}', '{tv}')", False)
-    text_re_template           = ("        contains(field('{f}'), '{v}', '{tv}', '{fv}')", False)
-
-    templates = {
-        'text.mult'            : ("        str_in_list(field('{f}'), '{mult}', '{v}', '{tv}', '{fv}')", False),
-        'text.mult.re'         : ("        in_list(field('{f}'), '{mult}', '^{v}$', '{tv}', '{fv}')", False),
-        'text.mult.empty'      : ("        test(field('{f}'), '{fv}', '{tv}')", False),
-        'text'                 : text_template,
-        'text.re'              : text_re_template,
-        'text.empty'           : text_empty_template,
-        'rating'               : ("        cmp(raw_field('{f}'), '{v}', '{ltv}', '{eqv}', '{gtv}')", True),
-        'rating.empty'         : text_empty_template,
-        'int'                  : ("        cmp(raw_field('{f}'), '{v}', '{ltv}', '{eqv}', '{gtv}')", True),
-        'int.empty'            : text_empty_template,
-        'float'                : ("        cmp(raw_field('{f}'), '{v}', '{ltv}', '{eqv}', '{gtv}')", True),
-        'float.empty'          : text_empty_template,
-        'bool'                 : ("        strcmp(field('{f}'), '{v}', '{ltv}', '{eqv}', '{gtv}')", True),
-        'bool.empty'           : text_empty_template,
-        'datetime'             : ("        strcmp(format_date(raw_field('{f}'), 'yyyyMMdd'), format_date('{v}', 'yyyyMMdd'), '{ltv}', '{eqv}', '{gtv}')", True),
-        'datetime.empty'       : text_empty_template,
-        'series'               : text_template,
-        'series.re'            : text_re_template,
-        'series.empty'         : text_empty_template,
-        'composite'            : text_template,
-        'composite.re'         : text_re_template,
-        'composite.empty'      : text_empty_template,
-        'enumeration'          : text_template,
-        'enumeration.re'       : text_re_template,
-        'enumeration.empty'    : text_empty_template,
-        'comments'             : text_template,
-        'comments.re'          : text_re_template,
-        'comments.empty'       : text_empty_template,
-    }
-
-    relationals = ('=', '!=', '<', '>', '<=', '>=')
-    relational_truth_vals = {
-        '=':        ('', '1', ''),
-        '!=':       ('1', '', '1'),
-        '<':        ('1', '', ''),
-        '>':        ('', '', '1'),
-        '<=':       ('1', '1', ''),
-        '>=':       ('', '1', '1'),
-    }
-
-    @staticmethod
-    def uses_this_wizard(txt):
-        if not txt or txt.startswith('program:\n#tag wizard'):
-            return True
-        return False
-
-    def __init__(self, parent, db, txt, mi):
-        QDialog.__init__(self, parent)
-        self.setWindowTitle(_('Coloring Wizard'))
-        self.setWindowIcon(QIcon(I('wizard.png')))
-
-        self.mi = mi
-
-        self.columns = []
-        self.completion_values = defaultdict(dict)
-        for k in db.all_field_keys():
-            m = db.metadata_for_field(k)
-            if k.endswith('_index') or (
-                    m['kind'] == 'field' and m['name'] and
-                    k not in ('ondevice', 'path', 'size', 'sort')):
-                self.columns.append(k)
-                self.completion_values[k]['dt'] = m['datatype']
-                if m['is_custom']:
-                    if m['datatype'] in ('int', 'float'):
-                        self.completion_values[k]['v'] = []
-                    elif m['datatype'] == 'bool':
-                        self.completion_values[k]['v'] = [_('Yes'), _('No')]
-                    else:
-                        self.completion_values[k]['v'] = db.all_custom(m['label'])
-                elif k == 'tags':
-                    self.completion_values[k]['v'] = db.all_tags()
-                elif k == 'formats':
-                    self.completion_values[k]['v'] = db.all_formats()
-                else:
-                    if k in ('publisher'):
-                        ck = k + 's'
-                    else:
-                        ck = k
-                    f = getattr(db, 'all_' + ck, None)
-                    if f:
-                        if k == 'authors':
-                            self.completion_values[k]['v'] = [v[1].\
-                                                replace('|', ',') for v in f()]
-                        else:
-                            self.completion_values[k]['v'] = [v[1] for v in f()]
-                    else:
-                            self.completion_values[k]['v'] = []
-
-                if k in self.completion_values:
-                    if k == 'authors':
-                        mult = '&'
-                    else:
-                        mult = ',' if m['is_multiple'] == '|' else m['is_multiple']
-                    self.completion_values[k]['m'] = mult
-
-        self.columns.sort(key=sort_key)
-        self.columns.insert(0, '')
-
-        l = QGridLayout()
-        self.setLayout(l)
-        l.setColumnStretch(2, 10)
-        l.setColumnMinimumWidth(5, 300)
-
-        h = QLabel(_('And'))
-        h.setToolTip('<p>' +
-             _('Set this box to indicate that the two conditions must both '
-               'be true to use the color. For example, you '
-               'can check if two tags are present, if the book has a tag '
-               'and a #read custom column is checked, or if a book has '
-               'some tag and has a particular format.'))
-        l.addWidget(h, 0, 0, 1, 1)
-
-        h = QLabel(_('Column'))
-        h.setAlignment(Qt.AlignCenter)
-        l.addWidget(h, 0, 1, 1, 1)
-
-        h = QLabel(_('is'))
-        h.setAlignment(Qt.AlignCenter)
-        l.addWidget(h, 0, 2, 1, 1)
-
-        h = QLabel(_('op'))
-        h.setToolTip('<p>' +
-             _('Use this box to tell what comparison operation to use. Some '
-               'comparisons cannot be used with certain options. For example, '
-               'if regular expressions are used, only equals and not equals '
-               'are valid.') + '</p>')
-        h.setAlignment(Qt.AlignCenter)
-        l.addWidget(h, 0, 3, 1, 1)
-
-        c = QLabel(_('empty'))
-        c.setToolTip('<p>' +
-             _('Check this box to check if the column is empty') + '</p>')
-        l.addWidget(c, 0, 4, 1, 1)
-
-        h = QLabel(_('Values'))
-        h.setAlignment(Qt.AlignCenter)
-        h.setToolTip('<p>' +
-             _('You can enter more than one value per box, separated by commas. '
-               'The comparison ignores letter case. Special note: authors are '
-               'separated by ampersands (&).<br>'
-               'A value can be a regular expression. Check the box to turn '
-               'them on. When using regular expressions, note that the wizard '
-               'puts anchors (^ and $) around the expression, so you '
-               'must ensure your expression matches from the beginning '
-               'to the end of the column/value you are checking.<br>'
-               'Regular expression examples:') + '<ul>' +
-             _('<li><code><b>.*</b></code> matches anything in the column.</li>'
-               '<li><code><b>A.*</b></code> matches anything beginning with A</li>'
-               '<li><code><b>.*mystery.*</b></code> matches anything containing '
-               'the word "mystery"</li>') + '</ul></p>')
-        l.addWidget(h , 0, 5, 1, 1)
-
-        c = QLabel(_('is RE'))
-        c.setToolTip('<p>' +
-             _('Check this box if the values box contains regular expressions') + '</p>')
-        l.addWidget(c, 0, 6, 1, 1)
-
-        c = QLabel(_('color'))
-        c.setAlignment(Qt.AlignCenter)
-        c.setToolTip('<p>' +
-             _('Use this color if the column matches the tests.') + '</p>')
-        l.addWidget(c, 0, 7, 1, 1)
-
-        self.andboxes       = []
-        self.opboxes        = []
-        self.tagboxes       = []
-        self.colorboxes     = []
-        self.reboxes        = []
-        self.colboxes       = []
-        self.emptyboxes     = []
-
-        self.colors = [unicode(s) for s in list(QColor.colorNames())]
-        self.colors.insert(0, '')
-
-        def create_widget(klass, box, layout, row, col, items,
-                          align=Qt.AlignCenter, rowspan=False):
-            w = klass(self)
-            if box is not None:
-                box.append(w)
-            if rowspan:
-                layout.addWidget(w, row, col, 2, 1, alignment=Qt.Alignment(align))
-            else:
-                layout.addWidget(w, row, col, 1, 1, alignment=Qt.Alignment(align))
-            if items:
-                w.addItems(items)
-            return w
-
-        maxlines = 10
-        for i in range(1, maxlines+1):
-            w = create_widget(QCheckBox, self.andboxes, l, i, 0, None, rowspan=True)
-            w.stateChanged.connect(partial(self.and_box_changed, line=i-1))
-            if i == maxlines:
-                # last box is invisible
-                w.setVisible(False)
-
-            w = create_widget(QComboBox, self.colboxes, l, i, 1, self.columns)
-            w.currentIndexChanged[str].connect(partial(self.column_changed, line=i-1))
-
-            w = QLabel(self)
-            w.setText(_('is'))
-            l.addWidget(w, i, 2, 1, 1)
-
-            w = create_widget(QComboBox, self.opboxes, l, i, 3, None)
-            w.setMaximumWidth(40)
-
-            w = create_widget(QCheckBox, self.emptyboxes, l, i, 4, None)
-            w.stateChanged.connect(partial(self.empty_box_changed, line=i-1))
-
-            create_widget(MultiCompleteLineEdit, self.tagboxes, l, i, 5, None, align=0)
-
-            w = create_widget(QCheckBox, self.reboxes, l, i, 6, None)
-            w.stateChanged.connect(partial(self.re_box_changed, line=i-1))
-
-            create_widget(QComboBox, self.colorboxes, l, i, 7, self.colors)
-
-        w = create_widget(QLabel, None, l, maxlines+1, 5, None)
-        w.setText(_('If none of the tests match, set the color to'))
-        self.elsebox = create_widget(QComboBox, None, l, maxlines+1, 7, self.colors)
-        self.elsebox.setToolTip('<p>' +
-                    _('If this box contains a color, it will be used if none '
-                      'of the above rules match.')  + '</p>')
-
-        if txt:
-            lines = txt.split('\n')[3:]
-            i = 0
-            for line in lines:
-                if line.startswith('#'):
-                    vals = line[1:].split(':|:')
-                    if len(vals) == 1 and line.startswith('#else:'):
-                        try:
-                            self.elsebox.setCurrentIndex(self.elsebox.findText(line[6:]))
-                        except:
-                            pass
-                        continue
-                    if len(vals) == 2:
-                        t, c = vals
-                        f = 'tags'
-                        a = re = e = 0
-                        op = '='
-                    else:
-                        t,c,f,re,a,op,e = vals
-                    try:
-                        self.colboxes[i].setCurrentIndex(self.colboxes[i].findText(f))
-                        self.colorboxes[i].setCurrentIndex(
-                                                self.colorboxes[i].findText(c))
-                        self.tagboxes[i].setText(t)
-                        self.reboxes[i].setChecked(re == '2')
-                        self.emptyboxes[i].setChecked(e == '2')
-                        self.andboxes[i].setChecked(a == '2')
-                        self.opboxes[i].setCurrentIndex(self.opboxes[i].findText(op))
-                        i += 1
-                    except:
-                        import traceback
-                        traceback.print_exc()
-                        pass
-
-        w = QLabel(_('Preview'))
-        l.addWidget(w, 99, 1, 1, 1)
-        w = self.test_box = QLineEdit(self)
-        w.setReadOnly(True)
-        l.addWidget(w, 99, 2, 1, 5)
-        w = QPushButton(_('Test'))
-        w.setToolTip('<p>' +
-                     _('Press this button to see what color this template will '
-                       'produce for the book that was selected when you '
-                       'entered the preferences dialog.'))
-        l.addWidget(w, 99, 7, 1, 1)
-        w.clicked.connect(self.preview)
-
-        bb = QDialogButtonBox(QDialogButtonBox.Ok|QDialogButtonBox.Cancel, parent=self)
-        l.addWidget(bb, 100, 5, 1, 3)
-        bb.accepted.connect(self.accepted)
-        bb.rejected.connect(self.reject)
-        self.template = ''
-
-    def preview(self):
-        if not self.generate_program():
-            return
-        t = composite_formatter.safe_format(self.template, self.mi,
-                                            _('EXCEPTION'), self.mi)
-        self.test_box.setText(t)
-
-    def generate_program(self):
-        res = ("program:\n#tag wizard -- do not directly edit\n"
-               "  first_non_empty(\n")
-        lines = []
-        was_and = had_line = False
-
-        line = 0
-        for tb, cb, fb, reb, ab, ob, eb in zip(
-                self.tagboxes, self.colorboxes, self.colboxes,
-                self.reboxes, self.andboxes, self.opboxes, self.emptyboxes):
-            f = unicode(fb.currentText())
-            if not f:
-                continue
-            m = self.completion_values[f]['m']
-            dt = self.completion_values[f]['dt']
-            c = unicode(cb.currentText()).strip()
-            re = reb.checkState()
-            a  = ab.checkState()
-            op = unicode(ob.currentText())
-            e  = eb.checkState()
-            line += 1
-
-            if m:
-                tags = [t.strip() for t in unicode(tb.text()).split(m) if t.strip()]
-                if re == 2:
-                    tags = '$|^'.join(tags)
-                else:
-                    tags = m.join(tags)
-                if m == '&':
-                    tags = tags.replace(',', '|')
-            else:
-                tags = unicode(tb.text()).strip()
-
-            if (tags or f) and not ((tags or e) and f and (a == 2 or c)):
-                error_dialog(self, _('Invalid line'),
-                             _('Line number {0} is not valid').format(line),
-                             show=True, show_copy_button=False)
-                return False
-
-            if not was_and:
-                if had_line:
-                    lines[-1] += ','
-                had_line = True
-                lines.append("    test(and(")
-            else:
-                lines[-1] += ','
-
-            key = dt + ('.mult' if m else '') + ('.empty' if e else '') + ('.re' if re else '')
-            tval = '1' if op == '=' else ''
-            fval = ''  if op == '=' else '1'
-            template, is_relational = self.templates[key]
-            if is_relational:
-                ltv, eqv, gtv = self.relational_truth_vals[op]
-            else:
-                ltv, eqv, gtv = (None, None, None)
-            lines.append(template.format(v=tags, f=f, tv=tval, fv=fval, mult=m,
-                                         ltv=ltv, eqv=eqv, gtv=gtv))
-
-            if a == 2:
-                was_and = True
-            else:
-                was_and = False
-                lines.append("    ), '{0}', '')".format(c))
-
-        res += '\n'.join(lines)
-        else_txt = unicode(self.elsebox.currentText())
-        if else_txt:
-            res += ",\n  '" + else_txt + "'"
-        res += ')\n'
-        self.template = res
-        res = ''
-        for tb, cb, fb, reb, ab, ob, eb in zip(
-                self.tagboxes, self.colorboxes, self.colboxes,
-                self.reboxes, self.andboxes, self.opboxes, self.emptyboxes):
-            t = unicode(tb.text()).strip()
-            if t.endswith(','):
-                t = t[:-1]
-            c = unicode(cb.currentText()).strip()
-            f = unicode(fb.currentText())
-            re = unicode(reb.checkState())
-            a = unicode(ab.checkState())
-            op = unicode(ob.currentText())
-            e = unicode(eb.checkState())
-            if f and (t or e) and (a == '2' or c):
-                res += '#' + t + ':|:' + c  + ':|:' + f + ':|:' + re + ':|:' + \
-                             a + ':|:' + op + ':|:' + e + '\n'
-        res += '#else:' + else_txt + '\n'
-        self.template += res
-        return True
-
-    def column_changed(self, s, line=None):
-        k = unicode(s)
-        valbox = self.tagboxes[line]
-        if k in self.completion_values:
-            valbox.update_items_cache(self.completion_values[k]['v'])
-            if self.completion_values[k]['m']:
-                valbox.set_separator(', ')
-            else:
-                valbox.set_separator(None)
-
-            dt = self.completion_values[k]['dt']
-            if dt in ('int', 'float', 'rating', 'bool'):
-                self.reboxes[line].setChecked(0)
-                self.reboxes[line].setEnabled(False)
-            else:
-                self.reboxes[line].setEnabled(True)
-            self.fill_in_opbox(line)
-        else:
-            valbox.update_items_cache([])
-            valbox.set_separator(None)
-
-    def fill_in_opbox(self, line):
-        opbox = self.opboxes[line]
-        opbox.clear()
-        k = unicode(self.colboxes[line].currentText())
-        if not k:
-            return
-        if k in self.completion_values:
-            rebox = self.reboxes[line]
-            ebox = self.emptyboxes[line]
-            idx = opbox.currentIndex()
-            if self.completion_values[k]['m'] or \
-                    rebox.checkState() == 2 or ebox.checkState() == 2:
-                opbox.addItems(self.relationals[0:2])
-                idx = idx if idx < 2 else 0
-            else:
-                opbox.addItems(self.relationals)
-            opbox.setCurrentIndex(max(idx, 0))
-
-    def re_box_changed(self, state, line=None):
-        self.fill_in_opbox(line)
-
-    def empty_box_changed(self, state, line=None):
-        if state == 2:
-            self.tagboxes[line].setText('')
-            self.tagboxes[line].setEnabled(False)
-            self.reboxes[line].setChecked(0)
-            self.reboxes[line].setEnabled(False)
-        else:
-            self.reboxes[line].setEnabled(True)
-            self.tagboxes[line].setEnabled(True)
-        self.fill_in_opbox(line)
-
-    def and_box_changed(self, state, line=None):
-        if state == 2:
-            self.colorboxes[line].setCurrentIndex(0)
-            self.colorboxes[line].setEnabled(False)
-        else:
-            self.colorboxes[line].setEnabled(True)
-
-    def accepted(self):
-        if self.generate_program():
-            self.accept()
-        else:
-            self.template = ''

src/calibre/gui2/dialogs/tweak_epub.py

@@ -7,7 +7,7 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import os, shutil
-from zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED
+from calibre.utils.zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED
 
 from PyQt4.Qt import QDialog
 

src/calibre/gui2/jobs.py

@@ -197,9 +197,11 @@ class JobManager(QAbstractTableModel): # {{{
     def row_to_job(self, row):
         return self.jobs[row]
 
-    def has_device_jobs(self):
+    def has_device_jobs(self, queued_also=False):
         for job in self.jobs:
-            if job.is_running and isinstance(job, DeviceJob):
+            if isinstance(job, DeviceJob):
+                if job.duration is None: # Running or waiting
+                    if (job.is_running or queued_also):
                         return True
         return False
 

src/calibre/gui2/library/coloring.py

@@ -1,502 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import (unicode_literals, division, absolute_import,
-                        print_function)
-from future_builtins import map
-
-__license__   = 'GPL v3'
-__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
-__docformat__ = 'restructuredtext en'
-
-import json, binascii, re
-from textwrap import dedent
-
-from PyQt4.Qt import (QWidget, QDialog, QLabel, QGridLayout, QComboBox,
-        QLineEdit, QIntValidator, QDoubleValidator, QFrame, QColor, Qt, QIcon,
-        QScrollArea, QPushButton, QVBoxLayout, QDialogButtonBox)
-
-from calibre.utils.icu import sort_key
-from calibre.gui2 import error_dialog
-
-class Rule(object): # {{{
-
-    SIGNATURE = '# BasicColorRule():'
-
-    def __init__(self, fm):
-        self.color = None
-        self.fm = fm
-        self.conditions = []
-
-    def add_condition(self, col, action, val):
-        if col not in self.fm:
-            raise ValueError('%r is not a valid column name'%col)
-        v = self.validate_condition(col, action, val)
-        if v:
-            raise ValueError(v)
-        self.conditions.append((col, action, val))
-
-    def validate_condition(self, col, action, val):
-        m = self.fm[col]
-        dt = m['datatype']
-        if (dt in ('int', 'float', 'rating') and action in ('lt', 'eq', 'gt')):
-            try:
-                int(val) if dt == 'int' else float(val)
-            except:
-                return '%r is not a valid numerical value'%val
-
-        if (dt in ('comments', 'series', 'text', 'enumeration') and 'pattern'
-                in action):
-            try:
-                re.compile(val)
-            except:
-                return '%r is not a valid regular expression'%val
-
-    @property
-    def signature(self):
-        args = (self.color, self.conditions)
-        sig = json.dumps(args, ensure_ascii=False)
-        return self.SIGNATURE + binascii.hexlify(sig.encode('utf-8'))
-
-    @property
-    def template(self):
-        if not self.color or not self.conditions:
-            return None
-        conditions = map(self.apply_condition, self.conditions)
-        conditions = (',\n' + ' '*9).join(conditions)
-        return dedent('''\
-                program:
-                {sig}
-                test(and(
-                         {conditions}
-                    ), {color}, '');
-                ''').format(sig=self.signature, conditions=conditions,
-                        color=self.color)
-
-    def apply_condition(self, condition):
-        col, action, val = condition
-        m = self.fm[col]
-        dt = m['datatype']
-
-        if dt == 'bool':
-            return self.bool_condition(col, action, val)
-
-        if dt in ('int', 'float', 'rating'):
-            return self.number_condition(col, action, val)
-
-        if dt == 'datetime':
-            return self.date_condition(col, action, val)
-
-        if dt in ('comments', 'series', 'text', 'enumeration'):
-            ism = m.get('is_multiple', False)
-            if ism:
-                return self.multiple_condition(col, action, val, ism)
-            return self.text_condition(col, action, val)
-
-    def bool_condition(self, col, action, val):
-        test = {'is true': 'True',
-                'is false': 'False',
-                'is undefined': 'None'}[action]
-        return "strcmp('%s', raw_field('%s'), '', '1', '')"%(test, col)
-
-    def number_condition(self, col, action, val):
-        lt, eq, gt = {
-                'eq': ('', '1', ''),
-                'lt': ('1', '', ''),
-                'gt': ('', '', '1')
-        }[action]
-        lt, eq, gt = '', '1', ''
-        return "cmp(field('%s'), %s, '%s', '%s', '%s')" % (col, val, lt, eq, gt)
-
-    def date_condition(self, col, action, val):
-        lt, eq, gt = {
-                'eq': ('', '1', ''),
-                'lt': ('1', '', ''),
-                'gt': ('', '', '1')
-        }[action]
-        return "cmp(format_date(raw_field('%s'), 'yyyy-MM-dd'), %s, '%s', '%s', '%s')" % (col,
-                val, lt, eq, gt)
-
-    def multiple_condition(self, col, action, val, sep):
-        if action == 'is set':
-            return "test('%s', '1', '')"%col
-        if action == 'is not set':
-            return "test('%s', '', '1')"%col
-        if action == 'has':
-            return "str_in_list(field('%s'), '%s', \"%s\", '1', '')"%(col, sep, val)
-        if action == 'does not have':
-            return "str_in_list(field('%s'), '%s', \"%s\", '', '1')"%(col, sep, val)
-        if action == 'has pattern':
-            return "in_list(field('%s'), '%s', \"%s\", '1', '')"%(col, sep, val)
-        if action == 'does not have pattern':
-            return "in_list(field('%s'), '%s', \"%s\", '', '1')"%(col, sep, val)
-
-    def text_condition(self, col, action, val):
-        if action == 'is set':
-            return "test('%s', '1', '')"%col
-        if action == 'is not set':
-            return "test('%s', '', '1')"%col
-        if action == 'is':
-            return "strcmp(field('%s'), \"%s\", '', '1', '')"%(col, val)
-        if action == 'is not':
-            return "strcmp(field('%s'), \"%s\", '1', '', '1')"%(col, val)
-        if action == 'matches pattern':
-            return "contains(field('%s'), \"%s\", '1', '')"%(col, val)
-        if action == 'does not match pattern':
-            return "contains(field('%s'), \"%s\", '', '1')"%(col, val)
-
-# }}}
-
-def rule_from_template(fm, template):
-    ok_lines = []
-    for line in template.splitlines():
-        if line.startswith(Rule.SIGNATURE):
-            raw = line[len(Rule.SIGNATURE):].strip()
-            try:
-                color, conditions = json.loads(binascii.unhexlify(raw).decode('utf-8'))
-            except:
-                continue
-            r = Rule(fm)
-            r.color = color
-            for c in conditions:
-                try:
-                    r.add_condition(*c)
-                except:
-                    continue
-            if r.color and r.conditions:
-                return r
-        else:
-            ok_lines.append(line)
-    return '\n'.join(ok_lines)
-
-def conditionable_columns(fm):
-    for key in fm:
-        m = fm[key]
-        dt = m['datatype']
-        if m.get('name', False) and dt in ('bool', 'int', 'float', 'rating', 'series',
-                'comments', 'text', 'enumeration', 'datetime'):
-            yield key
-
-
-def displayable_columns(fm):
-    for key in fm.displayable_field_keys():
-        if key not in ('sort', 'author_sort', 'comments', 'formats',
-                'identifiers', 'path'):
-            yield key
-
-class ConditionEditor(QWidget):
-
-    def __init__(self, fm, parent=None):
-        QWidget.__init__(self, parent)
-        self.fm = fm
-
-        self.action_map = {
-            'bool' : (
-                    (_('is true'), 'is true',),
-                    (_('is false'), 'is false'),
-                    (_('is undefined'), 'is undefined')
-            ),
-            'int' : (
-                (_('is equal to'), 'eq'),
-                (_('is less than'), 'lt'),
-                (_('is greater than'), 'gt')
-            ),
-            'multiple' : (
-                (_('has'), 'has'),
-                (_('does not have'), 'does not have'),
-                (_('has pattern'), 'has pattern'),
-                (_('does not have pattern'), 'does not have pattern'),
-                (_('is set'), 'is set'),
-                (_('is not set'), 'is not set'),
-            ),
-            'single'   : (
-                (_('is'), 'is'),
-                (_('is not'), 'is not'),
-                (_('matches pattern'), 'matches pattern'),
-                (_('does not match pattern'), 'does not match pattern'),
-                (_('is set'), 'is set'),
-                (_('is not set'), 'is not set'),
-            ),
-        }
-
-        for x in ('float', 'rating', 'datetime'):
-            self.action_map[x] = self.action_map['int']
-
-        self.l = l = QGridLayout(self)
-        self.setLayout(l)
-
-        self.l1 = l1 = QLabel(_('If the '))
-        l.addWidget(l1, 0, 0)
-
-        self.column_box = QComboBox(self)
-        l.addWidget(self.column_box, 0, 1)
-
-        self.l2 = l2 = QLabel(_(' column '))
-        l.addWidget(l2, 0, 2)
-
-        self.action_box = QComboBox(self)
-        l.addWidget(self.action_box, 0, 3)
-
-        self.l3 = l3 = QLabel(_(' the value '))
-        l.addWidget(l3, 0, 4)
-
-        self.value_box = QLineEdit(self)
-        l.addWidget(self.value_box, 0, 5)
-
-        self.column_box.addItem('', '')
-        for key in sorted(
-                conditionable_columns(fm),
-                key=lambda x:sort_key(fm[x]['name'])):
-            self.column_box.addItem(key, key)
-        self.column_box.setCurrentIndex(0)
-
-        self.column_box.currentIndexChanged.connect(self.init_action_box)
-        self.action_box.currentIndexChanged.connect(self.init_value_box)
-
-        for b in (self.column_box, self.action_box):
-            b.setSizeAdjustPolicy(b.AdjustToMinimumContentsLengthWithIcon)
-            b.setMinimumContentsLength(15)
-
-    @dynamic_property
-    def current_col(self):
-        def fget(self):
-            idx = self.column_box.currentIndex()
-            return unicode(self.column_box.itemData(idx).toString())
-        def fset(self, val):
-            for idx in range(self.column_box.count()):
-                c = unicode(self.column_box.itemData(idx).toString())
-                if c == val:
-                    self.column_box.setCurrentIndex(idx)
-                    return
-            raise ValueError('Column %r not found'%val)
-        return property(fget=fget, fset=fset)
-
-    @dynamic_property
-    def current_action(self):
-        def fget(self):
-            idx = self.action_box.currentIndex()
-            return unicode(self.action_box.itemData(idx).toString())
-        def fset(self, val):
-            for idx in range(self.action_box.count()):
-                c = unicode(self.action_box.itemData(idx).toString())
-                if c == val:
-                    self.action_box.setCurrentIndex(idx)
-                    return
-            raise ValueError('Action %r not valid for current column'%val)
-        return property(fget=fget, fset=fset)
-
-    @property
-    def current_val(self):
-        return unicode(self.value_box.text()).strip()
-
-    @dynamic_property
-    def condition(self):
-
-        def fget(self):
-            c, a, v = (self.current_col, self.current_action,
-                    self.current_val)
-            if not c or not a:
-                return None
-            return (c, a, v)
-
-        def fset(self, condition):
-            c, a, v = condition
-            if not v:
-                v = ''
-            v = v.strip()
-            self.current_col = c
-            self.current_action = a
-            self.value_box.setText(v)
-
-        return property(fget=fget, fset=fset)
-
-    def init_action_box(self):
-        self.action_box.blockSignals(True)
-        self.action_box.clear()
-        self.action_box.addItem('', '')
-        col = self.current_col
-        m = self.fm[col]
-        dt = m['datatype']
-        if dt in self.action_map:
-            actions = self.action_map[dt]
-        else:
-            k = 'multiple' if m['is_multiple'] else 'single'
-            actions = self.action_map[k]
-
-        for text, key in actions:
-            self.action_box.addItem(text, key)
-        self.action_box.setCurrentIndex(0)
-        self.action_box.blockSignals(False)
-        self.init_value_box()
-
-    def init_value_box(self):
-        self.value_box.setEnabled(True)
-        self.value_box.setText('')
-        self.value_box.setInputMask('')
-        self.value_box.setValidator(None)
-        col = self.current_col
-        m = self.fm[col]
-        dt = m['datatype']
-        action = self.current_action
-        if not col or not action:
-            return
-        tt = ''
-        if dt in ('int', 'float', 'rating'):
-            tt = _('Enter a number')
-            v = QIntValidator if dt == 'int' else QDoubleValidator
-            self.value_box.setValidator(v(self.value_box))
-        elif dt == 'datetime':
-            self.value_box.setInputMask('9999-99-99')
-            tt = _('Enter a date in the format YYYY-MM-DD')
-        else:
-            tt = _('Enter a string')
-            if 'pattern' in action:
-                tt = _('Enter a regular expression')
-        self.value_box.setToolTip(tt)
-        if action in ('is set', 'is not set', 'is true', 'is false',
-                'is undefined'):
-            self.value_box.setEnabled(False)
-
-
-class RuleEditor(QDialog):
-
-    def __init__(self, fm, parent=None):
-        QDialog.__init__(self, parent)
-        self.fm = fm
-
-        self.setWindowIcon(QIcon(I('format-fill-color.png')))
-        self.setWindowTitle(_('Create/edit a column coloring rule'))
-
-        self.l = l = QGridLayout(self)
-        self.setLayout(l)
-
-        self.l1 = l1 = QLabel(_('Create a coloring rule by'
-            ' filling in the boxes below'))
-        l.addWidget(l1, 0, 0, 1, 4)
-
-        self.f1 = QFrame(self)
-        self.f1.setFrameShape(QFrame.HLine)
-        l.addWidget(self.f1, 1, 0, 1, 4)
-
-        self.l2 = l2 = QLabel(_('Set the color of the column:'))
-        l.addWidget(l2, 2, 0)
-
-        self.column_box = QComboBox(self)
-        l.addWidget(self.column_box, 2, 1)
-
-        self.l3 = l3 = QLabel(_('to'))
-        l3.setAlignment(Qt.AlignHCenter)
-        l.addWidget(l3, 2, 2)
-
-        self.color_box = QComboBox(self)
-        l.addWidget(self.color_box, 2, 3)
-
-        self.l4 = l4 = QLabel(
-            _('Only if the following conditions are all satisfied:'))
-        l4.setAlignment(Qt.AlignHCenter)
-        l.addWidget(l4, 3, 0, 1, 4)
-
-        self.scroll_area = sa = QScrollArea(self)
-        sa.setMinimumHeight(300)
-        sa.setMinimumWidth(950)
-        sa.setWidgetResizable(True)
-        l.addWidget(sa, 4, 0, 1, 4)
-
-        self.add_button = b = QPushButton(QIcon(I('plus.png')),
-                _('Add another condition'))
-        l.addWidget(b, 5, 0, 1, 4)
-        b.clicked.connect(self.add_blank_condition)
-
-        self.l5 = l5 = QLabel(_('You can disable a condition by'
-            ' blanking all of its boxes'))
-        l.addWidget(l5, 6, 0, 1, 4)
-
-        self.bb = bb = QDialogButtonBox(
-                QDialogButtonBox.Ok|QDialogButtonBox.Cancel)
-        bb.accepted.connect(self.accept)
-        bb.rejected.connect(self.reject)
-        l.addWidget(bb, 7, 0, 1, 4)
-
-        self.conditions_widget = QWidget(self)
-        sa.setWidget(self.conditions_widget)
-        self.conditions_widget.setLayout(QVBoxLayout())
-        self.conditions_widget.layout().setAlignment(Qt.AlignTop)
-        self.conditions = []
-
-        for b in (self.column_box, self.color_box):
-            b.setSizeAdjustPolicy(b.AdjustToMinimumContentsLengthWithIcon)
-            b.setMinimumContentsLength(15)
-
-        for key in sorted(
-                displayable_columns(fm),
-                key=lambda x:sort_key(fm[x]['name'])):
-            name = fm[key]['name']
-            if name:
-                self.column_box.addItem(key, key)
-        self.column_box.setCurrentIndex(0)
-
-        self.color_box.addItems(QColor.colorNames())
-        self.color_box.setCurrentIndex(0)
-
-        self.resize(self.sizeHint())
-
-    def add_blank_condition(self):
-        c = ConditionEditor(self.fm, parent=self.conditions_widget)
-        self.conditions.append(c)
-        self.conditions_widget.layout().addWidget(c)
-
-    def accept(self):
-        if self.validate():
-            QDialog.accept(self)
-
-    def validate(self):
-        r = Rule(self.fm)
-        for c in self.conditions:
-            condition = c.condition
-            if condition is not None:
-                try:
-                    r.add_condition(*condition)
-                except Exception as e:
-                    import traceback
-                    error_dialog(self, _('Invalid condition'),
-                            _('One of the conditions for this rule is'
-                                ' invalid: <b>%s</b>')%e,
-                            det_msg=traceback.format_exc(), show=True)
-                    return False
-        if len(r.conditions) < 1:
-            error_dialog(self, _('No conditions'),
-                    _('You must specify at least one non-empty condition'
-                        ' for this rule'), show=True)
-            return False
-        return True
-
-    @property
-    def rule(self):
-        r = Rule(self.fm)
-        r.color = unicode(self.color_box.currentText())
-        idx = self.column_box.currentIndex()
-        col = unicode(self.column_box.itemData(idx).toString())
-        for c in self.conditions:
-            condition = c.condition
-            if condition is not None:
-                r.add_condition(*condition)
-
-        return col, r
-
-
-
-if __name__ == '__main__':
-    from PyQt4.Qt import QApplication
-    app = QApplication([])
-
-    from calibre.library import db
-
-    d = RuleEditor(db().field_metadata)
-    d.add_blank_condition()
-    d.exec_()
-
-    col, r = d.rule
-
-    print ('Column to be colored:', col)
-    print ('Template:')
-    print (r.template)
-

src/calibre/gui2/library/delegates.py

@@ -288,6 +288,8 @@ class CcNumberDelegate(QStyledItemDelegate): # {{{
     def setEditorData(self, editor, index):
         m = index.model()
         val = m.db.data[index.row()][m.custom_columns[m.column_map[index.column()]]['rec_index']]
+        if val is None:
+            val = 0
         editor.setValue(val)
 
 # }}}
@@ -426,7 +428,7 @@ class CcTemplateDelegate(QStyledItemDelegate): # {{{
         editor.textbox.setTabStopWidth(20)
         d = editor.exec_()
         if d:
-            m.setData(index, QVariant(editor.textbox.toPlainText()), Qt.EditRole)
+            m.setData(index, QVariant(editor.rule[1]), Qt.EditRole)
         return None
 
     def setModelData(self, editor, model, index):

src/calibre/gui2/library/models.py

@@ -99,8 +99,7 @@ class BooksModel(QAbstractTableModel): # {{{
         self.ids_to_highlight_set = set()
         self.current_highlighted_idx = None
         self.highlight_only = False
-        self.column_color_list = []
+        self.colors = frozenset([unicode(c) for c in QColor.colorNames()])
-        self.colors = [unicode(c) for c in QColor.colorNames()]
         self.read_config()
 
     def change_alignment(self, colname, alignment):
@@ -156,7 +155,6 @@ class BooksModel(QAbstractTableModel): # {{{
                 self.headers[col] = self.custom_columns[col]['name']
 
         self.build_data_convertors()
-        self.set_color_templates(reset=False)
         self.reset()
         self.database_changed.emit(db)
         self.stop_metadata_backup()
@@ -545,16 +543,6 @@ class BooksModel(QAbstractTableModel): # {{{
             img = self.default_image
         return img
 
-    def set_color_templates(self, reset=True):
-        self.column_color_list = []
-        for i in range(1,self.db.column_color_count+1):
-            name = self.db.prefs.get('column_color_name_'+str(i))
-            if name:
-                self.column_color_list.append((name,
-                        self.db.prefs.get('column_color_template_'+str(i))))
-        if reset:
-            self.reset()
-
     def build_data_convertors(self):
         def authors(r, idx=-1):
             au = self.db.data[r][idx]
@@ -620,10 +608,11 @@ class BooksModel(QAbstractTableModel): # {{{
 
         def text_type(r, mult=None, idx=-1):
             text = self.db.data[r][idx]
-            if text and mult is not None:
+            if text and mult:
-                if mult:
+                jv = mult['list_to_ui']
-                    return QVariant(u' & '.join(text.split('|')))
+                sv = mult['cache_to_list']
-                return QVariant(u', '.join(sorted(text.split('|'),key=sort_key)))
+                return QVariant(jv.join(
+                    sorted([t.strip() for t in text.split(sv)], key=sort_key)))
             return QVariant(text)
 
         def decorated_text_type(r, idx=-1):
@@ -677,8 +666,6 @@ class BooksModel(QAbstractTableModel): # {{{
             datatype = self.custom_columns[col]['datatype']
             if datatype in ('text', 'comments', 'composite', 'enumeration'):
                 mult=self.custom_columns[col]['is_multiple']
-                if mult is not None:
-                    mult = self.custom_columns[col]['display'].get('is_names', False)
                 self.dc[col] = functools.partial(text_type, idx=idx, mult=mult)
                 if datatype in ['text', 'composite', 'enumeration'] and not mult:
                     if self.custom_columns[col]['display'].get('use_decorations', False):
@@ -726,15 +713,17 @@ class BooksModel(QAbstractTableModel): # {{{
                 return QVariant(QColor('lightgreen'))
         elif role == Qt.ForegroundRole:
             key = self.column_map[col]
-            for k,fmt in self.column_color_list:
+            mi = None
+            for k, fmt in self.db.prefs['column_color_rules']:
                 if k != key:
                     continue
                 id_ = self.id(index)
                 if id_ in self.color_cache:
                     if key in self.color_cache[id_]:
                         return self.color_cache[id_][key]
-                mi = self.db.get_metadata(self.id(index), index_is_id=True)
                 try:
+                    if mi is None:
+                        mi = self.db.get_metadata(id_, index_is_id=True)
                     color = composite_formatter.safe_format(fmt, mi, '', mi)
                     if color in self.colors:
                         color = QColor(color)
@@ -743,7 +732,7 @@ class BooksModel(QAbstractTableModel): # {{{
                             self.color_cache[id_][key] = color
                             return color
                 except:
-                    return NONE
+                    continue
             if self.is_custom_column(key) and \
                         self.custom_columns[key]['datatype'] == 'enumeration':
                 cc = self.custom_columns[self.column_map[col]]['display']

src/calibre/gui2/metadata/single_download.py

@@ -35,8 +35,9 @@ from calibre import force_unicode
 
 class RichTextDelegate(QStyledItemDelegate): # {{{
 
-    def __init__(self, parent=None):
+    def __init__(self, parent=None, max_width=160):
         QStyledItemDelegate.__init__(self, parent)
+        self.max_width = max_width
 
     def to_doc(self, index):
         doc = QTextDocument()
@@ -46,8 +47,8 @@ class RichTextDelegate(QStyledItemDelegate): # {{{
     def sizeHint(self, option, index):
         doc = self.to_doc(index)
         ans = doc.size().toSize()
-        if ans.width() > 150:
+        if ans.width() > self.max_width - 10:
-            ans.setWidth(160)
+            ans.setWidth(self.max_width)
         ans.setHeight(ans.height()+10)
         return ans
 

src/calibre/gui2/preferences/coloring.py

@@ -0,0 +1,621 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from PyQt4.Qt import (QWidget, QDialog, QLabel, QGridLayout, QComboBox, QSize,
+        QLineEdit, QIntValidator, QDoubleValidator, QFrame, QColor, Qt, QIcon,
+        QScrollArea, QPushButton, QVBoxLayout, QDialogButtonBox, QToolButton,
+        QListView, QAbstractListModel, pyqtSignal, QSizePolicy, QSpacerItem)
+
+from calibre import prepare_string_for_xml
+from calibre.utils.icu import sort_key
+from calibre.gui2 import error_dialog
+from calibre.gui2.dialogs.template_dialog import TemplateDialog
+from calibre.gui2.metadata.single_download import RichTextDelegate
+from calibre.library.coloring import (Rule, conditionable_columns,
+    displayable_columns, rule_from_template)
+
+class ConditionEditor(QWidget): # {{{
+
+    def __init__(self, fm, parent=None):
+        QWidget.__init__(self, parent)
+        self.fm = fm
+
+        self.action_map = {
+            'bool' : (
+                    (_('is true'), 'is true',),
+                    (_('is false'), 'is false'),
+                    (_('is undefined'), 'is undefined')
+            ),
+            'ondevice' : (
+                    (_('is true'), 'is set',),
+                    (_('is false'), 'is not set'),
+            ),
+            'identifiers' : (
+                (_('has id'), 'has id'),
+                (_('does not have id'), 'does not have id'),
+            ),
+            'int' : (
+                (_('is equal to'), 'eq'),
+                (_('is less than'), 'lt'),
+                (_('is greater than'), 'gt')
+            ),
+            'multiple' : (
+                (_('has'), 'has'),
+                (_('does not have'), 'does not have'),
+                (_('has pattern'), 'has pattern'),
+                (_('does not have pattern'), 'does not have pattern'),
+                (_('is set'), 'is set'),
+                (_('is not set'), 'is not set'),
+            ),
+            'single'   : (
+                (_('is'), 'is'),
+                (_('is not'), 'is not'),
+                (_('matches pattern'), 'matches pattern'),
+                (_('does not match pattern'), 'does not match pattern'),
+                (_('is set'), 'is set'),
+                (_('is not set'), 'is not set'),
+            ),
+        }
+
+        for x in ('float', 'rating', 'datetime'):
+            self.action_map[x] = self.action_map['int']
+
+        self.l = l = QGridLayout(self)
+        self.setLayout(l)
+
+        self.l1 = l1 = QLabel(_('If the '))
+        l.addWidget(l1, 0, 0)
+
+        self.column_box = QComboBox(self)
+        l.addWidget(self.column_box, 0, 1)
+
+        self.l2 = l2 = QLabel(_(' column '))
+        l.addWidget(l2, 0, 2)
+
+        self.action_box = QComboBox(self)
+        l.addWidget(self.action_box, 0, 3)
+
+        self.l3 = l3 = QLabel(_(' value '))
+        l.addWidget(l3, 0, 4)
+
+        self.value_box = QLineEdit(self)
+        l.addWidget(self.value_box, 0, 5)
+
+        self.column_box.addItem('', '')
+        for key in sorted(
+                conditionable_columns(fm),
+                key=sort_key):
+            self.column_box.addItem(key, key)
+        self.column_box.setCurrentIndex(0)
+
+        self.column_box.currentIndexChanged.connect(self.init_action_box)
+        self.action_box.currentIndexChanged.connect(self.init_value_box)
+
+        for b in (self.column_box, self.action_box):
+            b.setSizeAdjustPolicy(b.AdjustToMinimumContentsLengthWithIcon)
+            b.setMinimumContentsLength(15)
+
+    @dynamic_property
+    def current_col(self):
+        def fget(self):
+            idx = self.column_box.currentIndex()
+            return unicode(self.column_box.itemData(idx).toString())
+        def fset(self, val):
+            for idx in range(self.column_box.count()):
+                c = unicode(self.column_box.itemData(idx).toString())
+                if c == val:
+                    self.column_box.setCurrentIndex(idx)
+                    return
+            raise ValueError('Column %r not found'%val)
+        return property(fget=fget, fset=fset)
+
+    @dynamic_property
+    def current_action(self):
+        def fget(self):
+            idx = self.action_box.currentIndex()
+            return unicode(self.action_box.itemData(idx).toString())
+        def fset(self, val):
+            for idx in range(self.action_box.count()):
+                c = unicode(self.action_box.itemData(idx).toString())
+                if c == val:
+                    self.action_box.setCurrentIndex(idx)
+                    return
+            raise ValueError('Action %r not valid for current column'%val)
+        return property(fget=fget, fset=fset)
+
+    @property
+    def current_val(self):
+        return unicode(self.value_box.text()).strip()
+
+    @dynamic_property
+    def condition(self):
+
+        def fget(self):
+            c, a, v = (self.current_col, self.current_action,
+                    self.current_val)
+            if not c or not a:
+                return None
+            return (c, a, v)
+
+        def fset(self, condition):
+            c, a, v = condition
+            if not v:
+                v = ''
+            v = v.strip()
+            self.current_col = c
+            self.current_action = a
+            self.value_box.setText(v)
+
+        return property(fget=fget, fset=fset)
+
+    def init_action_box(self):
+        self.action_box.blockSignals(True)
+        self.action_box.clear()
+        self.action_box.addItem('', '')
+        col = self.current_col
+        if col:
+            m = self.fm[col]
+            dt = m['datatype']
+            if dt in self.action_map:
+                actions = self.action_map[dt]
+            else:
+                if col == 'ondevice':
+                    k = 'ondevice'
+                elif col == 'identifiers':
+                    k = 'identifiers'
+                else:
+                    k = 'multiple' if m['is_multiple'] else 'single'
+                actions = self.action_map[k]
+
+            for text, key in actions:
+                self.action_box.addItem(text, key)
+        self.action_box.setCurrentIndex(0)
+        self.action_box.blockSignals(False)
+        self.init_value_box()
+
+    def init_value_box(self):
+        self.value_box.setEnabled(True)
+        self.value_box.setText('')
+        self.value_box.setInputMask('')
+        self.value_box.setValidator(None)
+        col = self.current_col
+        if not col:
+            return
+        m = self.fm[col]
+        dt = m['datatype']
+        action = self.current_action
+        if not action:
+            return
+        m = self.fm[col]
+        dt = m['datatype']
+        tt = ''
+        if col == 'identifiers':
+            tt = _('Enter either an identifier type or an '
+                    'identifier type and value of the form identifier:value')
+        elif dt in ('int', 'float', 'rating'):
+            tt = _('Enter a number')
+            v = QIntValidator if dt == 'int' else QDoubleValidator
+            self.value_box.setValidator(v(self.value_box))
+        elif dt == 'datetime':
+            self.value_box.setInputMask('9999-99-99')
+            tt = _('Enter a date in the format YYYY-MM-DD')
+        else:
+            tt = _('Enter a string.')
+            if 'pattern' in action:
+                tt = _('Enter a regular expression')
+            elif m.get('is_multiple', False):
+                tt += '\n' + _('You can match multiple values by separating'
+                        ' them with %s')%m['is_multiple']['ui_to_list']
+        self.value_box.setToolTip(tt)
+        if action in ('is set', 'is not set', 'is true', 'is false',
+                'is undefined'):
+            self.value_box.setEnabled(False)
+# }}}
+
+class RuleEditor(QDialog): # {{{
+
+    def __init__(self, fm, parent=None):
+        QDialog.__init__(self, parent)
+        self.fm = fm
+
+        self.setWindowIcon(QIcon(I('format-fill-color.png')))
+        self.setWindowTitle(_('Create/edit a column coloring rule'))
+
+        self.l = l = QGridLayout(self)
+        self.setLayout(l)
+
+        self.l1 = l1 = QLabel(_('Create a coloring rule by'
+            ' filling in the boxes below'))
+        l.addWidget(l1, 0, 0, 1, 5)
+
+        self.f1 = QFrame(self)
+        self.f1.setFrameShape(QFrame.HLine)
+        l.addWidget(self.f1, 1, 0, 1, 5)
+
+        self.l2 = l2 = QLabel(_('Set the color of the column:'))
+        l.addWidget(l2, 2, 0)
+
+        self.column_box = QComboBox(self)
+        l.addWidget(self.column_box, 2, 1)
+
+        self.l3 = l3 = QLabel(_('to'))
+        l.addWidget(l3, 2, 2)
+
+        self.color_box = QComboBox(self)
+        l.addWidget(self.color_box, 2, 3)
+        l.addItem(QSpacerItem(10, 10, QSizePolicy.Expanding), 2, 4)
+
+        self.l4 = l4 = QLabel(
+            _('Only if the following conditions are all satisfied:'))
+        l.addWidget(l4, 3, 0, 1, 5)
+
+        self.scroll_area = sa = QScrollArea(self)
+        sa.setMinimumHeight(300)
+        sa.setMinimumWidth(950)
+        sa.setWidgetResizable(True)
+        l.addWidget(sa, 4, 0, 1, 5)
+
+        self.add_button = b = QPushButton(QIcon(I('plus.png')),
+                _('Add another condition'))
+        l.addWidget(b, 5, 0, 1, 5)
+        b.clicked.connect(self.add_blank_condition)
+
+        self.l5 = l5 = QLabel(_('You can disable a condition by'
+            ' blanking all of its boxes'))
+        l.addWidget(l5, 6, 0, 1, 5)
+
+        self.bb = bb = QDialogButtonBox(
+                QDialogButtonBox.Ok|QDialogButtonBox.Cancel)
+        bb.accepted.connect(self.accept)
+        bb.rejected.connect(self.reject)
+        l.addWidget(bb, 7, 0, 1, 5)
+
+        self.conditions_widget = QWidget(self)
+        sa.setWidget(self.conditions_widget)
+        self.conditions_widget.setLayout(QVBoxLayout())
+        self.conditions_widget.layout().setAlignment(Qt.AlignTop)
+        self.conditions = []
+
+        for b in (self.column_box, self.color_box):
+            b.setSizeAdjustPolicy(b.AdjustToMinimumContentsLengthWithIcon)
+            b.setMinimumContentsLength(15)
+
+        for key in sorted(
+                displayable_columns(fm),
+                key=sort_key):
+            name = fm[key]['name']
+            if name:
+                self.column_box.addItem(key, key)
+        self.column_box.setCurrentIndex(0)
+
+        self.color_box.addItems(QColor.colorNames())
+        self.color_box.setCurrentIndex(0)
+
+        self.resize(self.sizeHint())
+
+    def add_blank_condition(self):
+        c = ConditionEditor(self.fm, parent=self.conditions_widget)
+        self.conditions.append(c)
+        self.conditions_widget.layout().addWidget(c)
+
+    def apply_rule(self, col, rule):
+        for i in range(self.column_box.count()):
+            c = unicode(self.column_box.itemData(i).toString())
+            if col == c:
+                self.column_box.setCurrentIndex(i)
+                break
+        if rule.color:
+            idx = self.color_box.findText(rule.color)
+            if idx >= 0:
+                self.color_box.setCurrentIndex(idx)
+        for c in rule.conditions:
+            ce = ConditionEditor(self.fm, parent=self.conditions_widget)
+            self.conditions.append(ce)
+            self.conditions_widget.layout().addWidget(ce)
+            try:
+                ce.condition = c
+            except:
+                import traceback
+                traceback.print_exc()
+
+
+    def accept(self):
+        if self.validate():
+            QDialog.accept(self)
+
+    def validate(self):
+        r = Rule(self.fm)
+        for c in self.conditions:
+            condition = c.condition
+            if condition is not None:
+                try:
+                    r.add_condition(*condition)
+                except Exception as e:
+                    import traceback
+                    error_dialog(self, _('Invalid condition'),
+                            _('One of the conditions for this rule is'
+                                ' invalid: <b>%s</b>')%e,
+                            det_msg=traceback.format_exc(), show=True)
+                    return False
+        if len(r.conditions) < 1:
+            error_dialog(self, _('No conditions'),
+                    _('You must specify at least one non-empty condition'
+                        ' for this rule'), show=True)
+            return False
+        return True
+
+    @property
+    def rule(self):
+        r = Rule(self.fm)
+        r.color = unicode(self.color_box.currentText())
+        idx = self.column_box.currentIndex()
+        col = unicode(self.column_box.itemData(idx).toString())
+        for c in self.conditions:
+            condition = c.condition
+            if condition is not None:
+                r.add_condition(*condition)
+
+        return col, r
+# }}}
+
+class RulesModel(QAbstractListModel): # {{{
+
+    def __init__(self, prefs, fm, parent=None):
+        QAbstractListModel.__init__(self, parent)
+
+        self.fm = fm
+        rules = list(prefs['column_color_rules'])
+        self.rules = []
+        for col, template in rules:
+            try:
+                rule = rule_from_template(self.fm, template)
+            except:
+                rule = template
+            self.rules.append((col, rule))
+
+    def rowCount(self, *args):
+        return len(self.rules)
+
+    def data(self, index, role):
+        row = index.row()
+        try:
+            col, rule = self.rules[row]
+        except:
+            return None
+
+        if role == Qt.DisplayRole:
+            return self.rule_to_html(col, rule)
+        if role == Qt.UserRole:
+            return (col, rule)
+
+    def add_rule(self, col, rule):
+        self.rules.append((col, rule))
+        self.reset()
+        return self.index(len(self.rules)-1)
+
+    def replace_rule(self, index, col, r):
+        self.rules[index.row()] = (col, r)
+        self.dataChanged.emit(index, index)
+
+    def remove_rule(self, index):
+        self.rules.remove(self.rules[index.row()])
+        self.reset()
+
+    def commit(self, prefs):
+        rules = []
+        for col, r in self.rules:
+            if isinstance(r, Rule):
+                r = r.template
+            if r is not None:
+                rules.append((col, r))
+        prefs['column_color_rules'] = rules
+
+    def move(self, idx, delta):
+        row = idx.row() + delta
+        if row >= 0 and row < len(self.rules):
+            t = self.rules[row]
+            self.rules[row] = self.rules[row-delta]
+            self.rules[row-delta] = t
+            self.dataChanged.emit(idx, idx)
+            idx = self.index(row)
+            self.dataChanged.emit(idx, idx)
+            return idx
+
+    def clear(self):
+        self.rules = []
+        self.reset()
+
+    def rule_to_html(self, col, rule):
+        if not isinstance(rule, Rule):
+            return _('''
+            <p>Advanced Rule for column <b>%s</b>:
+            <pre>%s</pre>
+            ''')%(col, prepare_string_for_xml(rule))
+        conditions = [self.condition_to_html(c) for c in rule.conditions]
+        return _('''\
+            <p>Set the color of <b>%s</b> to <b>%s</b> if the following
+            conditions are met:</p>
+            <ul>%s</ul>
+            ''') % (col, rule.color, ''.join(conditions))
+
+    def condition_to_html(self, condition):
+        c, a, v = condition
+        return (
+            _('<li>If the <b>%s</b> column <b>%s</b> value: <b>%s</b>') %
+                (c, a, prepare_string_for_xml(v)))
+
+# }}}
+
+class EditRules(QWidget): # {{{
+
+    changed = pyqtSignal()
+
+    def __init__(self, parent=None):
+        QWidget.__init__(self, parent)
+
+        self.l = l = QGridLayout(self)
+        self.setLayout(l)
+
+        self.l1 = l1 = QLabel(_(
+            'You can control the color of columns in the'
+            ' book list by creating "rules" that tell calibre'
+            ' what color to use. Click the Add Rule button below'
+            ' to get started. You can change an existing rule by double'
+            ' clicking it.'))
+        l1.setWordWrap(True)
+        l.addWidget(l1, 0, 0, 1, 2)
+
+        self.add_button = QPushButton(QIcon(I('plus.png')), _('Add Rule'),
+                self)
+        self.remove_button = QPushButton(QIcon(I('minus.png')),
+                _('Remove Rule'), self)
+        self.add_button.clicked.connect(self.add_rule)
+        self.remove_button.clicked.connect(self.remove_rule)
+        l.addWidget(self.add_button, 1, 0)
+        l.addWidget(self.remove_button, 1, 1)
+
+        self.g = g = QGridLayout()
+        self.rules_view = QListView(self)
+        self.rules_view.doubleClicked.connect(self.edit_rule)
+        self.rules_view.setSelectionMode(self.rules_view.SingleSelection)
+        self.rules_view.setAlternatingRowColors(True)
+        self.rtfd = RichTextDelegate(parent=self.rules_view, max_width=400)
+        self.rules_view.setItemDelegate(self.rtfd)
+        g.addWidget(self.rules_view, 0, 0, 2, 1)
+
+        self.up_button = b = QToolButton(self)
+        b.setIcon(QIcon(I('arrow-up.png')))
+        b.setToolTip(_('Move the selected rule up'))
+        b.clicked.connect(self.move_up)
+        g.addWidget(b, 0, 1, 1, 1, Qt.AlignTop)
+        self.down_button = b = QToolButton(self)
+        b.setIcon(QIcon(I('arrow-down.png')))
+        b.setToolTip(_('Move the selected rule down'))
+        b.clicked.connect(self.move_down)
+        g.addWidget(b, 1, 1, 1, 1, Qt.AlignBottom)
+
+        l.addLayout(g, 2, 0, 1, 2)
+        l.setRowStretch(2, 10)
+
+        self.add_advanced_button = b = QPushButton(QIcon(I('plus.png')),
+                _('Add Advanced Rule'), self)
+        b.clicked.connect(self.add_advanced)
+        l.addWidget(b, 3, 0, 1, 2)
+
+    def initialize(self, fm, prefs, mi):
+        self.model = RulesModel(prefs, fm)
+        self.rules_view.setModel(self.model)
+        self.fm = fm
+        self.mi = mi
+
+    def _add_rule(self, dlg):
+        if dlg.exec_() == dlg.Accepted:
+            col, r = dlg.rule
+            if r and col:
+                idx = self.model.add_rule(col, r)
+                self.rules_view.scrollTo(idx)
+                self.changed.emit()
+
+    def add_rule(self):
+            d = RuleEditor(self.model.fm)
+            d.add_blank_condition()
+            self._add_rule(d)
+
+    def add_advanced(self):
+        td = TemplateDialog(self, '', mi=self.mi, fm=self.fm, color_field='')
+        self._add_rule(td)
+
+    def edit_rule(self, index):
+        try:
+            col, rule = self.model.data(index, Qt.UserRole)
+        except:
+            return
+        if isinstance(rule, Rule):
+            d = RuleEditor(self.model.fm)
+            d.apply_rule(col, rule)
+        else:
+            d = TemplateDialog(self, rule, mi=self.mi, fm=self.fm, color_field=col)
+        if d.exec_() == d.Accepted:
+            col, r = d.rule
+            if r is not None and col:
+                self.model.replace_rule(index, col, r)
+                self.rules_view.scrollTo(index)
+                self.changed.emit()
+
+    def get_selected_row(self, txt):
+        sm = self.rules_view.selectionModel()
+        rows = list(sm.selectedRows())
+        if not rows:
+            error_dialog(self, _('No rule selected'),
+                    _('No rule selected for %s.')%txt, show=True)
+            return None
+        return rows[0]
+
+    def remove_rule(self):
+        row = self.get_selected_row(_('removal'))
+        if row is not None:
+            self.model.remove_rule(row)
+            self.changed.emit()
+
+    def move_up(self):
+        idx = self.rules_view.currentIndex()
+        if idx.isValid():
+            idx = self.model.move(idx, -1)
+            if idx is not None:
+                sm = self.rules_view.selectionModel()
+                sm.select(idx, sm.ClearAndSelect)
+                self.rules_view.setCurrentIndex(idx)
+                self.changed.emit()
+
+    def move_down(self):
+        idx = self.rules_view.currentIndex()
+        if idx.isValid():
+            idx = self.model.move(idx, 1)
+            if idx is not None:
+                sm = self.rules_view.selectionModel()
+                sm.select(idx, sm.ClearAndSelect)
+                self.rules_view.setCurrentIndex(idx)
+                self.changed.emit()
+
+    def clear(self):
+        self.model.clear()
+        self.changed.emit()
+
+    def commit(self, prefs):
+        self.model.commit(prefs)
+
+# }}}
+
+if __name__ == '__main__':
+    from PyQt4.Qt import QApplication
+    app = QApplication([])
+
+    from calibre.library import db
+
+    db = db()
+
+    if False:
+        d = RuleEditor(db.field_metadata)
+        d.add_blank_condition()
+        d.exec_()
+
+        col, r = d.rule
+
+        print ('Column to be colored:', col)
+        print ('Template:')
+        print (r.template)
+    else:
+        d = EditRules()
+        d.resize(QSize(800, 600))
+        d.initialize(db.field_metadata, db.prefs, None)
+        d.show()
+        app.exec_()
+        d.commit(db.prefs)
+
+

src/calibre/gui2/preferences/create_custom_column.py

@@ -13,6 +13,9 @@ from calibre.gui2 import error_dialog
 
 class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
 
+    # Note: in this class, we are treating is_multiple as the boolean that
+    # custom_columns expects to find in its structure. It does not use the dict
+
     column_types = {
                     0:{'datatype':'text',
                         'text':_('Text, column shown in the tag browser'),

src/calibre/gui2/preferences/look_feel.py

@@ -5,21 +5,19 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-from functools import partial
-
 from PyQt4.Qt import (QApplication, QFont, QFontInfo, QFontDialog,
-        QAbstractListModel, Qt, QColor, QIcon, QToolButton, QComboBox)
+        QAbstractListModel, Qt, QIcon)
 
 from calibre.gui2.preferences import ConfigWidgetBase, test_widget, CommaSeparatedList
 from calibre.gui2.preferences.look_feel_ui import Ui_Form
 from calibre.gui2 import config, gprefs, qt_app
-from calibre.gui2.dialogs.template_line_editor import TemplateLineEditor
 from calibre.utils.localization import (available_translations,
     get_language, get_lang)
 from calibre.utils.config import prefs
 from calibre.utils.icu import sort_key
 from calibre.gui2 import NONE
 from calibre.gui2.book_details import get_field_list
+from calibre.gui2.preferences.coloring import EditRules
 
 class DisplayedFields(QAbstractListModel): # {{{
 
@@ -162,117 +160,11 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.df_up_button.clicked.connect(self.move_df_up)
         self.df_down_button.clicked.connect(self.move_df_down)
 
-        self.color_help_text.setText('<p>' +
+        self.edit_rules = EditRules(self.tabWidget)
-                _('Here you can specify coloring rules for columns shown in the '
+        self.edit_rules.changed.connect(self.changed_signal)
-                  'library view. Choose the column you wish to color, then '
+        self.tabWidget.addTab(self.edit_rules,
-                  'supply a template that specifies the color to use based on '
+                QIcon(I('format-fill-color.png')), _('Column coloring'))
-                  'the values in the column. There is a '
+        self.tabWidget.setCurrentIndex(0)
-                  '<a href="http://manual.calibre-ebook.com/template_lang.html">'
-                  'tutorial</a> on using templates.') +
-                 '</p><p>' +
-                _('If you want to color a field based on contents of columns, '
-                  'then click the button next to an empty line to open the wizard. '
-                  'It will build a template for you. You can later edit that '
-                  'template with the same wizard. This is by far the easiest '
-                  'way to specify a template.') +
-                 '</p><p>' +
-                _('If you manually construct a template, then the template must '
-                  'evaluate to a valid color name shown in the color names box.'
-                  'You can use any legal template expression. '
-                  'For example, you can set the title to always display in '
-                  'green using the template "green" (without the quotes). '
-                  'To show the title in the color named in the custom column '
-                  '#column, use "{#column}". To show the title in blue if the '
-                  'custom column #column contains the value "foo", in red if the '
-                  'column contains the value "bar", otherwise in black, use '
-                  '<pre>{#column:switch(foo,blue,bar,red,black)}</pre>'
-                  'To show the title in blue if the book has the exact tag '
-                  '"Science Fiction", red if the book has the exact tag '
-                  '"Mystery", or black if the book has neither tag, use'
-                  "<pre>program: \n"
-                  "    t = field('tags'); \n"
-                  "    first_non_empty(\n"
-                  "        in_list(t, ',', '^Science Fiction$', 'blue', ''), \n"
-                  "        in_list(t, ',', '^Mystery$', 'red', 'black'))</pre>"
-                  'To show the title in green if it has one format, blue if it '
-                  'two formats, and red if more, use'
-                  "<pre>program:cmp(count(field('formats'),','), 2, 'green', 'blue', 'red')</pre>") +
-                               '</p><p>' +
-                _('You can access a multi-line template editor from the '
-                  'context menu (right-click).') + '</p><p>' +
-                _('<b>Note:</b> if you want to color a "custom column with a fixed set '
-                  'of values", it is often easier to specify the '
-                  'colors in the column definition dialog. There you can '
-                  'provide a color for each value without using a template.')+ '</p>')
-        self.color_help_scrollArea.setVisible(False)
-        self.color_help_button.clicked.connect(self.change_help_text)
-        self.colors_scrollArea.setVisible(False)
-        self.colors_label.setVisible(False)
-        self.colors_button.clicked.connect(self.change_colors_text)
-
-        choices = db.field_metadata.displayable_field_keys()
-        choices.sort(key=sort_key)
-        choices.insert(0, '')
-        self.column_color_count = db.column_color_count+1
-
-        mi=None
-        try:
-            idx = gui.library_view.currentIndex().row()
-            mi = db.get_metadata(idx, index_is_id=False)
-        except:
-            pass
-
-        l = self.column_color_layout
-        for i in range(1, self.column_color_count):
-            ccn = QComboBox(parent=self)
-            setattr(self, 'opt_column_color_name_'+str(i), ccn)
-            l.addWidget(ccn, i, 0, 1, 1)
-
-            wtb = QToolButton(parent=self)
-            setattr(self, 'opt_column_color_wizard_'+str(i), wtb)
-            wtb.setIcon(QIcon(I('wizard.png')))
-            l.addWidget(wtb, i, 1, 1, 1)
-
-            ttb = QToolButton(parent=self)
-            setattr(self, 'opt_column_color_tpledit_'+str(i), ttb)
-            ttb.setIcon(QIcon(I('edit_input.png')))
-            l.addWidget(ttb, i, 2, 1, 1)
-
-            tpl = TemplateLineEditor(parent=self)
-            setattr(self, 'opt_column_color_template_'+str(i), tpl)
-            tpl.textChanged.connect(partial(self.tpl_edit_text_changed, ctrl=i))
-            tpl.set_db(db)
-            tpl.set_mi(mi)
-            l.addWidget(tpl, i, 3, 1, 1)
-
-            wtb.clicked.connect(tpl.tag_wizard)
-            ttb.clicked.connect(tpl.open_editor)
-
-            r('column_color_name_'+str(i), db.prefs, choices=choices)
-            r('column_color_template_'+str(i), db.prefs)
-            txt = db.prefs.get('column_color_template_'+str(i), None)
-
-            wtb.setEnabled(tpl.enable_wizard_button(txt))
-            ttb.setEnabled(not tpl.enable_wizard_button(txt) or not txt)
-
-        all_colors = [unicode(s) for s in list(QColor.colorNames())]
-        self.colors_box.setText(', '.join(all_colors))
-
-    def change_help_text(self):
-        self.color_help_scrollArea.setVisible(not self.color_help_scrollArea.isVisible())
-
-    def change_colors_text(self):
-        self.colors_scrollArea.setVisible(not self.colors_scrollArea.isVisible())
-        self.colors_label.setVisible(not self.colors_label.isVisible())
-
-    def tpl_edit_text_changed(self, ign, ctrl=None):
-        tpl = getattr(self, 'opt_column_color_template_'+str(ctrl))
-        txt = unicode(tpl.text())
-        wtb = getattr(self, 'opt_column_color_wizard_'+str(ctrl))
-        ttb = getattr(self, 'opt_column_color_tpledit_'+str(ctrl))
-        wtb.setEnabled(tpl.enable_wizard_button(txt))
-        ttb.setEnabled(not tpl.enable_wizard_button(txt) or not txt)
-        tpl.setFocus()
 
     def initialize(self):
         ConfigWidgetBase.initialize(self)
@@ -283,6 +175,13 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.current_font = self.initial_font = font
         self.update_font_display()
         self.display_model.initialize()
+        db = self.gui.current_db
+        try:
+            idx = self.gui.library_view.currentIndex().row()
+            mi = db.get_metadata(idx, index_is_id=False)
+        except:
+            mi=None
+        self.edit_rules.initialize(db.field_metadata, db.prefs, mi)
 
     def restore_defaults(self):
         ConfigWidgetBase.restore_defaults(self)
@@ -292,6 +191,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
             self.changed_signal.emit()
             self.update_font_display()
         self.display_model.restore_defaults()
+        self.edit_rules.clear()
         self.changed_signal.emit()
 
     def build_font_obj(self):
@@ -341,12 +241,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
             self.changed_signal.emit()
 
     def commit(self, *args):
-        for i in range(1, self.column_color_count):
-            col = getattr(self, 'opt_column_color_name_'+str(i))
-            tpl = getattr(self, 'opt_column_color_template_'+str(i))
-            if not col.currentIndex() or not unicode(tpl.text()).strip():
-                col.setCurrentIndex(0)
-                tpl.setText('')
         rr = ConfigWidgetBase.commit(self, *args)
         if self.current_font != self.initial_font:
             gprefs['font'] = (self.current_font[:4] if self.current_font else
@@ -356,10 +250,11 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
             QApplication.setFont(self.font_display.font())
             rr = True
         self.display_model.commit()
+        self.edit_rules.commit(self.gui.current_db.prefs)
         return rr
 
     def refresh_gui(self, gui):
-        gui.library_view.model().set_color_templates()
+        gui.library_view.model().reset()
         self.update_font_display()
         gui.tags_view.reread_collapse_parameters()
         gui.library_view.refresh_book_details()

src/calibre/gui2/preferences/look_feel.ui

@@ -6,7 +6,7 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>717</width>
+    <width>820</width>
     <height>519</height>
    </rect>
   </property>
@@ -407,161 +407,6 @@ then the tags will be displayed each on their own line.</string>
        </item>
       </layout>
      </widget>
-     <widget class="QWidget" name="tab_5">
-      <attribute name="icon">
-       <iconset resource="../../../../resources/images.qrc">
-        <normaloff>:/images/format-fill-color.png</normaloff>:/images/format-fill-color.png</iconset>
-      </attribute>
-      <attribute name="title">
-       <string>Column Coloring</string>
-      </attribute>
-      <layout class="QGridLayout" name="column_color_layout">
-       <item row="0" column="0">
-        <widget class="QLabel" name="label">
-         <property name="text">
-          <string>Column to color</string>
-         </property>
-        </widget>
-       </item>
-       <item row="0" column="3">
-        <layout class="QHBoxLayout">
-         <item>
-          <widget class="QLabel" name="label">
-           <property name="text">
-            <string>Color selection template</string>
-           </property>
-           <property name="sizePolicy">
-            <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
-             <horstretch>10</horstretch>
-             <verstretch>0</verstretch>
-            </sizepolicy>
-           </property>
-          </widget>
-         </item>
-         <item>
-          <widget class="QLabel" name="label">
-           <property name="text">
-            <string>The template wizard is easiest to use</string>
-           </property>
-          </widget>
-         </item>
-         <item>
-          <widget class="QPushButton" name="color_help_button">
-           <property name="text">
-            <string>Show/hide help text</string>
-           </property>
-          </widget>
-         </item>
-         <item>
-          <widget class="QPushButton" name="colors_button">
-           <property name="text">
-            <string>Show/hide colors</string>
-           </property>
-          </widget>
-         </item>
-        </layout>
-       </item>
-       <item row="20" column="0">
-        <widget class="QLabel" name="colors_label">
-         <property name="text">
-          <string>Color names</string>
-         </property>
-        </widget>
-       </item>
-       <item row="21" column="0" colspan="8">
-        <widget class="QScrollArea" name="colors_scrollArea">
-         <property name="maximumSize">
-          <size>
-           <width>16777215</width>
-           <height>300</height>
-          </size>
-         </property>
-         <property name="widgetResizable">
-          <bool>true</bool>
-         </property>
-         <widget class="QWidget" name="scrollAreaWidgetContents_2">
-          <property name="geometry">
-           <rect>
-            <x>0</x>
-            <y>0</y>
-            <width>687</width>
-            <height>61</height>
-           </rect>
-          </property>
-          <layout class="QVBoxLayout" name="verticalLayout_2">
-           <item>
-            <widget class="QLabel" name="colors_box">
-             <property name="wordWrap">
-              <bool>true</bool>
-             </property>
-             <property name="alignment">
-              <set>Qt::AlignLeft|Qt::AlignTop</set>
-             </property>
-            </widget>
-           </item>
-          </layout>
-         </widget>
-        </widget>
-       </item>
-       <item row="30" column="0" colspan="8">
-        <widget class="QScrollArea" name="color_help_scrollArea">
-         <property name="minimumSize">
-          <size>
-           <width>0</width>
-           <height>200</height>
-          </size>
-         </property>
-         <property name="widgetResizable">
-          <bool>true</bool>
-         </property>
-         <property name="alignment">
-          <set>Qt::AlignLeft|Qt::AlignTop</set>
-         </property>
-         <widget class="QWidget" name="scrollAreaWidgetContents">
-          <property name="geometry">
-           <rect>
-            <x>0</x>
-            <y>0</y>
-            <width>687</width>
-            <height>194</height>
-           </rect>
-          </property>
-          <layout class="QVBoxLayout" name="verticalLayout">
-           <item>
-            <widget class="QLabel" name="color_help_text">
-             <property name="wordWrap">
-              <bool>true</bool>
-             </property>
-             <property name="openExternalLinks">
-              <bool>true</bool>
-             </property>
-            </widget>
-           </item>
-          </layout>
-         </widget>
-        </widget>
-       </item>
-       <item row="40" column="0">
-        <spacer>
-         <property name="sizePolicy">
-          <sizepolicy vsizetype="Expanding" hsizetype="Fixed">
-           <horstretch>0</horstretch>
-           <verstretch>10</verstretch>
-          </sizepolicy>
-         </property>
-         <property name="orientation">
-          <enum>Qt::Vertical</enum>
-         </property>
-         <property name="sizeHint" stdset="0">
-          <size>
-           <width>0</width>
-           <height>0</height>
-          </size>
-         </property>
-        </spacer>
-       </item>
-      </layout>
-     </widget>
     </widget>
    </item>
   </layout>
@@ -572,11 +417,6 @@ then the tags will be displayed each on their own line.</string>
    <extends>QLineEdit</extends>
    <header>calibre/gui2/complete.h</header>
   </customwidget>
-  <customwidget>
-   <class>TemplateLineEditor</class>
-   <extends>QLineEdit</extends>
-   <header>calibre/gui2/dialogs/template_line_editor.h</header>
-  </customwidget>
  </customwidgets>
  <resources>
   <include location="../../../../resources/images.qrc"/>

src/calibre/gui2/preferences/metadata_sources.py

@@ -283,7 +283,9 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.fields_model.dataChanged.connect(self.changed_signal)
 
         self.select_all_button.clicked.connect(self.fields_model.select_all)
+        self.select_all_button.clicked.connect(self.changed_signal)
         self.clear_all_button.clicked.connect(self.fields_model.clear_all)
+        self.clear_all_button.clicked.connect(self.changed_signal)
 
     def configure_plugin(self):
         for index in self.sources_view.selectionModel().selectedRows():

src/calibre/gui2/preferences/template_functions.py

@@ -7,7 +7,9 @@ __docformat__ = 'restructuredtext en'
 
 import json, traceback
 
-from calibre.gui2 import error_dialog
+from PyQt4.Qt import QDialogButtonBox
+
+from calibre.gui2 import error_dialog, warning_dialog
 from calibre.gui2.preferences import ConfigWidgetBase, test_widget
 from calibre.gui2.preferences.template_functions_ui import Ui_Form
 from calibre.gui2.widgets import PythonHighlighter
@@ -152,9 +154,14 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                          _('Name already used'), show=True)
             return
         if self.argument_count.value() == 0:
-            error_dialog(self.gui, _('Template functions'),
+            box = warning_dialog(self.gui, _('Template functions'),
-                         _('Argument count must be -1 or greater than zero'),
+                         _('Argument count should be -1 or greater than zero.'
-                         show=True)
+                           'Setting it to zero means that this function cannot '
+                           'be used in single function mode.'), det_msg = '',
+                         show=False)
+            box.bb.setStandardButtons(box.bb.standardButtons() | QDialogButtonBox.Cancel)
+            box.det_msg_toggle.setVisible(False)
+            if not box.exec_():
                 return
         try:
             prog = unicode(self.program.toPlainText())

src/calibre/gui2/preferences/toolbar.ui

@@ -16,13 +16,28 @@
   <layout class="QGridLayout" name="gridLayout">
    <item row="0" column="0">
     <widget class="QLabel" name="label">
+     <property name="font">
+      <font>
+       <weight>75</weight>
+       <bold>true</bold>
+      </font>
+     </property>
      <property name="text">
-      <string>Customize the actions in:</string>
+      <string>Choose the &amp;toolbar to customize:</string>
+     </property>
+     <property name="buddy">
+      <cstring>what</cstring>
      </property>
     </widget>
    </item>
    <item row="0" column="1" colspan="3">
     <widget class="QComboBox" name="what">
+     <property name="font">
+      <font>
+       <weight>75</weight>
+       <bold>true</bold>
+      </font>
+     </property>
      <property name="sizeAdjustPolicy">
       <enum>QComboBox::AdjustToMinimumContentsLengthWithIcon</enum>
      </property>

src/calibre/gui2/store/amazon_de_plugin.py

@@ -6,21 +6,23 @@ __license__ = 'GPL 3'
 __copyright__ = '2011, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
+import re, urllib
+from contextlib import closing
+
+from lxml import html
+
 from PyQt4.Qt import QUrl
 
+from calibre import browser
 from calibre.gui2 import open_url
-from calibre.gui2.store.amazon_plugin import AmazonKindleStore
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.search_result import SearchResult
 
-class AmazonDEKindleStore(AmazonKindleStore):
+class AmazonDEKindleStore(StorePlugin):
     '''
     For comments on the implementation, please see amazon_plugin.py
     '''
 
-    search_url = 'http://www.amazon.de/s/?url=search-alias%3Ddigital-text&field-keywords='
-    details_url = 'http://amazon.de/dp/'
-    drm_search_text = u'Gleichzeitige Verwendung von Geräten'
-    drm_free_text = u'Keine Einschränkung'
-
     def open(self, parent=None, detail_item=None, external=False):
         aff_id = {'tag': 'charhale0a-21'}
         store_link = ('http://www.amazon.de/gp/redirect.html?ie=UTF8&site-redirect=de'
@@ -32,3 +34,94 @@ class AmazonDEKindleStore(AmazonKindleStore):
                           '&location=http://www.amazon.de/dp/%(asin)s&site-redirect=de'
                           '&tag=%(tag)s&linkCode=ur2&camp=1638&creative=6742') % aff_id
         open_url(QUrl(store_link))
+
+    def search(self, query, max_results=10, timeout=60):
+        search_url = 'http://www.amazon.de/s/?url=search-alias%3Ddigital-text&field-keywords='
+        url =  search_url + urllib.quote_plus(query)
+        br = browser()
+
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+
+            # Amazon has two results pages.
+            is_shot = doc.xpath('boolean(//div[@id="shotgunMainResults"])')
+            # Horizontal grid of books.
+            if is_shot:
+                data_xpath = '//div[contains(@class, "result")]'
+                format_xpath = './/div[@class="productTitle"]/text()'
+                cover_xpath = './/div[@class="productTitle"]//img/@src'
+            # Vertical list of books.
+            else:
+                data_xpath = '//div[@class="productData"]'
+                format_xpath = './/span[@class="format"]/text()'
+                cover_xpath = '../div[@class="productImage"]/a/img/@src'
+
+            for data in doc.xpath(data_xpath):
+                if counter <= 0:
+                    break
+
+                # Even though we are searching digital-text only Amazon will still
+                # put in results for non Kindle books (author pages). Se we need
+                # to explicitly check if the item is a Kindle book and ignore it
+                # if it isn't.
+                format = ''.join(data.xpath(format_xpath))
+                if 'kindle' not in format.lower():
+                    continue
+
+                # We must have an asin otherwise we can't easily reference the
+                # book later.
+                asin_href = None
+                asin_a = data.xpath('.//div[@class="productTitle"]/a[1]')
+                if asin_a:
+                    asin_href = asin_a[0].get('href', '')
+                    m = re.search(r'/dp/(?P<asin>.+?)(/|$)', asin_href)
+                    if m:
+                        asin = m.group('asin')
+                    else:
+                        continue
+                else:
+                    continue
+
+                cover_url = ''.join(data.xpath(cover_xpath))
+
+                title = ''.join(data.xpath('.//div[@class="productTitle"]/a/text()'))
+                price = ''.join(data.xpath('.//div[@class="newPrice"]/span/text()'))
+
+                if is_shot:
+                    author = format.split(' von ')[-1]
+                else:
+                    author = ''.join(data.xpath('.//div[@class="productTitle"]/span[@class="ptBrand"]/text()'))
+                    author = author.split(' von ')[-1]
+
+                counter -= 1
+
+                s = SearchResult()
+                s.cover_url = cover_url.strip()
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price.strip()
+                s.detail_item = asin.strip()
+                s.formats = 'Kindle'
+
+                yield s
+
+    def get_details(self, search_result, timeout):
+        drm_search_text = u'Gleichzeitige Verwendung von Geräten'
+        drm_free_text = u'Keine Einschränkung'
+        url = 'http://amazon.de/dp/'
+
+        br = browser()
+        with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf:
+            idata = html.fromstring(nf.read())
+            if idata.xpath('boolean(//div[@class="content"]//li/b[contains(text(), "' +
+                           drm_search_text + '")])'):
+                if idata.xpath('boolean(//div[@class="content"]//li[contains(., "' +
+                               drm_free_text + '") and contains(b, "' +
+                               drm_search_text + '")])'):
+                    search_result.drm = SearchResult.DRM_UNLOCKED
+                else:
+                    search_result.drm = SearchResult.DRM_UNKNOWN
+            else:
+                search_result.drm = SearchResult.DRM_LOCKED
+        return True

src/calibre/gui2/store/amazon_plugin.py

@@ -131,16 +131,22 @@ class AmazonKindleStore(StorePlugin):
             
             # Amazon has two results pages.
             is_shot = doc.xpath('boolean(//div[@id="shotgunMainResults"])')
-            # Horizontal grid of books.
+            # Horizontal grid of books. Search "Paolo Bacigalupi"
             if is_shot:
                 data_xpath = '//div[contains(@class, "result")]'
-                format_xpath = './/div[@class="productTitle"]/text()'
+                format_xpath = './/div[@class="productTitle"]//text()'
+                asin_xpath = './/div[@class="productTitle"]//a'
                 cover_xpath = './/div[@class="productTitle"]//img/@src'
-            # Vertical list of books.
+                title_xpath = './/div[@class="productTitle"]/a//text()'
+                price_xpath = './/div[@class="newPrice"]/span/text()'
+            # Vertical list of books. Search "martin"
             else:
-                data_xpath = '//div[@class="productData"]'
+                data_xpath = '//div[contains(@class, "results")]//div[contains(@class, "result")]'
-                format_xpath = './/span[@class="format"]/text()'
+                format_xpath = './/span[@class="binding"]//text()'
-                cover_xpath = '../div[@class="productImage"]/a/img/@src'
+                asin_xpath = './/div[@class="image"]/a[1]'
+                cover_xpath = './/img[@class="productImage"]/@src'
+                title_xpath = './/a[@class="title"]/text()'
+                price_xpath = './/span[@class="price"]/text()'
             
             for data in doc.xpath(data_xpath):
                 if counter <= 0:
@@ -157,7 +163,7 @@ class AmazonKindleStore(StorePlugin):
                 # We must have an asin otherwise we can't easily reference the
                 # book later.
                 asin_href = None
-                asin_a = data.xpath('.//div[@class="productTitle"]/a[1]')
+                asin_a = data.xpath(asin_xpath)
                 if asin_a:
                     asin_href = asin_a[0].get('href', '')
                     m = re.search(r'/dp/(?P<asin>.+?)(/|$)', asin_href)
@@ -170,13 +176,13 @@ class AmazonKindleStore(StorePlugin):
                 
                 cover_url = ''.join(data.xpath(cover_xpath))
 
-                title = ''.join(data.xpath('.//div[@class="productTitle"]/a/text()'))
+                title = ''.join(data.xpath(title_xpath))
-                price = ''.join(data.xpath('.//div[@class="newPrice"]/span/text()'))
+                price = ''.join(data.xpath(price_xpath))
                 
                 if is_shot:
                     author = format.split(' by ')[-1]
                 else:
-                    author = ''.join(data.xpath('.//div[@class="productTitle"]/span[@class="ptBrand"]/text()'))
+                    author = ''.join(data.xpath('.//span[@class="ptBrand"]/text()'))
                     author = author.split('by ')[-1]
                 
                 counter -= 1

src/calibre/gui2/store/amazon_uk_plugin.py

@@ -15,17 +15,14 @@ from PyQt4.Qt import QUrl
 
 from calibre import browser
 from calibre.gui2 import open_url
-from calibre.gui2.store.amazon_plugin import AmazonKindleStore
+from calibre.gui2.store import StorePlugin
 from calibre.gui2.store.search_result import SearchResult
 
-class AmazonUKKindleStore(AmazonKindleStore):
+class AmazonUKKindleStore(StorePlugin):
     '''
     For comments on the implementation, please see amazon_plugin.py
     '''
 
-    search_url = 'http://www.amazon.co.uk/s/?url=search-alias%3Ddigital-text&field-keywords='
-    details_url = 'http://amazon.co.uk/dp/'
-
     def open(self, parent=None, detail_item=None, external=False):
         aff_id = {'tag': 'calcharles-21'}
         store_link = 'http://www.amazon.co.uk/gp/redirect.html?ie=UTF8&location=http://www.amazon.co.uk/Kindle-eBooks/b?ie=UTF8&node=341689031&ref_=sa_menu_kbo2&tag=%(tag)s&linkCode=ur2&camp=1634&creative=19450' % aff_id
@@ -36,7 +33,8 @@ class AmazonUKKindleStore(AmazonKindleStore):
         open_url(QUrl(store_link))
 
     def search(self, query, max_results=10, timeout=60):
-        url =  self.search_url + urllib.quote_plus(query)
+        search_url = 'http://www.amazon.co.uk/s/?url=search-alias%3Ddigital-text&field-keywords='
+        url =  search_url + urllib.quote_plus(query)
         br = browser()
 
         counter = max_results
@@ -75,15 +73,18 @@ class AmazonUKKindleStore(AmazonKindleStore):
                 s.title = title.strip()
                 s.price = price.strip()
                 s.detail_item = asin.strip()
-                s.formats = 'Kindle'
+                s.formats = ''
 
                 if is_shot:
                     # Amazon UK does not include the author on the grid layout
                     s.author = ''
                     self.get_details(s, timeout)
+                    if s.formats != 'Kindle':
+                        continue
                 else:
                     author = ''.join(data.xpath('.//div[@class="productTitle"]/span[@class="ptBrand"]/text()'))
                     s.author = author.split(' by ')[-1].strip()
+                    s.formats = 'Kindle'
 
                 yield s
 
@@ -92,18 +93,23 @@ class AmazonUKKindleStore(AmazonKindleStore):
         if search_result.drm:
             return
 
-        url = self.details_url
+        url = 'http://amazon.co.uk/dp/'
+        drm_search_text = u'Simultaneous Device Usage'
+        drm_free_text = u'Unlimited'
 
         br = browser()
         with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf:
             idata = html.fromstring(nf.read())
             if not search_result.author:
                 search_result.author = ''.join(idata.xpath('//div[@class="buying" and contains(., "Author")]/a/text()'))
+                is_kindle = idata.xpath('boolean(//div[@class="buying"]/h1/span/span[contains(text(), "Kindle Edition")])')
+                if is_kindle:
+                    search_result.formats = 'Kindle'
             if idata.xpath('boolean(//div[@class="content"]//li/b[contains(text(), "' +
-                           self.drm_search_text + '")])'):
+                           drm_search_text + '")])'):
                 if idata.xpath('boolean(//div[@class="content"]//li[contains(., "' +
-                               self.drm_free_text + '") and contains(b, "' +
+                               drm_free_text + '") and contains(b, "' +
-                               self.drm_search_text + '")])'):
+                               drm_search_text + '")])'):
                     search_result.drm = SearchResult.DRM_UNLOCKED
                 else:
                     search_result.drm = SearchResult.DRM_UNKNOWN

src/calibre/gui2/store/bn_plugin.py

@@ -8,7 +8,6 @@ __docformat__ = 'restructuredtext en'
 
 import random
 import re
-import urllib
 from contextlib import closing
 
 from lxml import html
@@ -47,26 +46,26 @@ class BNStore(BasicStoreConfig, StorePlugin):
             d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
-        url = 'http://productsearch.barnesandnoble.com/search/results.aspx?STORE=EBOOK&SZE=%s&WRD=' % max_results
+        query = query.replace(' ', '-')
-        url += urllib.quote_plus(query)
+        url = 'http://www.barnesandnoble.com/s/%s?store=ebook&sze=%s' % (query, max_results)
         
         br = browser()
         
         counter = max_results
         with closing(br.open(url, timeout=timeout)) as f:
             doc = html.fromstring(f.read())
-            for data in doc.xpath('//ul[contains(@class, "wgt-search-results-display")]/li[contains(@class, "search-result-item") and contains(@class, "nook-result-item")]'):
+            for data in doc.xpath('//ul[contains(@class, "result-set")]/li[contains(@class, "result")]'):
                 if counter <= 0:
                     break
                 
-                id = ''.join(data.xpath('.//div[contains(@class, "wgt-product-image-module")]/a/@href'))
+                id = ''.join(data.xpath('.//div[contains(@class, "image")]/a/@href'))
                 if not id:
                     continue
-                cover_url = ''.join(data.xpath('.//div[contains(@class, "wgt-product-image-module")]/a/img/@src'))
+                cover_url = ''.join(data.xpath('.//div[contains(@class, "image")]//img/@src'))
                 
-                title = ''.join(data.xpath('.//span[@class="product-title"]/a/text()'))
+                title = ''.join(data.xpath('.//p[@class="title"]//span[@class="name"]/text()'))
-                author = ', '.join(data.xpath('.//span[@class="contributers-line"]/a/text()'))
+                author = ', '.join(data.xpath('.//ul[@class="contributors"]//li[position()>1]//a/text()'))
-                price = ''.join(data.xpath('.//span[contains(@class, "onlinePriceValue2")]/text()'))
+                price = ''.join(data.xpath('.//table[@class="displayed-formats"]//a[@class="subtle"]/text()'))
                 
                 counter -= 1
                 
@@ -74,7 +73,7 @@ class BNStore(BasicStoreConfig, StorePlugin):
                 s.cover_url = cover_url
                 s.title = title.strip()
                 s.author = author.strip()
-                s.price = price
+                s.price = price.strip()
                 s.detail_item = id.strip()
                 s.drm = SearchResult.DRM_UNKNOWN
                 s.formats = 'Nook'

src/calibre/gui2/store/declined.txt

@@ -2,7 +2,8 @@ This is a list of stores that objected, declined
 or asked not to be included in the store integration.
 
 * Borders (http://www.borders.com/)
-* WH Smith (http://www.whsmith.co.uk/)
+* Indigo (http://www.chapters.indigo.ca/)
-  Refused to permit signing up for the affiliate program
 * Libraria Rizzoli (http://libreriarizzoli.corriere.it/).
   No reply with two attempts over 2 weeks
+* WH Smith (http://www.whsmith.co.uk/)
+  Refused to permit signing up for the affiliate program

src/calibre/gui2/store/epubbud_plugin.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import urllib
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class EpubBudStore(BasicStoreConfig, StorePlugin):
+
+    def open(self, parent=None, detail_item=None, external=False):
+        url = 'http://epubbud.com/'
+
+        if external or self.config.get('open_external', False):
+            open_url(QUrl(url_slash_cleaner(detail_item if detail_item else url)))
+        else:
+            d = WebStoreDialog(self.gui, url, parent, detail_item)
+            d.setWindowTitle(self.name)
+            d.set_tags(self.config.get('tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        '''
+        OPDS based search.
+        
+        We really should get the catelog from http://pragprog.com/catalog.opds
+        and look for the application/opensearchdescription+xml entry.
+        Then get the opensearch description to get the search url and
+        format. However, we are going to be lazy and hard code it.
+        '''
+        url = 'http://www.epubbud.com/search.php?format=atom&q=' + urllib.quote_plus(query)
+        
+        br = browser()
+        
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            # Use html instead of etree as html allows us
+            # to ignore the namespace easily.
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//entry'):
+                if counter <= 0:
+                    break
+
+                id = ''.join(data.xpath('.//id/text()'))
+                if not id:
+                    continue
+
+                cover_url = ''.join(data.xpath('.//link[@rel="http://opds-spec.org/thumbnail"]/@href'))
+                
+                title = u''.join(data.xpath('.//title/text()'))
+                author = u''.join(data.xpath('.//author/name/text()'))
+
+                counter -= 1
+                
+                s = SearchResult()
+                s.cover_url = cover_url
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = '$0.00'
+                s.detail_item = id.strip()
+                s.drm = SearchResult.DRM_UNLOCKED
+                s.formats = 'EPUB'
+                
+                yield s

src/calibre/gui2/store/gandalf_plugin.py

@@ -37,7 +37,7 @@ class GandalfStore(BasicStoreConfig, StorePlugin):
     def search(self, query, max_results=10, timeout=60):
         url = 'http://www.gandalf.com.pl/s/'
         values={
-            'search': query.encode('iso8859_2'),
+            'search': query.decode('utf-8').encode('iso8859_2'),
             'dzialx':'11'
             }
 

src/calibre/gui2/store/gutenberg_plugin.py

@@ -6,7 +6,7 @@ __license__ = 'GPL 3'
 __copyright__ = '2011, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
-import urllib2
+import urllib
 from contextlib import closing
 
 from lxml import html
@@ -42,7 +42,7 @@ class GutenbergStore(BasicStoreConfig, StorePlugin):
     def search(self, query, max_results=10, timeout=60):
         # Gutenberg's website does not allow searching both author and title.
         # Using a google search so we can search on both fields at once.
-        url = 'http://www.google.com/xhtml?q=site:gutenberg.org+' + urllib2.quote(query)
+        url = 'http://www.google.com/xhtml?q=site:gutenberg.org+' + urllib.quote_plus(query)
         
         br = browser()
         

src/calibre/gui2/store/legimi_plugin.py

@@ -40,7 +40,7 @@ class LegimiStore(BasicStoreConfig, StorePlugin):
             d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
-        url = 'http://www.legimi.com/pl/ebooks/?price=any&lang=pl&search=' + urllib.quote_plus(query.encode('utf-8')) + '&sort=relevance'
+        url = 'http://www.legimi.com/pl/ebooks/?price=any&lang=pl&search=' + urllib.quote_plus(query) + '&sort=relevance'
         
         br = browser()
         

src/calibre/gui2/store/manybooks_plugin.py

@@ -7,7 +7,7 @@ __copyright__ = '2011, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
 import re
-import urllib2
+import urllib
 from contextlib import closing
 
 from lxml import html
@@ -43,7 +43,7 @@ class ManyBooksStore(BasicStoreConfig, StorePlugin):
         # It also doesn't do a clear job of references authors and
         # secondary titles. Google is also faster.
         # Using a google search so we can search on both fields at once.
-        url = 'http://www.google.com/xhtml?q=site:manybooks.net+' + urllib2.quote(query)
+        url = 'http://www.google.com/xhtml?q=site:manybooks.net+' + urllib.quote_plus(query)
         
         br = browser()
         

src/calibre/gui2/store/nexto_plugin.py

@@ -44,7 +44,7 @@ class NextoStore(BasicStoreConfig, StorePlugin):
             d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
-        url = 'http://www.nexto.pl/szukaj.xml?search-clause=' + urllib.quote_plus(query.encode('utf-8')) + '&scid=1015'
+        url = 'http://www.nexto.pl/szukaj.xml?search-clause=' + urllib.quote_plus(query) + '&scid=1015'
 
         br = browser()
 
@@ -71,7 +71,7 @@ class NextoStore(BasicStoreConfig, StorePlugin):
                 author = ''
                 with closing(br.open('http://www.nexto.pl/' + id.strip(), timeout=timeout/4)) as nf:
                     idata = html.fromstring(nf.read())
-                    author = ''.join(idata.xpath('//div[@class="basic_data"]/p[1]/b/a/text()'))
+                    author = ', '.join(idata.xpath('//div[@class="basic_data"]/p[1]/b/a/text()'))
 
                 counter -= 1
 

src/calibre/gui2/store/search/models.py

@@ -33,7 +33,7 @@ class Matches(QAbstractItemModel):
 
     total_changed = pyqtSignal(int)
 
-    HEADERS = [_('Cover'), _('Title'), _('Price'), _('DRM'), _('Store'), _('')]
+    HEADERS = [_('Cover'), _('Title'), _('Price'), _('DRM'), _('Store'), '']
     HTML_COLS = (1, 4)
 
     def __init__(self, cover_thread_count=2, detail_thread_count=4):

src/calibre/gui2/store/search/search.py

@@ -82,6 +82,8 @@ class SearchDialog(QDialog, Ui_Dialog):
         self.restore_state()
 
     def setup_store_checks(self):
+        first_run = self.config.get('first_run', True)
+        
         # Add check boxes for each store so the user
         # can disable searching specific stores on a
         # per search basis.
@@ -98,7 +100,7 @@ class SearchDialog(QDialog, Ui_Dialog):
         icon = QIcon(I('donate.png'))
         for i, x in enumerate(sorted(self.gui.istores.keys(), key=lambda x: x.lower())):
             cbox = QCheckBox(x)
-            cbox.setChecked(existing.get(x, False))
+            cbox.setChecked(existing.get(x, first_run))
             store_list_layout.addWidget(cbox, i, 0, 1, 1)
             if self.gui.istores[x].base_plugin.affiliate:
                 iw = QLabel(self)
@@ -109,6 +111,8 @@ class SearchDialog(QDialog, Ui_Dialog):
         store_list_layout.setRowStretch(store_list_layout.rowCount(), 10)
         self.store_list.setWidget(stores_check_widget)
         
+        self.config['first_run'] = False
+
     def build_adv_search(self):
         adv = AdvSearchBuilderDialog(self)
         if adv.exec_() == QDialog.Accepted:
@@ -186,7 +190,7 @@ class SearchDialog(QDialog, Ui_Dialog):
         # Remove excess whitespace.
         query = re.sub(r'\s{2,}', ' ', query)
         query = query.strip()
-        return query
+        return query.encode('utf-8')
 
     def save_state(self):
         self.config['geometry'] = bytearray(self.saveGeometry())

src/calibre/gui2/store/search/search.ui

@@ -6,15 +6,15 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>584</width>
+    <width>872</width>
-    <height>533</height>
+    <height>610</height>
    </rect>
   </property>
   <property name="windowTitle">
    <string>Get Books</string>
   </property>
   <property name="windowIcon">
-   <iconset>
+   <iconset resource="../../../../../resources/images.qrc">
     <normaloff>:/images/store.png</normaloff>:/images/store.png</iconset>
   </property>
   <property name="sizeGripEnabled">
@@ -82,8 +82,8 @@
            <rect>
             <x>0</x>
             <y>0</y>
-            <width>125</width>
+            <width>173</width>
-            <height>127</height>
+            <height>106</height>
            </rect>
           </property>
          </widget>
@@ -255,7 +255,7 @@
   </customwidget>
  </customwidgets>
  <resources>
-  <include location="../../../../resources/images.qrc"/>
+  <include location="../../../../../resources/images.qrc"/>
  </resources>
  <connections>
   <connection>

src/calibre/gui2/store/virtualo_plugin.py

@@ -35,7 +35,7 @@ class VirtualoStore(BasicStoreConfig, StorePlugin):
             d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
-        url = 'http://virtualo.pl/c2/?q=' + urllib.quote(query.encode('utf-8'))
+        url = 'http://virtualo.pl/c2/?q=' + urllib.quote(query)
 
         br = browser()
 

src/calibre/gui2/store/zixo_plugin.py

@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, Tomasz Długosz <tomek3d@gmail.com>'
+__docformat__ = 'restructuredtext en'
+
+import re
+import urllib
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class ZixoStore(BasicStoreConfig, StorePlugin):
+
+    def open(self, parent=None, detail_item=None, external=False):
+
+        url = 'http://zixo.pl/e_ksiazki/start/'
+
+        if external or self.config.get('open_external', False):
+            open_url(QUrl(url_slash_cleaner(detail_item if detail_item else url)))
+        else:
+            d = WebStoreDialog(self.gui, url, parent, detail_item)
+            d.setWindowTitle(self.name)
+            d.set_tags(self.config.get('tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        url = 'http://zixo.pl/wyszukiwarka/?search=' + urllib.quote(query.encode('utf-8')) + '&product_type=0'
+
+        br = browser()
+
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//div[@class="productInline"]'):
+                if counter <= 0:
+                    break
+
+                id = ''.join(data.xpath('.//a[@class="productThumb"]/@href'))
+                if not id:
+                    continue
+
+                cover_url = ''.join(data.xpath('.//a[@class="productThumb"]/img/@src'))
+                title = ''.join(data.xpath('.//a[@class="title"]/text()'))
+                author = ''.join(data.xpath('.//div[@class="productDescription"]/span[1]/a/text()'))
+                price = ''.join(data.xpath('.//div[@class="priceList"]/span/text()'))
+                price = re.sub('\.', ',', price)
+
+                counter -= 1
+
+                s = SearchResult()
+                s.cover_url = cover_url
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price
+                s.detail_item = 'http://zixo.pl' + id.strip()
+                s.drm = SearchResult.DRM_LOCKED
+
+                yield s
+
+    def get_details(self, search_result, timeout):
+        br = browser()
+        with closing(br.open(search_result.detail_item, timeout=timeout)) as nf:
+            idata = html.fromstring(nf.read())
+            formats = ''.join(idata.xpath('//ul[@class="prop"]/li[3]/text()'))
+            formats = re.sub(r'\(.*\)', '', formats)
+            formats = re.sub('Zixo Reader', 'ZIXO', formats)
+            search_result.formats = formats
+        return True

src/calibre/library/caches.py

@@ -509,7 +509,8 @@ class ResultCache(SearchQueryParser): # {{{
             valq_mkind, valq = self._matchkind(query)
 
         loc = self.field_metadata[location]['rec_index']
-        split_char = self.field_metadata[location]['is_multiple']
+        split_char = self.field_metadata[location]['is_multiple'].get(
+                'cache_to_list', ',')
         for id_ in candidates:
             item = self._data[id_]
             if item is None:
@@ -665,7 +666,8 @@ class ResultCache(SearchQueryParser): # {{{
                 if fm['is_multiple'] and \
                         len(query) > 1 and query.startswith('#') and \
                         query[1:1] in '=<>!':
-                    vf = lambda item, loc=fm['rec_index'], ms=fm['is_multiple']:\
+                    vf = lambda item, loc=fm['rec_index'], \
+                                ms=fm['is_multiple']['cache_to_list']:\
                             len(item[loc].split(ms)) if item[loc] is not None else 0
                     return self.get_numeric_matches(location, query[1:],
                                                     candidates, val_func=vf)
@@ -703,7 +705,8 @@ class ResultCache(SearchQueryParser): # {{{
                             ['composite', 'text', 'comments', 'series', 'enumeration']:
                         exclude_fields.append(db_col[x])
                     col_datatype[db_col[x]] = self.field_metadata[x]['datatype']
-                    is_multiple_cols[db_col[x]] = self.field_metadata[x]['is_multiple']
+                    is_multiple_cols[db_col[x]] = \
+                        self.field_metadata[x]['is_multiple'].get('cache_to_list', None)
 
             try:
                 rating_query = int(query) * 2
@@ -1045,13 +1048,14 @@ class SortKeyGenerator(object):
 
             elif dt in ('text', 'comments', 'composite', 'enumeration'):
                 if val:
-                    sep = fm['is_multiple']
+                    if fm['is_multiple']:
-                    if sep:
+                        jv = fm['is_multiple']['list_to_ui']
-                        if fm['display'].get('is_names', False):
+                        sv = fm['is_multiple']['cache_to_list']
-                            val = sep.join(
+                        if '&' in jv:
-                                [author_to_author_sort(v) for v in val.split(sep)])
+                            val = jv.join(
+                                [author_to_author_sort(v) for v in val.split(sv)])
                         else:
-                            val = sep.join(sorted(val.split(sep),
+                            val = jv.join(sorted(val.split(sv),
                                               key=self.string_sort_key))
                 val = self.string_sort_key(val)
 

src/calibre/library/coloring.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+from future_builtins import map
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import binascii, re, json
+from textwrap import dedent
+
+class Rule(object): # {{{
+
+    SIGNATURE = '# BasicColorRule():'
+
+    def __init__(self, fm, color=None):
+        self.color = color
+        self.fm = fm
+        self.conditions = []
+
+    def add_condition(self, col, action, val):
+        if col not in self.fm:
+            raise ValueError('%r is not a valid column name'%col)
+        v = self.validate_condition(col, action, val)
+        if v:
+            raise ValueError(v)
+        self.conditions.append((col, action, val))
+
+    def validate_condition(self, col, action, val):
+        m = self.fm[col]
+        dt = m['datatype']
+        if (dt in ('int', 'float', 'rating') and action in ('lt', 'eq', 'gt')):
+            try:
+                int(val) if dt == 'int' else float(val)
+            except:
+                return '%r is not a valid numerical value'%val
+
+        if (dt in ('comments', 'series', 'text', 'enumeration') and 'pattern'
+                in action):
+            try:
+                re.compile(val)
+            except:
+                return '%r is not a valid regular expression'%val
+
+    @property
+    def signature(self):
+        args = (self.color, self.conditions)
+        sig = json.dumps(args, ensure_ascii=False)
+        return self.SIGNATURE + binascii.hexlify(sig.encode('utf-8'))
+
+    @property
+    def template(self):
+        if not self.color or not self.conditions:
+            return None
+        conditions = map(self.apply_condition, self.conditions)
+        conditions = (',\n' + ' '*9).join(conditions)
+        return dedent('''\
+                program:
+                {sig}
+                test(and(
+                         {conditions}
+                    ), '{color}', '');
+                ''').format(sig=self.signature, conditions=conditions,
+                        color=self.color)
+
+    def apply_condition(self, condition):
+        col, action, val = condition
+        m = self.fm[col]
+        dt = m['datatype']
+
+        if col == 'ondevice':
+            return self.ondevice_condition(col, action, val)
+
+        if col == 'identifiers':
+            return self.identifiers_condition(col, action, val)
+
+        if dt == 'bool':
+            return self.bool_condition(col, action, val)
+
+        if dt in ('int', 'float'):
+            return self.number_condition(col, action, val)
+
+        if dt == 'rating':
+            return self.rating_condition(col, action, val)
+
+        if dt == 'datetime':
+            return self.date_condition(col, action, val)
+
+        if dt in ('comments', 'series', 'text', 'enumeration', 'composite'):
+            ism = m.get('is_multiple', False)
+            if ism:
+                return self.multiple_condition(col, action, val, ism['ui_to_list'])
+            return self.text_condition(col, action, val)
+
+    def identifiers_condition(self, col, action, val):
+        if action == 'has id':
+            return "identifier_in_list(field('identifiers'), '%s', '1', '')"%val
+        return "identifier_in_list(field('identifiers'), '%s', '', '1')"%val
+
+    def ondevice_condition(self, col, action, val):
+        if action == 'is set':
+            return "test(ondevice(), '1', '')"
+        if action == 'is not set':
+            return "test(ondevice(), '', '1')"
+
+    def bool_condition(self, col, action, val):
+        test = {'is true': 'True',
+                'is false': 'False',
+                'is undefined': 'None'}[action]
+        return "strcmp('%s', raw_field('%s'), '', '1', '')"%(test, col)
+
+    def number_condition(self, col, action, val):
+        lt, eq, gt = {
+                'eq': ('', '1', ''),
+                'lt': ('1', '', ''),
+                'gt': ('', '', '1')
+        }[action]
+        return "cmp(raw_field('%s'), %s, '%s', '%s', '%s')" % (col, val, lt, eq, gt)
+
+    def rating_condition(self, col, action, val):
+        lt, eq, gt = {
+                'eq': ('', '1', ''),
+                'lt': ('1', '', ''),
+                'gt': ('', '', '1')
+        }[action]
+        return "cmp(field('%s'), %s, '%s', '%s', '%s')" % (col, val, lt, eq, gt)
+
+    def date_condition(self, col, action, val):
+        lt, eq, gt = {
+                'eq': ('', '1', ''),
+                'lt': ('1', '', ''),
+                'gt': ('', '', '1')
+        }[action]
+        return "cmp(format_date(raw_field('%s'), 'yyyy-MM-dd'), %s, '%s', '%s', '%s')" % (col,
+                val, lt, eq, gt)
+
+    def multiple_condition(self, col, action, val, sep):
+        if not sep or sep == '|':
+            sep = ','
+        if action == 'is set':
+            return "test(field('%s'), '1', '')"%col
+        if action == 'is not set':
+            return "test(field('%s'), '', '1')"%col
+        if action == 'has':
+            return "str_in_list(field('%s'), '%s', \"%s\", '1', '')"%(col, sep, val)
+        if action == 'does not have':
+            return "str_in_list(field('%s'), '%s', \"%s\", '', '1')"%(col, sep, val)
+        if action == 'has pattern':
+            return "in_list(field('%s'), '%s', \"%s\", '1', '')"%(col, sep, val)
+        if action == 'does not have pattern':
+            return "in_list(field('%s'), '%s', \"%s\", '', '1')"%(col, sep, val)
+
+    def text_condition(self, col, action, val):
+        if action == 'is set':
+            return "test(field('%s'), '1', '')"%col
+        if action == 'is not set':
+            return "test(field('%s'), '', '1')"%col
+        if action == 'is':
+            return "strcmp(field('%s'), \"%s\", '', '1', '')"%(col, val)
+        if action == 'is not':
+            return "strcmp(field('%s'), \"%s\", '1', '', '1')"%(col, val)
+        if action == 'matches pattern':
+            return "contains(field('%s'), \"%s\", '1', '')"%(col, val)
+        if action == 'does not match pattern':
+            return "contains(field('%s'), \"%s\", '', '1')"%(col, val)
+
+# }}}
+
+def rule_from_template(fm, template):
+    ok_lines = []
+    for line in template.splitlines():
+        if line.startswith(Rule.SIGNATURE):
+            raw = line[len(Rule.SIGNATURE):].strip()
+            try:
+                color, conditions = json.loads(binascii.unhexlify(raw).decode('utf-8'))
+            except:
+                continue
+            r = Rule(fm)
+            r.color = color
+            for c in conditions:
+                try:
+                    r.add_condition(*c)
+                except:
+                    continue
+            if r.color and r.conditions:
+                return r
+        else:
+            ok_lines.append(line)
+    return '\n'.join(ok_lines)
+
+def conditionable_columns(fm):
+    for key in fm:
+        m = fm[key]
+        dt = m['datatype']
+        if m.get('name', False) and dt in ('bool', 'int', 'float', 'rating', 'series',
+                'comments', 'text', 'enumeration', 'datetime', 'composite'):
+            if key == 'sort':
+                yield 'title_sort'
+            else:
+                yield key
+
+def displayable_columns(fm):
+    for key in fm.displayable_field_keys():
+        if key not in ('sort', 'author_sort', 'comments', 'formats',
+                'identifiers', 'path'):
+            yield key
+
+def migrate_old_rule(fm, template):
+    if template.startswith('program:\n#tag wizard'):
+        rules = []
+        for line in template.splitlines():
+            if line.startswith('#') and ':|:' in line:
+                value, color = line[1:].split(':|:')
+                r = Rule(fm, color=color)
+                r.add_condition('tags', 'has', value)
+                rules.append(r.template)
+        return rules
+    return [template]
+

src/calibre/library/custom_columns.py

@@ -78,6 +78,18 @@ class CustomColumns(object):
                     }
             if data['display'] is None:
                 data['display'] = {}
+            # set up the is_multiple separator dict
+            if data['is_multiple']:
+                if data['display'].get('is_names', False):
+                    seps = {'cache_to_list': '|', 'ui_to_list': '&', 'list_to_ui': ' & '}
+                elif data['datatype'] == 'composite':
+                    seps = {'cache_to_list': ',', 'ui_to_list': ',', 'list_to_ui': ', '}
+                else:
+                    seps = {'cache_to_list': '|', 'ui_to_list': ',', 'list_to_ui': ', '}
+            else:
+                seps = {}
+            data['multiple_seps'] = seps
+
             table, lt = self.custom_table_names(data['num'])
             if table not in custom_tables or (data['normalized'] and lt not in
                     custom_tables):
@@ -119,7 +131,7 @@ class CustomColumns(object):
                 if x is None:
                     return []
                 if isinstance(x, (str, unicode, bytes)):
-                    x = x.split('&' if d['display'].get('is_names', False) else',')
+                    x = x.split(d['multiple_seps']['ui_to_list'])
                 x = [y.strip() for y in x if y.strip()]
                 x = [y.decode(preferred_encoding, 'replace') if not isinstance(y,
                     unicode) else y for y in x]
@@ -181,10 +193,7 @@ class CustomColumns(object):
                 is_category = True
             else:
                 is_category = False
-            if v['is_multiple']:
+            is_m = v['multiple_seps']
-                is_m = ',' if v['datatype'] == 'composite' else '|'
-            else:
-                is_m = None
             tn = 'custom_column_{0}'.format(v['num'])
             self.field_metadata.add_custom_field(label=v['label'],
                     table=tn, column='value', datatype=v['datatype'],
@@ -200,7 +209,7 @@ class CustomColumns(object):
         row = self.data._data[idx] if index_is_id else self.data[idx]
         ans = row[self.FIELD_MAP[data['num']]]
         if data['is_multiple'] and data['datatype'] == 'text':
-            ans = ans.split('|') if ans else []
+            ans = ans.split(data['multiple_seps']['cache_to_list']) if ans else []
             if data['display'].get('sort_alpha', False):
                 ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower()))
         return ans
@@ -566,14 +575,21 @@ class CustomColumns(object):
     def custom_columns_in_meta(self):
         lines = {}
         for data in self.custom_column_label_map.values():
-            display = data['display']
             table, lt = self.custom_table_names(data['num'])
             if data['normalized']:
                 query = '%s.value'
                 if data['is_multiple']:
-                    query = 'group_concat(%s.value, "|")'
+#                    query = 'group_concat(%s.value, "{0}")'.format(
-                    if not display.get('sort_alpha', False):
+#                                        data['multiple_seps']['cache_to_list'])
-                        query = 'sort_concat(link.id, %s.value)'
+#                    if not display.get('sort_alpha', False):
+                    if data['multiple_seps']['cache_to_list'] == '|':
+                        query = 'sortconcat_bar(link.id, %s.value)'
+                    elif data['multiple_seps']['cache_to_list'] == '&':
+                        query = 'sortconcat_amper(link.id, %s.value)'
+                    else:
+                        prints('WARNING: unknown value in multiple_seps',
+                               data['multiple_seps']['cache_to_list'])
+                        query = 'sortconcat_bar(link.id, %s.value)'
                 line = '''(SELECT {query} FROM {lt} AS link INNER JOIN
                     {table} ON(link.value={table}.id) WHERE link.book=books.id)
                     custom_{num}

src/calibre/library/database2.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 The database used to store ebook metadata
 '''
 import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, \
-        json, uuid
+        json, uuid, tempfile
 import threading, random
 from itertools import repeat
 from math import ceil
@@ -211,10 +211,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         defs = self.prefs.defaults
         defs['gui_restriction'] = defs['cs_restriction'] = ''
         defs['categories_using_hierarchy'] = []
-        self.column_color_count = 5
+        defs['column_color_rules'] = []
-        for i in range(1,self.column_color_count+1):
-            defs['column_color_name_'+str(i)] = ''
-            defs['column_color_template_'+str(i)] = ''
 
         # Migrate the bool tristate tweak
         defs['bools_are_tristate'] = \
@@ -222,6 +219,24 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if self.prefs.get('bools_are_tristate') is None:
             self.prefs.set('bools_are_tristate', defs['bools_are_tristate'])
 
+        # Migrate column coloring rules
+        if self.prefs.get('column_color_name_1', None) is not None:
+            from calibre.library.coloring import migrate_old_rule
+            old_rules = []
+            for i in range(1, 6):
+                col = self.prefs.get('column_color_name_'+str(i), None)
+                templ = self.prefs.get('column_color_template_'+str(i), None)
+                if col and templ:
+                    try:
+                        del self.prefs['column_color_name_'+str(i)]
+                        rules = migrate_old_rule(self.field_metadata, templ)
+                        for templ in rules:
+                            old_rules.append((col, templ))
+                    except:
+                        pass
+            if old_rules:
+                self.prefs['column_color_rules'] += old_rules
+
         # Migrate saved search and user categories to db preference scheme
         def migrate_preference(key, default):
             oldval = prefs[key]
@@ -576,10 +591,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                     f.write(cdata)
             for format in formats:
                 # Get data as string (can't use file as source and target files may be the same)
-                f = self.format(id, format, index_is_id=True, as_file=False)
+                f = self.format(id, format, index_is_id=True, as_file=True)
-                if not f:
+                if f is None:
                     continue
-                stream = cStringIO.StringIO(f)
+                with tempfile.SpooledTemporaryFile(max_size=100*(1024**2)) as stream:
+                    with f:
+                        shutil.copyfileobj(f, stream)
+                    stream.seek(0)
                     self.add_format(id, format, stream, index_is_id=True,
                         path=tpath, notify=False)
         self.conn.execute('UPDATE books SET path=? WHERE id=?', (path, id))
@@ -1232,7 +1250,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             dex = field['rec_index']
             for book in self.data.iterall():
                 if field['is_multiple']:
-                    vals = [v.strip() for v in book[dex].split(field['is_multiple'])
+                    vals = [v.strip() for v in
+                            book[dex].split(field['is_multiple']['cache_to_list'])
                             if v.strip()]
                     if id_ in vals:
                         ans.add(book[0])
@@ -1360,7 +1379,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             tcategories[category] = {}
             # create a list of category/field_index for the books scan to use.
             # This saves iterating through field_metadata for each book
-            md.append((category, cat['rec_index'], cat['is_multiple'], False))
+            md.append((category, cat['rec_index'],
+                       cat['is_multiple'].get('cache_to_list', None), False))
 
         for category in tb_cats.iterkeys():
             cat = tb_cats[category]
@@ -1368,7 +1388,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                                 cat['display'].get('make_category', False):
                 tids[category] = {}
                 tcategories[category] = {}
-                md.append((category, cat['rec_index'], cat['is_multiple'],
+                md.append((category, cat['rec_index'],
+                           cat['is_multiple'].get('cache_to_list', None),
                            cat['datatype'] == 'composite'))
         #print 'end phase "collection":', time.clock() - last, 'seconds'
         #last = time.clock()
@@ -1556,13 +1577,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             if ids is not None:
                 count = self.conn.get('''SELECT COUNT(id)
                                        FROM data
-                                       WHERE format="%s" AND
+                                       WHERE format=? AND
-                                       books_list_filter(book)'''%fmt,
+                                       books_list_filter(book)''', (fmt,),
                                        all=False)
             else:
                 count = self.conn.get('''SELECT COUNT(id)
                                        FROM data
-                                       WHERE format="%s"'''%fmt,
+                                       WHERE format=?''', (fmt,),
                                        all=False)
             if count > 0:
                 categories['formats'].append(Tag(fmt, count=count, icon=icon,
@@ -1584,13 +1605,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             if ids is not None:
                 count = self.conn.get('''SELECT COUNT(book)
                                        FROM identifiers
-                                       WHERE type="%s" AND
+                                       WHERE type=? AND
-                                       books_list_filter(book)'''%ident,
+                                       books_list_filter(book)''', (ident,),
                                        all=False)
             else:
                 count = self.conn.get('''SELECT COUNT(id)
                                        FROM identifiers
-                                       WHERE type="%s"'''%ident,
+                                       WHERE type=?''', (ident,),
                                        all=False)
             if count > 0:
                 categories['identifiers'].append(Tag(ident, count=count, icon=icon,

src/calibre/library/field_metadata.py

@@ -50,9 +50,16 @@ class FieldMetadata(dict):
 
     datatype: the type of information in the field. Valid values are listed in
     VALID_DATA_TYPES below.
-    is_multiple: valid for the text datatype. If None, the field is to be
+    is_multiple: valid for the text datatype. If {}, the field is to be
-    treated as a single term. If not None, it contains a string, and the field
+    treated as a single term. If not None, it contains a dict of the form
-    is assumed to contain a list of terms separated by that string
+            {'cache_to_list': ',',
+             'ui_to_list': ',',
+             'list_to_ui': ', '}
+    where the cache_to_list contains the character used to split the value in
+    the meta2 table, ui_to_list contains the character used to create a list
+    from a value shown in the ui (each resulting value must be strip()ed and
+    empty values removed), and list_to_ui contains the string used in join()
+    to create a displayable string from the list.
 
     kind == field: is a db field.
     kind == category: standard tag category that isn't a field. see news.
@@ -97,7 +104,9 @@ class FieldMetadata(dict):
                            'link_column':'author',
                            'category_sort':'sort',
                            'datatype':'text',
-                           'is_multiple':',',
+                           'is_multiple':{'cache_to_list': ',',
+                                          'ui_to_list': '&',
+                                          'list_to_ui': ' & '},
                            'kind':'field',
                            'name':_('Authors'),
                            'search_terms':['authors', 'author'],
@@ -109,7 +118,7 @@ class FieldMetadata(dict):
                            'link_column':'series',
                            'category_sort':'(title_sort(name))',
                            'datatype':'series',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Series'),
                            'search_terms':['series'],
@@ -119,7 +128,9 @@ class FieldMetadata(dict):
             ('formats',   {'table':None,
                            'column':None,
                            'datatype':'text',
-                           'is_multiple':',',
+                           'is_multiple':{'cache_to_list': ',',
+                                          'ui_to_list': ',',
+                                          'list_to_ui': ', '},
                            'kind':'field',
                            'name':_('Formats'),
                            'search_terms':['formats', 'format'],
@@ -131,7 +142,7 @@ class FieldMetadata(dict):
                            'link_column':'publisher',
                            'category_sort':'name',
                            'datatype':'text',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Publishers'),
                            'search_terms':['publisher'],
@@ -143,7 +154,7 @@ class FieldMetadata(dict):
                            'link_column':'rating',
                            'category_sort':'rating',
                            'datatype':'rating',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Ratings'),
                            'search_terms':['rating'],
@@ -154,7 +165,7 @@ class FieldMetadata(dict):
                            'column':'name',
                            'category_sort':'name',
                            'datatype':None,
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'category',
                            'name':_('News'),
                            'search_terms':[],
@@ -166,7 +177,9 @@ class FieldMetadata(dict):
                            'link_column': 'tag',
                            'category_sort':'name',
                            'datatype':'text',
-                           'is_multiple':',',
+                           'is_multiple':{'cache_to_list': ',',
+                                          'ui_to_list': ',',
+                                          'list_to_ui': ', '},
                            'kind':'field',
                            'name':_('Tags'),
                            'search_terms':['tags', 'tag'],
@@ -176,7 +189,9 @@ class FieldMetadata(dict):
             ('identifiers',   {'table':None,
                            'column':None,
                            'datatype':'text',
-                           'is_multiple':',',
+                           'is_multiple':{'cache_to_list': ',',
+                                          'ui_to_list': ',',
+                                          'list_to_ui': ', '},
                            'kind':'field',
                            'name':_('Identifiers'),
                            'search_terms':['identifiers', 'identifier', 'isbn'],
@@ -186,7 +201,7 @@ class FieldMetadata(dict):
             ('author_sort',{'table':None,
                             'column':None,
                             'datatype':'text',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Author Sort'),
                            'search_terms':['author_sort'],
@@ -196,7 +211,9 @@ class FieldMetadata(dict):
             ('au_map',    {'table':None,
                            'column':None,
                            'datatype':'text',
-                           'is_multiple':',',
+                           'is_multiple':{'cache_to_list': ',',
+                                          'ui_to_list': None,
+                                          'list_to_ui': None},
                            'kind':'field',
                            'name':None,
                            'search_terms':[],
@@ -206,7 +223,7 @@ class FieldMetadata(dict):
             ('comments',  {'table':None,
                            'column':None,
                            'datatype':'text',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Comments'),
                            'search_terms':['comments', 'comment'],
@@ -216,7 +233,7 @@ class FieldMetadata(dict):
             ('cover',     {'table':None,
                            'column':None,
                            'datatype':'int',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':None,
                            'search_terms':['cover'],
@@ -226,7 +243,7 @@ class FieldMetadata(dict):
             ('id',        {'table':None,
                            'column':None,
                            'datatype':'int',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':None,
                            'search_terms':[],
@@ -236,7 +253,7 @@ class FieldMetadata(dict):
             ('last_modified', {'table':None,
                            'column':None,
                            'datatype':'datetime',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Modified'),
                            'search_terms':['last_modified'],
@@ -246,7 +263,7 @@ class FieldMetadata(dict):
             ('ondevice',  {'table':None,
                            'column':None,
                            'datatype':'text',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('On Device'),
                            'search_terms':['ondevice'],
@@ -256,7 +273,7 @@ class FieldMetadata(dict):
             ('path',      {'table':None,
                            'column':None,
                            'datatype':'text',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Path'),
                            'search_terms':[],
@@ -266,7 +283,7 @@ class FieldMetadata(dict):
             ('pubdate',   {'table':None,
                            'column':None,
                            'datatype':'datetime',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Published'),
                            'search_terms':['pubdate'],
@@ -276,7 +293,7 @@ class FieldMetadata(dict):
             ('marked',    {'table':None,
                            'column':None,
                            'datatype':'text',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name': None,
                            'search_terms':['marked'],
@@ -286,7 +303,7 @@ class FieldMetadata(dict):
             ('series_index',{'table':None,
                              'column':None,
                              'datatype':'float',
-                             'is_multiple':None,
+                             'is_multiple':{},
                              'kind':'field',
                              'name':None,
                              'search_terms':['series_index'],
@@ -296,7 +313,7 @@ class FieldMetadata(dict):
             ('sort',      {'table':None,
                            'column':None,
                            'datatype':'text',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Title Sort'),
                            'search_terms':['title_sort'],
@@ -306,7 +323,7 @@ class FieldMetadata(dict):
             ('size',      {'table':None,
                            'column':None,
                            'datatype':'float',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Size'),
                            'search_terms':['size'],
@@ -316,7 +333,7 @@ class FieldMetadata(dict):
             ('timestamp', {'table':None,
                            'column':None,
                            'datatype':'datetime',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Date'),
                            'search_terms':['date'],
@@ -326,7 +343,7 @@ class FieldMetadata(dict):
             ('title',     {'table':None,
                            'column':None,
                            'datatype':'text',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Title'),
                            'search_terms':['title'],
@@ -336,7 +353,7 @@ class FieldMetadata(dict):
             ('uuid',      {'table':None,
                            'column':None,
                            'datatype':'text',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':None,
                            'search_terms':[],
@@ -374,6 +391,8 @@ class FieldMetadata(dict):
         self.get = self._tb_cats.get
 
     def __getitem__(self, key):
+        if key == 'title_sort':
+            return self._tb_cats['sort']
         return self._tb_cats[key]
 
     def __setitem__(self, key, val):
@@ -390,6 +409,8 @@ class FieldMetadata(dict):
         return self.has_key(key)
 
     def has_key(self, key):
+        if key == 'title_sort':
+            return True
         return key in self._tb_cats
 
     def keys(self):
@@ -504,7 +525,7 @@ class FieldMetadata(dict):
         if datatype == 'series':
             key += '_index'
             self._tb_cats[key] = {'table':None,        'column':None,
-                                 'datatype':'float',   'is_multiple':None,
+                                 'datatype':'float',   'is_multiple':{},
                                  'kind':'field',       'name':'',
                                  'search_terms':[key], 'label':label+'_index',
                                  'colnum':None,        'display':{},
@@ -556,7 +577,7 @@ class FieldMetadata(dict):
         if icu_lower(label) != label:
             st.append(icu_lower(label))
         self._tb_cats[label] = {'table':None,          'column':None,
-                                'datatype':None,       'is_multiple':None,
+                                'datatype':None,       'is_multiple':{},
                                 'kind':'user',         'name':name,
                                 'search_terms':st,     'is_custom':False,
                                 'is_category':True,    'is_csp': False}
@@ -566,7 +587,7 @@ class FieldMetadata(dict):
         if label in self._tb_cats:
             raise ValueError('Duplicate user field [%s]'%(label))
         self._tb_cats[label] = {'table':None,        'column':None,
-                                'datatype':None,     'is_multiple':None,
+                                'datatype':None,     'is_multiple':{},
                                 'kind':'search',     'name':name,
                                 'search_terms':[],   'is_custom':False,
                                 'is_category':True,  'is_csp': False}

src/calibre/library/restore.py

@@ -171,7 +171,7 @@ class Restore(Thread):
                 for x in fields:
                     if x in cfm:
                         if x == 'is_multiple':
-                            args.append(cfm[x] is not None)
+                            args.append(bool(cfm[x]))
                         else:
                             args.append(cfm[x])
                 if len(args) == len(fields):

src/calibre/library/server/browse.py

@@ -770,7 +770,8 @@ class BrowseServer(object):
             summs.append(self.browse_summary_template.format(**args))
 
 
-        return json.dumps('\n'.join(summs), ensure_ascii=False)
+        raw = json.dumps('\n'.join(summs), ensure_ascii=False)
+        return raw
 
     def browse_render_details(self, id_):
         try:

src/calibre/library/server/mobile.py

@@ -231,7 +231,8 @@ class MobileServer(object):
             book['size'] = human_readable(book['size'])
 
             aus = record[FM['authors']] if record[FM['authors']] else __builtin__._('Unknown')
-            authors = '|'.join([i.replace('|', ',') for i in aus.split(',')])
+            aut_is = CFM['authors']['is_multiple']
+            authors = aut_is['list_to_ui'].join([i.replace('|', ',') for i in aus.split(',')])
             book['authors'] = authors
             book['series_index'] = fmt_sidx(float(record[FM['series_index']]))
             book['series'] = record[FM['series']]
@@ -254,8 +255,10 @@ class MobileServer(object):
                     continue
                 if datatype == 'text' and CFM[key]['is_multiple']:
                     book[key] = concat(name,
-                                       format_tag_string(val, ',',
+                                       format_tag_string(val,
-                                                         no_tag_count=True))
+                                           CFM[key]['is_multiple']['ui_to_list'],
+                                           no_tag_count=True,
+                                           joinval=CFM[key]['is_multiple']['list_to_ui']))
                 else:
                     book[key] = concat(name, val)
 

src/calibre/library/server/opds.py

@@ -180,9 +180,12 @@ def ACQUISITION_ENTRY(item, version, db, updated, CFM, CKEYS, prefix):
         if val:
             datatype = CFM[key]['datatype']
             if datatype == 'text' and CFM[key]['is_multiple']:
-                extra.append('%s: %s<br />'%(xml(name), xml(format_tag_string(val, ',',
+                extra.append('%s: %s<br />'%
-                                                           ignore_max=True,
+                             (xml(name),
-                                                           no_tag_count=True))))
+                              xml(format_tag_string(val,
+                                    CFM[key]['is_multiple']['ui_to_list'],
+                                    ignore_max=True, no_tag_count=True,
+                                    joinval=CFM[key]['is_multiple']['list_to_ui']))))
             elif datatype == 'comments':
                 extra.append('%s: %s<br />'%(xml(name), comments_to_html(unicode(val))))
             else: