Merge from main branch

.bzrignore

@@ -31,3 +31,4 @@ nbproject/
 .pydevproject
 .settings/
 *.DS_Store
+calibre_plugins/

Changelog.yaml

@@ -19,6 +19,72 @@
 #  new recipes:
 #    - title: 
 
+- version: 0.8.2
+  date: 2011-05-20
+
+  new features:
+    - title: "Various new ebook sources added to Get Books: Google Books, O'Reilly, archive.org, some Polish ebooks stores, etc."
+
+    - title: "Amazon metadata download: Allow user to configure Amazon plugin to use any of the US, UK, German, French and Italian Amazon websites"
+
+    - title: "When deleting large numbers of books, give the user the option to skip the Recycle Bin, since sending lots of files to the recycle bin can be very slow."
+      tickets: [784987]
+
+    - title: "OS X: The unified title+toolbar was disabled as it had various bugs. If you really want it you can turn it on again via Preferences->Tweaks, but be aware that you will see problems like the calibre windowd being too wide, weird animations when a device is detected, etc."
+
+    - title: "Add a tweak that controls what words are treated as suffixes when generating an author sort string from an author name."
+
+    - title: "Get Books: Store last few searches in history"
+  
+  bug fixes:
+    - title: "Fix a crash when a device is connected/disconnected while a modal dialog opened from the toolbar is visible"
+      tickets: [780484]
+
+    - title: "Fix incorrect results from ebooks.com when searching via Get Books"
+
+    - title: "Metadata plugboards: Add prioritization scheme to allow for using different settings for different locations"
+      tickets: [783229]
+
+    - title: "Fix manage authors dialog too wide"
+      tickets: [783065]
+
+    - title: "Fix multiple bracket types in author names not handled correctly when generating author sort string"
+      tickets: [782551]
+
+    - title: "MOBI Input: Don't error out when detecting TOC structure if one of the elements has an invalid margin unit"
+
+    - title: "More fixes for japanese language calibre on windows"
+      tickets: [782408]
+
+    - title: "Linux binaries: Always use either Cleanlook or Plastique styles for the GUI if no style can be loaded from the host computer"
+
+  improved recipes:
+    - Newsweek
+    - Economist
+    - Dvhn
+    - United Daily
+    - Dagens Nyheter
+    - GoComics
+    - faz.net
+    - golem.de
+
+  new recipes:
+    - title: National Geographic
+      author: gagsays
+    
+    - title: Various German news sources
+      author: schuster
+
+    - title: Dilema Veche
+      author: Silviu Cotoara
+
+    - title: "Glamour, Good to Know, Good Housekeeping and Men's Health"
+      author: Anonymous
+
+    - title: "Financial Sense and iProfessional"
+      author: Darko Miletic
+
+
 - version: 0.8.1
   date: 2011-05-13
 

recipes/borse_online.recipe

@@ -0,0 +1,33 @@
+from calibre.web.feeds.recipes import BasicNewsRecipe
+class AdvancedUserRecipe1303841067(BasicNewsRecipe):
+
+    title          = u'Börse-online'
+    __author__  = 'schuster'
+    oldest_article = 1
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    language               = 'de'
+    remove_javascript      = True
+    cover_url = 'http://www.dpv.de/images/1995/source.gif'
+    masthead_url = 'http://www.zeitschriften-cover.de/cover/boerse-online-cover-januar-2010-x1387.jpg'
+    extra_css = '''
+                    h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
+                    h4{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
+                    img {min-width:300px; max-width:600px; min-height:300px; max-height:800px}
+                    p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
+                    body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
+	'''
+    remove_tags_bevor = [dict(name='h3')]
+    remove_tags_after = [dict(name='div', attrs={'class':'artikelfuss'})]
+    remove_tags = [dict(attrs={'class':['moduleTopNav', 'moduleHeaderNav', 'text', 'blau', 'poll1150']}),
+                dict(id=['newsletterlayer', 'newsletterlayerClose', 'newsletterlayer_body', 'newsletterarray_error', 'newsletterlayer_emailadress', 'newsletterlayer_submit', 'kommentar']),
+                dict(name=['h2', 'Gesamtranking', 'h3',''])]
+
+    def print_version(self, url):
+        return url.replace('.html#nv=rss', '.html?mode=print')
+
+
+
+    feeds          = [(u'Börsennachrichten', u'http://www.boerse-online.de/rss/')]
+

recipes/capital_de.recipe

@@ -0,0 +1,61 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+class AdvancedUserRecipe1305470859(BasicNewsRecipe):
+    title                 = u'Capital.de'
+    language              = 'de'
+    __author__            = 'schuster'
+    oldest_article        =7
+    max_articles_per_feed = 35
+    no_stylesheets        = True
+    remove_javascript     = True
+    use_embedded_content  = False
+    masthead_url = 'http://www.wirtschaftsmedien-shop.de/media/stores/wirtschaftsmedien/capital/teaser_large_abo.jpg'
+    cover_url = 'http://d1kb9jvg6ylufe.cloudfront.net/WebsiteCMS/de/unternehmen/linktipps/mainColumn/08/image/DE_Capital_bis20mm_SW.jpg'
+
+    extra_css = '''
+                    h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
+                    h4{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
+                    img {min-width:300px; max-width:600px; min-height:300px; max-height:800px}
+                    p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
+                    body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
+	'''
+    def print_version(self, url):
+        return url.replace ('nv=rss#utm_source=rss2&utm_medium=rss_feed&utm_campaign=/', 'mode=print')
+    remove_tags_bevor = [dict(name='td', attrs={'class':'textcell'})]
+    remove_tags_after = [dict(name='div', attrs={'class':'artikelsplit'})]
+
+    feeds = [   (u'Wirtschaftsmagazin', u'http://www.capital.de/rss/'),
+	(u'Unternehmen', u'http://www.capital.de/rss/unternehmen'),
+	(u'Finanz & Geldanlage', u'http://www.capital.de/rss/finanzen/geldanlage')]
+
+    def append_page(self, soup, appendtag, position):
+        pager = soup.find('div',attrs={'class':'artikelsplit'})
+        if pager:
+           nexturl = self.INDEX + pager.a['href']
+           soup2 = self.index_to_soup(nexturl)
+           texttag = soup2.find('div', attrs={'class':'printable'})
+           for it in texttag.findAll(style=True):
+               del it['style']
+           newpos = len(texttag.contents)
+           self.append_page(soup2,texttag,newpos)
+           texttag.extract()
+           appendtag.insert(position,texttag)
+
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll('div', attrs={'class':'artikelsplit'}):
+            item.extract()
+        self.append_page(soup, soup.body, 3)
+        pager = soup.find('div',attrs={'class':'artikelsplit'})
+        if pager:
+           pager.extract()
+        return self.adeify_images(soup)
+
+
+
+    remove_tags = [dict(attrs={'class':['navSeitenAlle', 'kommentieren', 'teaserheader', 'teasercontent', 'info', 'zwischenhead', 'artikelsplit']}),
+                dict(id=['topNav', 'mainNav', 'subNav', 'socialmedia', 'footerRahmen', 'gatrixx_marktinformationen', 'pager', 'weitere']),
+                dict(span=['ratingtext', 'Gesamtranking', 'h3','']),
+	dict(rel=['canonical'])]
+

recipes/dilemaveche.recipe

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = u'2011, Silviu Cotoar\u0103'
+'''
+dilemaveche.ro
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class DilemaVeche(BasicNewsRecipe):
+    title                 = u'Dilema Veche'
+    __author__            = u'Silviu Cotoar\u0103'
+    description           = u'Sunt vechi, domnule!'
+    publisher             = u'Dilema Veche'
+    oldest_article        = 50
+    language              = 'ro'
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    category              = 'Ziare'
+    encoding              = 'utf-8'
+    cover_url             = 'http://www.dilemaveche.ro/sites/all/themes/dilema/theme/dilema_two/layouter/dilema_two_homepage/logo.png'
+
+    conversion_options = {
+                             'comments'   : description
+                            ,'tags'       : category
+                            ,'language'   : language
+                            ,'publisher'  : publisher
+                         }
+
+    keep_only_tags = [
+                          dict(name='h1', attrs={'class':'art_title'})
+                        , dict(name='h1', attrs={'class':'art_title online'})
+                        , dict(name='div', attrs={'class':'item'})
+                        , dict(name='div', attrs={'class':'art_content'})
+                     ]
+
+    remove_tags = [
+                         dict(name='div', attrs={'class':['article_details']})
+                        , dict(name='div', attrs={'class':['controale']})
+                        , dict(name='div', attrs={'class':['art_related_left']})
+                  ]
+
+    remove_tags_after = [
+                            dict(name='div', attrs={'class':['article_details']})
+                        ]
+
+    feeds          = [
+                        (u'Feeds', u'http://www.dilemaveche.ro/rss.xml')
+                     ]
+
+    def preprocess_html(self, soup):
+        return self.adeify_images(soup)

recipes/economist_free.recipe

@@ -14,7 +14,7 @@ class Economist(BasicNewsRecipe):
     description = ('Global news and current affairs from a European'
             ' perspective. Best downloaded on Friday mornings (GMT).'
             ' Much slower than the print edition based version.')
-
+    extra_css      = '.headline {font-size: x-large;} \n h2 { font-size: small;  } \n h1 { font-size: medium;  }'
     oldest_article = 7.0
     cover_url = 'http://www.economist.com/images/covers/currentcoverus_large.jpg'
     remove_tags = [

recipes/elmundo.recipe

@@ -1,6 +1,6 @@
 
 __license__   = 'GPL v3'
-__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2009-2011, Darko Miletic <darko.miletic at gmail.com>'
 '''
 elmundo.es
 '''
@@ -10,15 +10,24 @@ from calibre.web.feeds.news import BasicNewsRecipe
 class ElMundo(BasicNewsRecipe):
     title                 = 'El Mundo'
     __author__            = 'Darko Miletic'
-    description           = 'News from Spain'
-    publisher             = 'El Mundo'
+    description           = 'Lider de informacion en espaniol'
+    publisher             = 'Unidad Editorial Informacion General S.L.U.'
     category              = 'news, politics, Spain'
     oldest_article        = 2
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False
     encoding              = 'iso8859_15'
-    language              = 'es'
+    language              = 'es_ES'
+    masthead_url          = 'http://estaticos03.elmundo.es/elmundo/iconos/v4.x/v4.01/bg_h1.png'
+    publication_type      = 'newspaper'
+    extra_css             = """
+                               body{font-family: Arial,Helvetica,sans-serif}
+                               .metadata_noticia{font-size: small}
+                               h1,h2,h3,h4,h5,h6,.subtitulo {color: #3F5974}
+                               .hora{color: red}
+                               .update{color: gray}
+                            """
 
     conversion_options = {
                              'comments'  : description
@@ -30,22 +39,31 @@ class ElMundo(BasicNewsRecipe):
     keep_only_tags     = [dict(name='div', attrs={'class':'noticia'})]
     remove_tags_before = dict(attrs={'class':['titular','antetitulo']     })
     remove_tags_after  = dict(name='div' , attrs={'id':['desarrollo_noticia','tamano']})
-
+    remove_attributes  = ['lang','border']
     remove_tags = [
                      dict(name='div', attrs={'class':['herramientas','publicidad_google']})
                     ,dict(name='div', attrs={'id':'modulo_multimedia' })
                     ,dict(name='ul', attrs={'class':'herramientas' })
-                    ,dict(name=['object','link'])
+                    ,dict(name=['object','link','embed','iframe','base','meta'])
                   ]
 
     feeds = [
-              (u'Portada'         , u'http://rss.elmundo.es/rss/descarga.htm?data2=4' )
-             ,(u'Deportes'        , u'http://rss.elmundo.es/rss/descarga.htm?data2=14')
-             ,(u'Economia'        , u'http://rss.elmundo.es/rss/descarga.htm?data2=7' )
-             ,(u'Espana'          , u'http://rss.elmundo.es/rss/descarga.htm?data2=8' )
-             ,(u'Internacional'   , u'http://rss.elmundo.es/rss/descarga.htm?data2=9' )
-             ,(u'Cultura'         , u'http://rss.elmundo.es/rss/descarga.htm?data2=6' )
-             ,(u'Ciencia/Ecologia', u'http://rss.elmundo.es/rss/descarga.htm?data2=5' )
-             ,(u'Comunicacion'    , u'http://rss.elmundo.es/rss/descarga.htm?data2=26')
-             ,(u'Television'      , u'http://rss.elmundo.es/rss/descarga.htm?data2=76')
+              (u'Portada'         , u'http://estaticos.elmundo.es/elmundo/rss/portada.xml'       )
+             ,(u'Deportes'        , u'http://estaticos.elmundo.es/elmundodeporte/rss/portada.xml')
+             ,(u'Economia'        , u'http://estaticos.elmundo.es/elmundo/rss/economia.xml'      )
+             ,(u'Espana'          , u'http://estaticos.elmundo.es/elmundo/rss/espana.xml'        )
+             ,(u'Internacional'   , u'http://estaticos.elmundo.es/elmundo/rss/internacional.xml' )
+             ,(u'Cultura'         , u'http://estaticos.elmundo.es/elmundo/rss/cultura.xml'       )
+             ,(u'Ciencia/Ecologia', u'http://estaticos.elmundo.es/elmundo/rss/ciencia.xml'       )
+             ,(u'Comunicacion'    , u'http://estaticos.elmundo.es/elmundo/rss/comunicacion.xml'  )
+             ,(u'Television'      , u'http://estaticos.elmundo.es/elmundo/rss/television.xml'    )
             ]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        return soup
+
+    def get_article_url(self, article):
+        return article.get('guid',  None)
+

recipes/focus_de.recipe

@@ -0,0 +1,48 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1305567197(BasicNewsRecipe):
+    title          = u'Focus (DE)'
+    __author__  = 'Anonymous'
+    language = 'de'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    remove_javascript      = True
+
+    def print_version(self, url):
+        return url + '?drucken=1'
+
+    keep_only_tags = [
+                              dict(name='div', attrs={'id':['article']}) ]
+
+    remove_tags = [dict(name='div', attrs={'class':'sidebar'}),
+                            dict(name='div', attrs={'class':'commentForm'}),
+                            dict(name='div', attrs={'class':'comment clearfix oid-3534591 open'}),
+                            dict(name='div', attrs={'class':'similarityBlock'}),
+                            dict(name='div', attrs={'class':'footer'}),
+                            dict(name='div', attrs={'class':'getMoreComments'}),
+                            dict(name='div', attrs={'class':'moreComments'}),
+                            dict(name='div', attrs={'class':'ads'}),
+                            dict(name='div', attrs={'class':'articleContent'}),
+
+
+                            ]
+    remove_tags_after = [
+                            dict(name='div',attrs={'class':['commentForm','title', 'actions clearfix']})
+                                   ]
+
+
+    feeds          = [	(u'Eilmeldungen', u'http://rss2.focus.de/c/32191/f/533875/index.rss'),
+                                        (u'Auto-News', u'http://rss2.focus.de/c/32191/f/443320/index.rss'),
+                                        (u'Digital-News', u'http://rss2.focus.de/c/32191/f/443315/index.rss'),
+                                        (u'Finanzen-News', u'http://rss2.focus.de/c/32191/f/443317/index.rss'),
+                                        (u'Gesundheit-News', u'http://rss2.focus.de/c/32191/f/443314/index.rss'),
+                                        (u'Immobilien-News', u'http://rss2.focus.de/c/32191/f/443318/index.rss'),
+                                        (u'Kultur-News', u'http://rss2.focus.de/c/32191/f/443321/index.rss'),
+		(u'Panorama-News', u'http://rss2.focus.de/c/32191/f/533877/index.rss'),
+                                        (u'Politik-News', u'http://rss2.focus.de/c/32191/f/443313/index.rss'),
+                                        (u'Reisen-News', u'http://rss2.focus.de/c/32191/f/443316/index.rss'),
+                                        (u'Sport-News', u'http://rss2.focus.de/c/32191/f/443319/index.rss'),
+                                        (u'Wissen-News', u'http://rss2.focus.de/c/32191/f/533876/index.rss'),
+                         ]

recipes/go_comics.recipe

@@ -6,13 +6,13 @@ __copyright__ = 'Copyright 2010 Starson17'
 www.gocomics.com
 '''
 from calibre.web.feeds.news import BasicNewsRecipe
-import mechanize
+import mechanize, re
 
 class GoComics(BasicNewsRecipe):
     title               = 'GoComics'
     __author__          = 'Starson17'
-    __version__         = '1.03'
-    __date__            = '09 October 2010'
+    __version__         = '1.05'
+    __date__            = '19 may 2011'
     description         = u'200+ Comics - Customize for more days/comics: Defaults to 7 days, 25 comics - 20 general, 5 editorial.'
     category            = 'news, comics'
     language            = 'en'
@@ -20,6 +20,7 @@ class GoComics(BasicNewsRecipe):
     no_stylesheets      = True
     remove_javascript   = True
     cover_url           = 'http://paulbuckley14059.files.wordpress.com/2008/06/calvin-and-hobbes.jpg'
+    remove_attributes = ['style']
 
     ####### USER PREFERENCES - COMICS, IMAGE SIZE AND NUMBER OF COMICS TO RETRIEVE ########
     # num_comics_to_get - I've tried up to 99 on Calvin&Hobbes
@@ -40,6 +41,8 @@ class GoComics(BasicNewsRecipe):
 
     remove_tags = [dict(name='a', attrs={'class':['beginning','prev','cal','next','newest']}),
                    dict(name='div', attrs={'class':['tag-wrapper']}),
+                   dict(name='a', attrs={'href':re.compile(r'.*mutable_[0-9]+', re.IGNORECASE)}),
+                   dict(name='img', attrs={'src':re.compile(r'.*mutable_[0-9]+', re.IGNORECASE)}),
                    dict(name='ul', attrs={'class':['share-nav','feature-nav']}),
                    ]
 

recipes/grrm.recipe

@@ -0,0 +1,36 @@
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Darko Miletic <darko.miletic at gmail.com>'
+'''
+grrm.livejournal.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class NotABlog(BasicNewsRecipe):
+    title                 = 'Not A Blog - George R.R. Martin'
+    __author__            = 'Darko Miletic'
+    description           = 'George R.R. Martin'
+    oldest_article        = 15
+    max_articles_per_feed = 100
+    language              = 'en'
+    encoding              = 'utf-8'
+    no_stylesheets        = True
+    use_embedded_content  = True
+    publication_type      = 'blog'
+
+    conversion_options = {
+                          'comment'  : description
+                        , 'tags'     : 'sf, fantasy, game of thrones'
+                        , 'publisher': 'George R.R. Martin'
+                        , 'language' : language
+                        }
+
+    feeds = [(u'Posts', u'http://grrm.livejournal.com/data/rss')]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        return self.adeify_images(soup)
+
+

recipes/impulse_de.recipe

@@ -0,0 +1,32 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+class AdvancedUserRecipe1305470859(BasicNewsRecipe):
+    title                 = u'Impulse.de'
+    language              = 'de'
+    __author__            = 'schuster'
+    oldest_article        =14
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    remove_javascript     = True
+    use_embedded_content  = False
+    cover_url = 'http://www.bvk.de/files/image/bilder/Logo%20Impulse.jpg'
+
+    extra_css = '''
+                    h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
+                    h4{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
+                    img {min-width:300px; max-width:600px; min-height:300px; max-height:800px}
+                    p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
+                    body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
+	'''
+    def print_version(self, url):
+        return url.replace ('#utm_source=rss2&utm_medium=rss_feed&utm_campaign=/', '?mode=print')
+    remove_tags_bevor = [dict(name='h1', attrs={'class':'h2'})]
+    remove_tags_after = [dict(name='div', attrs={'class':'artikelfuss'})]
+
+    feeds = [   (u'impulstest', u'http://www.impulse.de/rss/')]
+
+
+    remove_tags = [dict(attrs={'class':['navSeitenAlle', 'kommentieren', 'teaserheader', 'teasercontent', 'info', 'zwischenhead', 'kasten_artikel']}),
+                dict(id=['metaNav', 'impKopf', 'impTopNav', 'impSubNav', 'footerRahmen', 'gatrixx_marktinformationen', 'pager', 'weitere', 'socialmedia', 'rating_open']),
+                dict(span=['ratingtext', 'Gesamtranking', 'h3','']),
+	dict(rel=['canonical'])]
+

recipes/kath_net.recipe

@@ -3,8 +3,9 @@ from calibre.web.feeds.news import BasicNewsRecipe
 class AdvancedUserRecipe1295262156(BasicNewsRecipe):
     title          = u'kath.net'
     __author__     = 'Bobus'
+    description    = u'Katholische Nachrichten'
     oldest_article = 7
-    language = 'en'
+    language = 'de'
     max_articles_per_feed = 100
 
     feeds          = [(u'kath.net', u'http://www.kath.net/2005/xml/index.xml')]

recipes/marca.recipe

@@ -1,14 +1,11 @@
-#!/usr/bin/env  python
 # -*- coding: utf-8 -*-
-
 __license__   = 'GPL v3'
-__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2009-2011, Darko Miletic <darko.miletic at gmail.com>'
 '''
 www.marca.com
 '''
 
 from calibre.web.feeds.news import BasicNewsRecipe
-from calibre.ebooks.BeautifulSoup import Tag
 
 class Marca(BasicNewsRecipe):
     title                 = 'Marca'
@@ -22,35 +19,30 @@ class Marca(BasicNewsRecipe):
     use_embedded_content  = False
     delay                 = 1
     encoding              = 'iso-8859-15'
-    language = 'es'
+    language              = 'es_ES'
+    publication_type      = 'newsportal'
+    masthead_url          = 'http://estaticos.marca.com/deporte/img/v3.0/img_marca-com.png'
+    extra_css             = """
+                                body{font-family: Tahoma,Geneva,sans-serif}
+                                h1,h2,h3,h4,h5,h6{font-family: 'LatoBlack',Tahoma,Geneva,sans-serif}
+                                .cab_articulo h4 {font-family: Georgia,"Times New Roman",Times,serif}
+                                .antetitulo{text-transform: uppercase}
+                            """
 
-    direction             = 'ltr'
-
-    html2lrf_options = [
-                          '--comment'  , description
-                        , '--category' , category
-                        , '--publisher', publisher
-                        ]
-
-    html2epub_options  = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"'
-
-    feeds              = [(u'Portada', u'http://rss.marca.com/rss/descarga.htm?data2=425')]
-
-    keep_only_tags = [dict(name='div', attrs={'class':['cab_articulo','col_izq']})]
+    feeds             = [(u'Portada', u'http://estaticos.marca.com/rss/portada.xml')]
 
+    keep_only_tags    = [dict(name='div', attrs={'class':['cab_articulo','cuerpo_articulo']})]
+    remove_attributes = ['lang']
     remove_tags       = [
-                             dict(name=['object','link','script'])
-                            ,dict(name='div', attrs={'class':['colC','peu']})
-                            ,dict(name='div', attrs={'class':['utilidades estirar','bloque_int_corr estirar']})
+                             dict(name=['object','link','script','embed','iframe','meta','base'])
+                            ,dict(name='div', attrs={'class':'tabs'})
                          ]
 
-    remove_tags_after = [dict(name='div', attrs={'class':'bloque_int_corr estirar'})]
-
     def preprocess_html(self, soup):
-        soup.html['dir' ] = self.direction
-        mcharset = Tag(soup,'meta',[("http-equiv","Content-Type"),("content","text/html; charset=utf-8")])
-        soup.head.insert(0,mcharset)
         for item in soup.findAll(style=True):
             del item['style']
         return soup
 
+    def get_article_url(self, article):
+        return article.get('guid',  None)
+

recipes/natgeo.recipe

@@ -0,0 +1,71 @@
+#!/usr/bin/env  python
+# -*- coding: utf-8 -*-
+__license__   = 'GPL v3'
+__copyright__ = '2011, gagsays <gagsays at gmail dot com>'
+'''
+nationalgeographic.com
+'''
+from calibre.web.feeds.news import BasicNewsRecipe
+class NatGeo(BasicNewsRecipe):
+    title          = u'National Geographic'
+    description = 'Daily news articles from The National Geographic'
+    language = 'en'
+    oldest_article = 20
+    max_articles_per_feed = 25
+    encoding              = 'utf8'
+    publisher              = 'nationalgeographic.com'
+    category               = 'science, nat geo'
+    __author__           = 'gagsays'
+    masthead_url        = 'http://s.ngeo.com/wpf/sites/themes/global/i/presentation/ng_logo_small.png'
+    description           = 'Inspiring people to care about the planet since 1888'
+    timefmt = ' [%a, %d %b, %Y]'
+    no_stylesheets        = True
+    use_embedded_content  = False
+
+    extra_css = '''
+                body {color: #000000;font-size: medium;}
+                h1 {color: #222222; font-size: large; font-weight:lighter; text-decoration:none; text-align: center;font-family:Georgia,Times New Roman,Times,serif;}
+          h2 {color: #454545; font-size: small; font-weight:lighter; text-decoration:none; text-align: justify; font-style:italic;font-family :Georgia,Times New Roman,Times,serif;}
+                h3 {color: #555555; font-size: small; font-style:italic; margin-top: 10px;}
+                img{margin-bottom: 0.25em;display:block;margin-left: auto;margin-right: auto;}
+                a:link,a,.a,href {text-decoration: none;color: #000000;}
+                .caption{color: #000000;font-size: xx-small;text-align: justify;font-weight:normal;}
+                .credit{color: #555555;font-size: xx-small;text-align: left;font-weight:lighter;}
+                p.author,p.publication{color: #000000;font-size: xx-small;text-align: left;display:inline;}
+                p.publication_time{color: #000000;font-size: xx-small;text-align: right;text-decoration: underline;}
+                p {margin-bottom: 0;}
+                p + p {text-indent: 1.5em;margin-top: 0;}
+                .hidden{display:none;}
+                #page_head{text-transform:uppercase;}
+               '''
+
+    def parse_feeds (self):
+      feeds = BasicNewsRecipe.parse_feeds(self)
+      for feed in feeds:
+        for article in feed.articles[:]:
+          if 'Presented' in article.title or 'Pictures' in article.title:
+            feed.articles.remove(article)
+      return feeds
+
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
+    remove_tags_before = dict(id='page_head')
+    keep_only_tags = [
+         dict(name='div',attrs={'id':['page_head','content_mainA']})
+    ]
+    remove_tags_after = [
+         dict(name='div',attrs={'class':['article_text','promo_collection']})
+    ]
+    remove_tags    = [
+                               dict(name='div', attrs={'class':['aside','primary full_width']})
+                               ,dict(name='div', attrs={'id':['header_search','navigation_mainB_wrap']})
+                                ]
+    feeds = [
+                    (u'Daily News', u'http://feeds.nationalgeographic.com/ng/News/News_Main')
+                    ]
+

recipes/national_geographic_de.recipe

@@ -0,0 +1,25 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1305567197(BasicNewsRecipe):
+    title          = u'National Geographic (DE)'
+    __author__  = 'Anonymous'
+    language = 'de'
+    oldest_article = 7
+    max_articles_per_feed = 1000
+    no_stylesheets         = True
+    use_embedded_content   = False
+    remove_javascript      = True
+    cover_url = 'http://www.nationalgeographic.de/images/national-geographic-logo.jpg'
+    keep_only_tags = [
+                              dict(name='div', attrs={'class':['contentbox_no_top_border']}) ]
+
+    remove_tags =  [
+                            dict(name='div', attrs={'class':'related'}),
+                            dict(name='li', attrs={'class':'first'}),
+                            dict(name='div', attrs={'class':'extrasbox_inner'}),
+
+                            ]
+
+    feeds          = [  (u'National Geographic', u'http://feeds.nationalgeographic.de/ng-neueste-artikel'),
+
+        ]

recipes/the_nation.recipe

@@ -1,5 +1,5 @@
 __license__   = 'GPL v3'
-__copyright__ = '2008 - 2010, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008 - 2011, Darko Miletic <darko.miletic at gmail.com>'
 '''
 thenation.com
 '''
@@ -19,7 +19,14 @@ class Thenation(BasicNewsRecipe):
     use_embedded_content  = False
     delay                 = 1
     masthead_url          = 'http://www.thenation.com/sites/default/themes/thenation/images/logo-main.gif'
-    exra_css               = ' body{font-family: Arial,Helvetica,sans-serif;} .print-created{font-size: small;} .caption{display: block; font-size: x-small;} '
+    login_url             = 'http://www.thenation.com/user?destination=%3Cfront%3E'
+    publication_type      = 'magazine'
+    needs_subscription    = 'optional'    
+    exra_css              = """ 
+                              body{font-family: Arial,Helvetica,sans-serif;} 
+                              .print-created{font-size: small;} 
+                              .caption{display: block; font-size: x-small;}
+                            """
 
     conversion_options = {
                           'comment'   : description
@@ -28,13 +35,30 @@ class Thenation(BasicNewsRecipe):
                         , 'language'  : language
                         }
 
-    keep_only_tags = [ dict(attrs={'class':['print-title','print-created','print-content','print-links']}) ]
-    remove_tags    = [dict(name='link')]
+    keep_only_tags    = [dict(attrs={'class':['print-title','print-created','print-content','print-links']})]
+    remove_tags       = [dict(name=['link','iframe','base','meta','object','embed'])]
+    remove_attributes = ['lang']
 
-    feeds       = [(u"Editor's Picks", u'http://www.thenation.com/rss/editors_picks')]
+    feeds       = [(u"Articles", u'http://www.thenation.com/rss/articles')]
 
     def print_version(self, url):
         return url.replace('.thenation.com/','.thenation.com/print/')
 
-    def preprocess_html(self, soup):
-        return self.adeify_images(soup)
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser()
+        br.open('http://www.thenation.com/')
+        if self.username is not None and self.password is not None:
+            br.open(self.login_url)
+            br.select_form(nr=1)
+            br['name'] = self.username
+            br['pass'] = self.password
+            br.submit()
+        return br
+
+    def get_cover_url(self):
+        soup = self.index_to_soup('http://www.thenation.com/issue/')
+        item = soup.find('div',attrs={'id':'cover-wrapper'})
+        if item:           
+           return item.img['src']
+        return None
+        

recipes/wash_post.recipe

@@ -1,64 +1,75 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.washingtonpost.com
+'''
+
+from calibre import strftime
 from calibre.web.feeds.news import BasicNewsRecipe
 
-
-class WashingtonPost(BasicNewsRecipe):
-
-    title = 'Washington Post'
-    description = 'US political news'
-    __author__ = 'Kovid Goyal'
-    use_embedded_content   = False
-    max_articles_per_feed = 20
-    language = 'en'
-    encoding = 'utf-8'
-
-
-    remove_javascript = True
+class TheWashingtonPost(BasicNewsRecipe):
+    title                 = 'The Washington Post'
+    __author__            = 'Darko Miletic'
+    description           = 'Leading source for news, video and opinion on politics, business, world and national news, science, travel, entertainment and more. Our local coverage includes reporting on education, crime, weather, traffic, real estate, jobs and cars for DC, Maryland and Virginia. Offering award-winning opinion writing, entertainment information and restaurant reviews.'
+    publisher             = 'The Washington Post Company'
+    category              = 'news, politics, USA'
+    oldest_article        = 2
+    max_articles_per_feed = 200
     no_stylesheets        = True
+    encoding              = 'utf8'
+    delay                 = 1
+    use_embedded_content  = False
+    language              = 'en'
+    remove_empty_feeds    = True
+    publication_type      = 'newspaper'
+    masthead_url          = 'http://www.washingtonpost.com/rw/sites/twpweb/img/logos/twp_logo_300.gif'
+    cover_url             = strftime('http://www.washingtonpost.com/rw/WashingtonPost/Content/Epaper/%Y-%m-%d/Ax1.pdf')
+    extra_css             = """
+                               body{font-family: Georgia,serif }
+                            """
+
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
+
+    keep_only_tags   = [dict(attrs={'id':['content','entryhead','entrytext']})]
+    remove_tags      = [
+                          dict(name=['meta','link','iframe','base'])
+                         ,dict(attrs={'id':'multimedia-leaf-page'})
+                       ]
+    remove_attributes= ['lang','property','epochtime','datetitle','pagetype','contenttype','comparetime']
+
 
     feeds = [
-            ('Politics', 'http://www.washingtonpost.com/rss/politics'),
-            ('Nation', 'http://www.washingtonpost.com/rss/national'),
-            ('World', 'http://www.washingtonpost.com/rss/world'),
-            ('Business', 'http://www.washingtonpost.com/rss/business'),
-            ('Lifestyle', 'http://www.washingtonpost.com/rss/lifestyle'),
-            ('Sports', 'http://www.washingtonpost.com/rss/sports'),
-            ('Redskins', 'http://www.washingtonpost.com/rss/sports/redskins'),
-            ('Opinions', 'http://www.washingtonpost.com/rss/opinions'),
-            ('Entertainment', 'http://www.washingtonpost.com/rss/entertainment'),
-            ('Local', 'http://www.washingtonpost.com/rss/local'),
-            ('Investigations',
-                'http://www.washingtonpost.com/rss/investigations'),
+              (u'World'          , u'http://feeds.washingtonpost.com/rss/world'                   )
+             ,(u'National'       , u'http://feeds.washingtonpost.com/rss/national'                )
+             ,(u'White House'    , u'http://feeds.washingtonpost.com/rss/politics/whitehouse'     )
+             ,(u'Business'       , u'http://feeds.washingtonpost.com/rss/business'                )
+             ,(u'Opinions'       , u'http://feeds.washingtonpost.com/rss/opinions'                )
+             ,(u'Investigations' , u'http://feeds.washingtonpost.com/rss/investigations'          )
+             ,(u'Local'          , u'http://feeds.washingtonpost.com/rss/local'                   )
+             ,(u'Entertainment'  , u'http://feeds.washingtonpost.com/rss/entertainment'           )
+             ,(u'Sports'         , u'http://feeds.washingtonpost.com/rss/sports'                  )
+             ,(u'Redskins'       , u'http://feeds.washingtonpost.com/rss/sports/redskins'         )
+             ,(u'Special Reports', u'http://feeds.washingtonpost.com/rss/national/special-reports')
             ]
 
-    remove_tags = [
-            {'class':lambda x: x and 'article-toolbar' in x},
-            {'class':lambda x: x and 'quick-comments' in x},
-            {'class':lambda x: x and 'tweet' in x},
-            {'class':lambda x: x and 'article-related' in x},
-            {'class':lambda x: x and 'hidden' in x.split()},
-            {'class':lambda x: x and 'also-read' in x.split()},
-            {'class':lambda x: x and 'partners-content' in x.split()},
-            {'class':['module share', 'module ads', 'comment-vars', 'hidden',
-                'share-icons-wrap', 'comments', 'flipper']},
-            {'id':['right-rail', 'save-and-share']},
-            {'width':'1', 'height':'1'},
-
-    ]
-
-    keep_only_tags = dict(id=['content', 'article'])
-
-    def get_article_url(self, *args):
-        ans = BasicNewsRecipe.get_article_url(self, *args)
-        ans = ans.rpartition('?')[0]
-        if ans.endswith('_video.html'):
-            return None
-        if 'ads.pheedo.com' in ans:
-            return None
-        #if not ans.endswith('_blog.html'):
-        #    return None
-        return ans
-
-
     def print_version(self, url):
-        return url.replace('_story.html', '_singlePage.html')
+        if '_story.html' in url:
+           return url.replace('_story.html','_print.html')
+        return url
+
+    def get_article_url(self, article):
+        link = BasicNewsRecipe.get_article_url(self,article)
+        if not 'washingtonpost.com' in link:
+           self.log('Skipping adds:', link)
+           return None
+        for it in ['_video.html','_gallery.html','_links.html']:
+            if it in link:
+               self.log('Skipping non-article:', link)
+               return None
+        return link
 

resources/default_tweaks.py

@@ -53,7 +53,8 @@ authors_completer_append_separator = False
 # periods are automatically handled.
 author_sort_copy_method = 'comma'
 author_name_suffixes = ('Jr', 'Sr', 'Inc', 'Ph.D', 'Phd',
-                        'MD', 'M.D', 'I', 'II', 'III', 'IV')
+                        'MD', 'M.D', 'I', 'II', 'III', 'IV',
+                        'Junior', 'Senior')
 
 #: Use author sort in Tag Browser
 # Set which author field to display in the tags pane (the list of authors,
@@ -350,3 +351,11 @@ send_news_to_device_location = "main"
 # work on all operating systems)
 server_listen_on = '0.0.0.0'
 
+#: Unified toolbar on OS X
+# If you enable this option and restart calibre, the toolbar will be 'unified'
+# with the titlebar as is normal for OS X applications. However, doing this has
+# various bugs, for instance the minimum width of the toolbar becomes twice
+# what it should be and it causes other random bugs on some systems, so turn it
+# on at your own risk!
+unified_title_toolbar_on_osx = False
+

resources/template-functions.json

@@ -3,10 +3,12 @@
     "divide": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x / y)\n", 
     "uppercase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return val.upper()\n", 
     "strcat": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    res = ''\n    for i in range(0, len(args)):\n        res += args[i]\n    return res\n", 
+    "in_list": "def evaluate(self, formatter, kwargs, mi, locals, val, sep, pat, fv, nfv):\n    l = [v.strip() for v in val.split(sep) if v.strip()]\n    for v in l:\n        if re.search(pat, v):\n            return fv\n    return nfv\n", 
     "substr": "def evaluate(self, formatter, kwargs, mi, locals, str_, start_, end_):\n    return str_[int(start_): len(str_) if int(end_) == 0 else int(end_)]\n", 
     "ifempty": "def evaluate(self, formatter, kwargs, mi, locals, val, value_if_empty):\n    if val:\n        return val\n    else:\n        return value_if_empty\n", 
     "booksize": "def evaluate(self, formatter, kwargs, mi, locals):\n    if mi.book_size is not None:\n        try:\n            return str(mi.book_size)\n        except:\n            pass\n    return ''\n", 
     "select": "def evaluate(self, formatter, kwargs, mi, locals, val, key):\n    if not val:\n        return ''\n    vals = [v.strip() for v in val.split(',')]\n    for v in vals:\n        if v.startswith(key+':'):\n            return v[len(key)+1:]\n    return ''\n", 
+    "first_non_empty": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    while i < len(args):\n        if args[i]:\n            return args[i]\n        i += 1\n    return ''\n", 
     "field": "def evaluate(self, formatter, kwargs, mi, locals, name):\n    return formatter.get_value(name, [], kwargs)\n", 
     "subtract": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x - y)\n", 
     "list_item": "def evaluate(self, formatter, kwargs, mi, locals, val, index, sep):\n    if not val:\n        return ''\n    index = int(index)\n    val = val.split(sep)\n    try:\n        return val[index]\n    except:\n        return ''\n", 

setup/__init__.py

@@ -12,7 +12,9 @@ is64bit = platform.architecture()[0] == '64bit'
 iswindows = re.search('win(32|64)', sys.platform)
 isosx = 'darwin' in sys.platform
 isfreebsd = 'freebsd' in sys.platform
-islinux = not isosx and not iswindows and not isfreebsd
+isnetbsd = 'netbsd' in sys.platform
+isbsd = isnetbsd or isfreebsd
+islinux = not isosx and not iswindows and not isbsd
 SRC = os.path.abspath('src')
 sys.path.insert(0, SRC)
 sys.resources_location = os.path.join(os.path.dirname(SRC), 'resources')

setup/extensions.py

@@ -11,7 +11,7 @@ from distutils import sysconfig
 
 from PyQt4.pyqtconfig import QtGuiModuleMakefile
 
-from setup import Command, islinux, isfreebsd, isosx, SRC, iswindows
+from setup import Command, islinux, isfreebsd, isbsd, isosx, SRC, iswindows
 from setup.build_environment import fc_inc, fc_lib, chmlib_inc_dirs, \
         fc_error, poppler_libs, poppler_lib_dirs, poppler_inc_dirs, podofo_inc, \
         podofo_lib, podofo_error, poppler_error, pyqt, OSX_SDK, NMAKE, \
@@ -21,7 +21,7 @@ from setup.build_environment import fc_inc, fc_lib, chmlib_inc_dirs, \
         jpg_lib_dirs, chmlib_lib_dirs, sqlite_inc_dirs, icu_inc_dirs, \
         icu_lib_dirs
 MT
-isunix = islinux or isosx or isfreebsd
+isunix = islinux or isosx or isbsd
 
 make = 'make' if isunix else NMAKE
 
@@ -205,7 +205,7 @@ if islinux:
     ldflags.append('-lpython'+sysconfig.get_python_version())
 
 
-if isfreebsd:
+if isbsd:
     cflags.append('-pthread')
     ldflags.append('-shared')
     cflags.append('-I'+sysconfig.get_python_inc())

setup/install.py

@@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
 
 import sys, os, textwrap, subprocess, shutil, tempfile, atexit, stat, shlex
 
-from setup import Command, islinux, isfreebsd, basenames, modules, functions, \
+from setup import Command, islinux, isfreebsd, isbsd, basenames, modules, functions, \
         __appname__, __version__
 
 HEADER = '''\
@@ -116,7 +116,7 @@ class Develop(Command):
 
 
     def pre_sub_commands(self, opts):
-        if not (islinux or isfreebsd):
+        if not (islinux or isbsd):
             self.info('\nSetting up a source based development environment is only '
                     'supported on linux. On other platforms, see the User Manual'
                     ' for help with setting up a development environment.')
@@ -156,7 +156,7 @@ class Develop(Command):
             self.warn('Failed to compile mount helper. Auto mounting of',
                 ' devices will not work')
 
-        if not isfreebsd and os.geteuid() != 0:
+        if not isbsd and os.geteuid() != 0:
             return self.warn('Must be run as root to compile mount helper. Auto '
                     'mounting of devices will not work.')
         src = os.path.join(self.SRC, 'calibre', 'devices', 'linux_mount_helper.c')
@@ -168,7 +168,7 @@ class Develop(Command):
         ret = p.wait()
         if ret != 0:
             return warn()
-        if not isfreebsd:
+        if not isbsd:
             os.chown(dest, 0, 0)
             os.chmod(dest, stat.S_ISUID|stat.S_ISGID|stat.S_IRUSR|stat.S_IWUSR|\
                     stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH)

setup/installer/windows/wix-template.xml

@@ -26,6 +26,11 @@
                </Upgrade>
                <CustomAction Id="PreventDowngrading" Error="Newer version already installed."/>
 
+               <Property Id="APPLICATIONFOLDER">
+                  <RegistrySearch Id='calibreInstDir' Type='raw'
+                      Root='HKLM' Key="Software\{app}\Installer" Name="InstallPath" />
+               </Property>
+
                 <Directory Id='TARGETDIR' Name='SourceDir'>
                         <Merge Id="VCRedist" SourceFile="{crt_msm}" DiskId="1" Language="0"/>
 						<Directory Id='ProgramFilesFolder' Name='PFiles'>
@@ -43,6 +48,9 @@
                             <Environment Id='UpdatePath' Name='PATH' Action='set' System='yes' Part='last' Value='[APPLICATIONFOLDER]' />
                             <RegistryValue Root="HKCU" Key="Software\Microsoft\{app}" Name="system_path_updated" Type="integer" Value="1" KeyPath="yes"/>
                         </Component>
+                        <Component Id="RememberInstallDir" Guid="*">
+                            <RegistryValue Root="HKLM" Key="Software\{app}\Installer" Name="InstallPath" Type="string" Value="[APPLICATIONFOLDER]" KeyPath="yes"/>
+                        </Component>
                 </DirectoryRef>
 
                 <DirectoryRef Id="ApplicationProgramsFolder">
@@ -87,7 +95,8 @@
 						ConfigurableDirectory="APPLICATIONFOLDER">
 
 						<Feature Id="MainApplication" Title="Program Files" Level="1"
-                            Description="All the files need to run {app}" Absent="disallow">
+                            Description="All the files needed to run {app}" Absent="disallow">
+                            <ComponentRef Id="RememberInstallDir"/>
                         </Feature>
 
                         <Feature Id="VCRedist" Title="Visual C++ 8.0 Runtime" AllowAdvertise="no" Display="hidden" Level="1">
@@ -115,7 +124,7 @@
 				<Property Id="ARPPRODUCTICON" Value="main_icon" />
 
                 <Condition
-                    Message="This application is only supported on Windows XP SP2, or higher.">
+                    Message="This application is only supported on Windows XP SP3, or higher.">
                     <![CDATA[Installed OR (VersionNT >= 501)]]>
                 </Condition>
                 <InstallExecuteSequence>

src/calibre/__init__.py

@@ -12,8 +12,8 @@ from functools import partial
 warnings.simplefilter('ignore', DeprecationWarning)
 
 
-from calibre.constants import (iswindows, isosx, islinux, isfreebsd, isfrozen,
-                              preferred_encoding, __appname__, __version__, __author__,
+from calibre.constants import (iswindows, isosx, islinux, isfrozen,
+        isbsd, preferred_encoding, __appname__, __version__, __author__,
         win32event, win32api, winerror, fcntl,
         filesystem_encoding, plugins, config_dir)
 from calibre.startup import winutil, winutilerror
@@ -31,7 +31,7 @@ if False:
     # Prevent pyflakes from complaining
     winutil, winutilerror, __appname__, islinux, __version__
     fcntl, win32event, isfrozen, __author__
-    winerror, win32api, isfreebsd
+    winerror, win32api, isbsd
 
 _mt_inited = False
 def _init_mimetypes():

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 8, 1)
+numeric_version = (0, 8, 2)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 
@@ -27,7 +27,9 @@ iswindows = 'win32' in _plat or 'win64' in _plat
 isosx     = 'darwin' in _plat
 isnewosx  = isosx and getattr(sys, 'new_app_bundle', False)
 isfreebsd = 'freebsd' in _plat
-islinux   = not(iswindows or isosx or isfreebsd)
+isnetbsd = 'netbsd' in _plat
+isbsd = isfreebsd or isnetbsd
+islinux   = not(iswindows or isosx or isbsd)
 isfrozen  = hasattr(sys, 'frozen')
 isunix = isosx or islinux
 

src/calibre/customize/__init__.py

@@ -607,10 +607,23 @@ class StoreBase(Plugin): # {{{
     supported_platforms = ['windows', 'osx', 'linux']
     author         = 'John Schember'
     type = _('Store')
+    # Information about the store. Should be in the primary language
+    # of the store. This should not be translatable when set by
+    # a subclass.
+    description = _('An ebook store.')
     minimum_calibre_version = (0, 8, 0)
+    version        = (1, 0, 1)
 
     actual_plugin = None
     
+    # Does the store only distribute ebooks without DRM.
+    drm_free_only = False
+    # This is the 2 letter country code for the corporate
+    # headquarters of the store.
+    headquarters = ''
+    # All formats the store distributes ebooks in.
+    formats = []
+
     def load_actual_plugin(self, gui):
         '''
         This method must return the actual interface action plugin object.

src/calibre/customize/builtins.py

@@ -855,6 +855,17 @@ class ActionStore(InterfaceActionBase):
     author = 'John Schember'
     actual_plugin = 'calibre.gui2.actions.store:StoreAction'
     
+    def customization_help(self, gui=False):
+        return 'Customize the behavior of the store search.'
+    
+    def config_widget(self):
+        from calibre.gui2.store.config.store import config_widget as get_cw
+        return get_cw()
+    
+    def save_settings(self, config_widget):
+        from calibre.gui2.store.config.store import save_settings as save
+        save(config_widget)
+
 plugins += [ActionAdd, ActionFetchAnnotations, ActionGenerateCatalog,
         ActionConvert, ActionDelete, ActionEditMetadata, ActionView,
         ActionFetchNews, ActionSaveToDisk, ActionShowBookDetails,
@@ -1095,138 +1106,304 @@ plugins += [LookAndFeel, Behavior, Columns, Toolbar, Search, InputOptions,
 # Store plugins {{{
 class StoreAmazonKindleStore(StoreBase):
     name = 'Amazon Kindle'
-    description = _('Kindle books from Amazon.')
+    description = u'Kindle books from Amazon.'
     actual_plugin = 'calibre.gui2.store.amazon_plugin:AmazonKindleStore'
     
+    drm_free_only = False
+    headquarters = 'US'
+    formats = ['KINDLE']
+
 class StoreAmazonDEKindleStore(StoreBase):
     name = 'Amazon DE Kindle'
-    description = _('Kindle books from Amazon.de.')
+    author = 'Charles Haley'
+    description = u'Kindle Bücher von Amazon.'
     actual_plugin = 'calibre.gui2.store.amazon_de_plugin:AmazonDEKindleStore'
     
+    drm_free_only = False
+    headquarters = 'DE'
+    formats = ['KINDLE']
+
 class StoreAmazonUKKindleStore(StoreBase):
     name = 'Amazon UK Kindle'
-    description = _('Kindle books from Amazon.uk.')
+    author = 'Charles Haley'
+    description = u'Kindle books from Amazon\'s UK web site. Also, includes French language ebooks.'
     actual_plugin = 'calibre.gui2.store.amazon_uk_plugin:AmazonUKKindleStore'
     
+    drm_free_only = False
+    headquarters = 'UK'
+    formats = ['KINDLE']
+
 class StoreArchiveOrgStore(StoreBase):
     name = 'Archive.org'
-    description = _('Free Books : Download & Streaming : Ebook  and Texts Archive : Internet Archive.')
+    description = u'An Internet library offering permanent access for researchers, historians, scholars, people with disabilities, and the general public to historical collections that exist in digital format.'
     actual_plugin = 'calibre.gui2.store.archive_org_plugin:ArchiveOrgStore'
 
+    drm_free_only = True
+    headquarters = 'US'
+    formats = ['DAISY', 'DJVU', 'EPUB', 'MOBI', 'PDF', 'TXT']
 
 class StoreBaenWebScriptionStore(StoreBase):
     name = 'Baen WebScription'
-    description = _('Ebooks for readers.')
+    description = u'Sci-Fi & Fantasy brought to you by Jim Baen.'
     actual_plugin = 'calibre.gui2.store.baen_webscription_plugin:BaenWebScriptionStore'
     
+    drm_free_only = True
+    headquarters = 'US'
+    formats = ['EPUB', 'LIT', 'LRF', 'MOBI', 'RB', 'RTF', 'ZIP']
+
 class StoreBNStore(StoreBase):
     name = 'Barnes and Noble'
-    description = _('Books, Textbooks, eBooks, Toys, Games and More.')
+    description = u'The world\'s largest book seller. As the ultimate destination for book lovers, Barnes & Noble.com offers an incredible array of content.'
     actual_plugin = 'calibre.gui2.store.bn_plugin:BNStore'
     
+    drm_free_only = False
+    headquarters = 'US'
+    formats = ['NOOK']
+
 class StoreBeamEBooksDEStore(StoreBase):
     name = 'Beam EBooks DE'
-    description = _('Der eBook Shop.')
+    author = 'Charles Haley'
+    description = u'Der eBook Shop.'
     actual_plugin = 'calibre.gui2.store.beam_ebooks_de_plugin:BeamEBooksDEStore'
     
+    drm_free_only = True
+    headquarters = 'DE'
+    formats = ['EPUB', 'MOBI', 'PDF']
+
 class StoreBeWriteStore(StoreBase):
     name = 'BeWrite Books'
-    description = _('Publishers of fine books.')
+    description = u'Publishers of fine books. Highly selective and editorially driven. Does not offer: books for children or exclusively YA, erotica, swords-and-sorcery fantasy and space-opera-style science fiction. All other genres are represented.'
     actual_plugin = 'calibre.gui2.store.bewrite_plugin:BeWriteStore'
     
+    drm_free_only = True
+    headquarters = 'US'
+    formats = ['EPUB', 'MOBI', 'PDF']
+
 class StoreDieselEbooksStore(StoreBase):
     name = 'Diesel eBooks'
-    description = _('World Famous eBook Store.')
+    description = u'Instant access to over 2.4 million titles from hundreds of publishers including Harlequin, HarperCollins, John Wiley & Sons, McGraw-Hill, Simon & Schuster and Random House.'
     actual_plugin = 'calibre.gui2.store.diesel_ebooks_plugin:DieselEbooksStore'
     
+    drm_free_only = False
+    headquarters = 'US'
+    formats = ['EPUB', 'PDF']
+
 class StoreEbookscomStore(StoreBase):
     name = 'eBooks.com'
-    description = _('The digital bookstore.')
+    description = u'Sells books in multiple electronic formats in all categories. Technical infrastructure is cutting edge, robust and scalable, with servers in the US and Europe.'
     actual_plugin = 'calibre.gui2.store.ebooks_com_plugin:EbookscomStore'
     
+    drm_free_only = False
+    headquarters = 'US'
+    formats = ['EPUB', 'LIT', 'MOBI', 'PDF']
+
 class StoreEPubBuyDEStore(StoreBase):
     name = 'EPUBBuy DE'
-    description = _('EPUBReaders eBook Shop.')
+    author = 'Charles Haley'
+    description = u'Deutsch epub-Spezialisten.'
     actual_plugin = 'calibre.gui2.store.epubbuy_de_plugin:EPubBuyDEStore'
     
+    drm_free_only = True
+    headquarters = 'DE'
+    formats = ['EPUB']
+
 class StoreEHarlequinStore(StoreBase):
     name = 'eHarlequin'
-    description = _('Entertain, enrich, inspire.')
+    description = u'A global leader in series romance and one of the world\'s leading publishers of books for women. Offers women a broad range of reading from romance to bestseller fiction, from young adult novels to erotic literature, from nonfiction to fantasy, from African-American novels to inspirational romance, and more.'
     actual_plugin = 'calibre.gui2.store.eharlequin_plugin:EHarlequinStore'
     
+    drm_free_only = False
+    headquarters = 'CA'
+    formats = ['EPUB', 'PDF']
+
 class StoreFeedbooksStore(StoreBase):
     name = 'Feedbooks'
-    description = _('Read anywhere.')
+    description = u'Feedbooks is a cloud publishing and distribution service, connected to a large ecosystem of reading systems and social networks. Provides a variety of genres from independent and classic books.'
     actual_plugin = 'calibre.gui2.store.feedbooks_plugin:FeedbooksStore'
     
+    drm_free_only = False
+    headquarters = 'FR'
+    formats = ['EPUB', 'MOBI', 'PDF']
+
 class StoreFoylesUKStore(StoreBase):
     name = 'Foyles UK'
-    description = _('Foyles of London, online.')
+    author = 'Charles Haley'
+    description = u'Foyles of London\'s ebook store. Provides extensive range covering all subjects.'
     actual_plugin = 'calibre.gui2.store.foyles_uk_plugin:FoylesUKStore'
 
+    drm_free_only = False
+    headquarters = 'UK'
+    formats = ['EPUB', 'PDF']
+
+class StoreGandalfStore(StoreBase):
+    name = 'Gandalf'
+    author = u'Tomasz Długosz'
+    description = u'Księgarnia internetowa Gandalf.'
+    actual_plugin = 'calibre.gui2.store.gandalf_plugin:GandalfStore'
+
+    drm_free_only = False
+    headquarters = 'PL'
+    formats = ['EPUB', 'PDF']
+
 class StoreGoogleBooksStore(StoreBase):
     name = 'Google Books'
-    description = _('Google Books')
+    description = u'Google Books'
     actual_plugin = 'calibre.gui2.store.google_books_plugin:GoogleBooksStore'
     
+    drm_free_only = False
+    headquarters = 'US'
+    formats = ['EPUB', 'PDF', 'TXT']
+
 class StoreGutenbergStore(StoreBase):
     name = 'Project Gutenberg'
-    description = _('The first producer of free ebooks.')
+    description = u'The first producer of free ebooks. Free in the United States because their copyright has expired. They may not be free of copyright in other countries. Readers outside of the United States must check the copyright laws of their countries before downloading or redistributing our ebooks.'
     actual_plugin = 'calibre.gui2.store.gutenberg_plugin:GutenbergStore'
     
+    drm_free_only = True
+    headquarters = 'US'
+    formats = ['EPUB', 'HTML', 'MOBI', 'PDB', 'TXT']
+
 class StoreKoboStore(StoreBase):
     name = 'Kobo'
-    description = _('eReading: anytime. anyplace.')
+    description = u'With over 2.3 million eBooks to browse we have engaged readers in over 200 countries in Kobo eReading. Our eBook listings include New York Times Bestsellers, award winners, classics and more!'
     actual_plugin = 'calibre.gui2.store.kobo_plugin:KoboStore'
     
+    drm_free_only = False
+    headquarters = 'CA'
+    formats = ['EPUB']
+
 class StoreManyBooksStore(StoreBase):
     name = 'ManyBooks'
-    description = _('The best ebooks at the best price: free!')
+    description = u'Public domain and creative commons works from many sources.'
     actual_plugin = 'calibre.gui2.store.manybooks_plugin:ManyBooksStore'
     
+    drm_free_only = True
+    headquarters = 'US'
+    formats = ['EPUB', 'FB2', 'JAR', 'LIT', 'LRF', 'MOBI', 'PDB', 'PDF', 'RB', 'RTF', 'TCR', 'TXT', 'ZIP']
+
 class StoreMobileReadStore(StoreBase):
     name = 'MobileRead'
-    description = _('Ebooks handcrafted with the utmost care.')
+    description = u'Ebooks handcrafted with the utmost care.'
     actual_plugin = 'calibre.gui2.store.mobileread.mobileread_plugin:MobileReadStore'
 
+    drm_free_only = True
+    headquarters = 'CH'
+    formats = ['EPUB', 'IMP', 'LRF', 'LIT', 'MOBI', 'PDF']
+
 class StoreNextoStore(StoreBase):
     name = 'Nexto'
-    description = _('Audiobooki mp3, ebooki, prasa - księgarnia internetowa.')
+    author = u'Tomasz Długosz'
+    description = u'Największy w Polsce sklep internetowy z audiobookami mp3, ebookami pdf oraz prasą do pobrania on-line.'
     actual_plugin = 'calibre.gui2.store.nexto_plugin:NextoStore'
     
+    drm_free_only = False
+    headquarters = 'PL'
+    formats = ['EPUB', 'PDF']
+
 class StoreOpenLibraryStore(StoreBase):
     name = 'Open Library'
-    description = _('One web page for every book.')
+    description = u'One web page for every book ever published. The goal is to be a true online library. Over 20 million records from a variety of large catalogs as well as single contributions, with more on the way.'
     actual_plugin = 'calibre.gui2.store.open_library_plugin:OpenLibraryStore'
     
+    drm_free_only = True
+    headquarters = ['US']
+    formats = ['DAISY', 'DJVU', 'EPUB', 'MOBI', 'PDF', 'TXT']
+
+class StoreOReillyStore(StoreBase):
+    name = 'OReilly'
+    description = u'Programming and tech ebooks from OReilly.'
+    actual_plugin = 'calibre.gui2.store.oreilly_plugin:OReillyStore'
+    
+    drm_free_only = True
+    headquarters = 'US'
+    formats = ['APK', 'DAISY', 'EPUB', 'MOBI', 'PDF']
+
+class StorePragmaticBookshelfStore(StoreBase):
+    name = 'Pragmatic Bookshelf'
+    description = u'The Pragmatic Bookshelf\'s collection of programming and tech books avaliable as ebooks.'
+    actual_plugin = 'calibre.gui2.store.pragmatic_bookshelf_plugin:PragmaticBookshelfStore'
+
+    drm_free_only = True
+    headquarters = 'US'
+    formats = ['EPUB', 'MOBI', 'PDF']
+
 class StoreSmashwordsStore(StoreBase):
     name = 'Smashwords'
-    description = _('Your ebook. Your way.')
+    description = u'An ebook publishing and distribution platform for ebook authors, publishers and readers. Covers many genres and formats.'
     actual_plugin = 'calibre.gui2.store.smashwords_plugin:SmashwordsStore'
     
+    drm_free_only = True
+    headquarters = 'US'
+    formats = ['EPUB', 'HTML', 'LRF', 'MOBI', 'PDB', 'RTF', 'TXT']
+
 class StoreWaterstonesUKStore(StoreBase):
     name = 'Waterstones UK'
-    description = _('Feel every word.')
+    author = 'Charles Haley'
+    description = u'Waterstone\'s mission is to be the leading Bookseller on the High Street and online providing customers the widest choice, great value and expert advice from a team passionate about Bookselling.'
     actual_plugin = 'calibre.gui2.store.waterstones_uk_plugin:WaterstonesUKStore'
     
+    drm_free_only = False
+    headquarters = 'UK'
+    formats = ['EPUB', 'PDF']
+
 class StoreWeightlessBooksStore(StoreBase):
     name = 'Weightless Books'
-    description = '(e)Books That Don\'t Weigh You Down.'
+    description = u'An independent DRM-free ebooksite devoted to ebooks of all sorts.'
     actual_plugin = 'calibre.gui2.store.weightless_books_plugin:WeightlessBooksStore'
 
+    drm_free_only = True
+    headquarters = 'US'
+    formats = ['EPUB', 'HTML', 'LIT', 'MOBI', 'PDF']
+
 class StoreWizardsTowerBooksStore(StoreBase):
     name = 'Wizards Tower Books'
-    description = 'Wizard\'s Tower Press.'
+    description = u'A science fiction and fantasy publisher. Concentrates mainly on making out-of-print works available once more as e-books, and helping other small presses exploit the e-book market. Also publishes a small number of limited-print-run anthologies with a view to encouraging diversity in the science fiction and fantasy field.'
     actual_plugin = 'calibre.gui2.store.wizards_tower_books_plugin:WizardsTowerBooksStore'
     
-plugins += [StoreArchiveOrgStore, StoreAmazonKindleStore, StoreAmazonDEKindleStore,
-    StoreAmazonUKKindleStore, StoreBaenWebScriptionStore, StoreBNStore,
-    StoreBeamEBooksDEStore, StoreBeWriteStore,
-    StoreDieselEbooksStore, StoreEbookscomStore, StoreEPubBuyDEStore,
-    StoreEHarlequinStore, StoreFeedbooksStore,
-    StoreFoylesUKStore, StoreGoogleBooksStore, StoreGutenbergStore,
-    StoreKoboStore, StoreManyBooksStore,
-    StoreMobileReadStore, StoreNextoStore, StoreOpenLibraryStore, StoreSmashwordsStore,
-    StoreWaterstonesUKStore, StoreWeightlessBooksStore, StoreWizardsTowerBooksStore]
+    drm_free_only = True
+    headquarters = 'UK'
+    formats = ['EPUB', 'MOBI']
+
+class StoreWoblinkStore(StoreBase):
+    name = 'Woblink'
+    author = 'Tomasz Długosz'
+    description = u'Czytanie zdarza się wszędzie!'
+    actual_plugin = 'calibre.gui2.store.woblink_plugin:WoblinkStore'
+
+    drm_free_only = False
+    location = 'PL'
+    formats = ['EPUB']
+
+plugins += [
+    StoreArchiveOrgStore,
+    StoreAmazonKindleStore,
+    StoreAmazonDEKindleStore,
+    StoreAmazonUKKindleStore,
+    StoreBaenWebScriptionStore,
+    StoreBNStore,
+    StoreBeamEBooksDEStore,
+    StoreBeWriteStore,
+    StoreDieselEbooksStore,
+    StoreEbookscomStore,
+    StoreEPubBuyDEStore,
+    StoreEHarlequinStore,
+    StoreFeedbooksStore,
+    StoreFoylesUKStore,
+    StoreGandalfStore, 
+    StoreGoogleBooksStore,
+    StoreGutenbergStore,
+    StoreKoboStore,
+    StoreManyBooksStore,
+    StoreMobileReadStore,
+    StoreNextoStore,
+    StoreOpenLibraryStore,
+    StoreOReillyStore,
+    StorePragmaticBookshelfStore,
+    StoreSmashwordsStore,
+    StoreWaterstonesUKStore,
+    StoreWeightlessBooksStore,
+    StoreWizardsTowerBooksStore,
+    StoreWoblinkStore
+]
 
 # }}}

src/calibre/customize/ui.py

@@ -216,9 +216,26 @@ def store_plugins():
     customization = config['plugin_customization']
     for plugin in _initialized_plugins:
         if isinstance(plugin, Store):
-            if not is_disabled(plugin):
             plugin.site_customization = customization.get(plugin.name, '')
             yield plugin
+
+def available_store_plugins():
+    for plugin in store_plugins():
+        if not is_disabled(plugin):
+            yield plugin
+
+def stores():
+    stores = set([])
+    for plugin in store_plugins():
+        stores.add(plugin.name)
+    return stores
+
+def available_stores():
+    stores = set([])
+    for plugin in available_store_plugins():
+        stores.add(plugin.name)
+    return stores
+
 # }}}
 
 # Metadata read/write {{{

src/calibre/devices/apple/driver.py

@@ -941,7 +941,7 @@ class ITUNES(DriverBase):
         # declared in use_plugboard_ext and a device name of ITUNES
         if DEBUG:
             self.log.info("ITUNES.set_plugboard()")
-            #self.log.info('  using plugboard %s' % plugboards)
+            #self.log.info('  plugboard: %s' % plugboards)
         self.plugboards = plugboards
         self.plugboard_func = pb_func
 
@@ -1052,7 +1052,6 @@ class ITUNES(DriverBase):
                     'title': metadata[i].title,
                      'uuid': metadata[i].uuid }
 
-
                 # Report progress
                 if self.report_progress is not None:
                     self.report_progress((i+1)/file_count, _('%d of %d') % (i+1, file_count))
@@ -2744,7 +2743,7 @@ class ITUNES(DriverBase):
         # Update metadata from plugboard
         # If self.plugboard is None (no transforms), original metadata is returned intact
         metadata_x = self._xform_metadata_via_plugboard(metadata, this_book.format)
-
+        self.log("metadata.title_sort: %s  metadata_x.title_sort: %s" % (metadata.title_sort, metadata_x.title_sort))
         if isosx:
             if lb_added:
                 lb_added.name.set(metadata_x.title)
@@ -2754,8 +2753,7 @@ class ITUNES(DriverBase):
                 lb_added.description.set("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S')))
                 lb_added.enabled.set(True)
                 lb_added.sort_artist.set(icu_title(metadata_x.author_sort))
-                lb_added.sort_name.set(metadata.title_sort)
-
+                lb_added.sort_name.set(metadata_x.title_sort)
 
             if db_added:
                 db_added.name.set(metadata_x.title)
@@ -2765,7 +2763,7 @@ class ITUNES(DriverBase):
                 db_added.description.set("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S')))
                 db_added.enabled.set(True)
                 db_added.sort_artist.set(icu_title(metadata_x.author_sort))
-                db_added.sort_name.set(metadata.title_sort)
+                db_added.sort_name.set(metadata_x.title_sort)
 
             if metadata_x.comments:
                 if lb_added:
@@ -2785,6 +2783,7 @@ class ITUNES(DriverBase):
 
             # Set genre from series if available, else first alpha tag
             # Otherwise iTunes grabs the first dc:subject from the opf metadata
+            # If title_sort applied in plugboard, that overrides using series/index as title_sort
             if metadata_x.series and self.settings().extra_customization[self.USE_SERIES_AS_CATEGORY]:
                 if DEBUG:
                     self.log.info(" ITUNES._update_iTunes_metadata()")
@@ -2796,6 +2795,8 @@ class ITUNES(DriverBase):
                 fraction = index-integer
                 series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0'))
                 if lb_added:
+                    # If no title_sort plugboard tweak, create sort_name from series/index
+                    if metadata.title_sort == metadata_x.title_sort:
                         lb_added.sort_name.set("%s %s" % (self.title_sorter(metadata_x.series), series_index))
                     lb_added.episode_ID.set(metadata_x.series)
                     lb_added.episode_number.set(metadata_x.series_index)
@@ -2810,6 +2811,8 @@ class ITUNES(DriverBase):
                                 break
 
                 if db_added:
+                    # If no title_sort plugboard tweak, create sort_name from series/index
+                    if metadata.title_sort == metadata_x.title_sort:
                         db_added.sort_name.set("%s %s" % (self.title_sorter(metadata_x.series), series_index))
                     db_added.episode_ID.set(metadata_x.series)
                     db_added.episode_number.set(metadata_x.series_index)
@@ -2845,7 +2848,7 @@ class ITUNES(DriverBase):
                 lb_added.Description = ("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S')))
                 lb_added.Enabled = True
                 lb_added.SortArtist = icu_title(metadata_x.author_sort)
-                lb_added.SortName = metadata.title_sort
+                lb_added.SortName = metadata_x.title_sort
 
             if db_added:
                 db_added.Name = metadata_x.title
@@ -2855,7 +2858,7 @@ class ITUNES(DriverBase):
                 db_added.Description = ("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S')))
                 db_added.Enabled = True
                 db_added.SortArtist = icu_title(metadata_x.author_sort)
-                db_added.SortName = metadata.title_sort
+                db_added.SortName = metadata_x.title_sort
 
             if metadata_x.comments:
                 if lb_added:
@@ -2888,6 +2891,8 @@ class ITUNES(DriverBase):
                 fraction = index-integer
                 series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0'))
                 if lb_added:
+                    # If no title_sort plugboard tweak, create sort_name from series/index
+                    if metadata.title_sort == metadata_x.title_sort:
                         lb_added.SortName = "%s %s" % (self.title_sorter(metadata_x.series), series_index)
                     lb_added.EpisodeID = metadata_x.series
 
@@ -2914,6 +2919,8 @@ class ITUNES(DriverBase):
                                 break
 
                 if db_added:
+                    # If no title_sort plugboard tweak, create sort_name from series/index
+                    if metadata.title_sort == metadata_x.title_sort:
                         db_added.SortName = "%s %s" % (self.title_sorter(metadata_x.series), series_index)
                     db_added.EpisodeID = metadata_x.series
 
@@ -2975,6 +2982,9 @@ class ITUNES(DriverBase):
                                            newmi.publisher if book.publisher != newmi.publisher else ''))
                 self.log.info("        tags: %s %s" % (book.tags, ">>> %s" %
                                            newmi.tags if book.tags != newmi.tags else ''))
+            else:
+                self.log("  matching plugboard not found")
+
         else:
             newmi = book
         return newmi

src/calibre/devices/eb600/driver.py

@@ -95,9 +95,8 @@ class POCKETBOOK360(EB600):
 
     FORMATS = ['epub', 'fb2', 'prc', 'mobi', 'pdf', 'djvu', 'rtf', 'chm', 'txt']
 
-    VENDOR_NAME = 'PHILIPS'
-    WINDOWS_MAIN_MEM = 'MASS_STORGE'
-    WINDOWS_CARD_A_MEM = 'MASS_STORGE'
+    VENDOR_NAME = ['PHILIPS', '__POCKET']
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['MASS_STORGE', 'BOOK_USB_STORAGE']
 
     OSX_MAIN_MEM   = 'Philips Mass Storge Media'
     OSX_CARD_A_MEM = 'Philips Mass Storge Media'

src/calibre/devices/libusb.py

@@ -8,10 +8,10 @@ from ctypes import cdll, POINTER, byref, pointer, Structure as _Structure, \
                    c_ubyte, c_ushort, c_int, c_char, c_void_p, c_byte, c_uint
 from errno import EBUSY, ENOMEM
 
-from calibre import iswindows, isosx, isfreebsd, load_library
+from calibre import iswindows, isosx, isbsd, load_library
 
 _libusb_name = 'libusb'
-PATH_MAX = 511 if iswindows else 1024 if (isosx or isfreebsd) else 4096
+PATH_MAX = 511 if iswindows else 1024 if (isosx or isbsd) else 4096
 if iswindows:
     class Structure(_Structure):
         _pack_ = 1

src/calibre/devices/misc.py

@@ -269,8 +269,8 @@ class NEXTBOOK(USBMS):
 
     EBOOK_DIR_MAIN = ''
 
-    VENDOR_NAME = 'NEXT2'
-    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = '1.0.14'
+    VENDOR_NAME = ['NEXT2', 'BK7005']
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['1.0.14', 'PLAYER']
     SUPPORTS_SUB_DIRS = True
     THUMBNAIL_HEIGHT = 120
 

src/calibre/devices/usbms/device.py

@@ -926,8 +926,8 @@ class Device(DeviceConfig, DevicePlugin):
         if not isinstance(template, unicode):
             template = template.decode('utf-8')
         app_id = str(getattr(mdata, 'application_id', ''))
-        # The db id will be in the created filename
-        extra_components = get_components(template, mdata, fname,
+        id_ = mdata.get('id', fname)
+        extra_components = get_components(template, mdata, id_,
                 timefmt=opts.send_timefmt, length=maxlen-len(app_id)-1)
         if not extra_components:
             extra_components.append(sanitize(self.filename_callback(fname,

src/calibre/ebooks/html/input.py

@@ -20,7 +20,7 @@ from itertools import izip
 from calibre.customize.conversion import InputFormatPlugin
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.customize.conversion import OptionRecommendation
-from calibre.constants import islinux, isfreebsd, iswindows
+from calibre.constants import islinux, isbsd, iswindows
 from calibre import unicode_path, as_unicode
 from calibre.utils.localization import get_lang
 from calibre.utils.filenames import ascii_filename
@@ -302,7 +302,7 @@ class HTMLInput(InputFormatPlugin):
         if getattr(self, '_is_case_sensitive', None) is not None:
             return self._is_case_sensitive
         if not path or not os.path.exists(path):
-            return islinux or isfreebsd
+            return islinux or isbsd
         self._is_case_sensitive = not (os.path.exists(path.lower()) \
                 and os.path.exists(path.upper()))
         return self._is_case_sensitive

src/calibre/ebooks/metadata/sources/amazon.py

@@ -549,7 +549,8 @@ class Amazon(Source):
                     r'//div[@id="Results"]/descendant::td[starts-with(@id, "search:Td:")]'):
                     for a in td.xpath(r'descendant::td[@class="dataColumn"]/descendant::a[@href]/span[@class="srTitle"]/..'):
                         title = tostring(a, method='text', encoding=unicode).lower()
-                        if 'bulk pack' not in title:
+                        if ('bulk pack' not in title and '[audiobook]' not in
+                                title and '[audio cd]' not in title):
                             matches.append(a.get('href'))
                         break
 

src/calibre/ebooks/metadata/sources/base.py

@@ -313,7 +313,7 @@ class Source(Plugin):
             title_patterns = [(re.compile(pat, re.IGNORECASE), repl) for pat, repl in
             [
                 # Remove things like: (2010) (Omnibus) etc.
-                (r'(?i)[({\[](\d{4}|omnibus|anthology|hardcover|paperback|turtleback|mass\s*market|edition|ed\.)[\])}]', ''),
+                (r'(?i)[({\[](\d{4}|omnibus|anthology|hardcover|audiobook|audio\scd|paperback|turtleback|mass\s*market|edition|ed\.)[\])}]', ''),
                 # Remove any strings that contain the substring edition inside
                 # parentheses
                 (r'(?i)[({\[].*?(edition|ed.).*?[\]})]', ''),

src/calibre/ebooks/pdf/pdftohtml.py

@@ -13,7 +13,7 @@ from functools import partial
 
 from calibre.ebooks import ConversionError, DRMError
 from calibre.ptempfile import PersistentTemporaryFile
-from calibre.constants import isosx, iswindows, islinux, isfreebsd
+from calibre.constants import isosx, iswindows, islinux, isbsd
 from calibre import CurrentDir
 
 PDFTOHTML = 'pdftohtml'
@@ -23,7 +23,7 @@ if isosx and hasattr(sys, 'frameworks_dir'):
 if iswindows and hasattr(sys, 'frozen'):
     PDFTOHTML = os.path.join(os.path.dirname(sys.executable), 'pdftohtml.exe')
     popen = partial(subprocess.Popen, creationflags=0x08) # CREATE_NO_WINDOW=0x08 so that no ugly console is popped up
-if (islinux or isfreebsd) and getattr(sys, 'frozen', False):
+if (islinux or isbsd) and getattr(sys, 'frozen', False):
     PDFTOHTML = os.path.join(sys.executables_location, 'bin', 'pdftohtml')
 
 def pdftohtml(output_dir, pdf_path, no_images):
@@ -43,7 +43,7 @@ def pdftohtml(output_dir, pdf_path, no_images):
         # This is neccessary as pdftohtml doesn't always (linux) respect absolute paths
         pdf_path = os.path.abspath(pdf_path)
         cmd = [PDFTOHTML, '-enc', 'UTF-8', '-noframes', '-p', '-nomerge', '-nodrm', '-q', pdf_path, os.path.basename(index)]
-        if isfreebsd:
+        if isbsd:
             cmd.remove('-nodrm')
         if no_images:
             cmd.append('-i')

src/calibre/gui2/__init__.py

@@ -12,7 +12,7 @@ from PyQt4.Qt import (QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt,
 
 ORG_NAME = 'KovidsBrain'
 APP_UID  = 'libprs500'
-from calibre.constants import islinux, iswindows, isfreebsd, isfrozen, isosx
+from calibre.constants import islinux, iswindows, isbsd, isfrozen, isosx
 from calibre.utils.config import Config, ConfigProxy, dynamic, JSONConfig
 from calibre.utils.localization import set_qt_translator
 from calibre.ebooks.metadata import MetaInformation
@@ -628,7 +628,7 @@ class Application(QApplication):
         st = self.style()
         if st is not None:
             st = unicode(st.objectName()).lower()
-        if (islinux or isfreebsd) and st in ('windows', 'motif', 'cde'):
+        if (islinux or isbsd) and st in ('windows', 'motif', 'cde'):
             from PyQt4.Qt import QStyleFactory
             styles = set(map(unicode, QStyleFactory.keys()))
             if 'Plastique' in styles and os.environ.get('KDE_FULL_SESSION',
@@ -697,7 +697,7 @@ def open_local_file(path):
 
 def is_ok_to_use_qt():
     global gui_thread, _store_app
-    if (islinux or isfreebsd) and ':' not in os.environ.get('DISPLAY', ''):
+    if (islinux or isbsd) and ':' not in os.environ.get('DISPLAY', ''):
         return False
     if _store_app is None and QApplication.instance() is None:
         _store_app = QApplication([])

src/calibre/gui2/actions/copy_to_library.py

@@ -16,7 +16,7 @@ from calibre.gui2 import error_dialog, Dispatcher, warning_dialog
 from calibre.gui2.dialogs.progress import ProgressDialog
 from calibre.utils.config import prefs, tweaks
 
-class Worker(Thread):
+class Worker(Thread): # {{{
 
     def __init__(self, ids, db, loc, progress, done, delete_after):
         Thread.__init__(self)
@@ -75,7 +75,7 @@ class Worker(Thread):
                 if co is not None:
                     newdb.set_conversion_options(x, 'PIPE', co)
             self.processed.add(x)
-
+# }}}
 
 class CopyToLibraryAction(InterfaceAction):
 

src/calibre/gui2/actions/delete.py

@@ -9,11 +9,12 @@ from functools import partial
 
 from PyQt4.Qt import QMenu, QObject, QTimer
 
-from calibre.gui2 import error_dialog
+from calibre.gui2 import error_dialog, question_dialog
 from calibre.gui2.dialogs.delete_matching_from_device import DeleteMatchingFromDeviceDialog
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.dialogs.confirm_delete_location import confirm_location
 from calibre.gui2.actions import InterfaceAction
+from calibre.utils.recycle_bin import can_recycle
 
 single_shot = partial(QTimer.singleShot, 10)
 
@@ -24,6 +25,15 @@ class MultiDeleter(QObject):
         QObject.__init__(self, gui)
         self.model = gui.library_view.model()
         self.ids = ids
+        self.permanent = False
+        if can_recycle and len(ids) > 100:
+            if question_dialog(gui, _('Are you sure?'), '<p>'+
+                _('You are trying to delete %d books. '
+                    'Sending so many files to the Recycle'
+                    ' Bin <b>can be slow</b>. Should calibre skip the'
+                    ' Recycle Bin? If you click Yes the files'
+                    ' will be <b>permanently deleted</b>.')%len(ids)):
+                self.permanent = True
         self.gui = gui
         self.failures = []
         self.deleted_ids = []
@@ -44,7 +54,8 @@ class MultiDeleter(QObject):
             title_ = self.model.db.title(id_, index_is_id=True)
             if title_:
                 title = title_
-            self.model.db.delete_book(id_, notify=False, commit=False)
+            self.model.db.delete_book(id_, notify=False, commit=False,
+                    permanent=self.permanent)
             self.deleted_ids.append(id_)
         except:
             import traceback

src/calibre/gui2/bars.py

@@ -0,0 +1,316 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+from PyQt4.Qt import (QObject, QToolBar, Qt, QSize, QToolButton, QVBoxLayout,
+        QLabel, QWidget, QAction, QMenuBar, QMenu)
+
+from calibre.constants import isosx
+from calibre.gui2 import gprefs
+
+class ToolBar(QToolBar): # {{{
+
+    def __init__(self, donate, location_manager, parent):
+        QToolBar.__init__(self, parent)
+        self.setContextMenuPolicy(Qt.PreventContextMenu)
+        self.setMovable(False)
+        self.setFloatable(False)
+        self.setOrientation(Qt.Horizontal)
+        self.setAllowedAreas(Qt.TopToolBarArea|Qt.BottomToolBarArea)
+        self.setStyleSheet('QToolButton:checked { font-weight: bold }')
+        self.preferred_width = self.sizeHint().width()
+        self.gui = parent
+        self.donate_button = donate
+        self.added_actions = []
+
+        self.location_manager = location_manager
+        donate.setAutoRaise(True)
+        donate.setCursor(Qt.PointingHandCursor)
+        self.setAcceptDrops(True)
+        self.showing_donate = False
+
+    def resizeEvent(self, ev):
+        QToolBar.resizeEvent(self, ev)
+        style = self.get_text_style()
+        self.setToolButtonStyle(style)
+        if hasattr(self, 'd_widget') and hasattr(self.d_widget, 'filler'):
+            self.d_widget.filler.setVisible(style != Qt.ToolButtonIconOnly)
+
+    def get_text_style(self):
+        style = Qt.ToolButtonTextUnderIcon
+        s = gprefs['toolbar_icon_size']
+        if s != 'off':
+            p = gprefs['toolbar_text']
+            if p == 'never':
+                style = Qt.ToolButtonIconOnly
+            elif p == 'auto' and self.preferred_width > self.width()+35:
+                style = Qt.ToolButtonIconOnly
+        return style
+
+    def contextMenuEvent(self, *args):
+        pass
+
+    def update_lm_actions(self):
+        for ac in self.added_actions:
+            if ac in self.location_manager.all_actions:
+                ac.setVisible(ac in self.location_manager.available_actions)
+
+    def init_bar(self, actions):
+        self.showing_donate = False
+        for ac in self.added_actions:
+            m = ac.menu()
+            if m is not None:
+                m.setVisible(False)
+
+        self.clear()
+        self.added_actions = []
+
+        bar = self
+
+        for what in actions:
+            if what is None:
+                bar.addSeparator()
+            elif what == 'Location Manager':
+                for ac in self.location_manager.all_actions:
+                    bar.addAction(ac)
+                    bar.added_actions.append(ac)
+                    bar.setup_tool_button(bar, ac, QToolButton.MenuButtonPopup)
+                    ac.setVisible(False)
+            elif what == 'Donate':
+                self.d_widget = QWidget()
+                self.d_widget.setLayout(QVBoxLayout())
+                self.d_widget.layout().addWidget(self.donate_button)
+                if isosx:
+                    self.d_widget.setStyleSheet('QWidget, QToolButton {background-color: none; border: none; }')
+                    self.d_widget.layout().setContentsMargins(0,0,0,0)
+                    self.d_widget.setContentsMargins(0,0,0,0)
+                    self.d_widget.filler = QLabel(u'\u00a0')
+                    self.d_widget.layout().addWidget(self.d_widget.filler)
+                bar.addWidget(self.d_widget)
+                self.showing_donate = True
+            elif what in self.gui.iactions:
+                action = self.gui.iactions[what]
+                bar.addAction(action.qaction)
+                self.added_actions.append(action.qaction)
+                self.setup_tool_button(bar, action.qaction, action.popup_type)
+        self.preferred_width = self.sizeHint().width()
+
+    def setup_tool_button(self, bar, ac, menu_mode=None):
+        ch = bar.widgetForAction(ac)
+        if ch is None:
+            ch = self.child_bar.widgetForAction(ac)
+        ch.setCursor(Qt.PointingHandCursor)
+        ch.setAutoRaise(True)
+        if ac.menu() is not None and menu_mode is not None:
+            ch.setPopupMode(menu_mode)
+        return ch
+
+    #support drag&drop from/to library from/to reader/card
+    def dragEnterEvent(self, event):
+        md = event.mimeData()
+        if md.hasFormat("application/calibre+from_library") or \
+           md.hasFormat("application/calibre+from_device"):
+            event.setDropAction(Qt.CopyAction)
+            event.accept()
+        else:
+            event.ignore()
+
+    def dragMoveEvent(self, event):
+        allowed = False
+        md = event.mimeData()
+        #Drop is only allowed in the location manager widget's different from the selected one
+        for ac in self.location_manager.available_actions:
+            w = self.widgetForAction(ac)
+            if w is not None:
+                if ( md.hasFormat("application/calibre+from_library") or \
+                     md.hasFormat("application/calibre+from_device") ) and \
+                        w.geometry().contains(event.pos()) and \
+                        isinstance(w, QToolButton) and not w.isChecked():
+                    allowed = True
+                    break
+        if allowed:
+            event.acceptProposedAction()
+        else:
+            event.ignore()
+
+    def dropEvent(self, event):
+        data = event.mimeData()
+
+        mime = 'application/calibre+from_library'
+        if data.hasFormat(mime):
+            ids = list(map(int, str(data.data(mime)).split()))
+            tgt = None
+            for ac in self.location_manager.available_actions:
+                w = self.widgetForAction(ac)
+                if w is not None and w.geometry().contains(event.pos()):
+                    tgt = ac.calibre_name
+            if tgt is not None:
+                if tgt == 'main':
+                    tgt = None
+                self.gui.sync_to_device(tgt, False, send_ids=ids)
+                event.accept()
+
+        mime = 'application/calibre+from_device'
+        if data.hasFormat(mime):
+            paths = [unicode(u.toLocalFile()) for u in data.urls()]
+            if paths:
+                self.gui.iactions['Add Books'].add_books_from_device(
+                        self.gui.current_view(), paths=paths)
+                event.accept()
+
+# }}}
+
+class MenuAction(QAction): # {{{
+
+    def __init__(self, clone, parent):
+        QAction.__init__(self, clone.text(), parent)
+        self.clone = clone
+        clone.changed.connect(self.clone_changed)
+
+    def clone_changed(self):
+        self.setText(self.clone.text())
+# }}}
+
+class MenuBar(QMenuBar): # {{{
+
+    def __init__(self, location_manager, parent):
+        QMenuBar.__init__(self, parent)
+        self.gui = parent
+        self.setNativeMenuBar(True)
+
+        self.location_manager = location_manager
+        self.added_actions = []
+
+        self.donate_action = QAction(_('Donate'), self)
+        self.donate_menu = QMenu()
+        self.donate_menu.addAction(self.gui.donate_action)
+        self.donate_action.setMenu(self.donate_menu)
+
+    def update_lm_actions(self):
+        for ac in self.added_actions:
+            if ac in self.location_manager.all_actions:
+                ac.setVisible(ac in self.location_manager.available_actions)
+
+    def init_bar(self, actions):
+        for ac in self.added_actions:
+            m = ac.menu()
+            if m is not None:
+                m.setVisible(False)
+
+        self.clear()
+        self.added_actions = []
+
+        for what in actions:
+            if what is None:
+                continue
+            elif what == 'Location Manager':
+                for ac in self.location_manager.all_actions:
+                    ac = self.build_menu(ac)
+                    self.addAction(ac)
+                    self.added_actions.append(ac)
+                    ac.setVisible(False)
+            elif what == 'Donate':
+                self.addAction(self.donate_action)
+            elif what in self.gui.iactions:
+                action = self.gui.iactions[what]
+                ac = self.build_menu(action.qaction)
+                self.addAction(ac)
+                self.added_actions.append(ac)
+
+    def build_menu(self, action):
+        m = action.menu()
+        ac = MenuAction(action, self)
+        if m is None:
+            m = QMenu()
+            m.addAction(action)
+        ac.setMenu(m)
+        return ac
+
+# }}}
+
+class BarsManager(QObject):
+
+    def __init__(self, donate_button, location_manager, parent):
+        QObject.__init__(self, parent)
+        self.donate_button, self.location_manager = (donate_button,
+                location_manager)
+
+        bars = [ToolBar(donate_button, location_manager, parent) for i in
+                range(3)]
+        self.main_bars = tuple(bars[:2])
+        self.child_bars = tuple(bars[2:])
+
+        self.menu_bar = MenuBar(self.location_manager, self.parent())
+        self.parent().setMenuBar(self.menu_bar)
+
+        self.apply_settings()
+        self.init_bars()
+
+    def database_changed(self, db):
+        pass
+
+    @property
+    def bars(self):
+        for x in self.main_bars + self.child_bars:
+            yield x
+
+    @property
+    def showing_donate(self):
+        for b in self.bars:
+            if b.isVisible() and b.showing_donate:
+                return True
+        return False
+
+    def init_bars(self):
+        self.bar_actions = tuple(
+            [gprefs['action-layout-toolbar'+x] for x in ('', '-device')] +
+            [gprefs['action-layout-toolbar-child']] +
+            [gprefs['action-layout-menubar']] +
+            [gprefs['action-layout-menubar-device']]
+        )
+
+        for bar, actions in zip(self.bars, self.bar_actions[:3]):
+            bar.init_bar(actions)
+
+    def update_bars(self):
+        '''
+        This shows the correct main toolbar and rebuilds the menubar based on
+        whether a device is connected or not. Note that the toolbars are
+        explicitly not rebuilt, this is to workaround a Qt limitation iwth
+        QToolButton's popup menus and modal dialogs. If you want the toolbars
+        rebuilt, call init_bars().
+        '''
+        showing_device = self.location_manager.has_device
+        main_bar = self.main_bars[1 if showing_device else 0]
+        child_bar = self.child_bars[0]
+        for bar in self.bars:
+            bar.setVisible(False)
+            bar.update_lm_actions()
+        if main_bar.added_actions:
+            main_bar.setVisible(True)
+        if child_bar.added_actions:
+            child_bar.setVisible(True)
+
+        self.menu_bar.init_bar(self.bar_actions[4 if showing_device else 3])
+        self.menu_bar.update_lm_actions()
+        self.menu_bar.setVisible(bool(self.menu_bar.added_actions))
+
+    def apply_settings(self):
+        sz = gprefs['toolbar_icon_size']
+        sz = {'off':0, 'small':24, 'medium':48, 'large':64}[sz]
+        style = Qt.ToolButtonTextUnderIcon
+        if sz > 0 and gprefs['toolbar_text'] == 'never':
+            style = Qt.ToolButtonIconOnly
+
+        for bar in self.bars:
+            bar.setIconSize(QSize(sz, sz))
+            bar.setToolButtonStyle(style)
+        self.donate_button.set_normal_icon_size(sz, sz)
+
+

src/calibre/gui2/device.py

@@ -751,6 +751,7 @@ class DeviceMixin(object): # {{{
             if self.current_view() != self.library_view:
                 self.book_details.reset_info()
             self.location_manager.update_devices()
+            self.bars_manager.update_bars()
             self.library_view.set_device_connected(self.device_connected)
             self.refresh_ondevice()
         device_signals.device_connection_changed.emit(connected)
@@ -764,6 +765,7 @@ class DeviceMixin(object): # {{{
         info, cp, fs = job.result
         self.location_manager.update_devices(cp, fs,
                 self.device_manager.device.icon)
+        self.bars_manager.update_bars()
         self.status_bar.device_connected(info[0])
         self.device_manager.books(Dispatcher(self.metadata_downloaded))
 

src/calibre/gui2/dialogs/edit_authors_dialog.ui

@@ -78,22 +78,22 @@
     </widget>
    </item>
    <item>
-    <layout class="QHBoxLayout" name="horizontalLayout">
-     <item>
+    <layout class="QGridLayout">
+     <item row="0" column="0">
       <widget class="QPushButton" name="sort_by_author">
        <property name="text">
         <string>Sort by author</string>
        </property>
       </widget>
      </item>
-     <item>
+     <item row="0" column="1">
       <widget class="QPushButton" name="sort_by_author_sort">
        <property name="text">
         <string>Sort by author sort</string>
        </property>
       </widget>
      </item>
-     <item>
+     <item row="1" column="0">
       <widget class="QPushButton" name="recalc_author_sort">
        <property name="toolTip">
         <string>Reset all the author sort values to a value automatically
@@ -105,7 +105,7 @@ generated can be controlled via Preferences-&gt;Advanced-&gt;Tweaks</string>
        </property>
       </widget>
      </item>
-     <item>
+     <item row="1" column="1">
       <widget class="QPushButton" name="auth_sort_to_author">
        <property name="toolTip">
         <string>Copy author sort to author for every author. You typically use this button
@@ -116,20 +116,7 @@ after changing Preferences-&gt;Advanced-&gt;Tweaks-&gt;Author sort name algorith
        </property>
       </widget>
      </item>
-     <item>
-      <spacer name="horizontalSpacer_3">
-       <property name="orientation">
-        <enum>Qt::Horizontal</enum>
-       </property>
-       <property name="sizeHint" stdset="0">
-        <size>
-         <width>40</width>
-         <height>20</height>
-        </size>
-       </property>
-      </spacer>
-     </item>
-     <item>
+     <item row="1" column="2">
       <widget class="QDialogButtonBox" name="buttonBox">
        <property name="sizePolicy">
         <sizepolicy hsizetype="Minimum" vsizetype="Fixed">

src/calibre/gui2/dialogs/template_dialog.py

@@ -5,10 +5,193 @@ __license__   = 'GPL v3'
 
 import json
 
-from PyQt4.Qt import Qt, QDialog, QDialogButtonBox
+from PyQt4.Qt import (Qt, QDialog, QDialogButtonBox, QSyntaxHighlighter,
+                      QRegExp, QApplication,
+                      QTextCharFormat, QFont, QColor, QCursor)
+
 from calibre.gui2.dialogs.template_dialog_ui import Ui_TemplateDialog
 from calibre.utils.formatter_functions import formatter_functions
 
+class ParenPosition:
+
+    def __init__(self, block, pos, paren):
+        self.block = block
+        self.pos = pos
+        self.paren = paren
+        self.highlight = False
+
+    def set_highlight(self, to_what):
+        self.highlight = to_what
+
+class TemplateHighlighter(QSyntaxHighlighter):
+
+    Config = {}
+    Rules = []
+    Formats = {}
+    BN_FACTOR = 1000
+
+    KEYWORDS = ["program"]
+
+    def __init__(self, parent=None):
+        super(TemplateHighlighter, self).__init__(parent)
+
+        self.initializeFormats()
+
+        TemplateHighlighter.Rules.append((QRegExp(
+                "|".join([r"\b%s\b" % keyword for keyword in self.KEYWORDS])),
+                "keyword"))
+        TemplateHighlighter.Rules.append((QRegExp(
+                "|".join([r"\b%s\b" % builtin for builtin in
+                          formatter_functions.get_builtins()])),
+                "builtin"))
+
+        TemplateHighlighter.Rules.append((QRegExp(
+                r"\b[+-]?[0-9]+[lL]?\b"
+                r"|\b[+-]?0[xX][0-9A-Fa-f]+[lL]?\b"
+                r"|\b[+-]?[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?\b"),
+                "number"))
+
+        stringRe = QRegExp(r"""(?:[^:]'[^']*'|"[^"]*")""")
+        stringRe.setMinimal(True)
+        TemplateHighlighter.Rules.append((stringRe, "string"))
+
+        lparenRe = QRegExp(r'\(')
+        lparenRe.setMinimal(True)
+        TemplateHighlighter.Rules.append((lparenRe, "lparen"))
+        rparenRe = QRegExp(r'\)')
+        rparenRe.setMinimal(True)
+        TemplateHighlighter.Rules.append((rparenRe, "rparen"))
+
+        self.regenerate_paren_positions()
+        self.highlighted_paren = False
+
+    def initializeFormats(self):
+        Config = self.Config
+        Config["fontfamily"] = "monospace"
+        #Config["fontsize"] = 10
+        for name, color, bold, italic in (
+                ("normal", "#000000", False, False),
+                ("keyword", "#000080", True, False),
+                ("builtin", "#0000A0", False, False),
+                ("comment", "#007F00", False, True),
+                ("string", "#808000", False, False),
+                ("number", "#924900", False, False),
+                ("lparen", "#000000", True, True),
+                ("rparen", "#000000", True, True)):
+            Config["%sfontcolor" % name] = color
+            Config["%sfontbold" % name] = bold
+            Config["%sfontitalic" % name] = italic
+
+        baseFormat = QTextCharFormat()
+        baseFormat.setFontFamily(Config["fontfamily"])
+        #baseFormat.setFontPointSize(Config["fontsize"])
+
+        for name in ("normal", "keyword", "builtin", "comment",
+                     "string", "number", "lparen", "rparen"):
+            format = QTextCharFormat(baseFormat)
+            format.setForeground(QColor(Config["%sfontcolor" % name]))
+            if Config["%sfontbold" % name]:
+                format.setFontWeight(QFont.Bold)
+            format.setFontItalic(Config["%sfontitalic" % name])
+            self.Formats[name] = format
+
+    def find_paren(self, bn, pos):
+        dex = bn * self.BN_FACTOR + pos
+        return self.paren_pos_map.get(dex, None)
+
+    def highlightBlock(self, text):
+        bn = self.currentBlock().blockNumber()
+        textLength = text.length()
+
+        self.setFormat(0, textLength, self.Formats["normal"])
+
+        if text.isEmpty():
+            pass
+        elif text[0] == "#":
+            self.setFormat(0, text.length(), self.Formats["comment"])
+            return
+
+        for regex, format in TemplateHighlighter.Rules:
+            i = regex.indexIn(text)
+            while i >= 0:
+                length = regex.matchedLength()
+                if format in ['lparen', 'rparen']:
+                    pp = self.find_paren(bn, i)
+                    if pp and pp.highlight:
+                        self.setFormat(i, length, self.Formats[format])
+                else:
+                    self.setFormat(i, length, self.Formats[format])
+                i = regex.indexIn(text, i + length)
+
+        if self.generate_paren_positions:
+            t = unicode(text)
+            i = 0
+            foundQuote = False
+            while i < len(t):
+                c = t[i]
+                if c == ':':
+                    # Deal with the funky syntax of template program mode.
+                    # This won't work if there are more than one template
+                    # expression in the document.
+                    if not foundQuote and i+1 < len(t) and t[i+1] == "'":
+                        i += 2
+                elif c in ["'", '"']:
+                    foundQuote = True
+                    i += 1
+                    j = t[i:].find(c)
+                    if j < 0:
+                        i = len(t)
+                    else:
+                        i = i + j
+                elif c in ['(', ')']:
+                    pp = ParenPosition(bn, i, c)
+                    self.paren_positions.append(pp)
+                    self.paren_pos_map[bn*self.BN_FACTOR+i] = pp
+                i += 1
+
+    def rehighlight(self):
+        QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
+        QSyntaxHighlighter.rehighlight(self)
+        QApplication.restoreOverrideCursor()
+
+    def check_cursor_pos(self, chr, block, pos_in_block):
+        found_pp = -1
+        for i, pp in enumerate(self.paren_positions):
+            pp.set_highlight(False)
+            if pp.block == block and pp.pos == pos_in_block:
+                found_pp = i
+
+        if chr not in ['(', ')']:
+            if self.highlighted_paren:
+                self.rehighlight()
+                self.highlighted_paren = False
+            return
+
+        if found_pp >= 0:
+            stack = 0
+            if chr == '(':
+                list = self.paren_positions[found_pp+1:]
+            else:
+                list = reversed(self.paren_positions[0:found_pp])
+            for pp in list:
+                if pp.paren == chr:
+                    stack += 1;
+                elif stack:
+                    stack -= 1
+                else:
+                    pp.set_highlight(True)
+                    self.paren_positions[found_pp].set_highlight(True)
+                    break
+        self.highlighted_paren = True
+        self.rehighlight()
+
+    def regenerate_paren_positions(self):
+        self.generate_paren_positions = True
+        self.paren_positions = []
+        self.paren_pos_map = {}
+        self.rehighlight()
+        self.generate_paren_positions = False
+
 class TemplateDialog(QDialog, Ui_TemplateDialog):
 
     def __init__(self, parent, text):
@@ -20,6 +203,11 @@ class TemplateDialog(QDialog, Ui_TemplateDialog):
         self.setWindowFlags(self.windowFlags()&(~Qt.WindowContextHelpButtonHint))
         self.setWindowIcon(icon)
 
+        self.last_text = ''
+        self.highlighter = TemplateHighlighter(self.textbox.document())
+        self.textbox.cursorPositionChanged.connect(self.text_cursor_changed)
+        self.textbox.textChanged.connect(self.textbox_changed)
+
         self.textbox.setTabStopWidth(10)
         self.source_code.setTabStopWidth(10)
         self.documentation.setReadOnly(True)
@@ -46,6 +234,22 @@ class TemplateDialog(QDialog, Ui_TemplateDialog):
         self.function.setCurrentIndex(0)
         self.function.currentIndexChanged[str].connect(self.function_changed)
 
+    def textbox_changed(self):
+        cur_text = unicode(self.textbox.toPlainText())
+        if self.last_text != cur_text:
+            self.last_text = cur_text
+            self.highlighter.regenerate_paren_positions()
+
+    def text_cursor_changed(self):
+        cursor = self.textbox.textCursor()
+        block_number = cursor.blockNumber()
+        pos_in_block = cursor.positionInBlock()
+        position = cursor.position()
+        t = unicode(self.textbox.toPlainText())
+        if position < len(t):
+            self.highlighter.check_cursor_pos(t[position], block_number,
+                                              pos_in_block)
+
     def function_changed(self, toWhat):
         name = unicode(toWhat)
         self.source_code.clear()

src/calibre/gui2/dialogs/template_line_editor.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from PyQt4.Qt import (QLineEdit, QDialog, QGridLayout, QLabel,
+                      QDialogButtonBox, QColor, QComboBox, QIcon)
+
+from calibre.gui2.dialogs.template_dialog import TemplateDialog
+from calibre.gui2.complete import MultiCompleteLineEdit
+from calibre.gui2 import error_dialog
+
+class TemplateLineEditor(QLineEdit):
+
+    '''
+    Extend the context menu of a QLineEdit to include more actions.
+    '''
+
+    def __init__(self, parent):
+        QLineEdit.__init__(self, parent)
+        self.tags = None
+
+    def set_tags(self, tags):
+        self.tags = tags
+
+    def contextMenuEvent(self, event):
+        menu = self.createStandardContextMenu()
+        menu.addSeparator()
+
+        action_open_editor = menu.addAction(_('Open Template Editor'))
+        action_open_editor.triggered.connect(self.open_editor)
+        if self.tags:
+            action_tag_wizard = menu.addAction(_('Open Tag Wizard'))
+            action_tag_wizard.triggered.connect(self.tag_wizard)
+        menu.exec_(event.globalPos())
+
+    def open_editor(self):
+        t = TemplateDialog(self, self.text())
+        t.setWindowTitle(_('Edit template'))
+        if t.exec_():
+            self.setText(t.textbox.toPlainText())
+
+    def tag_wizard(self):
+        txt = unicode(self.text())
+        if txt and not txt.startswith('program:\n#tag wizard'):
+            error_dialog(self, _('Invalid text'),
+                 _('The text in the box was not generated by this wizard'),
+                 show=True, show_copy_button=False)
+            return
+        d = TagWizard(self, self.tags, unicode(self.text()))
+        if d.exec_():
+            self.setText(d.template)
+
+class TagWizard(QDialog):
+
+    def __init__(self, parent, tags, txt):
+        QDialog.__init__(self, parent)
+        self.setWindowTitle(_('Tag Wizard'))
+        self.setWindowIcon(QIcon(I('wizard.png')))
+
+        self.tags = tags
+        l = QGridLayout()
+        self.setLayout(l)
+        l.setColumnStretch(0, 1)
+        l.setColumnMinimumWidth(0, 300)
+        l.addWidget(QLabel(_('Tags (more than one per box permitted)')), 0, 0, 1, 1)
+        l.addWidget(QLabel(_('Color')), 0, 1, 1, 1)
+        self.tagboxes = []
+        self.colorboxes = []
+        self.colors = [unicode(s) for s in list(QColor.colorNames())]
+        self.colors.insert(0, '')
+        for i in range(0, 10):
+            tb = MultiCompleteLineEdit(self)
+            tb.set_separator(', ')
+            tb.update_items_cache(self.tags)
+            self.tagboxes.append(tb)
+            l.addWidget(tb, i+1, 0, 1, 1)
+            cb = QComboBox(self)
+            cb.addItems(self.colors)
+            self.colorboxes.append(cb)
+            l.addWidget(cb, i+1, 1, 1, 1)
+
+        if txt:
+            lines = txt.split('\n')[3:]
+            i = 0
+            for line in lines:
+                if line.startswith('#'):
+                    t,c = line[1:].split(':|:')
+                    try:
+                        self.colorboxes[i].setCurrentIndex(self.colorboxes[i].findText(c))
+                        self.tagboxes[i].setText(t)
+                    except:
+                        pass
+                    i += 1
+
+        bb = QDialogButtonBox(QDialogButtonBox.Ok|QDialogButtonBox.Cancel, parent=self)
+        l.addWidget(bb, 100, 1, 1, 1)
+        bb.accepted.connect(self.accepted)
+        bb.rejected.connect(self.reject)
+        self.template = ''
+
+    def accepted(self):
+        res = ("program:\n#tag wizard -- do not directly edit\n"
+               "  t = field('tags');\n  first_non_empty(\n")
+        lines = []
+        for tb, cb in zip(self.tagboxes, self.colorboxes):
+            tags = [t.strip() for t in unicode(tb.text()).split(',') if t.strip()]
+            tags = '$|^'.join(tags)
+            c = unicode(cb.currentText()).strip()
+            if not tags or not c:
+                continue
+            if c not in self.colors:
+                error_dialog(self, _('Invalid color'),
+                             _('The color {0} is not valid').format(c),
+                             show=True, show_copy_button=False)
+                return False
+            lines.append("    in_list(t, ',', '^{0}$', '{1}', '')".format(tags, c))
+        res += ',\n'.join(lines)
+        res += ')\n'
+        self.template = res
+        res = ''
+        for tb, cb in zip(self.tagboxes, self.colorboxes):
+            t = unicode(tb.text()).strip()
+            if t.endswith(','):
+                t = t[:-1]
+            c = unicode(cb.currentText()).strip()
+            if t and c:
+                res += '#' + t + ':|:' + c  + '\n'
+        self.template += res
+        self.accept()

src/calibre/gui2/layout.py

@@ -7,16 +7,17 @@ __docformat__ = 'restructuredtext en'
 
 from functools import partial
 
-from PyQt4.Qt import (QIcon, Qt, QWidget, QToolBar, QSize,
-    pyqtSignal, QToolButton, QMenu, QMenuBar, QAction,
+from PyQt4.Qt import (QIcon, Qt, QWidget, QSize,
+    pyqtSignal, QToolButton, QMenu,
     QObject, QVBoxLayout, QSizePolicy, QLabel, QHBoxLayout, QActionGroup)
 
 
-from calibre.constants import __appname__, isosx
+from calibre.constants import __appname__
 from calibre.gui2.search_box import SearchBox2, SavedSearchBox
 from calibre.gui2.throbber import ThrobbingButton
-from calibre.gui2 import gprefs
+from calibre.gui2.bars import BarsManager
 from calibre.gui2.widgets import ComboBoxWithHelp
+from calibre.utils.config_base import tweaks
 from calibre import human_readable
 
 class LocationManager(QObject): # {{{
@@ -35,6 +36,8 @@ class LocationManager(QObject): # {{{
         self._mem = []
         self.tooltips = {}
 
+        self.all_actions = []
+
         def ac(name, text, icon, tooltip):
             icon = QIcon(I(icon))
             ac = self.location_actions.addAction(icon, text)
@@ -59,6 +62,7 @@ class LocationManager(QObject): # {{{
             ac.setMenu(m)
             ac.calibre_name = name
 
+            self.all_actions.append(ac)
             return ac
 
         self.library_action = ac('library', _('Library'), 'lt.png',
@@ -234,259 +238,6 @@ class Spacer(QWidget): # {{{
         self.l.addStretch(10)
 # }}}
 
-class MenuAction(QAction): # {{{
-
-    def __init__(self, clone, parent):
-        QAction.__init__(self, clone.text(), parent)
-        self.clone = clone
-        clone.changed.connect(self.clone_changed)
-
-    def clone_changed(self):
-        self.setText(self.clone.text())
-# }}}
-
-class MenuBar(QMenuBar): # {{{
-
-    def __init__(self, location_manager, parent):
-        QMenuBar.__init__(self, parent)
-        self.gui = parent
-        self.setNativeMenuBar(True)
-
-        self.location_manager = location_manager
-        self.location_manager.locations_changed.connect(self.build_bar)
-        self.added_actions = []
-
-        self.donate_action = QAction(_('Donate'), self)
-        self.donate_menu = QMenu()
-        self.donate_menu.addAction(self.gui.donate_action)
-        self.donate_action.setMenu(self.donate_menu)
-        self.build_bar()
-
-    def build_bar(self, changed_action=None):
-        showing_device = self.location_manager.has_device
-        actions = '-device' if showing_device else ''
-        actions = gprefs['action-layout-menubar'+actions]
-
-        show_main = len(actions) > 0
-        self.setVisible(show_main)
-
-        for ac in self.added_actions:
-            m = ac.menu()
-            if m is not None:
-                m.setVisible(False)
-
-        self.clear()
-        self.added_actions = []
-        self.action_map = {}
-
-        for what in actions:
-            if what is None:
-                continue
-            elif what == 'Location Manager':
-                for ac in self.location_manager.available_actions:
-                    ac = self.build_menu(ac)
-                    self.addAction(ac)
-                    self.added_actions.append(ac)
-            elif what == 'Donate':
-                self.addAction(self.donate_action)
-            elif what in self.gui.iactions:
-                action = self.gui.iactions[what]
-                ac = self.build_menu(action.qaction)
-                self.addAction(ac)
-                self.added_actions.append(ac)
-
-    def build_menu(self, action):
-        m = action.menu()
-        ac = MenuAction(action, self)
-        if m is None:
-            m = QMenu()
-            m.addAction(action)
-        ac.setMenu(m)
-        return ac
-
-# }}}
-
-class BaseToolBar(QToolBar): # {{{
-
-    def __init__(self, parent):
-        QToolBar.__init__(self, parent)
-        self.setContextMenuPolicy(Qt.PreventContextMenu)
-        self.setMovable(False)
-        self.setFloatable(False)
-        self.setOrientation(Qt.Horizontal)
-        self.setAllowedAreas(Qt.TopToolBarArea|Qt.BottomToolBarArea)
-        self.setStyleSheet('QToolButton:checked { font-weight: bold }')
-        self.preferred_width = self.sizeHint().width()
-
-    def resizeEvent(self, ev):
-        QToolBar.resizeEvent(self, ev)
-        style = self.get_text_style()
-        self.setToolButtonStyle(style)
-        if hasattr(self, 'd_widget') and hasattr(self.d_widget, 'filler'):
-            self.d_widget.filler.setVisible(style != Qt.ToolButtonIconOnly)
-
-    def get_text_style(self):
-        style = Qt.ToolButtonTextUnderIcon
-        s = gprefs['toolbar_icon_size']
-        if s != 'off':
-            p = gprefs['toolbar_text']
-            if p == 'never':
-                style = Qt.ToolButtonIconOnly
-            elif p == 'auto' and self.preferred_width > self.width()+35:
-                style = Qt.ToolButtonIconOnly
-        return style
-
-    def contextMenuEvent(self, *args):
-        pass
-
-# }}}
-
-class ToolBar(BaseToolBar): # {{{
-
-    def __init__(self, donate, location_manager, child_bar, parent):
-        BaseToolBar.__init__(self, parent)
-        self.gui = parent
-        self.child_bar = child_bar
-        self.donate_button = donate
-        self.apply_settings()
-
-        self.location_manager = location_manager
-        self.location_manager.locations_changed.connect(self.build_bar)
-        donate.setAutoRaise(True)
-        donate.setCursor(Qt.PointingHandCursor)
-        self.added_actions = []
-        self.build_bar()
-        self.setAcceptDrops(True)
-
-    def apply_settings(self):
-        sz = gprefs['toolbar_icon_size']
-        sz = {'off':0, 'small':24, 'medium':48, 'large':64}[sz]
-        self.setIconSize(QSize(sz, sz))
-        self.child_bar.setIconSize(QSize(sz, sz))
-        style = Qt.ToolButtonTextUnderIcon
-        if sz > 0 and gprefs['toolbar_text'] == 'never':
-            style = Qt.ToolButtonIconOnly
-        self.setToolButtonStyle(style)
-        self.child_bar.setToolButtonStyle(style)
-        self.donate_button.set_normal_icon_size(sz, sz)
-
-    def build_bar(self):
-        self.showing_donate = False
-        showing_device = self.location_manager.has_device
-        mactions = '-device' if showing_device else ''
-        mactions = gprefs['action-layout-toolbar'+mactions]
-        cactions = gprefs['action-layout-toolbar-child']
-
-        show_main = len(mactions) > 0
-        self.setVisible(show_main)
-        show_child = len(cactions) > 0
-        self.child_bar.setVisible(show_child)
-
-        for ac in self.added_actions:
-            m = ac.menu()
-            if m is not None:
-                m.setVisible(False)
-
-        self.clear()
-        self.child_bar.clear()
-        self.added_actions = []
-
-        for bar, actions in ((self, mactions), (self.child_bar, cactions)):
-            for what in actions:
-                if what is None:
-                    bar.addSeparator()
-                elif what == 'Location Manager':
-                    for ac in self.location_manager.available_actions:
-                        bar.addAction(ac)
-                        bar.added_actions.append(ac)
-                        bar.setup_tool_button(bar, ac, QToolButton.MenuButtonPopup)
-                elif what == 'Donate':
-                    self.d_widget = QWidget()
-                    self.d_widget.setLayout(QVBoxLayout())
-                    self.d_widget.layout().addWidget(self.donate_button)
-                    if isosx:
-                        self.d_widget.setStyleSheet('QWidget, QToolButton {background-color: none; border: none; }')
-                        self.d_widget.layout().setContentsMargins(0,0,0,0)
-                        self.d_widget.setContentsMargins(0,0,0,0)
-                        self.d_widget.filler = QLabel(u'\u00a0')
-                        self.d_widget.layout().addWidget(self.d_widget.filler)
-                    bar.addWidget(self.d_widget)
-                    self.showing_donate = True
-                elif what in self.gui.iactions:
-                    action = self.gui.iactions[what]
-                    bar.addAction(action.qaction)
-                    self.added_actions.append(action.qaction)
-                    self.setup_tool_button(bar, action.qaction, action.popup_type)
-        self.preferred_width = self.sizeHint().width()
-        self.child_bar.preferred_width = self.child_bar.sizeHint().width()
-
-    def setup_tool_button(self, bar, ac, menu_mode=None):
-        ch = bar.widgetForAction(ac)
-        if ch is None:
-            ch = self.child_bar.widgetForAction(ac)
-        ch.setCursor(Qt.PointingHandCursor)
-        ch.setAutoRaise(True)
-        if ac.menu() is not None and menu_mode is not None:
-            ch.setPopupMode(menu_mode)
-        return ch
-
-    def database_changed(self, db):
-        pass
-
-    #support drag&drop from/to library from/to reader/card
-    def dragEnterEvent(self, event):
-        md = event.mimeData()
-        if md.hasFormat("application/calibre+from_library") or \
-           md.hasFormat("application/calibre+from_device"):
-            event.setDropAction(Qt.CopyAction)
-            event.accept()
-        else:
-            event.ignore()
-
-    def dragMoveEvent(self, event):
-        allowed = False
-        md = event.mimeData()
-        #Drop is only allowed in the location manager widget's different from the selected one
-        for ac in self.location_manager.available_actions:
-            w = self.widgetForAction(ac)
-            if w is not None:
-                if ( md.hasFormat("application/calibre+from_library") or \
-                     md.hasFormat("application/calibre+from_device") ) and \
-                        w.geometry().contains(event.pos()) and \
-                        isinstance(w, QToolButton) and not w.isChecked():
-                    allowed = True
-                    break
-        if allowed:
-            event.acceptProposedAction()
-        else:
-            event.ignore()
-
-    def dropEvent(self, event):
-        data = event.mimeData()
-
-        mime = 'application/calibre+from_library'
-        if data.hasFormat(mime):
-            ids = list(map(int, str(data.data(mime)).split()))
-            tgt = None
-            for ac in self.location_manager.available_actions:
-                w = self.widgetForAction(ac)
-                if w is not None and w.geometry().contains(event.pos()):
-                    tgt = ac.calibre_name
-            if tgt is not None:
-                if tgt == 'main':
-                    tgt = None
-                self.gui.sync_to_device(tgt, False, send_ids=ids)
-                event.accept()
-
-        mime = 'application/calibre+from_device'
-        if data.hasFormat(mime):
-            paths = [unicode(u.toLocalFile()) for u in data.urls()]
-            if paths:
-                self.gui.iactions['Add Books'].add_books_from_device(
-                        self.gui.current_view(), paths=paths)
-                event.accept()
-
-# }}}
 
 class MainWindowMixin(object): # {{{
 
@@ -507,13 +258,16 @@ class MainWindowMixin(object): # {{{
         self.iactions['Fetch News'].init_scheduler(db)
 
         self.search_bar = SearchBar(self)
-        self.child_bar = BaseToolBar(self)
-        self.tool_bar = ToolBar(self.donate_button,
-                self.location_manager, self.child_bar, self)
-        self.addToolBar(Qt.TopToolBarArea, self.tool_bar)
-        self.addToolBar(Qt.BottomToolBarArea, self.child_bar)
-        self.menu_bar = MenuBar(self.location_manager, self)
-        self.setMenuBar(self.menu_bar)
+        self.bars_manager = BarsManager(self.donate_button,
+                self.location_manager, self)
+        for bar in self.bars_manager.main_bars:
+            self.addToolBar(Qt.TopToolBarArea, bar)
+        for bar in self.bars_manager.child_bars:
+            self.addToolBar(Qt.BottomToolBarArea, bar)
+        self.bars_manager.update_bars()
+        # This is disabled because it introduces various toolbar related bugs
+        # The width of the toolbar becomes the sum of both toolbars
+        if tweaks['unified_title_toolbar_on_osx']:
             self.setUnifiedTitleAndToolBarOnMac(True)
 
         l = self.centralwidget.layout()

src/calibre/gui2/library/delegates.py

@@ -7,11 +7,12 @@ __docformat__ = 'restructuredtext en'
 
 from math import cos, sin, pi
 
-from PyQt4.Qt import QColor, Qt, QModelIndex, QSize, \
-                     QPainterPath, QLinearGradient, QBrush, \
-                     QPen, QStyle, QPainter, QStyleOptionViewItemV4, \
-                     QIcon,  QDoubleSpinBox, QVariant, QSpinBox, \
-                     QStyledItemDelegate, QComboBox, QTextDocument
+from PyQt4.Qt import (QColor, Qt, QModelIndex, QSize, QApplication,
+                     QPainterPath, QLinearGradient, QBrush,
+                     QPen, QStyle, QPainter, QStyleOptionViewItemV4,
+                     QIcon,  QDoubleSpinBox, QVariant, QSpinBox,
+                     QStyledItemDelegate, QComboBox, QTextDocument,
+                     QAbstractTextDocumentLayout)
 
 from calibre.gui2 import UNDEFINED_QDATE, error_dialog
 from calibre.gui2.widgets import EnLineEdit
@@ -27,7 +28,6 @@ from calibre.gui2.dialogs.template_dialog import TemplateDialog
 class RatingDelegate(QStyledItemDelegate): # {{{
     COLOR    = QColor("blue")
     SIZE     = 16
-    PEN      = QPen(COLOR, 1, Qt.SolidLine, Qt.RoundCap, Qt.RoundJoin)
 
     def __init__(self, parent):
         QStyledItemDelegate.__init__(self, parent)
@@ -40,10 +40,7 @@ class RatingDelegate(QStyledItemDelegate): # {{{
                                   50 + 40 * sin(0.8 * i * pi))
         self.star_path.closeSubpath()
         self.star_path.setFillRule(Qt.WindingFill)
-        gradient = QLinearGradient(0, 0, 0, 100)
-        gradient.setColorAt(0.0, self.COLOR)
-        gradient.setColorAt(1.0, self.COLOR)
-        self.brush = QBrush(gradient)
+        self.gradient = QLinearGradient(0, 0, 0, 100)
         self.factor = self.SIZE/100.
 
     def sizeHint(self, option, index):
@@ -53,7 +50,8 @@ class RatingDelegate(QStyledItemDelegate): # {{{
     def paint(self, painter, option, index):
         style = self._parent.style()
         option = QStyleOptionViewItemV4(option)
-        self.initStyleOption(option, self.dummy)
+        self.initStyleOption(option, index)
+        option.text = u''
         num = index.model().data(index, Qt.DisplayRole).toInt()[0]
         def draw_star():
             painter.save()
@@ -70,13 +68,23 @@ class RatingDelegate(QStyledItemDelegate): # {{{
                     painter, self._parent)
         elif option.state & QStyle.State_Selected:
             painter.fillRect(option.rect, option.palette.highlight())
+        else:
+            painter.fillRect(option.rect, option.backgroundBrush)
+
         try:
             painter.setRenderHint(QPainter.Antialiasing)
             painter.setClipRect(option.rect)
             y = option.rect.center().y()-self.SIZE/2.
             x = option.rect.left()
-            painter.setPen(self.PEN)
-            painter.setBrush(self.brush)
+            color = index.data(Qt.ForegroundRole)
+            if color.isNull() or not color.isValid():
+                color = self.COLOR
+            else:
+                color = QColor(color)
+            painter.setPen(QPen(color,  1, Qt.SolidLine, Qt.RoundCap, Qt.RoundJoin))
+            self.gradient.setColorAt(0.0, color)
+            self.gradient.setColorAt(1.0, color)
+            painter.setBrush(QBrush(self.gradient))
             painter.translate(x, y)
             i = 0
             while i < num:
@@ -311,17 +319,21 @@ class CcCommentsDelegate(QStyledItemDelegate): # {{{
         self.document = QTextDocument()
 
     def paint(self, painter, option, index):
-        style = self.parent().style()
-        self.document.setHtml(index.data(Qt.DisplayRole).toString())
-        painter.save()
+        self.initStyleOption(option, index)
+        style = QApplication.style() if option.widget is None \
+                                                else option.widget.style()
+        self.document.setHtml(option.text)
+        option.text = u''
         if hasattr(QStyle, 'CE_ItemViewItem'):
-            style.drawControl(QStyle.CE_ItemViewItem, option,
-                    painter, self.parent())
-        elif option.state & QStyle.State_Selected:
-            painter.fillRect(option.rect, option.palette.highlight())
-        painter.setClipRect(option.rect)
-        painter.translate(option.rect.topLeft())
-        self.document.drawContents(painter)
+            style.drawControl(QStyle.CE_ItemViewItem, option, painter)
+        ctx = QAbstractTextDocumentLayout.PaintContext()
+        ctx.palette = option.palette #.setColor(QPalette.Text, QColor("red"));
+        if hasattr(QStyle, 'SE_ItemViewItemText'):
+            textRect = style.subElementRect(QStyle.SE_ItemViewItemText, option)
+            painter.save()
+            painter.translate(textRect.topLeft())
+            painter.setClipRect(textRect.translated(-textRect.topLeft()))
+            self.document.documentLayout().draw(painter, ctx)
             painter.restore()
 
     def createEditor(self, parent, option, index):

src/calibre/gui2/library/models.py

@@ -14,6 +14,7 @@ from PyQt4.Qt import (QAbstractTableModel, Qt, pyqtSignal, QIcon, QImage,
 from calibre.gui2 import NONE, UNDEFINED_QDATE
 from calibre.utils.pyparsing import ParseException
 from calibre.ebooks.metadata import fmt_sidx, authors_to_string, string_to_authors
+from calibre.ebooks.metadata.book.base import composite_formatter
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.utils.config import tweaks, prefs
 from calibre.utils.date import dt_factory, qt_to_dt
@@ -96,6 +97,8 @@ class BooksModel(QAbstractTableModel): # {{{
         self.ids_to_highlight_set = set()
         self.current_highlighted_idx = None
         self.highlight_only = False
+        self.column_color_map = {}
+        self.colors = [unicode(c) for c in QColor.colorNames()]
         self.read_config()
 
     def change_alignment(self, colname, alignment):
@@ -151,6 +154,7 @@ class BooksModel(QAbstractTableModel): # {{{
                 self.headers[col] = self.custom_columns[col]['name']
 
         self.build_data_convertors()
+        self.set_color_templates(reset=False)
         self.reset()
         self.database_changed.emit(db)
         self.stop_metadata_backup()
@@ -532,6 +536,15 @@ class BooksModel(QAbstractTableModel): # {{{
             img = self.default_image
         return img
 
+    def set_color_templates(self, reset=True):
+        self.column_color_map = {}
+        for i in range(1,self.db.column_color_count+1):
+            name = self.db.prefs.get('column_color_name_'+str(i))
+            if name:
+                self.column_color_map[name] = \
+                        self.db.prefs.get('column_color_template_'+str(i))
+        if reset:
+            self.reset()
 
     def build_data_convertors(self):
         def authors(r, idx=-1):
@@ -693,9 +706,36 @@ class BooksModel(QAbstractTableModel): # {{{
             return NONE
         if role in (Qt.DisplayRole, Qt.EditRole):
             return self.column_to_dc_map[col](index.row())
-        elif role == Qt.BackgroundColorRole:
+        elif role == Qt.BackgroundRole:
             if self.id(index) in self.ids_to_highlight_set:
-                return QColor('lightgreen')
+                return QVariant(QColor('lightgreen'))
+        elif role == Qt.ForegroundRole:
+            key = self.column_map[col]
+            if key in self.column_color_map:
+                mi = self.db.get_metadata(self.id(index), index_is_id=True)
+                fmt = self.column_color_map[key]
+                try:
+                    color = composite_formatter.safe_format(fmt, mi, '', mi)
+                    if color in self.colors:
+                        color = QColor(color)
+                        if color.isValid():
+                            return QVariant(color)
+                except:
+                    return NONE
+            elif self.is_custom_column(key) and \
+                        self.custom_columns[key]['datatype'] == 'enumeration':
+                cc = self.custom_columns[self.column_map[col]]['display']
+                colors = cc.get('enum_colors', [])
+                values = cc.get('enum_values', [])
+                txt = unicode(index.data(Qt.DisplayRole).toString())
+                if len(colors) > 0 and txt in values:
+                    try:
+                        color = QColor(colors[values.index(txt)])
+                        if color.isValid():
+                            return QVariant(color)
+                    except:
+                        pass
+            return NONE
         elif role == Qt.DecorationRole:
             if self.column_to_dc_decorator_map[col] is not None:
                 return self.column_to_dc_decorator_map[index.column()](index.row())

src/calibre/gui2/lrf_renderer/main.py

@@ -5,7 +5,7 @@ import sys, logging, os, traceback, time
 from PyQt4.QtGui import QKeySequence, QPainter, QDialog, QSpinBox, QSlider, QIcon
 from PyQt4.QtCore import Qt, QObject, SIGNAL, QCoreApplication, QThread
 
-from calibre import __appname__, setup_cli_handlers, islinux, isfreebsd
+from calibre import __appname__, setup_cli_handlers, islinux, isbsd
 from calibre.ebooks.lrf.lrfparser import LRFDocument
 
 from calibre.gui2 import ORG_NAME, APP_UID, error_dialog, \
@@ -258,7 +258,7 @@ def file_renderer(stream, opts, parent=None, logger=None):
         level = logging.DEBUG if opts.verbose else logging.INFO
         logger = logging.getLogger('lrfviewer')
         setup_cli_handlers(logger, level)
-    if islinux or isfreebsd:
+    if islinux or isbsd:
         try: # Set lrfviewer as the default for LRF files for this user
             from subprocess import call
             call('xdg-mime default calibre-lrfviewer.desktop application/lrf', shell=True)
@@ -307,7 +307,7 @@ def main(args=sys.argv, logger=None):
     if hasattr(opts, 'help'):
         parser.print_help()
         return 1
-    pid = os.fork() if (islinux or isfreebsd) else -1
+    pid = os.fork() if (islinux or isbsd) else -1
     if pid <= 0:
         app = Application(args)
         app.setWindowIcon(QIcon(I('viewer.png')))

src/calibre/gui2/metadata/basic_widgets.py

@@ -88,11 +88,22 @@ class TitleEdit(EnLineEdit):
 
     def commit(self, db, id_):
         title = self.current_val
+        try:
             if self.COMMIT:
                 getattr(db, 'set_'+ self.TITLE_ATTR)(id_, title, notify=False)
             else:
                 getattr(db, 'set_'+ self.TITLE_ATTR)(id_, title, notify=False,
                         commit=False)
+        except (IOError, OSError) as err:
+            if getattr(err, 'errno', -1) == 13: # Permission denied
+                import traceback
+                fname = err.filename if err.filename else 'file'
+                error_dialog(self, _('Permission denied'),
+                        _('Could not open %s. Is it being used by another'
+                        ' program?')%fname, det_msg=traceback.format_exc(),
+                        show=True)
+                return False
+            raise
         return True
 
     @dynamic_property
@@ -225,8 +236,19 @@ class AuthorsEdit(MultiCompleteComboBox):
 
     def commit(self, db, id_):
         authors = self.current_val
+        try:
             self.books_to_refresh |= db.set_authors(id_, authors, notify=False,
                 allow_case_change=True)
+        except (IOError, OSError) as err:
+            if getattr(err, 'errno', -1) == 13: # Permission denied
+                import traceback
+                fname = err.filename if err.filename else 'file'
+                error_dialog(self, _('Permission denied'),
+                        _('Could not open %s. Is it being used by another'
+                        ' program?')%fname, det_msg=traceback.format_exc(),
+                        show=True)
+                return False
+            raise
         return True
 
     @dynamic_property

src/calibre/gui2/preferences/create_custom_column.py

@@ -6,7 +6,7 @@ __copyright__ = '2010, Kovid Goyal <kovid at kovidgoyal.net>'
 import re
 from functools import partial
 
-from PyQt4.Qt import QDialog, Qt, QListWidgetItem, QVariant
+from PyQt4.Qt import QDialog, Qt, QListWidgetItem, QVariant, QColor
 
 from calibre.gui2.preferences.create_custom_column_ui import Ui_QCreateCustomColumn
 from calibre.gui2 import error_dialog
@@ -126,11 +126,15 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
                                 c['display'].get('make_category', False))
         elif ct == 'enumeration':
             self.enum_box.setText(','.join(c['display'].get('enum_values', [])))
+            self.enum_colors.setText(','.join(c['display'].get('enum_colors', [])))
         self.datatype_changed()
         if ct in ['text', 'composite', 'enumeration']:
             self.use_decorations.setChecked(c['display'].get('use_decorations', False))
         elif ct == '*text':
             self.is_names.setChecked(c['display'].get('is_names', False))
+
+        all_colors = [unicode(s) for s in list(QColor.colorNames())]
+        self.enum_colors_label.setToolTip('<p>' + ', '.join(all_colors) + '</p>')
         self.exec_()
 
     def shortcut_activated(self, url):
@@ -170,7 +174,7 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
         for x in ('box', 'default_label', 'label', 'sort_by', 'sort_by_label',
                   'make_category'):
             getattr(self, 'composite_'+x).setVisible(col_type in ['composite', '*composite'])
-        for x in ('box', 'default_label', 'label'):
+        for x in ('box', 'default_label', 'label', 'colors', 'colors_label'):
             getattr(self, 'enum_'+x).setVisible(col_type == 'enumeration')
         self.use_decorations.setVisible(col_type in ['text', 'composite', 'enumeration'])
         self.is_names.setVisible(col_type == '*text')
@@ -247,7 +251,20 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
                 if l[i] in l[i+1:]:
                     return self.simple_error('', _('The value "{0}" is in the '
                     'list more than once').format(l[i]))
-            display_dict = {'enum_values': l}
+            c = unicode(self.enum_colors.text())
+            if c:
+                c = [v.strip() for v in unicode(self.enum_colors.text()).split(',')]
+            else:
+                c = []
+            if len(c) != 0 and len(c) != len(l):
+                return self.simple_error('', _('The colors box must be empty or '
+                'contain the same number of items as the value box'))
+            for tc in c:
+                if tc not in QColor.colorNames():
+                    return self.simple_error('',
+                            _('The color {0} is unknown').format(tc))
+
+            display_dict = {'enum_values': l, 'enum_colors': c}
         elif col_type == 'text' and is_multiple:
             display_dict = {'is_names': self.is_names.isChecked()}
 

src/calibre/gui2/preferences/create_custom_column.ui

@@ -304,8 +304,8 @@ Everything else will show nothing.</string>
         </widget>
        </item>
        <item row="6" column="2">
-        <layout class="QHBoxLayout" name="horizontalLayout_2">
-         <item>
+        <layout class="QGridLayout" name="horizontalLayout_2">
+         <item row="0" column="0">
           <widget class="QLineEdit" name="enum_box">
            <property name="sizePolicy">
             <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
@@ -320,13 +320,34 @@ four values, the first of them being the empty value.</string>
            </property>
           </widget>
          </item>
-         <item>
+         <item row="0" column="1">
           <widget class="QLabel" name="enum_default_label">
            <property name="toolTip">
             <string>The empty string is always the first value</string>
            </property>
            <property name="text">
-            <string>Default: (nothing)</string>
+            <string>Values</string>
+           </property>
+          </widget>
+         </item>
+         <item row="1" column="0">
+          <widget class="QLineEdit" name="enum_colors">
+           <property name="sizePolicy">
+            <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
+             <horstretch>0</horstretch>
+             <verstretch>0</verstretch>
+            </sizepolicy>
+           </property>
+           <property name="toolTip">
+            <string>A list of color names to use when displaying an item. The
+list must be empty or contain a color for each value.</string>
+           </property>
+          </widget>
+         </item>
+         <item row="1" column="1">
+          <widget class="QLabel" name="enum_colors_label">
+           <property name="text">
+            <string>Colors</string>
            </property>
           </widget>
          </item>

src/calibre/gui2/preferences/device_user_defined.py

@@ -76,7 +76,7 @@ class UserDefinedDevice(QDialog):
                     for i,d in enumerate(sorted(new_drives,
                                     key=lambda x: after['drive_details'][x][0])):
                         if i == 0:
-                            res +=  _('Windows main memory ID string') + ': ' + \
+                            res +=  _('Windows main memory vendor string') + ': ' + \
                                     after['drive_details'][d][1] + '\n'
                             res += _('Windows main memory ID string') + ': ' + \
                                     after['drive_details'][d][2] + '\n'

src/calibre/gui2/preferences/look_feel.py

@@ -6,7 +6,7 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 from PyQt4.Qt import (QApplication, QFont, QFontInfo, QFontDialog,
-        QAbstractListModel, Qt)
+        QAbstractListModel, Qt, QColor)
 
 from calibre.gui2.preferences import ConfigWidgetBase, test_widget, CommaSeparatedList
 from calibre.gui2.preferences.look_feel_ui import Ui_Form
@@ -159,6 +159,62 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.df_up_button.clicked.connect(self.move_df_up)
         self.df_down_button.clicked.connect(self.move_df_down)
 
+        self.color_help_text.setText('<p>' +
+                _('Here you can specify coloring rules for columns shown in the '
+                  'library view. Choose the column you wish to color, then '
+                  'supply a template that specifies the color to use based on '
+                  'the values in the column. There is a '
+                  '<a href="http://calibre-ebook.com/user_manual/template_lang.html">'
+                  'tutorial</a> on using templates.') +
+                 '</p><p>' +
+                _('If you want to color a field based on tags, then click the '
+                  'button next to an empty line to open the tags wizard. '
+                  'It will build a template for you. You can later edit that '
+                  'template with the same wizard. If you edit it by hand, the '
+                  'wizard might not work or might restore old values.') +
+                 '</p><p>' +
+                _('The template must evaluate to one of the color names shown '
+                  'below. You can use any legal template expression. '
+                  'For example, you can set the title to always display in '
+                  'green using the template "green" (without the quotes). '
+                  'To show the title in the color named in the custom column '
+                  '#column, use "{#column}". To show the title in blue if the '
+                  'custom column #column contains the value "foo", in red if the '
+                  'column contains the value "bar", otherwise in black, use '
+                  '<pre>{#column:switch(foo,blue,bar,red,black)}</pre>'
+                  'To show the title in blue if the book has the exact tag '
+                  '"Science Fiction", red if the book has the exact tag '
+                  '"Mystery", or black if the book has neither tag, use'
+                  "<pre>program: \n"
+                  "    t = field('tags'); \n"
+                  "    first_non_empty(\n"
+                  "        in_list(t, ',', '^Science Fiction$', 'blue', ''), \n"
+                  "        in_list(t, ',', '^Mystery$', 'red', 'black'))</pre>"
+                  'To show the title in green if it has one format, blue if it '
+                  'two formats, and red if more, use'
+                  "<pre>program:cmp(count(field('formats'),','), 2, 'green', 'blue', 'red')</pre>") +
+                               '</p><p>' +
+                _('You can access a multi-line template editor from the '
+                  'context menu (right-click).') + '</p><p>' +
+                _('<b>Note:</b> if you want to color a "custom column with a fixed set '
+                  'of values", it is often easier to specify the '
+                  'colors in the column definition dialog. There you can '
+                  'provide a color for each value without using a template.')+ '</p>')
+        choices = db.field_metadata.displayable_field_keys()
+        choices.sort(key=sort_key)
+        choices.insert(0, '')
+        self.column_color_count = db.column_color_count+1
+        tags = db.all_tags()
+        for i in range(1, self.column_color_count):
+            r('column_color_name_'+str(i), db.prefs, choices=choices)
+            r('column_color_template_'+str(i), db.prefs)
+            tpl = getattr(self, 'opt_column_color_template_'+str(i))
+            tpl.set_tags(tags)
+            toolbutton = getattr(self, 'opt_column_color_wizard_'+str(i))
+            toolbutton.clicked.connect(tpl.tag_wizard)
+        all_colors = [unicode(s) for s in list(QColor.colorNames())]
+        self.colors_box.setText(', '.join(all_colors))
+
     def initialize(self):
         ConfigWidgetBase.initialize(self)
         font = gprefs['font']
@@ -226,6 +282,12 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
             self.changed_signal.emit()
 
     def commit(self, *args):
+        for i in range(1, self.column_color_count):
+            col = getattr(self, 'opt_column_color_name_'+str(i))
+            tpl = getattr(self, 'opt_column_color_template_'+str(i))
+            if not col.currentIndex() or not unicode(tpl.text()).strip():
+                col.setCurrentIndex(0)
+                tpl.setText('')
         rr = ConfigWidgetBase.commit(self, *args)
         if self.current_font != self.initial_font:
             gprefs['font'] = (self.current_font[:4] if self.current_font else
@@ -238,6 +300,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         return rr
 
     def refresh_gui(self, gui):
+        gui.library_view.model().set_color_templates()
         self.update_font_display()
         gui.tags_view.reread_collapse_parameters()
         gui.library_view.refresh_book_details()

src/calibre/gui2/preferences/look_feel.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>717</width>
-    <height>390</height>
+    <height>519</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -407,6 +407,193 @@ then the tags will be displayed each on their own line.</string>
        </item>
       </layout>
      </widget>
+     <widget class="QWidget" name="tab_5">
+      <attribute name="icon">
+       <iconset resource="../../../../resources/images.qrc">
+        <normaloff>:/images/format-fill-color.png</normaloff>:/images/format-fill-color.png</iconset>
+      </attribute>
+      <attribute name="title">
+       <string>Column Coloring</string>
+      </attribute>
+      <layout class="QGridLayout" name="column_color_layout">
+       <item row="1" column="0">
+        <widget class="QLabel" name="label">
+         <property name="text">
+          <string>Column to color</string>
+         </property>
+        </widget>
+       </item>
+       <item row="1" column="1">
+        <widget class="QLabel" name="label">
+         <property name="text">
+          <string>Color selection template</string>
+         </property>
+        </widget>
+       </item>
+       <item row="2" column="0">
+        <widget class="QComboBox" name="opt_column_color_name_1"/>
+       </item>
+       <item row="2" column="1">
+        <widget class="TemplateLineEditor" name="opt_column_color_template_1"/>
+       </item>
+       <item row="2" column="2">
+        <widget class="QToolButton" name="opt_column_color_wizard_1">
+         <property name="icon">
+          <iconset resource="../../../../resources/images.qrc">
+           <normaloff>:/images/wizard.png</normaloff>:/images/wizard.png</iconset>
+         </property>
+         <property name="toolTip">
+          <string>Open the tags wizard.</string>
+         </property>
+        </widget>
+       </item>
+       <item row="3" column="0">
+        <widget class="QComboBox" name="opt_column_color_name_2"/>
+       </item>
+       <item row="3" column="1">
+        <widget class="TemplateLineEditor" name="opt_column_color_template_2"/>
+       </item>
+       <item row="3" column="2">
+        <widget class="QToolButton" name="opt_column_color_wizard_2">
+         <property name="icon">
+          <iconset resource="../../../../resources/images.qrc">
+           <normaloff>:/images/wizard.png</normaloff>:/images/wizard.png</iconset>
+         </property>
+         <property name="toolTip">
+          <string>Open the tags wizard.</string>
+         </property>
+        </widget>
+       </item>
+       <item row="4" column="0">
+        <widget class="QComboBox" name="opt_column_color_name_3"/>
+       </item>
+       <item row="4" column="1">
+        <widget class="TemplateLineEditor" name="opt_column_color_template_3"/>
+       </item>
+       <item row="4" column="2">
+        <widget class="QToolButton" name="opt_column_color_wizard_3">
+         <property name="icon">
+          <iconset resource="../../../../resources/images.qrc">
+           <normaloff>:/images/wizard.png</normaloff>:/images/wizard.png</iconset>
+         </property>
+         <property name="toolTip">
+          <string>Open the tags wizard.</string>
+         </property>
+        </widget>
+       </item>
+       <item row="5" column="0">
+        <widget class="QComboBox" name="opt_column_color_name_4"/>
+       </item>
+       <item row="5" column="1">
+        <widget class="TemplateLineEditor" name="opt_column_color_template_4"/>
+       </item>
+       <item row="5" column="2">
+        <widget class="QToolButton" name="opt_column_color_wizard_4">
+         <property name="icon">
+          <iconset resource="../../../../resources/images.qrc">
+           <normaloff>:/images/wizard.png</normaloff>:/images/wizard.png</iconset>
+         </property>
+         <property name="toolTip">
+          <string>Open the tags wizard.</string>
+         </property>
+        </widget>
+       </item>
+       <item row="6" column="0">
+        <widget class="QComboBox" name="opt_column_color_name_5"/>
+       </item>
+       <item row="6" column="1">
+        <widget class="TemplateLineEditor" name="opt_column_color_template_5"/>
+       </item>
+       <item row="6" column="2">
+        <widget class="QToolButton" name="opt_column_color_wizard_5">
+         <property name="icon">
+          <iconset resource="../../../../resources/images.qrc">
+           <normaloff>:/images/wizard.png</normaloff>:/images/wizard.png</iconset>
+         </property>
+         <property name="toolTip">
+          <string>Open the tags wizard.</string>
+         </property>
+        </widget>
+       </item>
+       <item row="20" column="0">
+        <widget class="QLabel" name="label">
+         <property name="text">
+          <string>Color names</string>
+         </property>
+        </widget>
+       </item>
+       <item row="0" column="0" colspan="3">
+        <widget class="QScrollArea" name="scrollArea">
+         <property name="minimumSize">
+          <size>
+           <width>0</width>
+           <height>200</height>
+          </size>
+         </property>
+         <property name="widgetResizable">
+          <bool>true</bool>
+         </property>
+         <property name="alignment">
+          <set>Qt::AlignCenter</set>
+         </property>
+         <widget class="QWidget" name="scrollAreaWidgetContents">
+          <property name="geometry">
+           <rect>
+            <x>0</x>
+            <y>0</y>
+            <width>687</width>
+            <height>194</height>
+           </rect>
+          </property>
+          <layout class="QVBoxLayout" name="verticalLayout">
+           <item>
+            <widget class="QLabel" name="color_help_text">
+             <property name="wordWrap">
+              <bool>true</bool>
+             </property>
+             <property name="openExternalLinks">
+              <bool>true</bool>
+             </property>
+            </widget>
+           </item>
+          </layout>
+         </widget>
+        </widget>
+       </item>
+       <item row="21" column="0" colspan="3">
+        <widget class="QScrollArea" name="scrollArea_2">
+         <property name="maximumSize">
+          <size>
+           <width>16777215</width>
+           <height>120</height>
+          </size>
+         </property>
+         <property name="widgetResizable">
+          <bool>true</bool>
+         </property>
+         <widget class="QWidget" name="scrollAreaWidgetContents_2">
+          <property name="geometry">
+           <rect>
+            <x>0</x>
+            <y>0</y>
+            <width>687</width>
+            <height>61</height>
+           </rect>
+          </property>
+          <layout class="QVBoxLayout" name="verticalLayout_2">
+           <item>
+            <widget class="QLabel" name="colors_box">
+             <property name="wordWrap">
+              <bool>true</bool>
+             </property>
+            </widget>
+           </item>
+          </layout>
+         </widget>
+        </widget>
+       </item>
+      </layout>
+     </widget>
     </widget>
    </item>
   </layout>
@@ -417,6 +604,11 @@ then the tags will be displayed each on their own line.</string>
    <extends>QLineEdit</extends>
    <header>calibre/gui2/complete.h</header>
   </customwidget>
+  <customwidget>
+   <class>TemplateLineEditor</class>
+   <extends>QLineEdit</extends>
+   <header>calibre/gui2/dialogs/template_line_editor.h</header>
+  </customwidget>
  </customwidgets>
  <resources>
   <include location="../../../../resources/images.qrc"/>

src/calibre/gui2/preferences/main.py

@@ -361,10 +361,9 @@ class Preferences(QMainWindow):
             self.gui.tags_view.recount()
             self.gui.create_device_menu()
             self.gui.set_device_menu_items_state(bool(self.gui.device_connected))
-            self.gui.tool_bar.build_bar()
-            self.gui.menu_bar.build_bar()
+            self.gui.bars_manager.apply_settings()
+            self.gui.bars_manager.update_bars()
             self.gui.build_context_menus()
-            self.gui.tool_bar.apply_settings()
 
         return QMainWindow.closeEvent(self, *args)
 

src/calibre/gui2/preferences/plugboard.py

@@ -5,50 +5,30 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-from PyQt4.Qt import Qt, QLineEdit, QComboBox, SIGNAL, QListWidgetItem
+import copy
+
+from PyQt4.Qt import Qt, QComboBox, QListWidgetItem
 
 from calibre.customize.ui import is_disabled
-from calibre.gui2 import error_dialog
+from calibre.gui2 import error_dialog, question_dialog
 from calibre.gui2.device import device_name_for_plugboards
-from calibre.gui2.dialogs.template_dialog import TemplateDialog
+from calibre.gui2.dialogs.template_line_editor import TemplateLineEditor
 from calibre.gui2.preferences import ConfigWidgetBase, test_widget
 from calibre.gui2.preferences.plugboard_ui import Ui_Form
 from calibre.customize.ui import metadata_writers, device_plugins
 from calibre.library.save_to_disk import plugboard_any_format_value, \
-                        plugboard_any_device_value, plugboard_save_to_disk_value
+                    plugboard_any_device_value, plugboard_save_to_disk_value, \
+                    find_plugboard
 from calibre.library.server.content import plugboard_content_server_value, \
                                         plugboard_content_server_formats
 from calibre.utils.formatter import validation_formatter
 
 
-class LineEditWithTextBox(QLineEdit):
-
-    '''
-    Extend the context menu of a QLineEdit to include more actions.
-    '''
-
-    def contextMenuEvent(self, event):
-        menu = self.createStandardContextMenu()
-        menu.addSeparator()
-
-        action_open_editor = menu.addAction(_('Open Editor'))
-
-        self.connect(action_open_editor, SIGNAL('triggered()'), self.open_editor)
-        menu.exec_(event.globalPos())
-
-    def open_editor(self):
-        t = TemplateDialog(self, self.text())
-        if t.exec_():
-            self.setText(t.textbox.toPlainText())
-
 class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
     def genesis(self, gui):
         self.gui = gui
         self.db = gui.library_view.model().db
-        self.current_plugboards = self.db.prefs.get('plugboards',{})
-        self.current_device = None
-        self.current_format = None
 
     def initialize(self):
         def field_cmp(x, y):
@@ -64,6 +44,10 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
         ConfigWidgetBase.initialize(self)
 
+        self.current_plugboards = copy.deepcopy(self.db.prefs.get('plugboards',{}))
+        self.current_device = None
+        self.current_format = None
+
         if self.gui.device_manager.connected_device is not None:
             self.device_label.setText(_('Device currently connected: ') +
                     self.gui.device_manager.connected_device.__class__.__name__)
@@ -103,7 +87,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.source_widgets = []
         self.dest_widgets = []
         for i in range(0, len(self.dest_fields)-1):
-            w = LineEditWithTextBox(self)
+            w = TemplateLineEditor(self)
             self.source_widgets.append(w)
             self.fields_layout.addWidget(w, 5+i, 0, 1, 1)
             w = QComboBox(self)
@@ -196,51 +180,66 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
             return
         self.clear_fields(edit_boxes=True)
         self.current_device = unicode(txt)
-        error = False
-        if self.current_format == plugboard_any_format_value:
-            # user specified any format.
-            for f in self.current_plugboards:
-                devs = set(self.current_plugboards[f])
-                if self.current_device != plugboard_save_to_disk_value and \
-                        plugboard_any_device_value in devs:
-                    # specific format/any device in list. conflict.
-                    # note: any device does not match save_to_disk
-                    error = True
-                    break
-                if self.current_device in devs:
-                    # specific format/current device in list. conflict
-                    error = True
-                    break
-                if self.current_device == plugboard_any_device_value:
-                    # any device and a specific device already there. conflict
-                    error = True
-                    break
-        else:
-            # user specified specific format.
-            for f in self.current_plugboards:
-                devs = set(self.current_plugboards[f])
-                if f == plugboard_any_format_value and \
-                                self.current_device in devs:
-                    # any format/same device in list. conflict.
-                    error = True
-                    break
-                if f == self.current_format and self.current_device in devs:
-                    # current format/current device in list. conflict
-                    error = True
-                    break
-                if f == self.current_format and plugboard_any_device_value in devs:
-                    # current format/any device in list. conflict
-                    error = True
-                    break
 
-        if error:
+        if self.current_format in self.current_plugboards and \
+                self.current_device in self.current_plugboards[self.current_format]:
             error_dialog(self, '',
-                     _('That format and device already has a plugboard or '
-                       'conflicts with another plugboard.'),
+                     _('That format and device already has a plugboard.'),
                      show=True)
             self.new_device.setCurrentIndex(0)
             return
-        if self.current_device in self.device_to_formats_map:
+
+        # If we have a specific format/device combination, check if a more
+        # general combination matches.
+        if self.current_format != plugboard_any_format_value and \
+                self.current_device != plugboard_any_device_value:
+            if find_plugboard(self.current_device, self.current_format,
+                      self.current_plugboards):
+                if not question_dialog(self.gui,
+                        _('Possibly override plugboard?'),
+                        _('A more general plugboard already exists for '
+                          'that format and device. '
+                          'Are you sure you want to add the new plugboard?')):
+                    self.new_device.setCurrentIndex(0)
+                    return
+
+        # If we have a specific format, check if we are adding a possibly-
+        # covered plugboard
+        if self.current_format != plugboard_any_format_value:
+            if self.current_format in self.current_plugboards:
+                if self.current_device == plugboard_any_device_value:
+                    if not question_dialog(self.gui,
+                               _('Add possibly overridden plugboard?'),
+                               _('More specific device plugboards exist for '
+                                 'that format. '
+                                 'Are you sure you want to add the new plugboard?')):
+                        self.new_device.setCurrentIndex(0)
+                        return
+        # We are adding an 'any format' entry. Check if we are adding a specific
+        # device and if so, does some other plugboard match that device.
+        elif self.current_device != plugboard_any_device_value:
+            for fmt in self.current_plugboards:
+                if find_plugboard(self.current_device, fmt, self.current_plugboards):
+                    if not question_dialog(self.gui,
+                            _('Really add plugboard?'),
+                            _('A different plugboard matches that format and '
+                              'device combination. '
+                              'Are you sure you want to add the new plugboard?')):
+                        self.new_device.setCurrentIndex(0)
+                        return
+        # We are adding an any format/any device entry, which will be overridden
+        # by any other entry. Ask if such entries exist.
+        elif len(self.current_plugboards):
+            if not question_dialog(self.gui,
+                       _('Add possibly overridden plugboard?'),
+                       _('More specific format and device plugboards '
+                         'already exist. '
+                         'Are you sure you want to add the new plugboard?')):
+                self.new_device.setCurrentIndex(0)
+                return
+
+        if self.current_format != plugboard_any_format_value and \
+                    self.current_device in self.device_to_formats_map:
             allowable_formats = self.device_to_formats_map[self.current_device]
             if self.current_format not in allowable_formats:
                 error_dialog(self, '',

src/calibre/gui2/preferences/toolbar.py

@@ -317,6 +317,10 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
             am.restore_defaults()
         self.changed_signal.emit()
 
+    def refresh_gui(self, gui):
+        gui.bars_manager.init_bars()
+        gui.bars_manager.update_bars()
+
 
 if __name__ == '__main__':
     from PyQt4.Qt import QApplication

src/calibre/gui2/store/amazon_de_plugin.py

@@ -16,7 +16,7 @@ class AmazonDEKindleStore(AmazonKindleStore):
     For comments on the implementation, please see amazon_plugin.py
     '''
 
-    search_url = 'http://www.amazon.de/s/url=search-alias%3Ddigital-text&field-keywords='
+    search_url = 'http://www.amazon.de/s/?url=search-alias%3Ddigital-text&field-keywords='
     details_url = 'http://amazon.de/dp/'
     drm_search_text = u'Gleichzeitige Verwendung von Geräten'
     drm_free_text = u'Keine Einschränkung'

src/calibre/gui2/store/amazon_plugin.py

@@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
 
 import random
 import re
-import urllib2
+import urllib
 from contextlib import closing
 
 from lxml import html
@@ -22,7 +22,7 @@ from calibre.gui2.store.search_result import SearchResult
 
 class AmazonKindleStore(StorePlugin):
 
-    search_url = 'http://www.amazon.com/s/url=search-alias%3Ddigital-text&field-keywords='
+    search_url = 'http://www.amazon.com/s/?url=search-alias%3Ddigital-text&field-keywords='
     details_url = 'http://amazon.com/dp/'
     drm_search_text = u'Simultaneous Device Usage'
     drm_free_text = u'Unlimited'
@@ -122,13 +122,27 @@ class AmazonKindleStore(StorePlugin):
         open_url(QUrl(store_link))
 
     def search(self, query, max_results=10, timeout=60):
-        url =  self.search_url + urllib2.quote(query)
+        url =  self.search_url + urllib.quote_plus(query)
         br = browser()
 
         counter = max_results
         with closing(br.open(url, timeout=timeout)) as f:
             doc = html.fromstring(f.read())
-            for data in doc.xpath('//div[@class="productData"]'):
+            
+            # Amazon has two results pages.
+            is_shot = doc.xpath('boolean(//div[@id="shotgunMainResults"])')
+            # Horizontal grid of books.
+            if is_shot:
+                data_xpath = '//div[contains(@class, "result")]'
+                format_xpath = './/div[@class="productTitle"]/text()'
+                cover_xpath = './/div[@class="productTitle"]//img/@src'
+            # Vertical list of books.
+            else:
+                data_xpath = '//div[@class="productData"]'
+                format_xpath = './/span[@class="format"]/text()'
+                cover_xpath = '../div[@class="productImage"]/a/img/@src'
+            
+            for data in doc.xpath(data_xpath):
                 if counter <= 0:
                     break
                 
@@ -136,14 +150,14 @@ class AmazonKindleStore(StorePlugin):
                 # put in results for non Kindle books (author pages). Se we need
                 # to explicitly check if the item is a Kindle book and ignore it
                 # if it isn't.
-                type = ''.join(data.xpath('//span[@class="format"]/text()'))
-                if 'kindle' not in type.lower():
+                format = ''.join(data.xpath(format_xpath))
+                if 'kindle' not in format.lower():
                     continue
                 
                 # We must have an asin otherwise we can't easily reference the
                 # book later.
                 asin_href = None
-                asin_a = data.xpath('div[@class="productTitle"]/a[1]')
+                asin_a = data.xpath('.//div[@class="productTitle"]/a[1]')
                 if asin_a:
                     asin_href = asin_a[0].get('href', '')
                     m = re.search(r'/dp/(?P<asin>.+?)(/|$)', asin_href)
@@ -154,28 +168,21 @@ class AmazonKindleStore(StorePlugin):
                 else:
                     continue
                 
-                cover_url = ''
-                if asin_href:
-                    cover_img = data.xpath('//div[@class="productImage"]/a[@href="%s"]/img/@src' % asin_href)
-                    if cover_img:
-                        cover_url = cover_img[0]
-                        parts = cover_url.split('/')
-                        bn = parts[-1]
-                        f, _, ext = bn.rpartition('.')
-                        if '_' in f:
-                            bn = f.partition('_')[0]+'_SL160_.'+ext
-                            parts[-1] = bn
-                            cover_url = '/'.join(parts)
+                cover_url = ''.join(data.xpath(cover_xpath))
 
-                title = ''.join(data.xpath('div[@class="productTitle"]/a/text()'))
-                author = ''.join(data.xpath('div[@class="productTitle"]/span[@class="ptBrand"]/text()'))
-                author = author.split('by')[-1]
-                price = ''.join(data.xpath('div[@class="newPrice"]/span/text()'))
+                title = ''.join(data.xpath('.//div[@class="productTitle"]/a/text()'))
+                price = ''.join(data.xpath('.//div[@class="newPrice"]/span/text()'))
+                
+                if is_shot:
+                    author = format.split(' by ')[-1]
+                else:
+                    author = ''.join(data.xpath('.//div[@class="productTitle"]/span[@class="ptBrand"]/text()'))
+                    author = author.split(' by ')[-1]
                 
                 counter -= 1
     
                 s = SearchResult()
-                s.cover_url = cover_url
+                s.cover_url = cover_url.strip()
                 s.title = title.strip()
                 s.author = author.strip()
                 s.price = price.strip()

src/calibre/gui2/store/amazon_uk_plugin.py

@@ -17,7 +17,7 @@ class AmazonUKKindleStore(AmazonKindleStore):
     For comments on the implementation, please see amazon_plugin.py
     '''
 
-    search_url = 'http://www.amazon.co.uk/s/url=search-alias%3Ddigital-text&field-keywords='
+    search_url = 'http://www.amazon.co.uk/s/?url=search-alias%3Ddigital-text&field-keywords='
     details_url = 'http://amazon.co.uk/dp/'
 
     def open(self, parent=None, detail_item=None, external=False):

src/calibre/gui2/store/config/search_widget.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+from PyQt4.Qt import QWidget
+
+from calibre.gui2 import JSONConfig
+from calibre.gui2.store.config.search_widget_ui import Ui_Form
+
+class StoreConfigWidget(QWidget, Ui_Form):
+
+    def __init__(self, config=None):
+        QWidget.__init__(self)
+        self.setupUi(self)
+
+        self.config = JSONConfig('store/search') if not config else config
+
+        # These default values should be the same as in
+        # calibre.gui2.store.search.search:SearchDialog.load_settings
+        # Seconds
+        self.opt_timeout.setValue(self.config.get('timeout', 75))
+        self.opt_hang_time.setValue(self.config.get('hang_time', 75))
+
+        self.opt_max_results.setValue(self.config.get('max_results', 10))
+        self.opt_open_external.setChecked(self.config.get('open_external', True))
+
+        # Number of threads to run for each type of operation
+        self.opt_search_thread_count.setValue(self.config.get('search_thread_count', 4))
+        self.opt_cache_thread_count.setValue(self.config.get('cache_thread_count', 2))
+        self.opt_cover_thread_count.setValue(self.config.get('cover_thread_count', 2))
+        self.opt_details_thread_count.setValue(self.config.get('details_thread_count', 4))
+
+    def save_settings(self):
+        self.config['timeout'] = self.opt_timeout.value()
+        self.config['hang_time'] = self.opt_hang_time.value()
+        self.config['max_results'] = self.opt_max_results.value()
+        self.config['open_external'] = self.opt_open_external.isChecked()
+        self.config['search_thread_count'] = self.opt_search_thread_count.value()
+        self.config['cache_thread_count'] = self.opt_cache_thread_count.value()
+        self.config['cover_thread_count'] = self.opt_cover_thread_count.value()
+        self.config['details_thread_count'] = self.opt_details_thread_count.value()

src/calibre/gui2/store/config/search_widget.ui

@@ -0,0 +1,162 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Form</class>
+ <widget class="QWidget" name="Form">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>465</width>
+    <height>396</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Form</string>
+  </property>
+  <layout class="QVBoxLayout" name="verticalLayout">
+   <item>
+    <widget class="QGroupBox" name="groupBox">
+     <property name="title">
+      <string>Time</string>
+     </property>
+     <layout class="QFormLayout" name="formLayout">
+      <item row="0" column="0">
+       <widget class="QLabel" name="label">
+        <property name="text">
+         <string>Number of seconds to wait for a store to respond</string>
+        </property>
+       </widget>
+      </item>
+      <item row="0" column="1">
+       <widget class="QSpinBox" name="opt_timeout">
+        <property name="minimum">
+         <number>1</number>
+        </property>
+       </widget>
+      </item>
+      <item row="1" column="0">
+       <widget class="QLabel" name="label_2">
+        <property name="text">
+         <string>Number of seconds to let a store process results</string>
+        </property>
+       </widget>
+      </item>
+      <item row="1" column="1">
+       <widget class="QSpinBox" name="opt_hang_time">
+        <property name="minimum">
+         <number>1</number>
+        </property>
+        <property name="maximum">
+         <number>99</number>
+        </property>
+        <property name="singleStep">
+         <number>1</number>
+        </property>
+        <property name="value">
+         <number>1</number>
+        </property>
+       </widget>
+      </item>
+     </layout>
+    </widget>
+   </item>
+   <item>
+    <widget class="QGroupBox" name="groupBox_2">
+     <property name="title">
+      <string>Display</string>
+     </property>
+     <layout class="QFormLayout" name="formLayout_2">
+      <item row="0" column="0">
+       <widget class="QLabel" name="label_3">
+        <property name="text">
+         <string>Maximum number of results to show per store</string>
+        </property>
+       </widget>
+      </item>
+      <item row="0" column="1">
+       <widget class="QSpinBox" name="opt_max_results">
+        <property name="minimum">
+         <number>1</number>
+        </property>
+       </widget>
+      </item>
+      <item row="1" column="0">
+       <widget class="QCheckBox" name="opt_open_external">
+        <property name="text">
+         <string>Open search result in system browser</string>
+        </property>
+       </widget>
+      </item>
+     </layout>
+    </widget>
+   </item>
+   <item>
+    <widget class="QGroupBox" name="groupBox_3">
+     <property name="title">
+      <string>Performance</string>
+     </property>
+     <layout class="QFormLayout" name="formLayout_3">
+      <item row="0" column="0">
+       <widget class="QLabel" name="label_4">
+        <property name="text">
+         <string>Number of simultaneous searches</string>
+        </property>
+       </widget>
+      </item>
+      <item row="0" column="1">
+       <widget class="QSpinBox" name="opt_search_thread_count">
+        <property name="minimum">
+         <number>1</number>
+        </property>
+       </widget>
+      </item>
+      <item row="1" column="0">
+       <widget class="QLabel" name="label_5">
+        <property name="text">
+         <string>Number of simultaneous cache updates</string>
+        </property>
+       </widget>
+      </item>
+      <item row="1" column="1">
+       <widget class="QSpinBox" name="opt_cache_thread_count">
+        <property name="minimum">
+         <number>1</number>
+        </property>
+       </widget>
+      </item>
+      <item row="2" column="0">
+       <widget class="QLabel" name="label_6">
+        <property name="text">
+         <string>Number of simultaneous cover downloads</string>
+        </property>
+       </widget>
+      </item>
+      <item row="2" column="1">
+       <widget class="QSpinBox" name="opt_cover_thread_count">
+        <property name="minimum">
+         <number>1</number>
+        </property>
+       </widget>
+      </item>
+      <item row="3" column="0">
+       <widget class="QLabel" name="label_7">
+        <property name="text">
+         <string>Number of simultaneous details downloads</string>
+        </property>
+       </widget>
+      </item>
+      <item row="3" column="1">
+       <widget class="QSpinBox" name="opt_details_thread_count">
+        <property name="minimum">
+         <number>1</number>
+        </property>
+       </widget>
+      </item>
+     </layout>
+    </widget>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections/>
+</ui>

src/calibre/gui2/store/config/store.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+'''
+Config widget access functions for configuring the store action.
+'''
+
+def config_widget():
+    from calibre.gui2.store.config.search_widget import StoreConfigWidget
+    return StoreConfigWidget()
+
+def save_settings(config_widget):
+    config_widget.save_settings()

src/calibre/gui2/store/ebooks_com_plugin.py

@@ -59,9 +59,10 @@ class EbookscomStore(BasicStoreConfig, StorePlugin):
                     break
 
                 id = ''.join(data.xpath('.//a[1]/@href'))
-                id = id.split('=')[-1]
-                if not id:
+                mo = re.search('\d+', id)
+                if not mo:
                     continue
+                id = mo.group()
                 
                 cover_url = ''.join(data.xpath('.//img[1]/@src'))
                 

src/calibre/gui2/store/gandalf_plugin.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, Tomasz Długosz <tomek3d@gmail.com>'
+__docformat__ = 'restructuredtext en'
+
+import re
+import urllib
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class GandalfStore(BasicStoreConfig, StorePlugin):
+
+    def open(self, parent=None, detail_item=None, external=False):
+        url = 'http://www.gandalf.com.pl/ebooks/'
+
+        if external or self.config.get('open_external', False):
+            open_url(QUrl(url_slash_cleaner(detail_item if detail_item else url)))
+        else:
+            d = WebStoreDialog(self.gui, url, parent, detail_item)
+            d.setWindowTitle(self.name)
+            d.set_tags(self.config.get('tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        url = 'http://www.gandalf.com.pl/s/'
+        values={
+            'search': query.encode('iso8859_2'),
+            'dzialx':'11'
+            }
+
+        br = browser()
+
+        counter = max_results
+        with closing(br.open(url, data=urllib.urlencode(values), timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//div[@class="box"]'):
+                if counter <= 0:
+                    break
+
+                id = ''.join(data.xpath('.//div[@class="info"]/h3/a/@href'))
+                if not id:
+                    continue
+
+                cover_url = ''.join(data.xpath('.//img/@src'))
+                title = ''.join(data.xpath('.//div[@class="info"]/h3/a/@title'))
+                formats = title.split()
+                formats = formats[-1]
+                author = ''.join(data.xpath('.//div[@class="info"]/h4/text() | .//div[@class="info"]/h4/span/text()'))
+                price = ''.join(data.xpath('.//h3[@class="promocja"]/text()'))
+                price = re.sub('PLN', 'zł', price)
+                price = re.sub('\.', ',', price)
+
+                counter -= 1
+
+                s = SearchResult()
+                s.cover_url = cover_url
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price
+                s.detail_item = id.strip()
+                s.drm = SearchResult.DRM_UNKNOWN
+                s.formats = formats.upper().strip()
+
+                yield s

src/calibre/gui2/store/google_books_plugin.py

@@ -51,7 +51,7 @@ class GoogleBooksStore(BasicStoreConfig, StorePlugin):
 
                 title = ''.join(data.xpath('.//h3/a//text()'))
                 authors = data.xpath('.//span[@class="gl"]//a//text()')
-                if authors[-1].strip().lower() == 'preview':
+                if authors[-1].strip().lower() in ('preview', 'read'):
                     authors = authors[:-1]
                 else:
                     continue

src/calibre/gui2/store/nexto_plugin.py

@@ -63,6 +63,7 @@ class NextoStore(BasicStoreConfig, StorePlugin):
 
                 cover_url = ''.join(data.xpath('.//img[@class="cover"]/@src'))
                 title = ''.join(data.xpath('.//a[@class="title"]/text()'))
+                title = re.sub(r' - ebook$', '', title)
                 formats = ', '.join(data.xpath('.//ul[@class="formats_available"]/li//b/text()'))
                 DrmFree = re.search(r'bez.DRM', formats)
                 formats = re.sub(r'\(.+\)', '', formats)

src/calibre/gui2/store/oreilly_plugin.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import re
+import urllib
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class OReillyStore(BasicStoreConfig, StorePlugin):
+
+    def open(self, parent=None, detail_item=None, external=False):
+        url = 'http://oreilly.com/ebooks/'
+
+        if detail_item:
+            detail_item = 'https://epoch.oreilly.com/shop/cart.orm?prod=%s.EBOOK&p=CALIBRE' % detail_item
+
+        if external or self.config.get('open_external', False):
+            open_url(QUrl(url_slash_cleaner(detail_item if detail_item else url)))
+        else:
+            d = WebStoreDialog(self.gui, url, parent, detail_item)
+            d.setWindowTitle(self.name)
+            d.set_tags(self.config.get('tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        url = 'http://search.oreilly.com/?t1=Books&t2=Format&t3=Ebook&q=' + urllib.quote_plus(query)
+        
+        br = browser()
+        
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//div[@id="results"]/div[@class="result"]'):
+                if counter <= 0:
+                    break
+
+                full_id = ''.join(data.xpath('.//div[@class="title"]/a/@href'))
+                mo = re.search('\d+', full_id)
+                if not mo:
+                    continue
+                id = mo.group()
+
+                cover_url = ''.join(data.xpath('.//div[@class="bigCover"]//img/@src'))
+
+                title = ''.join(data.xpath('.//div[@class="title"]/a/text()'))
+                author = ''.join(data.xpath('.//div[@class="author"]/text()'))
+                author = author.split('By ')[-1].strip()
+
+                # Get the detail here because we need to get the ebook id for the detail_item.
+                with closing(br.open(full_id, timeout=timeout)) as nf:
+                    idoc = html.fromstring(nf.read())
+                    
+                    price = ''.join(idoc.xpath('(//span[@class="price"])[1]/span//text()'))
+                    formats = ', '.join(idoc.xpath('//div[@class="ebook_formats"]//a/text()'))
+                    
+                    eid = ''.join(idoc.xpath('(//a[@class="product_buy_link" and contains(@href, ".EBOOK")])[1]/@href')).strip()
+                    mo = re.search('\d+', eid)
+                    if mo:
+                        id = mo.group()
+
+                counter -= 1
+                
+                s = SearchResult()
+                s.cover_url = cover_url.strip()
+                s.title = title.strip()
+                s.author = author.strip()
+                s.detail_item = id.strip()
+                s.price = price.strip()
+                s.drm = SearchResult.DRM_UNLOCKED
+                s.formats = formats.upper()
+                
+                yield s

src/calibre/gui2/store/pragmatic_bookshelf_plugin.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import urllib
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class PragmaticBookshelfStore(BasicStoreConfig, StorePlugin):
+
+    def open(self, parent=None, detail_item=None, external=False):
+        url = 'http://pragprog.com/'
+
+        if external or self.config.get('open_external', False):
+            open_url(QUrl(url_slash_cleaner(detail_item if detail_item else url)))
+        else:
+            d = WebStoreDialog(self.gui, url, parent, detail_item)
+            d.setWindowTitle(self.name)
+            d.set_tags(self.config.get('tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        '''
+        OPDS based search.
+        
+        We really should get the catelog from http://pragprog.com/catalog.opds
+        and look for the application/opensearchdescription+xml entry.
+        Then get the opensearch description to get the search url and
+        format. However, we are going to be lazy and hard code it.
+        '''
+        url = 'http://pragprog.com/catalog/search?q=' + urllib.quote_plus(query)
+        
+        br = browser()
+        
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            # Use html instead of etree as html allows us
+            # to ignore the namespace easily.
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//entry'):
+                if counter <= 0:
+                    break
+
+                id = ''.join(data.xpath('.//link[@rel="http://opds-spec.org/acquisition/buy"]/@href'))
+                if not id:
+                    continue
+
+                price = ''.join(data.xpath('.//price/@currencycode')).strip()
+                price += ' '
+                price += ''.join(data.xpath('.//price/text()')).strip()
+                if not price.strip():
+                    continue
+
+                cover_url = ''.join(data.xpath('.//link[@rel="http://opds-spec.org/cover"]/@href'))
+                
+                title = ''.join(data.xpath('.//title/text()'))
+                author = ''.join(data.xpath('.//author//text()'))
+
+                counter -= 1
+                
+                s = SearchResult()
+                s.cover_url = cover_url
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price.strip()
+                s.detail_item = id.strip()
+                s.drm = SearchResult.DRM_UNLOCKED
+                s.formats = 'EPUB, PDF, MOBI'
+                
+                yield s

src/calibre/gui2/store/search/download_thread.py

@@ -22,7 +22,7 @@ class GenericDownloadThreadPool(object):
     at the end of the function.
     '''
 
-    def __init__(self, thread_type, thread_count):
+    def __init__(self, thread_type, thread_count=1):
         self.thread_type = thread_type
         self.thread_count = thread_count
 
@@ -30,6 +30,9 @@ class GenericDownloadThreadPool(object):
         self.results = Queue()
         self.threads = []
 
+    def set_thread_count(self, thread_count):
+        self.thread_count = thread_count
+
     def add_task(self):
         '''
         This must be implemented in a sub class and this function
@@ -92,8 +95,8 @@ class SearchThreadPool(GenericDownloadThreadPool):
     def __init__(self, thread_count):
         GenericDownloadThreadPool.__init__(self, SearchThread, thread_count)
 
-    def add_task(self, query, store_name, store_plugin, timeout):
-        self.tasks.put((query, store_name, store_plugin, timeout))
+    def add_task(self, query, store_name, store_plugin, max_results, timeout):
+        self.tasks.put((query, store_name, store_plugin, max_results, timeout))
         GenericDownloadThreadPool.add_task(self)
 
 
@@ -112,8 +115,8 @@ class SearchThread(Thread):
     def run(self):
         while self._run and not self.tasks.empty():
             try:
-                query, store_name, store_plugin, timeout = self.tasks.get()
-                for res in store_plugin.search(query, timeout=timeout):
+                query, store_name, store_plugin, max_results, timeout = self.tasks.get()
+                for res in store_plugin.search(query, max_results=max_results, timeout=timeout):
                     if not self._run:
                         return
                     res.store_name = store_name

src/calibre/gui2/store/search/models.py

@@ -9,7 +9,8 @@ __docformat__ = 'restructuredtext en'
 import re
 from operator import attrgetter
 
-from PyQt4.Qt import (Qt, QAbstractItemModel, QVariant, QPixmap, QModelIndex, QSize)
+from PyQt4.Qt import (Qt, QAbstractItemModel, QVariant, QPixmap, QModelIndex, QSize,
+                      pyqtSignal)
 
 from calibre.gui2 import NONE
 from calibre.gui2.store.search_result import SearchResult
@@ -30,10 +31,12 @@ def comparable_price(text):
 
 class Matches(QAbstractItemModel):
 
+    total_changed = pyqtSignal(int)
+
     HEADERS = [_('Cover'), _('Title'), _('Price'), _('DRM'), _('Store')]
     HTML_COLS = (1, 4)
 
-    def __init__(self):
+    def __init__(self, cover_thread_count=2, detail_thread_count=4):
         QAbstractItemModel.__init__(self)
 
         self.DRM_LOCKED_ICON = QPixmap(I('drm-locked.png')).scaledToHeight(64,
@@ -51,8 +54,8 @@ class Matches(QAbstractItemModel):
         self.matches = []
         self.query = ''
         self.search_filter = SearchFilter()
-        self.cover_pool = CoverThreadPool(2)
-        self.details_pool = DetailsThreadPool(4)
+        self.cover_pool = CoverThreadPool(cover_thread_count)
+        self.details_pool = DetailsThreadPool(detail_thread_count)
 
         self.sort_col = 2
         self.sort_order = Qt.AscendingOrder
@@ -69,6 +72,7 @@ class Matches(QAbstractItemModel):
         self.query = ''
         self.cover_pool.abort()
         self.details_pool.abort()
+        self.total_changed.emit(self.rowCount())
         self.reset()
 
     def add_result(self, result, store_plugin):
@@ -101,6 +105,7 @@ class Matches(QAbstractItemModel):
             self.matches = list(self.search_filter.parse(self.query))
         else:
             self.matches = list(self.search_filter.universal_set())
+        self.total_changed.emit(self.rowCount())
         self.sort(self.sort_col, self.sort_order, False)
         self.layoutChanged.emit()
 

src/calibre/gui2/store/search/search.py

@@ -9,18 +9,17 @@ __docformat__ = 'restructuredtext en'
 import re
 from random import shuffle
 
-from PyQt4.Qt import (Qt, QDialog, QTimer, QCheckBox, QVBoxLayout, QIcon, QWidget) 
+from PyQt4.Qt import (Qt, QDialog, QDialogButtonBox, QTimer, QCheckBox,
+                      QVBoxLayout, QIcon, QWidget)
 
 from calibre.gui2 import JSONConfig, info_dialog
 from calibre.gui2.progress_indicator import ProgressIndicator
+from calibre.gui2.store.config.search_widget import StoreConfigWidget
 from calibre.gui2.store.search.adv_search_builder import AdvSearchBuilderDialog
 from calibre.gui2.store.search.download_thread import SearchThreadPool, \
     CacheUpdateThreadPool
 from calibre.gui2.store.search.search_ui import Ui_Dialog
 
-HANG_TIME = 75000 # milliseconds seconds
-TIMEOUT = 75 # seconds
-
 class SearchDialog(QDialog, Ui_Dialog):
 
     def __init__(self, istores, parent=None, query=''):
@@ -28,13 +27,22 @@ class SearchDialog(QDialog, Ui_Dialog):
         self.setupUi(self)
 
         self.config = JSONConfig('store/search')
-        
         self.search_edit.initialize('store_search_search')
 
+        # Loads variables that store various settings.
+        # This needs to be called soon in __init__ because
+        # the variables it sets up are used later.
+        self.load_settings()
+
         # We keep a cache of store plugins and reference them by name.
         self.store_plugins = istores
-        self.search_pool = SearchThreadPool(4)
-        self.cache_pool = CacheUpdateThreadPool(2)
+
+        # Setup our worker threads.
+        self.search_pool = SearchThreadPool(self.search_thread_count)
+        self.cache_pool = CacheUpdateThreadPool(self.cache_thread_count)
+        self.results_view.model().cover_pool.set_thread_count(self.cover_thread_count)
+        self.results_view.model().details_pool.set_thread_count(self.details_thread_count)
+
         # Check for results and hung threads.
         self.checker = QTimer()
         self.progress_checker = QTimer()
@@ -42,21 +50,21 @@ class SearchDialog(QDialog, Ui_Dialog):
 
         # Update store caches silently.
         for p in self.store_plugins.values():
-            self.cache_pool.add_task(p, 30)
+            self.cache_pool.add_task(p, self.timeout)
 
         # Add check boxes for each store so the user
         # can disable searching specific stores on a
         # per search basis.
         stores_check_widget = QWidget()
-        stores_group_layout = QVBoxLayout()
-        stores_check_widget.setLayout(stores_group_layout)
+        store_list_layout = QVBoxLayout()
+        stores_check_widget.setLayout(store_list_layout)
         for x in sorted(self.store_plugins.keys(), key=lambda x: x.lower()):
             cbox = QCheckBox(x)
             cbox.setChecked(False)
-            stores_group_layout.addWidget(cbox)
+            store_list_layout.addWidget(cbox)
             setattr(self, 'store_check_' + x, cbox)
-        stores_group_layout.addStretch()
-        self.stores_group.setWidget(stores_check_widget)
+        store_list_layout.addStretch()
+        self.store_list.setWidget(stores_check_widget)
 
         # Set the search query
         self.search_edit.setText(query)
@@ -66,15 +74,18 @@ class SearchDialog(QDialog, Ui_Dialog):
         self.top_layout.addWidget(self.pi)
 
         self.adv_search_button.setIcon(QIcon(I('search.png')))
+        self.configure.setIcon(QIcon(I('config.png')))
 
         self.adv_search_button.clicked.connect(self.build_adv_search)
         self.search.clicked.connect(self.do_search)
         self.checker.timeout.connect(self.get_results)
         self.progress_checker.timeout.connect(self.check_progress)
         self.results_view.activated.connect(self.open_store)
+        self.results_view.model().total_changed.connect(self.update_book_total)
         self.select_all_stores.clicked.connect(self.stores_select_all)
         self.select_invert_stores.clicked.connect(self.stores_select_invert)
         self.select_none_stores.clicked.connect(self.stores_select_none)
+        self.configure.clicked.connect(self.do_config)
         self.finished.connect(self.dialog_closed)
 
         self.progress_checker.start(100)
@@ -128,7 +139,7 @@ class SearchDialog(QDialog, Ui_Dialog):
         # Add plugins that the user has checked to the search pool's work queue.
         for n in store_names:
             if getattr(self, 'store_check_' + n).isChecked():
-                self.search_pool.add_task(query, n, self.store_plugins[n], TIMEOUT)
+                self.search_pool.add_task(query, n, self.store_plugins[n], self.max_results, self.timeout)
         self.hang_check = 0
         self.checker.start(100)
         self.pi.startAnimation()
@@ -190,7 +201,7 @@ class SearchDialog(QDialog, Ui_Dialog):
         else:
             self.resize_columns()
 
-        self.open_external.setChecked(self.config.get('open_external', False))
+        self.open_external.setChecked(self.should_open_external)
 
         store_check = self.config.get('store_checked', None)
         if store_check:
@@ -202,11 +213,56 @@ class SearchDialog(QDialog, Ui_Dialog):
         self.results_view.model().sort_order = self.config.get('sort_order', Qt.AscendingOrder)
         self.results_view.header().setSortIndicator(self.results_view.model().sort_col, self.results_view.model().sort_order)
 
+    def load_settings(self):
+        # Seconds
+        self.timeout = self.config.get('timeout', 75)
+        # Milliseconds
+        self.hang_time = self.config.get('hang_time', 75) * 1000
+
+        self.max_results = self.config.get('max_results', 10)
+        self.should_open_external = self.config.get('open_external', True)
+
+        # Number of threads to run for each type of operation
+        self.search_thread_count = self.config.get('search_thread_count', 4)
+        self.cache_thread_count = self.config.get('cache_thread_count', 2)
+        self.cover_thread_count = self.config.get('cover_thread_count', 2)
+        self.details_thread_count = self.config.get('details_thread_count', 4)
+
+    def do_config(self):
+        # Save values that need to be synced between the dialog and the
+        # search widget.
+        self.config['open_external'] = self.open_external.isChecked()
+
+        d = QDialog(self)
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        v = QVBoxLayout(d)
+        button_box.accepted.connect(d.accept)
+        button_box.rejected.connect(d.reject)
+        d.setWindowTitle(_('Customize Get Books'))
+        config_widget = StoreConfigWidget(self.config)
+        v.addWidget(config_widget)
+        v.addWidget(button_box)
+
+        d.exec_()
+
+        if d.result() == QDialog.Accepted:
+            config_widget.save_settings()
+            self.config_changed()
+
+    def config_changed(self):
+        self.load_settings()
+
+        self.open_external.setChecked(self.should_open_external)
+        self.search_pool.set_thread_count(self.search_thread_count)
+        self.cache_pool.set_thread_count(self.cache_thread_count)
+        self.results_view.model().cover_pool.set_thread_count(self.cover_thread_count)
+        self.results_view.model().details_pool.set_thread_count(self.details_thread_count)
+
     def get_results(self):
         # We only want the search plugins to run
         # a maximum set amount of time before giving up.
         self.hang_check += 1
-        if self.hang_check >= HANG_TIME:
+        if self.hang_check >= self.hang_time:
             self.search_pool.abort()
             self.checker.stop()
         else:
@@ -222,6 +278,8 @@ class SearchDialog(QDialog, Ui_Dialog):
         if not self.search_pool.threads_running() and not self.results_view.model().has_results():
             info_dialog(self, _('No matches'), _('Couldn\'t find any books matching your query.'), show=True, show_copy_button=False)
 
+    def update_book_total(self, total):
+        self.total.setText('%s' % total)
 
     def open_store(self, index):
         result = self.results_view.model().get_result(index)

src/calibre/gui2/store/search/search.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>937</width>
-    <height>669</height>
+    <width>584</width>
+    <height>533</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -20,7 +20,7 @@
   <property name="sizeGripEnabled">
    <bool>true</bool>
   </property>
-  <layout class="QVBoxLayout" name="verticalLayout">
+  <layout class="QVBoxLayout" name="verticalLayout_5">
    <item>
     <layout class="QHBoxLayout" name="top_layout">
      <item>
@@ -66,8 +66,14 @@
        <string>Stores</string>
       </property>
       <layout class="QVBoxLayout" name="verticalLayout_2">
+       <property name="spacing">
+        <number>0</number>
+       </property>
+       <property name="margin">
+        <number>0</number>
+       </property>
        <item>
-        <widget class="QScrollArea" name="stores_group">
+        <widget class="QScrollArea" name="store_list">
          <property name="widgetResizable">
           <bool>true</bool>
          </property>
@@ -76,15 +82,18 @@
            <rect>
             <x>0</x>
             <y>0</y>
-            <width>215</width>
-            <height>93</height>
+            <width>102</width>
+            <height>129</height>
            </rect>
           </property>
          </widget>
         </widget>
        </item>
        <item>
-        <layout class="QHBoxLayout" name="horizontalLayout_3">
+        <layout class="QVBoxLayout" name="verticalLayout">
+         <property name="spacing">
+          <number>0</number>
+         </property>
          <item>
           <widget class="QPushButton" name="select_all_stores">
            <property name="text">
@@ -108,32 +117,11 @@
          </item>
         </layout>
        </item>
-       <item>
-        <widget class="QCheckBox" name="open_external">
-         <property name="toolTip">
-          <string>Open a selected book in the system's web browser</string>
-         </property>
-         <property name="text">
-          <string>Open in &amp;external browser</string>
-         </property>
-        </widget>
-       </item>
       </layout>
      </widget>
-     <widget class="QSplitter" name="splitter_2">
-      <property name="sizePolicy">
-       <sizepolicy hsizetype="Expanding" vsizetype="Preferred">
-        <horstretch>2</horstretch>
-        <verstretch>0</verstretch>
-       </sizepolicy>
-      </property>
-      <property name="orientation">
-       <enum>Qt::Horizontal</enum>
-      </property>
-      <widget class="QSplitter" name="splitter">
-       <property name="orientation">
-        <enum>Qt::Horizontal</enum>
-       </property>
+     <widget class="QWidget" name="verticalLayoutWidget">
+      <layout class="QVBoxLayout" name="verticalLayout_4">
+       <item>
         <widget class="ResultsView" name="results_view">
          <property name="sizePolicy">
           <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
@@ -172,12 +160,61 @@
           <bool>false</bool>
          </property>
         </widget>
+       </item>
+       <item>
+        <layout class="QHBoxLayout" name="horizontalLayout">
+         <item>
+          <widget class="QToolButton" name="configure">
+           <property name="text">
+            <string>...</string>
+           </property>
           </widget>
+         </item>
+         <item>
+          <widget class="QCheckBox" name="open_external">
+           <property name="toolTip">
+            <string>Open a selected book in the system's web browser</string>
+           </property>
+           <property name="text">
+            <string>Open in &amp;external browser</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <spacer name="horizontalSpacer_2">
+           <property name="orientation">
+            <enum>Qt::Horizontal</enum>
+           </property>
+           <property name="sizeHint" stdset="0">
+            <size>
+             <width>40</width>
+             <height>20</height>
+            </size>
+           </property>
+          </spacer>
+         </item>
+        </layout>
+       </item>
+      </layout>
      </widget>
     </widget>
    </item>
    <item>
     <layout class="QHBoxLayout" name="bottom_layout">
+     <item>
+      <widget class="QLabel" name="label_2">
+       <property name="text">
+        <string>Books:</string>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QLabel" name="total">
+       <property name="text">
+        <string>0</string>
+       </property>
+      </widget>
+     </item>
      <item>
       <spacer name="horizontalSpacer">
        <property name="orientation">

src/calibre/gui2/store/woblink_plugin.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, Tomasz Długosz <tomek3d@gmail.com>'
+__docformat__ = 'restructuredtext en'
+
+import re
+import urllib
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class WoblinkStore(BasicStoreConfig, StorePlugin):
+
+    def open(self, parent=None, detail_item=None, external=False):
+
+        url = 'http://woblink.com/publication'
+        detail_url = None
+
+        if detail_item:
+            detail_url = 'http://woblink.com' + detail_item
+
+        if external or self.config.get('open_external', False):
+            open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url)))
+        else:
+            d = WebStoreDialog(self.gui, url, parent, detail_url)
+            d.setWindowTitle(self.name)
+            d.set_tags(self.config.get('tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        url = 'http://woblink.com/publication?query=' + urllib.quote_plus(query.encode('utf-8'))
+
+        br = browser()
+
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//div[@class="book-item"]'):
+                if counter <= 0:
+                    break
+
+                id = ''.join(data.xpath('.//td[@class="w10 va-t"]/a[1]/@href'))
+                if not id:
+                    continue
+
+                cover_url = ''.join(data.xpath('.//td[@class="w10 va-t"]/a[1]/img/@src'))
+                title = ''.join(data.xpath('.//h3[@class="title"]/a[1]/text()'))
+                author = ''.join(data.xpath('.//p[@class="author"]/a[1]/text()'))
+                price = ''.join(data.xpath('.//div[@class="prices"]/p[1]/span/text()'))
+                price = re.sub('PLN', ' zł', price)
+                price = re.sub('\.', ',', price)
+
+                counter -= 1
+
+                s = SearchResult()
+                s.cover_url = 'http://woblink.com' + cover_url
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price
+                s.detail_item = id.strip()
+                s.drm = SearchResult.DRM_LOCKED
+                s.formats = 'EPUB'
+
+                yield s

src/calibre/gui2/ui.py

@@ -23,7 +23,7 @@ from calibre.constants import __appname__, isosx
 from calibre.utils.config import prefs, dynamic
 from calibre.utils.ipc.server import Server
 from calibre.library.database2 import LibraryDatabase2
-from calibre.customize.ui import interface_actions, store_plugins
+from calibre.customize.ui import interface_actions, available_store_plugins
 from calibre.gui2 import error_dialog, GetMetadata, open_url, \
         gprefs, max_available_height, config, info_dialog, Dispatcher, \
         question_dialog
@@ -144,7 +144,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
 
     def load_store_plugins(self):
         self.istores = OrderedDict()
-        for store in store_plugins():
+        for store in available_store_plugins():
             if self.opts.ignore_plugins and store.plugin_path is not None:
                 continue
             try:
@@ -288,8 +288,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
                 self.db_images.reset()
 
         self.library_view.model().count_changed()
-        self.tool_bar.database_changed(self.library_view.model().db)
-        self.library_view.model().database_changed.connect(self.tool_bar.database_changed,
+        self.bars_manager.database_changed(self.library_view.model().db)
+        self.library_view.model().database_changed.connect(self.bars_manager.database_changed,
                 type=Qt.QueuedConnection)
 
         ########################### Tags Browser ##############################
@@ -324,7 +324,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
 
         self.read_settings()
         self.finalize_layout()
-        if self.tool_bar.showing_donate:
+        if self.bars_manager.showing_donate:
             self.donate_button.start_animation()
         self.set_window_title()
 

src/calibre/gui2/viewer/main.py

@@ -20,7 +20,7 @@ from calibre.gui2 import Application, ORG_NAME, APP_UID, choose_files, \
     info_dialog, error_dialog, open_url, available_height
 from calibre.ebooks.oeb.iterator import EbookIterator
 from calibre.ebooks import DRMError
-from calibre.constants import islinux, isfreebsd, isosx, filesystem_encoding
+from calibre.constants import islinux, isbsd, isosx, filesystem_encoding
 from calibre.utils.config import Config, StringConfig, JSONConfig
 from calibre.gui2.search_box import SearchBox2
 from calibre.ebooks.metadata import MetaInformation
@@ -801,7 +801,7 @@ def main(args=sys.argv):
 
     parser = option_parser()
     opts, args = parser.parse_args(args)
-    pid = os.fork() if False and (islinux or isfreebsd) else -1
+    pid = os.fork() if False and (islinux or isbsd) else -1
     if pid <= 0:
         app = Application(args)
         app.setWindowIcon(QIcon(I('viewer.png')))

src/calibre/gui2/widgets.py

@@ -833,7 +833,7 @@ class PythonHighlighter(QSyntaxHighlighter):
         Config["tabwidth"] = settings.value("tabwidth",
                 QVariant(4)).toInt()[0]
         Config["fontfamily"] = settings.value("fontfamily",
-                QVariant("Bitstream Vera Sans Mono")).toString()
+                QVariant("monospace")).toString()
         Config["fontsize"] = settings.value("fontsize",
                 QVariant(10)).toInt()[0]
         for name, color, bold, italic in (

src/calibre/gui2/wizard/send_email.py

@@ -138,8 +138,11 @@ class SendEmail(QWidget, Ui_Form):
                     'username': '',
                     'url': 'www.hotmail.com',
                     'extra': _('If you are setting up a new'
-                        ' hotmail account, you must log in to it '
-                        ' once before you will be able to send mails.'),
+                        ' hotmail account, Microsoft requires that you '
+                        ' verify your account periodically, before it'
+                        ' will let calibre send email. In this case, I'
+                        ' strongly suggest you setup a free gmail account'
+                        ' instead.'),
                 }
         }[service]
         d = QDialog(self)

src/calibre/library/database2.py

@@ -211,6 +211,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         defs = self.prefs.defaults
         defs['gui_restriction'] = defs['cs_restriction'] = ''
         defs['categories_using_hierarchy'] = []
+        self.column_color_count = 5
+        for i in range(1,self.column_color_count+1):
+            defs['column_color_name_'+str(i)] = ''
+            defs['column_color_template_'+str(i)] = ''
 
         # Migrate the bool tristate tweak
         defs['bools_are_tristate'] = \
@@ -1145,7 +1149,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             self.notify('metadata', [id])
         return True
 
-    def delete_book(self, id, notify=True, commit=True):
+    def delete_book(self, id, notify=True, commit=True, permanent=False):
         '''
         Removes book from the result cache and the underlying database.
         If you set commit to False, you must call clean() manually afterwards
@@ -1155,10 +1159,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         except:
             path = None
         if path and os.path.exists(path):
-            self.rmtree(path)
+            self.rmtree(path, permanent=permanent)
             parent = os.path.dirname(path)
             if len(os.listdir(parent)) == 0:
-                self.rmtree(parent)
+                self.rmtree(parent, permanent=permanent)
         self.conn.execute('DELETE FROM books WHERE id=?', (id,))
         if commit:
             self.conn.commit()

src/calibre/library/save_to_disk.py

@@ -56,16 +56,17 @@ for x in FORMAT_ARG_DESCS:
 def find_plugboard(device_name, format, plugboards):
     cpb = None
     if format in plugboards:
-        cpb = plugboards[format]
-    elif plugboard_any_format_value in plugboards:
-        cpb = plugboards[plugboard_any_format_value]
-    if cpb is not None:
-        if device_name in cpb:
-            cpb = cpb[device_name]
-        elif plugboard_any_device_value in cpb:
-            cpb = cpb[plugboard_any_device_value]
-        else:
-            cpb = None
+        pb = plugboards[format]
+        if device_name in pb:
+            cpb = pb[device_name]
+        elif plugboard_any_device_value in pb:
+            cpb = pb[plugboard_any_device_value]
+    if not cpb and plugboard_any_format_value in plugboards:
+        pb = plugboards[plugboard_any_format_value]
+        if device_name in pb:
+            cpb = pb[device_name]
+        elif plugboard_any_device_value in pb:
+            cpb = pb[plugboard_any_device_value]
     if DEBUG:
         prints('Device using plugboard', format, device_name, cpb)
     return cpb

src/calibre/library/server/base.py

@@ -218,7 +218,7 @@ class LibraryServer(ContentServer, MobileServer, XMLServer, OPDSServer, Cache,
                 cherrypy.engine.start()
             except:
                 ip = get_external_ip()
-                if not ip or ip == '127.0.0.1':
+                if not ip or ip.startswith('127.'):
                     raise
                 cherrypy.log('Trying to bind to single interface: '+ip)
                 cherrypy.config.update({'server.socket_host' : ip})

src/calibre/linux.py

@@ -7,7 +7,7 @@ import sys, os, cPickle, textwrap, stat, importlib
 from subprocess import check_call
 
 from calibre import  __appname__, prints, guess_type
-from calibre.constants import islinux, isfreebsd
+from calibre.constants import islinux, isnetbsd, isbsd
 from calibre.customize.ui import all_input_formats
 from calibre.ptempfile import TemporaryDirectory
 from calibre import CurrentDir
@@ -136,17 +136,17 @@ class PostInstall:
         self.icon_resources = []
         self.menu_resources = []
         self.mime_resources = []
-        if islinux or isfreebsd:
+        if islinux or isbsd:
             self.setup_completion()
         self.install_man_pages()
-        if islinux or isfreebsd:
+        if islinux or isbsd:
             self.setup_desktop_integration()
         self.create_uninstaller()
 
         from calibre.utils.config import config_dir
         if os.path.exists(config_dir):
             os.chdir(config_dir)
-            if islinux or isfreebsd:
+            if islinux or isbsd:
                 for f in os.listdir('.'):
                     if os.stat(f).st_uid == 0:
                         import shutil
@@ -195,6 +195,9 @@ class PostInstall:
                 'bash-completion')
             if os.path.exists(bc):
                 f = os.path.join(bc, 'calibre')
+            else:
+                if isnetbsd:
+                    f = os.path.join(self.opts.staging_root, 'share/bash_completion.d/calibre')
                 else:
                     f = os.path.join(self.opts.staging_etc, 'bash_completion.d/calibre')
             if not os.path.exists(os.path.dirname(f)):
@@ -300,7 +303,7 @@ class PostInstall:
     def install_man_pages(self): # {{{
         try:
             from calibre.utils.help2man import create_man_page
-            if isfreebsd:
+            if isbsd:
                 manpath = os.path.join(self.opts.staging_root, 'man/man1')
             else:
                 manpath = os.path.join(self.opts.staging_sharedir, 'man/man1')
@@ -316,7 +319,7 @@ class PostInstall:
                     continue
                 parser = parser()
                 raw = create_man_page(prog, parser)
-                if isfreebsd:
+                if isbsd:
                     manfile = os.path.join(manpath, prog+'.1')
                 else:
                     manfile = os.path.join(manpath, prog+'.1'+__appname__+'.bz2')
@@ -353,7 +356,7 @@ class PostInstall:
                     mimetypes = set([])
                     for x in all_input_formats():
                         mt = guess_type('dummy.'+x)[0]
-                        if mt and 'chemical' not in mt:
+                        if mt and 'chemical' not in mt and 'ctc-posml' not in mt:
                             mimetypes.add(mt)
 
                     def write_mimetypes(f):
@@ -373,11 +376,10 @@ class PostInstall:
                     des = ('calibre-gui.desktop', 'calibre-lrfviewer.desktop',
                             'calibre-ebook-viewer.desktop')
                     for x in des:
-                        cmd = ['xdg-desktop-menu', 'install', './'+x]
-                        if x != des[-1]:
-                            cmd.insert(2, '--noupdate')
+                        cmd = ['xdg-desktop-menu', 'install', '--noupdate', './'+x]
                         check_call(' '.join(cmd), shell=True)
                         self.menu_resources.append(x)
+                    check_call(['xdg-desktop-menu', 'forceupdate'])
                     f = open('calibre-mimetypes', 'wb')
                     f.write(MIME)
                     f.close()

src/calibre/manual/conversion.rst

@@ -655,6 +655,7 @@ Some limitations of PDF input are:
     * Some PDFs use special glyphs to represent ll or ff or fi, etc. Conversion of these may or may not work depending on just how they are represented internally in the PDF.
     * Some PDFs store their images upside down with a rotation instruction, |app| currently doesn't support that instruction, so the images will be rotated in the output as well. 
     * Links and Tables of Contents are not supported
+    * PDFs that use embedded non-unicode fonts to represent non-English characters will result in garbled output for those characters
 
 To re-iterate **PDF is a really, really bad** format to use as input. If you absolutely must use PDF, then be prepared for an
 output ranging anywhere from decent to unusable, depending on the input PDF.

src/calibre/manual/template_lang.rst

@@ -123,7 +123,8 @@ The functions available are:
     * ``contains(pattern, text if match, text if not match`` -- checks if field contains matches for the regular expression `pattern`. Returns `text if match` if matches are found, otherwise it returns `text if no match`.
     * ``count(separator)`` -- interprets the value as a list of items separated by `separator`, returning the number of items in the list. Most lists use a comma as the separator, but authors uses an ampersand. Examples: `{tags:count(,)}`, `{authors:count(&)}`
     * ``ifempty(text)``	-- if the field is not empty, return the value of the field. Otherwise return `text`.
-    * ``list_item(index, separator)`` -- interpret the value as a list of items separated by `separator`, returning the `index`th item. The first item is number zero. The last item can be returned using `list_item(-1,separator)`. If the item is not in the list, then the empty value is returned. The separator has the same meaning as in the `count` function.
+    * ``in_list(separator, pattern, found_val, not_found_val)`` -- interpret the field as a list of items separated by `separator`, comparing the `pattern` against each value in the list. If the pattern matches a value, return `found_val`, otherwise return `not_found_val`.
+    * ``list_item(index, separator)`` -- interpret the field as a list of items separated by `separator`, returning the `index`th item. The first item is number zero. The last item can be returned using `list_item(-1,separator)`. If the item is not in the list, then the empty value is returned. The separator has the same meaning as in the `count` function.
     * ``re(pattern, replacement)`` -- return the field after applying the regular expression. All instances of `pattern` are replaced with `replacement`. As in all of |app|, these are python-compatible regular expressions.
     * ``shorten(left chars, middle text, right chars)`` -- Return a shortened version of the field, consisting of `left chars` characters from the beginning of the field, followed by `middle text`, followed by `right chars` characters from the end of the string. `Left chars` and `right chars` must be integers. For example, assume the title of the book is `Ancient English Laws in the Times of Ivanhoe`, and you want it to fit in a space of at most 15 characters. If you use ``{title:shorten(9,-,5)}``, the result will be `Ancient E-nhoe`. If the field's length is less than ``left chars`` + ``right chars`` + the length of ``middle text``, then the field will be used intact. For example, the title `The Dome` would not be changed.
     * ``switch(pattern, value, pattern, value, ..., else_value)`` -- for each ``pattern, value`` pair, checks if the field matches the regular expression ``pattern`` and if so, returns that ``value``. If no ``pattern`` matches, then ``else_value`` is returned. You can have as many ``pattern, value`` pairs as you want.
@@ -234,6 +235,7 @@ The following functions are available in addition to those described in single-f
     * ``cmp(x, y, lt, eq, gt)`` -- compares x and y after converting both to numbers. Returns ``lt`` if x < y. Returns ``eq`` if x == y. Otherwise returns ``gt``.
     * ``divide(x, y)`` -- returns x / y. Throws an exception if either x or y are not numbers.
     * ``field(name)`` -- returns the metadata field named by ``name``.
+    * ``first_non_empty(value, value, ...) -- returns the first value that is not empty. If all values are empty, then the empty value is returned. You can have as many values as you want.
     * ``format_date(x, date_format)`` -- format_date(val, format_string) -- format the value, which must be a date field, using the format_string, returning a string. The formatting codes are::
     
         d    : the day as number without a leading zero (1 to 31)
@@ -374,13 +376,18 @@ To accomplish this, we:
 Templates and Plugboards
 ------------------------
 
-Plugboards are used for changing the metadata written into books during send-to-device and save-to-disk operations. A plugboard permits you to specify a template to provide the data to write into the book's metadata. You can use plugboards to modify the following fields: authors, author_sort, language, publisher, tags, title, title_sort. This feature should help those of you who want to use different metadata in your books on devices to solve sorting or display issues.
+Plugboards are used for changing the metadata written into books during send-to-device and save-to-disk operations. A plugboard permits you to specify a template to provide the data to write into the book's metadata. You can use plugboards to modify the following fields: authors, author_sort, language, publisher, tags, title, title_sort. This feature helps people who want to use different metadata in books on devices to solve sorting or display issues.
 
 When you create a plugboard, you specify the format and device for which the plugboard is to be used. A special device is provided, save_to_disk, that is used when saving formats (as opposed to sending them to a device). Once you have chosen the format and device, you choose the metadata fields to change, providing templates to supply the new values. These templates are `connected` to their destination fields, hence the name `plugboards`. You can, of course, use composite columns in these templates.
 
-The tags and authors fields have special treatment, because both of these fields can hold more than one item. After all, book can have many tags and many authors. When you specify that one of these two fields is to be changed, the result of evaluating the template is examined to see if more than one item is there.
+When a plugboard might apply (content server, save to disk, or send to device), |app| searches the defined plugboards to choose the correct one for the given format and device. For example, to find the appropriate plugboard for an EPUB book being sent to an ANDROID device, |app| searches the plugboards using the following search order:
 
-For tags, the result cut apart whereever |app| finds a comma. For example, if the template produces the value ``Thriller, Horror``, then the result will be two tags, ``Thriller`` and ``Horror``. There is no way to put a comma in the middle of a tag.
+    * a plugboard with an exact match on format and device, e.g., ``EPUB`` and ``ANDROID``
+    * a plugboard with an exact match on format and the special ``any device`` choice, e.g., ``EPUB`` and ``any device``
+    * a plugboard with the special ``any format`` choice and an exact match on device, e.g., ``any format`` and ``ANDROID``
+    * a plugboard with ``any format`` and ``any device``
+    
+The tags and authors fields have special treatment, because both of these fields can hold more than one item. A book can have many tags and many authors. When you specify that one of these two fields is to be changed, the template's result is examined to see if more than one item is there. For tags, the result is cut apart wherever |app| finds a comma. For example, if the template produces the value ``Thriller, Horror``, then the result will be two tags, ``Thriller`` and ``Horror``. There is no way to put a comma in the middle of a tag.
 
 The same thing happens for authors, but using a different character for the cut, a `&` (ampersand) instead of a comma. For example, if the template produces the value ``Blogs, Joe&Posts, Susan``, then the book will end up with two authors, ``Blogs, Joe`` and ``Posts, Susan``. If the template produces the value ``Blogs, Joe;Posts, Susan``, then the book will have one author with a rather strange name.
 

src/calibre/ptempfile.py

@@ -29,6 +29,10 @@ def remove_dir(x):
 
 def base_dir():
     global _base_dir
+    if _base_dir is not None and not os.path.exists(_base_dir):
+        # Some people seem to think that running temp file cleaners that
+        # delete the temp dirs of running programs is a good idea!
+        _base_dir = None
     if _base_dir is None:
         td = os.environ.get('CALIBRE_WORKER_TEMP_DIR', None)
         if td is not None:
@@ -39,8 +43,20 @@ def base_dir():
         if td and os.path.exists(td):
             _base_dir = td
         else:
-            _base_dir = tempfile.mkdtemp(prefix='%s_%s_tmp_'%(__appname__,
-                __version__), dir=os.environ.get('CALIBRE_TEMP_DIR', None))
+            base = os.environ.get('CALIBRE_TEMP_DIR', None)
+            prefix = u'%s_%s_tmp_'%(__appname__, __version__)
+            try:
+                # First try an ascii path as that is what was done historically
+                # and we dont want to break working code
+                # _base_dir will be a bytestring
+                _base_dir = tempfile.mkdtemp(prefix=prefix.encode('ascii'), dir=base)
+            except:
+                # Failed to create tempdir (probably localized windows)
+                # Try unicode. This means that all temp paths created by this
+                # module will be unicode, this may cause problems elsewhere, if
+                # so, hopefully people will open tickets and they can be fixed.
+                _base_dir = tempfile.mkdtemp(prefix=prefix, dir=base)
+
             atexit.register(remove_dir, _base_dir)
     return _base_dir