merge from trunk

resources/calibre-portable.bat

@@ -6,25 +6,37 @@ REM  - Calibre Library Files
 REM  - Calibre Config Files
 REM  - Calibre Metadata database
 REM  - Calibre Source files
+REM  - Calibre Temp Files
 REM By setting the paths correctly it can be used to run:
 REM  - A "portable calibre" off a USB stick.
 REM  - A network installation with local metadata database
 REM    (for performance) and books stored on a network share 
+REM  - A local installation using customised settings
 REM
-REM If trying to run off a USB stick then the following 
+REM If trying to run off a USB stick then the folder structure
-REM folder structure is recommended:
+REM shown below is recommended (relative to the location of 
+REM this batch file).  This can structure can also be used
+REM when running of a local hard disk if you want to get the
+REM level of control this batch file provides.
 REM  - Calibre2			Location of program files
 REM  - CalibreConfig		Location of Configuration files
 REM  - CalibreLibrary		Location of Books and metadata
+REM  - CalibreSource 		Location of Calibre Source files (Optional)
+REM
+REM This batch file is designed so that if you create the recommended
+REM folder structure then it can be used 'as is' without modification.
 
 
 REM -------------------------------------
 REM Set up Calibre Config folder
+REM
+REM This is where user specific settings
+REM are stored.
 REM -------------------------------------
 
 IF EXIST CalibreConfig (
 	SET CALIBRE_CONFIG_DIRECTORY=%cd%\CalibreConfig
-	ECHO CONFIG=%cd%\CalibreConfig
+	ECHO CONFIG FILES:    %cd%\CalibreConfig
 )
 
 
@@ -35,21 +47,18 @@ REM Location where Book files are located
 REM Either set explicit path, or if running from a USB stick
 REM a relative path can be used to avoid need to know the
 REM drive letter of the USB stick.
-
+REM
 REM Comment out any of the following that are not to be used
+REM (although leaving them in does not really matter)
 REM --------------------------------------------------------------
 
 IF EXIST U:\eBooks\CalibreLibrary (
 	SET CALIBRE_LIBRARY_DIRECTORY=U:\eBOOKS\CalibreLibrary
-	ECHO LIBRARY=U:\eBOOKS\CalibreLibrary
+	ECHO LIBRARY FILES:   U:\eBOOKS\CalibreLibrary
 )
 IF EXIST CalibreLibrary (
 	SET CALIBRE_LIBRARY_DIRECTORY=%cd%\CalibreLibrary
-	ECHO LIBRARY=%cd%\CalibreLibrary
+	ECHO LIBRARY FILES:   %cd%\CalibreLibrary
-)
-IF EXIST CalibreBooks (
-	SET CALIBRE_LIBRARY_DIRECTORY=%cd%\CalibreBooks
-	ECHO LIBRARY=%cd%\CalibreBooks
 )
 
 
@@ -60,7 +69,7 @@ REM Location where the metadata.db file is located.  If not set
 REM the same location as Books files will be assumed.  This.
 REM options is used to get better performance when the Library is
 REM on a (slow) network drive.  Putting the metadata.db file 
-REM locally makes gives a big performance improvement.
+REM locally then makes gives a big performance improvement.
 REM
 REM NOTE.  If you use this option, then the ability to switch
 REM        libraries within Calibre will be disabled.  Therefore
@@ -68,19 +77,10 @@ REM        you do not want to set it if the metadata.db file
 REM        is at the same location as the book files.
 REM --------------------------------------------------------------
 
-IF EXIST CalibreBooks (
+IF EXIST %cd%\CalibreMetadata\metadata.db (
-	IF NOT "%CALIBRE_LIBRARY_DIRECTORY%" == "%cd%\CalibreBooks" (
-		SET SET CALIBRE_OVERRIDE_DATABASE_PATH=%cd%\CalibreBooks\metadata.db
-		ECHO DATABASE=%cd%\CalibreBooks\metadata.db
-		ECHO '
-		ECHO ***CAUTION*** Library Switching will be disabled 
-		ECHO '
-	)
-)
-IF EXIST CalibreMetadata (
 	IF NOT "%CALIBRE_LIBRARY_DIRECTORY%" == "%cd%\CalibreMetadata" (
 		SET CALIBRE_OVERRIDE_DATABASE_PATH=%cd%\CalibreMetadata\metadata.db
-		ECHO DATABASE=%cd%\CalibreMetadata\metadata.db
+		ECHO DATABASE:        %cd%\CalibreMetadata\metadata.db
 		ECHO '
 		ECHO ***CAUTION*** Library Switching will be disabled 
 		ECHO '
@@ -96,37 +96,60 @@ REM When running from source the GUI will have a '*' after the version.
 REM number that is displayed at the bottom of the Calibre main screen.
 REM --------------------------------------------------------------
 
-IF EXIST Calibre\src (
+IF EXIST CalibreSource\src (
-	SET CALIBRE_DEVELOP_FROM=%cd%\Calibre\src
+	SET CALIBRE_DEVELOP_FROM=%cd%\CalibreSource\src
-	ECHO SOURCE=%cd%\Calibre\src
+	ECHO SOURCE FILES:    %cd%\CalibreSource\src
-)
-IF EXIST D:\Calibre\Calibre\src (
-	SET CALIBRE_DEVELOP_FROM=D:\Calibre\Calibre\src
-	ECHO SOURCE=D:\Calibre\Calibre\src
 )
 
+
 REM --------------------------------------------------------------
 REM Specify Location of calibre binaries (optional)
 REM
 REM To avoid needing Calibre to be set in the search path, ensure
 REM that Calibre Program Files is current directory when starting.
 REM The following test falls back to using search path .
-REM This folder can be populated by cpying the Calibre2 folder from
+REM This folder can be populated by copying the Calibre2 folder from
-REM an existing isntallation or by isntalling direct to here.
+REM an existing installation or by installing direct to here.
 REM --------------------------------------------------------------
 
-IF EXIST Calibre2 (
+IF EXIST %cd%\Calibre2 (
-	Calibre2 CD Calibre2
+	CD %cd%\Calibre2
-	ECHO PROGRAMS=%cd%
+	ECHO PROGRAM FILES:   %cd%
 )
 
+
+REM --------------------------------------------------------------
+REM Location of Calibre Temporary files  (optional)
+REM
+REM Calibre creates a lot of temproary files while running
+REM In theory these are removed when Calibre finishes, but
+REM in practise files can be left behind (particularily if
+REM any errors occur.  Using this option allows some
+REM explicit clean-up of these files.
+REM If not set Calibre uses the normal system TEMP location
+REM --------------------------------------------------------------
+
+SET CALIBRE_TEMP_DIR=%TEMP%\CALIBRE_TEMP
+ECHO TEMPORARY FILES: %CALIBRE_TEMP_DIR%
+
+IF NOT "%CALIBRE_TEMP_DIR%" == "" (
+	IF EXIST "%CALIBRE_TEMP_DIR%" RMDIR /s /q "%CALIBRE_TEMP_DIR%"
+	MKDIR "%CALIBRE_TEMP_DIR%"
+	REM set the following for any components that do
+	REM not obey the CALIBRE_TEMP_DIR setting
+	SET TMP=%CALIBRE_TEMP_DIR%
+	SET TEMP=%CALIBRE_TEMP_DIR%
+)
+
+
 REM ----------------------------------------------------------
 REM  The following gives a chance to check the settings before
 REM  starting Calibre.  It can be commented out if not wanted.
 REM ----------------------------------------------------------
 
-echo "Press CTRL-C if you do not want to continue"
+ECHO '
-pause
+ECHO "Press CTRL-C if you do not want to continue"
+PAUSE
 
 
 REM --------------------------------------------------------
@@ -141,5 +164,7 @@ REM If used without /WAIT opotion launches Calibre and contines batch file.
 REM Use with /WAIT to wait until Calibre completes to run a task on exit
 REM --------------------------------------------------------
 
-echo "Starting up Calibre"
+ECHO "Starting up Calibre"
-START /belownormal Calibre --with-library "%CALIBRE_LIBRARY_DIRECTORY%"
+ECHO OFF
+ECHO %cd%
+START /belownormal Calibre --with-library "%CALIBRE_LIBRARY_DIRECTORY%"

resources/catalog/stylesheet.css

@@ -62,6 +62,18 @@ div.description {
 	text-indent: 1em;
 	}
 
+/*
+* 	Attempt to minimize widows and orphans by logically grouping chunks
+* 	Recommend enabling for iPad
+*   Some reports of problems with Sony ereaders, presumably ADE engines
+*/
+/*
+div.logical_group {
+	display:inline-block;
+	width:100%;
+	}
+*/
+
 p.date_index {
 	font-size:x-large;
 	text-align:center;

resources/recipes/20_minutos.recipe

@@ -0,0 +1,67 @@
+# -*- coding: utf-8
+__license__   = 'GPL v3'
+__author__    = 'Luis Hernandez'
+__copyright__ = 'Luis Hernandez<tolyluis@gmail.com>'
+description   = 'Periódico gratuito en español - v0.5 - 25 Jan 2011'
+
+'''
+www.20minutos.es
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1294946868(BasicNewsRecipe):
+
+    title          = u'20 Minutos'
+    publisher      = u'Grupo 20 Minutos'
+
+    __author__            = u'Luis Hernández'
+    description           = u'Periódico gratuito en español'
+    cover_url     = 'http://estaticos.20minutos.es/mmedia/especiales/corporativo/css/img/logotipos_grupo20minutos.gif'
+
+    oldest_article = 5
+    max_articles_per_feed = 100
+
+    remove_javascript = True
+    no_stylesheets        = True
+    use_embedded_content  = False
+
+    encoding              = 'ISO-8859-1'
+    language              = 'es'
+    timefmt        = '[%a, %d %b, %Y]'
+
+    keep_only_tags     = [dict(name='div', attrs={'id':['content']})
+                                  ,dict(name='div', attrs={'class':['boxed','description','lead','article-content']})
+                                  ,dict(name='span', attrs={'class':['photo-bar']})
+                                  ,dict(name='ul', attrs={'class':['article-author']})
+                                ]
+
+    remove_tags_before = dict(name='ul' , attrs={'class':['servicios-sub']})
+    remove_tags_after  = dict(name='div' , attrs={'class':['related-news','col']})
+
+    remove_tags = [
+                     dict(name='ol', attrs={'class':['navigation',]})
+                    ,dict(name='span', attrs={'class':['action']})
+                    ,dict(name='div', attrs={'class':['twitter comments-list hidden','related-news','col']})
+                    ,dict(name='div', attrs={'id':['twitter-destacados']})
+                    ,dict(name='ul', attrs={'class':['article-user-actions','stripped-list']})
+                                          ]
+
+    feeds = [
+              (u'Portada'              , u'http://www.20minutos.es/rss/')
+             ,(u'Nacional'             , u'http://www.20minutos.es/rss/nacional/')
+             ,(u'Internacional'       , u'http://www.20minutos.es/rss/internacional/')
+             ,(u'Economia'           , u'http://www.20minutos.es/rss/economia/')
+             ,(u'Deportes'            , u'http://www.20minutos.es/rss/deportes/')
+             ,(u'Tecnologia'          , u'http://www.20minutos.es/rss/tecnologia/')
+             ,(u'Gente - TV'         , u'http://www.20minutos.es/rss/gente-television/')
+             ,(u'Motor'                 , u'http://www.20minutos.es/rss/motor/')
+             ,(u'Salud'                 , u'http://www.20minutos.es/rss/belleza-y-salud/')
+             ,(u'Viajes'                , u'http://www.20minutos.es/rss/viajes/')
+             ,(u'Vivienda'             , u'http://www.20minutos.es/rss/vivienda/')
+             ,(u'Empleo'              , u'http://www.20minutos.es/rss/empleo/')
+             ,(u'Cine'                  , u'http://www.20minutos.es/rss/cine/')
+             ,(u'Musica'               , u'http://www.20minutos.es/rss/musica/')
+             ,(u'Comunidad20'     , u'http://www.20minutos.es/rss/zona20/')
+            ]
+

resources/recipes/abc.recipe

@@ -0,0 +1,43 @@
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class ABCRecipe(BasicNewsRecipe):
+  title          = u'ABC Linuxu'
+  oldest_article = 5
+  max_articles_per_feed = 3#5
+  __author__ = 'Funthomas'
+  language = 'cs'
+
+  feeds = [
+    #(u'Blogy', u'http://www.abclinuxu.cz/auto/blogDigest.rss'),
+    (u'Články', u'http://www.abclinuxu.cz/auto/abc.rss'),
+    (u'Zprávičky','http://www.abclinuxu.cz/auto/zpravicky.rss')
+  ]
+
+  remove_javascript = True
+  no_stylesheets    = True
+  remove_attributes = ['width','height']
+
+  remove_tags_before = dict(name='h1')
+  remove_tags = [
+    dict(attrs={'class':['meta-vypis','page_tools','cl_perex']}),
+    dict(attrs={'class':['cl_nadpis-link','komix-nav']})
+  ]
+
+  remove_tags_after = [
+    dict(name='div',attrs={'class':['cl_perex','komix-nav']}),
+    dict(attrs={'class':['meta-vypis','page_tools']}),
+    dict(name='',attrs={'':''}),
+  ]
+
+
+  preprocess_regexps = [
+    (re.compile(r'</div>.*<p class="perex">', re.DOTALL),lambda match: '</div><p class="perex">')
+  ]
+  def print_version(self, url):
+    return url + '?varianta=print&noDiz'
+
+  extra_css = '''
+            h1 {font-size:130%; font-weight:bold}
+            h3 {font-size:111%; font-weight:bold}
+        '''

resources/recipes/dailytportal.recipe

@@ -0,0 +1,66 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011, Darko Miletic <darko.miletic at gmail.com>'
+'''
+daily.tportal.hr
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Pagina12(BasicNewsRecipe):
+    title                 = 'Daily tportal.h'
+    __author__            = 'Darko Miletic'
+    description           = 'News from Croatia'
+    publisher             = 'tportal.hr'
+    category              = 'news, politics, Croatia'
+    oldest_article        = 2
+    max_articles_per_feed = 200
+    no_stylesheets        = True
+    encoding              = 'utf-8'
+    use_embedded_content  = False
+    language              = 'en_HR'
+    remove_empty_feeds    = True
+    publication_type      = 'newsportal'
+    extra_css             = """
+                               body{font-family: Verdana,sans-serif }
+                               img{margin-bottom: 0.4em; display:block}
+                               h1,h2{color: #2D648A; font-family: Georgia,serif}
+                               .artAbstract{font-size: 1.2em; font-family: Georgia,serif}
+                            """
+
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
+
+    remove_tags = [
+                     dict(name=['meta','link','embed','object','iframe','base'])
+                    ,dict(name='div', attrs={'class':'artInfo'})
+                  ]
+    remove_attributes=['lang']
+
+    keep_only_tags=dict(attrs={'class':'articleDetails'})
+
+    feeds = [(u'News', u'http://daily.tportal.hr/rss/dailynaslovnicarss.xml')]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll('a'):
+            limg = item.find('img')
+            if item.string is not None:
+               str = item.string
+               item.replaceWith(str)
+            else:
+               if limg:
+                  item.name = 'div'
+                  item.attrs = []
+               else:
+                   str = self.tag_to_string(item)
+                   item.replaceWith(str)
+        for item in soup.findAll('img'):
+            if not item.has_key('alt'):
+               item['alt'] = 'image'
+        return soup
+

resources/recipes/economist.recipe

@@ -22,8 +22,11 @@ class Economist(BasicNewsRecipe):
 
     oldest_article = 7.0
     cover_url = 'http://www.economist.com/images/covers/currentcoverus_large.jpg'
-    remove_tags = [dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']),
+    remove_tags = [
-            dict(attrs={'class':['dblClkTrk', 'ec-article-info']})]
+            dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']),
+            dict(attrs={'class':['dblClkTrk', 'ec-article-info']}),
+            {'class': lambda x: x and 'share-links-header' in x},
+    ]
     keep_only_tags = [dict(id='ec-article-body')]
     needs_subscription = False
     no_stylesheets = True

resources/recipes/economist_free.recipe

@@ -16,8 +16,11 @@ class Economist(BasicNewsRecipe):
 
     oldest_article = 7.0
     cover_url = 'http://www.economist.com/images/covers/currentcoverus_large.jpg'
-    remove_tags = [dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']),
+    remove_tags = [
-            dict(attrs={'class':['dblClkTrk', 'ec-article-info']})]
+            dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']),
+            dict(attrs={'class':['dblClkTrk', 'ec-article-info']}),
+            {'class': lambda x: x and 'share-links-header' in x},
+    ]
     keep_only_tags = [dict(id='ec-article-body')]
     no_stylesheets = True
     preprocess_regexps = [(re.compile('</html>.*', re.DOTALL),

resources/recipes/heise.recipe

@@ -52,6 +52,7 @@ class heiseDe(BasicNewsRecipe):
 		   dict(id='navi_login'),
 		   dict(id='navigation'),
 		   dict(id='breadcrumb'),
+		   dict(id='adblockerwarnung'),
 		   dict(id=''),
 		   dict(id='sitemap'),
 		   dict(id='bannerzone'),
@@ -67,3 +68,4 @@ class heiseDe(BasicNewsRecipe):
 
 
 
+

resources/recipes/hna.recipe

@@ -21,7 +21,7 @@ class hnaDe(BasicNewsRecipe):
     max_articles_per_feed = 40
     no_stylesheets = True
     remove_javascript = True
-    encoding = 'iso-8859-1'
+    encoding = 'utf-8'
 
     remove_tags = [dict(id='topnav'),
 		   dict(id='nav_main'),
@@ -60,3 +60,4 @@ class hnaDe(BasicNewsRecipe):
     feeds =  [ ('hna_soehre', 'http://feeds2.feedburner.com/hna/soehre'),
 	       ('hna_kassel', 'http://feeds2.feedburner.com/hna/kassel') ]
 
+

resources/recipes/ibm_smarter_planet.recipe

@@ -1,17 +1,18 @@
+
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class AdvancedUserRecipe1293122276(BasicNewsRecipe):
-    title          = u'Smarter Planet | Tumblr for eReaders'
+    title          = u'Smarter Planet | Tumblr'
     __author__ = 'Jack Mason'
     author  = 'IBM Global Business Services'
     publisher             = 'IBM'
     language = 'en'
     category              = 'news, technology, IT, internet of things, analytics'
-    oldest_article = 7
+    oldest_article = 14
     max_articles_per_feed = 30
     no_stylesheets = True
     use_embedded_content = False
-    masthead_url          = 'http://30.media.tumblr.com/tumblr_l70dow9UmU1qzs4rbo1_r3_250.jpg'
+    masthead_url          = 'http://www.hellercd.com/wp-content/uploads/2010/09/hero.jpg'
     remove_tags_before = dict(id='item')
     remove_tags_after  = dict(id='item')
     remove_tags = [dict(attrs={'class':['sidebar', 'about', 'footer', 'description,' 'disqus', 'nav', 'notes', 'disqus_thread']}),
@@ -21,4 +22,3 @@ class AdvancedUserRecipe1293122276(BasicNewsRecipe):
 
 
     feeds          = [(u'Smarter Planet Tumblr', u'http://smarterplanet.tumblr.com/mobile/rss')]
-

resources/recipes/idnes.recipe

@@ -0,0 +1,54 @@
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class iHeuteRecipe(BasicNewsRecipe):
+    __author__  = 'FunThomas'
+    title = u'iDnes.cz'
+    publisher = u'MAFRA a.s.'
+    description = 'iDNES.cz Zprávy, Technet, Komiksy a další'
+    oldest_article = 3
+    max_articles_per_feed = 2
+
+    feeds = [
+            (u'Zprávy', u'http://servis.idnes.cz/rss.asp?c=zpravodaj'),
+            (u'Sport', u'http://servis.idnes.cz/rss.asp?c=sport'),
+            (u'Technet', u'http://servis.idnes.cz/rss.asp?c=technet'),
+            (u'Mobil', u'http://servis.idnes.cz/rss.asp?c=mobil'),
+            (u'Ekonomika', u'http://servis.idnes.cz/rss.asp?c=ekonomikah'),
+            #(u'Kultura', u'http://servis.idnes.cz/rss.asp?c=kultura'),
+            (u'Cestování', u'http://servis.idnes.cz/rss.asp?c=iglobe'),
+            #(u'Kavárna', u'http://servis.idnes.cz/rss.asp?r=kavarna'),
+            (u'Komixy', u'http://servis.idnes.cz/rss.asp?c=komiksy')
+            ]
+
+
+    encoding = 'cp1250'
+    language = 'cs'
+    cover_url = 'http://g.idnes.cz/u/loga-n4/idnes.gif'
+    remove_javascript = True
+    no_stylesheets = True
+
+    remove_attributes = ['width','height']
+    remove_tags = [dict(name='div',   attrs={'id':['zooming']}),
+                   dict(name='div',   attrs={'class':['related','mapa-wrapper']}),
+                   dict(name='table', attrs={'id':['opener-img','portal']}),
+                   dict(name='table', attrs={'class':['video-16ku9']})]
+    remove_tags_after  = [dict(name='div',attrs={'id':['related','related2']})]
+
+    keep_only_tags = [dict(name='div', attrs={'class':['art-full adwords-text','dil-day']})
+                      ,dict(name='table',attrs={'class':['kemel-box']})]
+
+    def print_version(self, url):
+        print_url = url
+        split_url = url.split("?")
+        if (split_url[0].rfind('dilbert.asp')  != -1):      #dilbert komix
+            print_url = print_url.replace('.htm','.gif&tisk=1')
+            print_url = print_url.replace('.asp','.aspx')
+        elif (split_url[0].rfind('kemel.asp')  == -1):      #not Kemel komix
+            print_url = 'http://zpravy.idnes.cz/tiskni.asp?' +  split_url[1]
+        #kemel                  kemel print page doesn't work
+        return print_url
+
+    extra_css = '''
+                  h1 {font-size:125%; font-weight:bold}
+                  h3 {font-size:110%; font-weight:bold}
+                '''

resources/recipes/la_tribuna.recipe

@@ -0,0 +1,29 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1294946868(BasicNewsRecipe):
+    title             = u'La Tribuna de Talavera'
+    __author__  = 'Luis Hernández'
+    description   = 'Diario de Talavera de la Reina'
+    cover_url     = 'http://www.latribunadetalavera.es/entorno/mancheta.gif'
+
+    oldest_article = 5
+    max_articles_per_feed = 50
+
+    remove_javascript = True
+    no_stylesheets        = True
+    use_embedded_content  = False
+
+    encoding              = 'utf-8'
+    language              = 'es'
+    timefmt        = '[%a, %d %b, %Y]'
+
+    keep_only_tags     = [dict(name='div', attrs={'id':['articulo']})
+                                  ,dict(name='div', attrs={'class':['foto']})
+                                  ,dict(name='p', attrs={'id':['texto']})
+                                ]
+
+    remove_tags_before = dict(name='div' , attrs={'class':['comparte']})
+    remove_tags_after  = dict(name='div' , attrs={'id':['relacionadas']})
+
+
+    feeds          = [(u'Portada', u'http://www.latribunadetalavera.es/rss.html')]

resources/recipes/new_yorker.recipe

@@ -1,5 +1,5 @@
 __license__   = 'GPL v3'
-__copyright__ = '2008-2010, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2011, Darko Miletic <darko.miletic at gmail.com>'
 '''
 newyorker.com
 '''
@@ -54,10 +54,10 @@ class NewYorker(BasicNewsRecipe):
                         ,dict(attrs={'id':['show-header','show-footer'] })
                      ]
     remove_attributes = ['lang']
-    feeds             = [(u'The New Yorker', u'http://feeds.newyorker.com/services/rss/feeds/everything.xml')]
+    feeds             = [(u'The New Yorker', u'http://www.newyorker.com/services/rss/feeds/everything.xml')]
 
     def print_version(self, url):
-        return url + '?printable=true'
+        return 'http://www.newyorker.com' + url + '?printable=true'
 
     def image_url_processor(self, baseurl, url):
         return url.strip()

resources/recipes/nytimes_sub.recipe

@@ -1,6 +1,5 @@
 #!/usr/bin/env  python
 # -*- coding: utf-8 -*-
-
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 '''
@@ -28,6 +27,10 @@ class NYTimes(BasicNewsRecipe):
     # previous paid versions of the new york times to best sent to the back issues folder on the kindle
     replaceKindleVersion = False
 
+    # download higher resolution images than the small thumbnails typically included in the article
+    # the down side of having large beautiful images is the file size is much larger, on the order of 7MB per paper
+    useHighResImages = True
+
     # includeSections: List of sections to include. If empty, all sections found will be included.
     # Otherwise, only the sections named will be included. For example,
     #
@@ -90,7 +93,6 @@ class NYTimes(BasicNewsRecipe):
                     (u'Sunday Magazine',u'magazine'),
                     (u'Week in Review',u'weekinreview')]
 
-
     if headlinesOnly:
         title='New York Times Headlines'
         description = 'Headlines from the New York Times'
@@ -127,7 +129,7 @@ class NYTimes(BasicNewsRecipe):
 
     earliest_date = date.today() - timedelta(days=oldest_article)
 
-    __author__  = 'GRiker/Kovid Goyal/Nick Redding'
+    __author__  = 'GRiker/Kovid Goyal/Nick Redding/Ben Collier'
     language = 'en'
     requires_version = (0, 7, 5)
 
@@ -149,7 +151,7 @@ class NYTimes(BasicNewsRecipe):
                             'dottedLine',
                             'entry-meta',
                             'entry-response module',
-                            'icon enlargeThis',
+                            #'icon enlargeThis', #removed to provide option for high res images
                             'leftNavTabs',
                             'metaFootnote',
                             'module box nav',
@@ -163,7 +165,23 @@ class NYTimes(BasicNewsRecipe):
                             'entry-tags', #added for DealBook
                             'footer promos clearfix', #added for DealBook
                             'footer links clearfix', #added for DealBook
-                            'inlineImage module', #added for DealBook
+                            'tabsContainer', #added for other blog downloads
+                            'column lastColumn', #added for other blog downloads
+                            'pageHeaderWithLabel', #added for other gadgetwise downloads
+                            'column two', #added for other blog downloads
+                            'column two last', #added for other blog downloads
+                            'column three', #added for other blog downloads
+                            'column three last', #added for other blog downloads
+                            'column four',#added for other blog downloads
+                            'column four last',#added for other blog downloads
+                            'column last', #added for other blog downloads
+                            'timestamp published', #added for other blog downloads
+                            'entry entry-related',
+                            'subNavigation tabContent active', #caucus blog navigation
+                            'columnGroup doubleRule',
+                            'mediaOverlay slideshow',
+                            'headlinesOnly multiline flush',
+                            'wideThumb',
                             re.compile('^subNavigation'),
                             re.compile('^leaderboard'),
                             re.compile('^module'),
@@ -254,7 +272,7 @@ class NYTimes(BasicNewsRecipe):
     def exclude_url(self,url):
         if not url.startswith("http"):
             return True
-        if not url.endswith(".html") and 'dealbook.nytimes.com' not in url: #added for DealBook
+        if not url.endswith(".html") and 'dealbook.nytimes.com' not in url and 'blogs.nytimes.com' not in url: #added for DealBook
             return True
         if 'nytimes.com' not in url:
             return True
@@ -592,19 +610,84 @@ class NYTimes(BasicNewsRecipe):
                     self.log("Skipping article dated %s" % date_str)
                     return None
 
-        kicker_tag = soup.find(attrs={'class':'kicker'})
+        #all articles are from today, no need to print the date on every page
-        if kicker_tag: # remove Op_Ed author head shots
+        try:
-            tagline = self.tag_to_string(kicker_tag)
+            if not self.webEdition:
-            if tagline=='Op-Ed Columnist':
+                date_tag = soup.find(True,attrs={'class': ['dateline','date']})
-                img_div = soup.find('div','inlineImage module')
+                if date_tag:
-                if img_div:
+                    date_tag.extract()
-                    img_div.extract()
+        except:
+            self.log("Error removing the published date")
 
+        if self.useHighResImages:
+            try:
+                #open up all the "Enlarge this Image" pop-ups and download the full resolution jpegs
+                enlargeThisList = soup.findAll('div',{'class':'icon enlargeThis'})
+                if enlargeThisList:
+                    for popupref in enlargeThisList:
+                        popupreflink = popupref.find('a')
+                        if popupreflink:
+                            reflinkstring = str(popupreflink['href'])
+                            refstart = reflinkstring.find("javascript:pop_me_up2('") + len("javascript:pop_me_up2('")
+                            refend = reflinkstring.find(".html", refstart) + len(".html")
+                            reflinkstring = reflinkstring[refstart:refend]
+
+                            popuppage = self.browser.open(reflinkstring)
+                            popuphtml = popuppage.read()
+                            popuppage.close()
+                            if popuphtml:
+                                st = time.localtime()
+                                year = str(st.tm_year)
+                                month = "%.2d" % st.tm_mon
+                                day = "%.2d" % st.tm_mday
+                                imgstartpos = popuphtml.find('http://graphics8.nytimes.com/images/' + year + '/' +  month +'/' + day +'/') + len('http://graphics8.nytimes.com/images/' + year + '/' +  month +'/' + day +'/')
+                                highResImageLink = 'http://graphics8.nytimes.com/images/' + year + '/' +  month +'/' + day +'/' + popuphtml[imgstartpos:popuphtml.find('.jpg',imgstartpos)+4]
+                                popupSoup = BeautifulSoup(popuphtml)
+                                highResTag = popupSoup.find('img', {'src':highResImageLink})
+                                if highResTag:
+                                    try:
+                                        newWidth = highResTag['width']
+                                        newHeight = highResTag['height']
+                                        imageTag = popupref.parent.find("img")
+                                    except:
+                                        self.log("Error: finding width and height of img")
+                                    popupref.extract()
+                                    if imageTag:
+                                        try:
+                                            imageTag['src'] = highResImageLink
+                                            imageTag['width'] = newWidth
+                                            imageTag['height'] = newHeight
+                                        except:
+                                            self.log("Error setting the src width and height parameters")
+            except Exception:
+                self.log("Error pulling high resolution images")
+
+            try:
+                #remove "Related content" bar
+                runAroundsFound = soup.findAll('div',{'class':['articleInline runaroundLeft','articleInline doubleRule runaroundLeft','articleInline runaroundLeft firstArticleInline']})
+                if runAroundsFound:
+                    for runAround in runAroundsFound:
+                        #find all section headers
+                        hlines = runAround.findAll(True ,{'class':['sectionHeader','sectionHeader flushBottom']})
+                        if hlines:
+                            for hline in hlines:
+                                hline.extract()
+            except:
+                self.log("Error removing related content bar")
+
+
+            try:
+                #in case pulling images failed, delete the enlarge this text
+                enlargeThisList = soup.findAll('div',{'class':'icon enlargeThis'})
+                if enlargeThisList:
+                    for popupref in enlargeThisList:
+                        popupref.extract()
+            except:
+                self.log("Error removing Enlarge this text")
 
         return self.strip_anchors(soup)
 
     def postprocess_html(self,soup, True):
-
         try:
                 if self.one_picture_per_article:
                         # Remove all images after first
@@ -766,6 +849,8 @@ class NYTimes(BasicNewsRecipe):
         try:
             if len(article.text_summary.strip()) == 0:
                 articlebodies = soup.findAll('div',attrs={'class':'articleBody'})
+                if not articlebodies: #added to account for blog formats
+                    articlebodies = soup.findAll('div', attrs={'class':'entry-content'}) #added to account for blog formats
                 if articlebodies:
                     for articlebody in articlebodies:
                         if articlebody:
@@ -774,13 +859,14 @@ class NYTimes(BasicNewsRecipe):
                                 refparagraph = self.massageNCXText(self.tag_to_string(p,use_alt=False)).strip()
                                 #account for blank paragraphs and short paragraphs by appending them to longer ones
                                 if len(refparagraph) > 0:
-                                    if len(refparagraph) > 70: #approximately one line of text
+                                    if len(refparagraph) > 140: #approximately two lines of text
                                         article.summary = article.text_summary = shortparagraph + refparagraph
                                         return
                                     else:
                                         shortparagraph = refparagraph + " "
                                         if shortparagraph.strip().find(" ") == -1 and not shortparagraph.strip().endswith(":"):
                                             shortparagraph = shortparagraph + "- "
+
         except:
             self.log("Error creating article descriptions")
             return

resources/recipes/root.recipe

@@ -0,0 +1,39 @@
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1289939440(BasicNewsRecipe):
+  __author__  = 'FunThomas'
+  title       = u'Root.cz'
+  description = u'Zprávičky a články z Root.cz'
+  publisher   = u'Internet Info, s.r.o'
+  oldest_article = 2               #max stari clanku ve dnech
+  max_articles_per_feed = 50       #max pocet clanku na feed
+
+  feeds = [
+    (u'Články', u'http://www.root.cz/rss/clanky/'),
+    (u'Zprávičky', u'http://www.root.cz/rss/zpravicky/')
+  ]
+
+  publication_type = u'magazine'
+  language = u'cs'
+  no_stylesheets = True
+  remove_javascript = True
+  cover_url = u'http://i.iinfo.cz/urs/logo-root-bila-oranzova-cerna-111089527143118.gif'
+
+  remove_attributes = ['width','height','href'] #,'href'
+  keep_only_tags = [
+    dict(name='h1'),
+    dict(name='a',attrs={'class':'author'}),
+    dict(name='p', attrs={'class':'intro'}),
+    dict(name='div',attrs={'class':'urs'})
+  ]
+
+  preprocess_regexps = [
+    (re.compile(u'<p class="perex[^"]*">[^<]*<img[^>]*>', re.DOTALL),lambda match: '<p class="intro">'),
+    (re.compile(u'<h3><a name="tucnak">Tričko tučňák.*</body>', re.DOTALL),lambda match: '<!--deleted-->')
+  ]
+
+  extra_css = '''
+                h1 {font-size:130%; font-weight:bold}
+                h3 {font-size:111%; font-weight:bold}
+              '''

resources/recipes/sinfest.recipe

@@ -0,0 +1,33 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Nadid <nadid.skywalker at gmail.com>'
+'''
+http://www.sinfest.net
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class SinfestBig(BasicNewsRecipe):
+    title                  = 'Sinfest'
+    __author__             = 'nadid'
+    description            = 'Sinfest'
+    reverse_article_order = False
+    oldest_article         = 5
+    max_articles_per_feed  = 100
+    no_stylesheets         = True
+    use_embedded_content   = True
+    encoding               = 'utf-8'
+    publisher              = 'Tatsuya Ishida/Museworks'
+    category               = 'comic'
+    language               = 'en'
+
+    conversion_options = {
+                             'comments'        : description
+                            ,'tags'            : category
+                            ,'language'        : language
+                            ,'publisher'       : publisher
+                         }
+
+    feeds = [(u'SinFest', u'http://henrik.nyh.se/scrapers/sinfest.rss' )]
+    def get_article_url(self, article):
+        return article.get('link')
+

resources/recipes/zdnet.recipe

@@ -27,12 +27,34 @@ class cdnet(BasicNewsRecipe):
 		   dict(id='header'),
 		   dict(id='search'),
 		   dict(id='nav'),
+		   dict(id='blog-author-info'),
+		   dict(id='post-tags'),
+		   dict(id='bio-naraine'),
+		   dict(id='bio-kennedy'),
+		   dict(id='author-short-disclosure-kennedy'),
 		   dict(id=''),
 		   dict(name='div', attrs={'class':'banner'}),
+		   dict(name='div', attrs={'class':'int'}),
+		   dict(name='div', attrs={'class':'talkback clear space-2'}),
+		   dict(name='div', attrs={'class':'content-1 clear'}),
+		   dict(name='div', attrs={'class':'space-2'}),
+		   dict(name='div', attrs={'class':'space-3'}),
+		   dict(name='div', attrs={'class':'thumb-2 left'}),
+		   dict(name='div', attrs={'class':'hotspot'}),
+		   dict(name='div', attrs={'class':'hed hed-1 space-1'}),
+		   dict(name='div', attrs={'class':'view-1 clear content-3 space-2'}),
+		   dict(name='div', attrs={'class':'hed hed-1 space-1'}),
+		   dict(name='div', attrs={'class':'hed hed-1'}),
+		   dict(name='div', attrs={'class':'post-header'}),
+		   dict(name='div', attrs={'class':'lvl-nav clear'}),
+		   dict(name='div', attrs={'class':'t-share-overlay overlay-pop contain-overlay-4'}),
 		   dict(name='p', attrs={'class':'tags'}),
+		   dict(name='span', attrs={'class':'follow'}),
+		   dict(name='span', attrs={'class':'int'}),
+		   dict(name='h4', attrs={'class':'h s-4'}),
 		   dict(name='a', attrs={'href':'http://www.twitter.com/ryanaraine'}),
 		   dict(name='div', attrs={'class':'special1'})]
-    remove_tags_after = [dict(name='div', attrs={'class':'bloggerDesc clear'})]
+    remove_tags_after = [dict(name='div', attrs={'class':'clear'})]
 
     feeds =  [ ('zdnet', 'http://feeds.feedburner.com/zdnet/security') ]
 
@@ -43,3 +65,4 @@ class cdnet(BasicNewsRecipe):
         return soup
 
 
+

src/calibre/devices/android/driver.py

@@ -54,7 +54,7 @@ class ANDROID(USBMS):
             0x1004 : { 0x61cc : [0x100] },
 
             # Archos
-            0x0e79 : { 0x1419: [0x0216], 0x1420 : [0x0216]},
+            0x0e79 : { 0x1419: [0x0216], 0x1420 : [0x0216], 0x1422 : [0x0216]},
 
             }
     EBOOK_DIR_MAIN = ['eBooks/import', 'wordplayer/calibretransfer', 'Books']
@@ -70,7 +70,7 @@ class ANDROID(USBMS):
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID',
             'SCH-I500_CARD', 'SPH-D700_CARD', 'MB810', 'GT-P1000', 'DESIRE',
-            'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT']
+            'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT']

src/calibre/devices/folder_device/driver.py

@@ -22,7 +22,7 @@ class FOLDER_DEVICE_FOR_CONFIG(USBMS):
     PRODUCT_ID  = [0xffff]
     BCD         = [0xffff]
     DEVICE_PLUGBOARD_NAME = 'FOLDER_DEVICE'
-
+    SUPPORTS_SUB_DIRS = True
 
 class FOLDER_DEVICE(USBMS):
     type = _('Device Interface')

src/calibre/devices/hanvon/driver.py

@@ -24,7 +24,7 @@ class N516(USBMS):
     supported_platforms = ['windows', 'osx', 'linux']
 
     # Ordered list of supported formats
-    FORMATS     = ['epub', 'prc', 'html', 'pdf', 'txt']
+    FORMATS     = ['epub', 'prc', 'mobi', 'html', 'pdf', 'txt']
 
     VENDOR_ID   = [0x0525]
     PRODUCT_ID  = [0xa4a5]

src/calibre/ebooks/conversion/plumber.py

@@ -576,10 +576,12 @@ OptionRecommendation(name='sr3_replace',
         if not input_fmt:
             raise ValueError('Input file must have an extension')
         input_fmt = input_fmt[1:].lower()
+        self.archive_input_tdir = None
         if input_fmt in ('zip', 'rar', 'oebzip'):
             self.log('Processing archive...')
-            tdir = PersistentTemporaryDirectory('_plumber')
+            tdir = PersistentTemporaryDirectory('_plumber_archive')
             self.input, input_fmt = self.unarchive(self.input, tdir)
+            self.archive_input_tdir = tdir
         if os.access(self.input, os.R_OK):
             nfp = run_plugins_on_preprocess(self.input, input_fmt)
             if nfp != self.input:

src/calibre/ebooks/conversion/utils.py

@@ -155,7 +155,7 @@ class HeuristicProcessor(object):
         ]
 
         for word in ITALICIZE_WORDS:
-            html = html.replace(word, '<i>%s</i>' % word)
+            html = re.sub(r'(?<=\s|>)' + word + r'(?=\s|<)', '<i>%s</i>' % word, html)
 
         for pat in ITALICIZE_STYLE_PATS:
             html = re.sub(pat, lambda mo: '<i>%s</i>' % mo.group('words'), html)

src/calibre/ebooks/fb2/fb2ml.py

@@ -99,7 +99,10 @@ class FB2MLizer(object):
         metadata['appname'] = __appname__
         metadata['version'] = __version__
         metadata['date'] = '%i.%i.%i' % (datetime.now().day, datetime.now().month, datetime.now().year)
-        metadata['lang'] = u''.join(self.oeb_book.metadata.lang) if self.oeb_book.metadata.lang else 'en'
+        if self.oeb_book.metadata.language:
+            metadata['lang'] = self.oeb_book.metadata.language[0].value
+        else:
+            metadata['lang'] = u'en'
         metadata['id'] = None
         metadata['cover'] = self.get_cover()
 

src/calibre/ebooks/metadata/covers.py

@@ -121,6 +121,7 @@ class LibraryThingCovers(CoverDownload): # {{{
     LIBRARYTHING = 'http://www.librarything.com/isbn/'
 
     def get_cover_url(self, isbn, br, timeout=5.):
+
         try:
             src = br.open_novisit('http://www.librarything.com/isbn/'+isbn,
                     timeout=timeout).read().decode('utf-8', 'replace')
@@ -129,6 +130,8 @@ class LibraryThingCovers(CoverDownload): # {{{
                 err = Exception(_('LibraryThing.com timed out. Try again later.'))
             raise err
         else:
+            if '/wiki/index.php/HelpThing:Verify' in src:
+                raise Exception('LibraryThing is blocking calibre.')
             s = BeautifulSoup(src)
             url = s.find('td', attrs={'class':'left'})
             if url is None:
@@ -142,9 +145,12 @@ class LibraryThingCovers(CoverDownload): # {{{
             return url
 
     def has_cover(self, mi, ans, timeout=5.):
-        if not mi.isbn:
+        if not mi.isbn or not self.site_customization:
             return False
-        br = browser()
+        from calibre.ebooks.metadata.library_thing import get_browser, login
+        br = get_browser()
+        un, _, pw = self.site_customization.partition(':')
+        login(br, un, pw)
         try:
             self.get_cover_url(mi.isbn, br, timeout=timeout)
             self.debug('cover for', mi.isbn, 'found')
@@ -153,9 +159,12 @@ class LibraryThingCovers(CoverDownload): # {{{
             self.debug(e)
 
     def get_covers(self, mi, result_queue, abort, timeout=5.):
-        if not mi.isbn:
+        if not mi.isbn or not self.site_customization:
             return
-        br = browser()
+        from calibre.ebooks.metadata.library_thing import get_browser, login
+        br = get_browser()
+        un, _, pw = self.site_customization.partition(':')
+        login(br, un, pw)
         try:
             url = self.get_cover_url(mi.isbn, br, timeout=timeout)
             cover_data = br.open_novisit(url).read()
@@ -164,6 +173,11 @@ class LibraryThingCovers(CoverDownload): # {{{
             result_queue.put((False, self.exception_to_string(e),
                 traceback.format_exc(), self.name))
 
+    def customization_help(self, gui=False):
+        ans = _('To use librarything.com you must sign up for a %sfree account%s '
+                'and enter your username and password separated by a : below.')
+        return '<p>'+ans%('<a href="http://www.librarything.com">', '</a>')
+
 # }}}
 
 def check_for_cover(mi, timeout=5.): # {{{

src/calibre/ebooks/metadata/fetch.py

@@ -251,19 +251,26 @@ class LibraryThing(MetadataSource): # {{{
 
     name = 'LibraryThing'
     metadata_type = 'social'
-    description = _('Downloads series/tags/rating information from librarything.com')
+    description = _('Downloads series/covers/rating information from librarything.com')
 
     def fetch(self):
-        if not self.isbn:
+        if not self.isbn or not self.site_customization:
             return
         from calibre.ebooks.metadata.library_thing import get_social_metadata
+        un, _, pw = self.site_customization.partition(':')
         try:
             self.results = get_social_metadata(self.title, self.book_author,
-                    self.publisher, self.isbn)
+                    self.publisher, self.isbn, username=un, password=pw)
         except Exception, e:
             self.exception = e
             self.tb = traceback.format_exc()
 
+    @property
+    def string_customization_help(self):
+        ans = _('To use librarything.com you must sign up for a %sfree account%s '
+                'and enter your username and password separated by a : below.')
+        return '<p>'+ans%('<a href="http://www.librarything.com">', '</a>')
+
     # }}}
 
 

src/calibre/ebooks/metadata/library_thing.py

@@ -4,14 +4,13 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Fetch cover from LibraryThing.com based on ISBN number.
 '''
 
-import sys, socket, os, re, random
+import sys, re, random
 
 from lxml import html
 import mechanize
 
 from calibre import browser, prints
 from calibre.utils.config import OptionParser
-from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.ebooks.chardet import strip_encoding_declarations
 
 OPENLIBRARY = 'http://covers.openlibrary.org/b/isbn/%s-L.jpg?default=false'
@@ -28,6 +27,12 @@ def get_ua():
     ]
     return choices[random.randint(0, len(choices)-1)]
 
+_lt_br = None
+def get_browser():
+    global _lt_br
+    if _lt_br is None:
+        _lt_br = browser(user_agent=get_ua())
+    return _lt_br.clone_browser()
 
 class HeadRequest(mechanize.Request):
 
@@ -35,7 +40,7 @@ class HeadRequest(mechanize.Request):
         return 'HEAD'
 
 def check_for_cover(isbn, timeout=5.):
-    br = browser(user_agent=get_ua())
+    br = get_browser()
     br.set_handle_redirect(False)
     try:
         br.open_novisit(HeadRequest(OPENLIBRARY%isbn), timeout=timeout)
@@ -54,46 +59,16 @@ class ISBNNotFound(LibraryThingError):
 class ServerBusy(LibraryThingError):
     pass
 
-def login(br, username, password, force=True):
+def login(br, username, password):
-    br.open('http://www.librarything.com')
+    raw = br.open('http://www.librarything.com').read()
+    if '>Sign out' in raw:
+        return
     br.select_form('signup')
     br['formusername'] = username
     br['formpassword'] = password
-    br.submit()
+    raw = br.submit().read()
-
+    if '>Sign out' not in raw:
-
+        raise ValueError('Failed to login as %r:%r'%(username, password))
-def cover_from_isbn(isbn, timeout=5., username=None, password=None):
-    src = None
-    br = browser(user_agent=get_ua())
-    try:
-        return br.open(OPENLIBRARY%isbn, timeout=timeout).read(), 'jpg'
-    except:
-        pass # Cover not found
-    if username and password:
-        try:
-            login(br, username, password, force=False)
-        except:
-            pass
-    try:
-        src = br.open_novisit('http://www.librarything.com/isbn/'+isbn,
-                timeout=timeout).read().decode('utf-8', 'replace')
-    except Exception, err:
-        if isinstance(getattr(err, 'args', [None])[0], socket.timeout):
-            err = LibraryThingError(_('LibraryThing.com timed out. Try again later.'))
-        raise err
-    else:
-        s = BeautifulSoup(src)
-        url = s.find('td', attrs={'class':'left'})
-        if url is None:
-            if s.find('div', attrs={'class':'highloadwarning'}) is not None:
-                raise ServerBusy(_('Could not fetch cover as server is experiencing high load. Please try again later.'))
-            raise ISBNNotFound('ISBN: '+isbn+_(' not found.'))
-        url = url.find('img')
-        if url is None:
-            raise LibraryThingError(_('LibraryThing.com server error. Try again later.'))
-        url = re.sub(r'_S[XY]\d+', '', url['src'])
-        cover_data = br.open_novisit(url).read()
-        return cover_data, url.rpartition('.')[-1]
 
 def option_parser():
     parser = OptionParser(usage=\
@@ -113,15 +88,16 @@ def get_social_metadata(title, authors, publisher, isbn, username=None,
     from calibre.ebooks.metadata import MetaInformation
     mi = MetaInformation(title, authors)
     if isbn:
-        br = browser(user_agent=get_ua())
+        br = get_browser()
-        if username and password:
+        try:
-            try:
+            login(br, username, password)
-                login(br, username, password, force=False)
-            except:
-                pass
 
-        raw = br.open_novisit('http://www.librarything.com/isbn/'
+            raw = br.open_novisit('http://www.librarything.com/isbn/'
-                    +isbn).read()
+                        +isbn).read()
+        except:
+            return mi
+        if '/wiki/index.php/HelpThing:Verify' in raw:
+            raise Exception('LibraryThing is blocking calibre.')
         if not raw:
             return mi
         raw = raw.decode('utf-8', 'replace')
@@ -172,15 +148,46 @@ def main(args=sys.argv):
         parser.print_help()
         return 1
     isbn = args[1]
-    mi = get_social_metadata('', [], '', isbn)
+    from calibre.customize.ui import metadata_sources, cover_sources
+    lt = None
+    for x in metadata_sources('social'):
+        if x.name == 'LibraryThing':
+            lt = x
+            break
+    lt('', '', '', isbn, True)
+    lt.join()
+    if lt.exception:
+        print lt.tb
+        return 1
+    mi = lt.results
     prints(mi)
-    cover_data, ext = cover_from_isbn(isbn, username=opts.username,
+    mi.isbn = isbn
-            password=opts.password)
+
-    if not ext:
+    lt = None
-        ext = 'jpg'
+    for x in cover_sources():
-    oname = os.path.abspath(isbn+'.'+ext)
+        if x.name == 'librarything.com covers':
-    open(oname, 'w').write(cover_data)
+            lt = x
-    print 'Cover saved to file', oname
+            break
+
+    from threading import Event
+    from Queue import Queue
+    ev = Event()
+    lt.has_cover(mi, ev)
+    hc = ev.is_set()
+    print 'Has cover:', hc
+    if hc:
+        abort = Event()
+        temp = Queue()
+        lt.get_covers(mi, temp, abort)
+
+        cover = temp.get_nowait()
+        if cover[0]:
+            open(isbn + '.jpg', 'wb').write(cover[1])
+            print 'Cover saved to:', isbn+'.jpg'
+        else:
+            print 'Cover download failed'
+            print cover[2]
+
     return 0
 
 if __name__ == '__main__':

src/calibre/ebooks/metadata/sources/__init__.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+

src/calibre/ebooks/oeb/reader.py

@@ -17,7 +17,7 @@ from lxml import etree
 import cssutils
 
 from calibre.ebooks.oeb.base import OPF1_NS, OPF2_NS, OPF2_NSMAP, DC11_NS, \
-    DC_NSES, OPF
+    DC_NSES, OPF, xml2text
 from calibre.ebooks.oeb.base import OEB_DOCS, OEB_STYLES, OEB_IMAGES, \
     PAGE_MAP_MIME, JPEG_MIME, NCX_MIME, SVG_MIME
 from calibre.ebooks.oeb.base import XMLDECL_RE, COLLAPSE_RE, \
@@ -423,7 +423,7 @@ class OEBReader(object):
             path, frag = urldefrag(href)
             if path not in self.oeb.manifest.hrefs:
                 continue
-            title = ' '.join(xpath(anchor, './/text()'))
+            title = xml2text(anchor)
             title = COLLAPSE_RE.sub(' ', title.strip())
             if href not in titles:
                 order.append(href)

src/calibre/ebooks/pml/pmlconverter.py

@@ -499,14 +499,15 @@ class PML_HTMLizer(object):
 
         indent_state = {'t': False, 'T': False}
         adv_indent_val = ''
+        # Keep track of the number of empty lines
+        # between paragraphs. When we reach a set number
+        # we assume it's a soft scene break.
+        empty_count = 0
 
         for s in self.STATES:
             self.state[s] = [False, ''];
 
         for line in pml.splitlines():
-            if not line:
-                continue
-
             parsed = []
             empty = True
             basic_indent = indent_state['t']
@@ -575,10 +576,15 @@ class PML_HTMLizer(object):
                         if indent_state[c]:
                             basic_indent = True
                     elif c == 'T':
-                        indent_state[c] = not indent_state[c]
+                        # Ensure we only store the value on the first T set for the line.
-                        if indent_state[c]:
+                        if not indent_state['T']:
                             adv_indent = True
                             adv_indent_val = self.code_value(line)
+                        else:
+                            # We detected a T previously on this line.
+                            # Don't replace the first detected value.
+                            self.code_value(line)
+                        indent_state['T'] = True
                     elif c == '-':
                         empty = False
                         text = '­'
@@ -592,7 +598,12 @@ class PML_HTMLizer(object):
                 parsed.append(text)
                 c = line.read(1)
 
-            if not empty:
+            if empty:
+                empty_count += 1
+                if empty_count == 3:
+                    output.append('<p>&nbsp;</p>')
+            else:
+                empty_count = 0
                 text = self.end_line()
                 parsed.append(text)
                 

src/calibre/gui2/__init__.py

@@ -8,12 +8,12 @@ from urllib import unquote
 from PyQt4.Qt import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, \
                          QByteArray, QTranslator, QCoreApplication, QThread, \
                          QEvent, QTimer, pyqtSignal, QDate, QDesktopServices, \
-                         QFileDialog, QMessageBox, QPixmap, QFileIconProvider, \
+                         QFileDialog, QFileIconProvider, \
-                         QIcon, QApplication, QDialog, QPushButton, QUrl, QFont
+                         QIcon, QApplication, QDialog, QUrl, QFont
 
 ORG_NAME = 'KovidsBrain'
 APP_UID  = 'libprs500'
-from calibre.constants import islinux, iswindows, isosx, isfreebsd, isfrozen
+from calibre.constants import islinux, iswindows, isfreebsd, isfrozen
 from calibre.utils.config import Config, ConfigProxy, dynamic, JSONConfig
 from calibre.utils.localization import set_qt_translator
 from calibre.ebooks.metadata.meta import get_metadata, metadata_from_formats
@@ -178,104 +178,40 @@ def is_widescreen():
 def extension(path):
     return os.path.splitext(path)[1][1:].lower()
 
-class CopyButton(QPushButton):
-
-    ACTION_KEYS = [Qt.Key_Enter, Qt.Key_Return, Qt.Key_Space]
-
-    def copied(self):
-        self.emit(SIGNAL('copy()'))
-        self.setDisabled(True)
-        self.setText(_('Copied'))
-
-
-    def keyPressEvent(self, ev):
-        try:
-            if ev.key() in self.ACTION_KEYS:
-                self.copied()
-                return
-        except:
-            pass
-        QPushButton.keyPressEvent(self, ev)
-
-
-    def keyReleaseEvent(self, ev):
-        try:
-            if ev.key() in self.ACTION_KEYS:
-                return
-        except:
-            pass
-        QPushButton.keyReleaseEvent(self, ev)
-
-    def mouseReleaseEvent(self, ev):
-        ev.accept()
-        self.copied()
-
-class MessageBox(QMessageBox):
-
-    def __init__(self, type_, title, msg, buttons, parent, det_msg=''):
-        QMessageBox.__init__(self, type_, title, msg, buttons, parent)
-        self.title = title
-        self.msg = msg
-        self.det_msg = det_msg
-        self.setDetailedText(det_msg)
-        # Cannot set keyboard shortcut as the event is not easy to filter
-        self.cb = CopyButton(_('Copy') if isosx else _('Copy to Clipboard'))
-        self.connect(self.cb, SIGNAL('copy()'), self.copy_to_clipboard)
-        self.addButton(self.cb, QMessageBox.ActionRole)
-        default_button = self.button(self.Ok)
-        if default_button is None:
-            default_button = self.button(self.Yes)
-        if default_button is not None:
-            self.setDefaultButton(default_button)
-
-    def copy_to_clipboard(self):
-        QApplication.clipboard().setText('%s: %s\n\n%s' %
-                (self.title, self.msg, self.det_msg))
-
-
 
 def warning_dialog(parent, title, msg, det_msg='', show=False,
         show_copy_button=True):
-    d = MessageBox(QMessageBox.Warning, 'WARNING: '+title, msg, QMessageBox.Ok,
+    from calibre.gui2.dialogs.message_box import MessageBox
-                    parent, det_msg)
+    d = MessageBox(MessageBox.WARNING, 'WARNING: '+title, msg, det_msg, parent=parent,
-    d.setEscapeButton(QMessageBox.Ok)
+            show_copy_button=show_copy_button)
-    d.setIconPixmap(QPixmap(I('dialog_warning.png')))
-    if not show_copy_button:
-        d.cb.setVisible(False)
     if show:
         return d.exec_()
     return d
 
 def error_dialog(parent, title, msg, det_msg='', show=False,
         show_copy_button=True):
-    d = MessageBox(QMessageBox.Critical, 'ERROR: '+title, msg, QMessageBox.Ok,
+    from calibre.gui2.dialogs.message_box import MessageBox
-                    parent, det_msg)
+    d = MessageBox(MessageBox.ERROR, 'ERROR: '+title, msg, det_msg, parent=parent,
-    d.setIconPixmap(QPixmap(I('dialog_error.png')))
+                    show_copy_button=show_copy_button)
-    d.setEscapeButton(QMessageBox.Ok)
-    if not show_copy_button:
-        d.cb.setVisible(False)
     if show:
         return d.exec_()
     return d
 
-def question_dialog(parent, title, msg, det_msg='', show_copy_button=True,
+def question_dialog(parent, title, msg, det_msg='', show_copy_button=False,
-        buttons=QMessageBox.Yes|QMessageBox.No, yes_button=QMessageBox.Yes):
+        buttons=None, yes_button=None):
-    d = MessageBox(QMessageBox.Question, title, msg, buttons,
+    from calibre.gui2.dialogs.message_box import MessageBox
-                    parent, det_msg)
+    d = MessageBox(MessageBox.QUESTION, title, msg, det_msg, parent=parent,
-    d.setIconPixmap(QPixmap(I('dialog_question.png')))
+                    show_copy_button=show_copy_button)
-    d.setEscapeButton(QMessageBox.No)
+    if buttons is not None:
-    if not show_copy_button:
+        d.bb.setStandardButtons(buttons)
-        d.cb.setVisible(False)
 
-    return d.exec_() == yes_button
+    return d.exec_() == d.Accepted
 
 def info_dialog(parent, title, msg, det_msg='', show=False,
         show_copy_button=True):
-    d = MessageBox(QMessageBox.Information, title, msg, QMessageBox.Ok,
+    from calibre.gui2.dialogs.message_box import MessageBox
-                    parent, det_msg)
+    d = MessageBox(MessageBox.INFO, title, msg, det_msg, parent=parent,
-    d.setIconPixmap(QPixmap(I('dialog_information.png')))
+                    show_copy_button=show_copy_button)
-    if not show_copy_button:
-        d.cb.setVisible(False)
 
     if show:
         return d.exec_()
@@ -550,6 +486,14 @@ def choose_dir(window, name, title, default_dir='~'):
     if dir:
         return dir[0]
 
+def choose_osx_app(window, name, title, default_dir='/Applications'):
+    fd = FileDialog(title=title, parent=window, name=name, mode=QFileDialog.ExistingFile,
+            default_dir=default_dir)
+    app = fd.get_files()
+    fd.setParent(None)
+    if app:
+        return app
+
 def choose_files(window, name, title,
                  filters=[], all_files=True, select_only_single_file=False):
     '''

src/calibre/gui2/actions/annotate.py

@@ -9,7 +9,7 @@ import os, datetime
 
 from PyQt4.Qt import pyqtSignal, QModelIndex, QThread, Qt
 
-from calibre.gui2 import error_dialog, gprefs
+from calibre.gui2 import error_dialog
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag, NavigableString
 from calibre import strftime
 from calibre.gui2.actions import InterfaceAction
@@ -165,10 +165,12 @@ class FetchAnnotationsAction(InterfaceAction):
                 ka_soup.insert(0,divTag)
                 return ka_soup
 
+            '''
             def mark_book_as_read(self,id):
                 read_tag = gprefs.get('catalog_epub_mobi_read_tag')
                 if read_tag:
                     self.db.set_tags(id, [read_tag], append=True)
+            '''
 
             def canceled(self):
                 self.pd.hide()
@@ -201,10 +203,12 @@ class FetchAnnotationsAction(InterfaceAction):
                         # Update library comments
                         self.db.set_comment(id, mi.comments)
 
+                        '''
                         # Update 'read' tag except for Catalogs/Clippings
                         if bm.value.percent_read >= self.FINISHED_READING_PCT_THRESHOLD:
                             if not set(mi.tags).intersection(ignore_tags):
                                 self.mark_book_as_read(id)
+                        '''
 
                         # Add bookmark file to id
                         self.db.add_format_with_hooks(id, bm.value.bookmark_extension,

src/calibre/gui2/actions/choose_library.py

@@ -343,7 +343,7 @@ class ChooseLibraryAction(InterfaceAction):
         db.dirtied(list(db.data.iterallids()))
         info_dialog(self.gui, _('Backup metadata'),
             _('Metadata will be backed up while calibre is running, at the '
-              'rate of approximately 1 book per second.'), show=True)
+              'rate of approximately 1 book every three seconds.'), show=True)
 
     def check_library(self):
         db = self.gui.library_view.model().db
@@ -390,7 +390,7 @@ class ChooseLibraryAction(InterfaceAction):
         #self.dbref = weakref.ref(self.gui.library_view.model().db)
         #self.before_mem = memory()/1024**2
         self.gui.library_moved(loc)
-        #QTimer.singleShot(1000, self.debug_leak)
+        #QTimer.singleShot(5000, self.debug_leak)
 
     def debug_leak(self):
         import gc
@@ -398,7 +398,7 @@ class ChooseLibraryAction(InterfaceAction):
         ref = self.dbref
         for i in xrange(3): gc.collect()
         if ref() is not None:
-            print 11111, ref()
+            print 'DB object alive:', ref()
             for r in gc.get_referrers(ref())[:10]:
                 print r
                 print

src/calibre/gui2/catalog/catalog_epub_mobi.py

@@ -335,7 +335,7 @@ class PluginWidget(QWidget,Ui_Form):
         '''
 
         return
-
+        '''
         if new_state == 0:
             # unchecked
             self.merge_source_field.setEnabled(False)
@@ -348,6 +348,7 @@ class PluginWidget(QWidget,Ui_Form):
             self.merge_before.setEnabled(True)
             self.merge_after.setEnabled(True)
             self.include_hr.setEnabled(True)
+        '''
 
     def header_note_source_field_changed(self,new_index):
         '''

src/calibre/gui2/convert/bulk.py

@@ -4,6 +4,8 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
+import shutil
+
 from PyQt4.Qt import QString, SIGNAL
 
 from calibre.gui2.convert.single import Config, sort_formats_by_preference, \
@@ -108,6 +110,11 @@ class BulkConfig(Config):
         idx = oidx if -1 < oidx < self._groups_model.rowCount() else 0
         self.groups.setCurrentIndex(self._groups_model.index(idx))
         self.stack.setCurrentIndex(idx)
+        try:
+            shutil.rmtree(self.plumber.archive_input_tdir, ignore_errors=True)
+        except:
+            pass
+
 
     def setup_output_formats(self, db, preferred_output_format):
         if preferred_output_format:

src/calibre/gui2/convert/look_and_feel.ui

@@ -43,7 +43,7 @@
       <double>0.000000000000000</double>
      </property>
      <property name="maximum">
-      <double>30.000000000000000</double>
+      <double>50.000000000000000</double>
      </property>
      <property name="singleStep">
       <double>1.000000000000000</double>

src/calibre/gui2/convert/regex_builder.py

@@ -6,7 +6,7 @@ __docformat__ = 'restructuredtext en'
 
 import re
 
-from PyQt4.QtCore import SIGNAL, Qt
+from PyQt4.QtCore import SIGNAL, Qt, pyqtSignal
 from PyQt4.QtGui import QDialog, QWidget, QDialogButtonBox, \
                         QBrush, QTextCursor, QTextEdit
 
@@ -19,8 +19,8 @@ from calibre.gui2.dialogs.choose_format import ChooseFormatDialog
 
 class RegexBuilder(QDialog, Ui_RegexBuilder):
 
-    def __init__(self, db, book_id, regex, *args):
+    def __init__(self, db, book_id, regex, doc=None, parent=None):
-        QDialog.__init__(self, *args)
+        QDialog.__init__(self, parent)
         self.setupUi(self)
 
         self.regex.setText(regex)
@@ -28,9 +28,13 @@ class RegexBuilder(QDialog, Ui_RegexBuilder):
 
         if not db or not book_id:
             self.button_box.addButton(QDialogButtonBox.Open)
-        elif not self.select_format(db, book_id):
+        elif not doc and not self.select_format(db, book_id):
             self.cancelled = True
             return
+
+        if doc:
+            self.preview.setPlainText(doc)
+
         self.cancelled = False
         self.connect(self.button_box, SIGNAL('clicked(QAbstractButton*)'), self.button_clicked)
         self.connect(self.regex, SIGNAL('textChanged(QString)'), self.regex_valid)
@@ -153,24 +157,36 @@ class RegexBuilder(QDialog, Ui_RegexBuilder):
         if button == self.button_box.button(QDialogButtonBox.Ok):
             self.accept()
 
+    def doc(self):
+        return unicode(self.preview.toPlainText())
+
 class RegexEdit(QWidget, Ui_Edit):
 
+    doc_update = pyqtSignal(unicode)
+
     def __init__(self, parent=None):
         QWidget.__init__(self, parent)
         self.setupUi(self)
 
         self.book_id = None
         self.db = None
+        self.doc_cache = None
 
         self.connect(self.button, SIGNAL('clicked()'), self.builder)
 
     def builder(self):
-        bld = RegexBuilder(self.db, self.book_id, self.edit.text(), self)
+        bld = RegexBuilder(self.db, self.book_id, self.edit.text(), self.doc_cache, self)
         if bld.cancelled:
             return
+        if not self.doc_cache:
+            self.doc_cache = bld.doc()
+            self.doc_update.emit(self.doc_cache)
         if bld.exec_() == bld.Accepted:
             self.edit.setText(bld.regex.text())
 
+    def doc(self):
+        return self.doc_cache
+
     def setObjectName(self, *args):
         QWidget.setObjectName(self, *args)
         if hasattr(self, 'edit'):
@@ -185,8 +201,11 @@ class RegexEdit(QWidget, Ui_Edit):
     def set_db(self, db):
         self.db = db
 
+    def set_doc(self, doc):
+        self.doc_cache = doc
+
     def break_cycles(self):
-        self.db = None
+        self.db = self.doc_cache = None
 
     @property
     def text(self):

src/calibre/gui2/convert/search_and_replace.py

@@ -35,13 +35,32 @@ class SearchAndReplaceWidget(Widget, Ui_Form):
         self.opt_sr3_search.set_book_id(book_id)
         self.opt_sr3_search.set_db(db)
 
+        self.opt_sr1_search.doc_update.connect(self.update_doc)
+        self.opt_sr2_search.doc_update.connect(self.update_doc)
+        self.opt_sr3_search.doc_update.connect(self.update_doc)
+
     def break_cycles(self):
         Widget.break_cycles(self)
 
+        def d(x):
+            try:
+                x.disconnect()
+            except:
+                pass
+
+        d(self.opt_sr1_search)
+        d(self.opt_sr2_search)
+        d(self.opt_sr3_search)
+
         self.opt_sr1_search.break_cycles()
         self.opt_sr2_search.break_cycles()
         self.opt_sr3_search.break_cycles()
 
+    def update_doc(self, doc):
+        self.opt_sr1_search.set_doc(doc)
+        self.opt_sr2_search.set_doc(doc)
+        self.opt_sr3_search.set_doc(doc)
+
     def pre_commit_check(self):
         for x in ('sr1_search', 'sr2_search', 'sr3_search'):
             x = getattr(self, 'opt_'+x)

src/calibre/gui2/convert/single.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import sys, cPickle
+import sys, cPickle, shutil
 
 from PyQt4.Qt import QString, SIGNAL, QAbstractListModel, Qt, QVariant, QFont
 
@@ -224,6 +224,10 @@ class Config(ResizableDialog, Ui_Dialog):
         idx = oidx if -1 < oidx < self._groups_model.rowCount() else 0
         self.groups.setCurrentIndex(self._groups_model.index(idx))
         self.stack.setCurrentIndex(idx)
+        try:
+            shutil.rmtree(self.plumber.archive_input_tdir, ignore_errors=True)
+        except:
+            pass
 
 
     def setup_input_output_formats(self, db, book_id, preferred_input_format,

src/calibre/gui2/dialogs/message_box.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+from PyQt4.Qt import QDialog, QIcon, QApplication, QSize, QKeySequence, \
+    QAction, Qt
+
+from calibre.constants import __version__
+from calibre.gui2.dialogs.message_box_ui import Ui_Dialog
+
+class MessageBox(QDialog, Ui_Dialog):
+
+    ERROR = 0
+    WARNING = 1
+    INFO = 2
+    QUESTION = 3
+
+    def __init__(self, type_, title, msg, det_msg='', show_copy_button=True,
+            parent=None):
+        QDialog.__init__(self, parent)
+        icon = {
+                self.ERROR : 'error',
+                self.WARNING: 'warning',
+                self.INFO:    'information',
+                self.QUESTION: 'question',
+        }[type_]
+        icon = 'dialog_%s.png'%icon
+        self.icon = QIcon(I(icon))
+        self.setupUi(self)
+
+        self.setWindowTitle(title)
+        self.setWindowIcon(self.icon)
+        self.icon_label.setPixmap(self.icon.pixmap(128, 128))
+        self.msg.setText(msg)
+        self.det_msg.setPlainText(det_msg)
+        self.det_msg.setVisible(False)
+
+        if det_msg:
+            self.show_det_msg = _('Show &details')
+            self.hide_det_msg = _('Hide &details')
+            self.det_msg_toggle = self.bb.addButton(self.show_det_msg, self.bb.ActionRole)
+            self.det_msg_toggle.clicked.connect(self.toggle_det_msg)
+            self.det_msg_toggle.setToolTip(
+                    _('Show detailed information about this error'))
+
+        if show_copy_button:
+            self.ctc_button = self.bb.addButton(_('&Copy to clipboard'),
+                    self.bb.ActionRole)
+            self.ctc_button.clicked.connect(self.copy_to_clipboard)
+
+
+        self.copy_action = QAction(self)
+        self.addAction(self.copy_action)
+        self.copy_action.setShortcuts(QKeySequence.Copy)
+        self.copy_action.triggered.connect(self.copy_to_clipboard)
+
+        self.is_question = type_ == self.QUESTION
+        if self.is_question:
+            self.bb.setStandardButtons(self.bb.Yes|self.bb.No)
+            self.bb.button(self.bb.Yes).setDefault(True)
+        else:
+            self.bb.button(self.bb.Ok).setDefault(True)
+
+        self.do_resize()
+
+    def toggle_det_msg(self, *args):
+        vis = self.det_msg.isVisible()
+        self.det_msg_toggle.setText(self.show_det_msg if vis else
+                self.hide_det_msg)
+        self.det_msg.setVisible(not vis)
+        self.do_resize()
+
+    def do_resize(self):
+        sz = self.sizeHint() + QSize(100, 0)
+        sz.setWidth(min(500, sz.width()))
+        sz.setHeight(min(500, sz.height()))
+        self.resize(sz)
+
+    def copy_to_clipboard(self, *args):
+        QApplication.clipboard().setText(
+                'calibre, version %s\n%s: %s\n\n%s' %
+                (__version__, unicode(self.windowTitle()),
+                    unicode(self.msg.text()),
+                    unicode(self.det_msg.toPlainText())))
+        self.ctc_button.setText(_('Copied'))
+
+    def showEvent(self, ev):
+        ret = QDialog.showEvent(self, ev)
+        if self.is_question:
+            self.bb.button(self.bb.Yes).setFocus(Qt.OtherFocusReason)
+        else:
+            self.bb.button(self.bb.Ok).setFocus(Qt.OtherFocusReason)
+        return ret
+
+if __name__ == '__main__':
+    app = QApplication([])
+    from calibre.gui2 import question_dialog
+    print question_dialog(None, 'title', 'msg <a href="http://google.com">goog</a> ',
+            det_msg='det '*1000,
+            show_copy_button=True)

src/calibre/gui2/dialogs/message_box.ui

@@ -0,0 +1,105 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Dialog</class>
+ <widget class="QDialog" name="Dialog">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>497</width>
+    <height>235</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Dialog</string>
+  </property>
+  <layout class="QGridLayout" name="gridLayout">
+   <item row="0" column="0">
+    <widget class="QLabel" name="icon_label">
+     <property name="maximumSize">
+      <size>
+       <width>68</width>
+       <height>68</height>
+      </size>
+     </property>
+     <property name="text">
+      <string/>
+     </property>
+     <property name="pixmap">
+      <pixmap resource="../../../../resources/images.qrc">:/images/dialog_warning.png</pixmap>
+     </property>
+     <property name="scaledContents">
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
+   <item row="0" column="1">
+    <widget class="QLabel" name="msg">
+     <property name="text">
+      <string/>
+     </property>
+     <property name="wordWrap">
+      <bool>true</bool>
+     </property>
+     <property name="openExternalLinks">
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
+   <item row="1" column="0" colspan="2">
+    <widget class="QPlainTextEdit" name="det_msg">
+     <property name="readOnly">
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="0" colspan="2">
+    <widget class="QDialogButtonBox" name="bb">
+     <property name="orientation">
+      <enum>Qt::Horizontal</enum>
+     </property>
+     <property name="standardButtons">
+      <set>QDialogButtonBox::Ok</set>
+     </property>
+    </widget>
+   </item>
+  </layout>
+ </widget>
+ <resources>
+  <include location="../../../../resources/images.qrc"/>
+ </resources>
+ <connections>
+  <connection>
+   <sender>bb</sender>
+   <signal>accepted()</signal>
+   <receiver>Dialog</receiver>
+   <slot>accept()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>248</x>
+     <y>254</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>157</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>bb</sender>
+   <signal>rejected()</signal>
+   <receiver>Dialog</receiver>
+   <slot>reject()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>316</x>
+     <y>260</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>286</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
+</ui>

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -6,7 +6,8 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import re, os
 
 from PyQt4.Qt import Qt, QDialog, QGridLayout, QVBoxLayout, QFont, QLabel, \
-                     pyqtSignal, QDialogButtonBox, QDate, QLineEdit
+                     pyqtSignal, QDialogButtonBox, QInputDialog, QLineEdit, \
+                     QMessageBox, QDate
 
 from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
 from calibre.gui2.dialogs.tag_editor import TagEditor
@@ -14,9 +15,9 @@ from calibre.ebooks.metadata import string_to_authors, authors_to_string
 from calibre.ebooks.metadata.book.base import composite_formatter
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.gui2.custom_column_widgets import populate_metadata_page
-from calibre.gui2 import error_dialog, ResizableDialog, UNDEFINED_QDATE
+from calibre.gui2 import error_dialog, ResizableDialog, UNDEFINED_QDATE, gprefs
 from calibre.gui2.progress_indicator import ProgressIndicator
-from calibre.utils.config import dynamic
+from calibre.utils.config import dynamic, JSONConfig
 from calibre.utils.titlecase import titlecase
 from calibre.utils.icu import sort_key, capitalize
 from calibre.utils.config import prefs, tweaks
@@ -320,8 +321,15 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
             'This operation cannot be canceled or undone'))
         self.do_again = False
         self.central_widget.setCurrentIndex(tab)
+        geom = gprefs.get('bulk_metadata_window_geometry', None)
+        if geom is not None:
+            self.restoreGeometry(bytes(geom))
         self.exec_()
 
+    def save_state(self, *args):
+        gprefs['bulk_metadata_window_geometry'] = \
+            bytearray(self.saveGeometry())
+
     def do_apply_pubdate(self, *args):
         self.apply_pubdate.setChecked(True)
 
@@ -451,6 +459,15 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         self.results_count.valueChanged[int].connect(self.s_r_display_bounds_changed)
         self.starting_from.valueChanged[int].connect(self.s_r_display_bounds_changed)
 
+        self.save_button.clicked.connect(self.s_r_save_query)
+        self.remove_button.clicked.connect(self.s_r_remove_query)
+
+        self.queries = JSONConfig("search_replace_queries")
+        self.query_field.addItem("")
+        self.query_field.addItems(sorted([q for q in self.queries], key=sort_key))
+        self.query_field.currentIndexChanged[str].connect(self.s_r_query_change)
+        self.query_field.setCurrentIndex(0)
+
     def s_r_get_field(self, mi, field):
         if field:
             if field == '{template}':
@@ -780,7 +797,12 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
             self.series_start_number.setEnabled(False)
             self.series_start_number.setValue(1)
 
+    def reject(self):
+        self.save_state()
+        ResizableDialog.reject(self)
+
     def accept(self):
+        self.save_state()
         if len(self.ids) < 1:
             return QDialog.accept(self)
 
@@ -862,3 +884,117 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
     def series_changed(self, *args):
         self.write_series = True
 
+    def s_r_remove_query(self, *args):
+        if self.query_field.currentIndex() == 0:
+            return
+
+        ret = QMessageBox.question(self, _("Delete saved search/replace"),
+                _("The selected saved search/replace will be deleted. "
+                  "Are you sure?"),
+                QMessageBox.Ok, QMessageBox.Cancel)
+
+        if ret == QMessageBox.Cancel:
+            return
+
+        item_id = self.query_field.currentIndex()
+        item_name = unicode(self.query_field.currentText())
+
+        self.query_field.blockSignals(True)
+        self.query_field.removeItem(item_id)
+        self.query_field.blockSignals(False)
+        self.query_field.setCurrentIndex(0)
+
+        if item_name in self.queries.keys():
+            del(self.queries[item_name])
+            self.queries.commit()
+
+    def s_r_save_query(self, *args):
+        name, ok =  QInputDialog.getText(self, _('Save search/replace'),
+                _('Search/replace name:'))
+        if not ok:
+            return
+
+        new = True
+        name = unicode(name)
+        if name in self.queries.keys():
+            ret = QMessageBox.question(self, _("Save search/replace"),
+                    _("That saved search/replace already exists and will be overwritten. "
+                      "Are you sure?"),
+                    QMessageBox.Ok, QMessageBox.Cancel)
+            if ret == QMessageBox.Cancel:
+                return
+            new = False
+
+        query = {}
+        query['name'] = name
+        query['search_field'] = unicode(self.search_field.currentText())
+        query['search_mode'] = unicode(self.search_mode.currentText())
+        query['s_r_template'] = unicode(self.s_r_template.text())
+        query['search_for'] = unicode(self.search_for.text())
+        query['case_sensitive'] = self.case_sensitive.isChecked()
+        query['replace_with'] = unicode(self.replace_with.text())
+        query['replace_func'] = unicode(self.replace_func.currentText())
+        query['destination_field'] = unicode(self.destination_field.currentText())
+        query['replace_mode'] = unicode(self.replace_mode.currentText())
+        query['comma_separated'] = self.comma_separated.isChecked()
+        query['results_count'] = self.results_count.value()
+        query['starting_from'] = self.starting_from.value()
+        query['multiple_separator'] = unicode(self.multiple_separator.text())
+
+        self.queries[name] = query
+        self.queries.commit()
+
+        if new:
+            self.query_field.blockSignals(True)
+            self.query_field.clear()
+            self.query_field.addItem('')
+            self.query_field.addItems(sorted([q for q in self.queries], key=sort_key))
+            self.query_field.blockSignals(False)
+        self.query_field.setCurrentIndex(self.query_field.findText(name))
+
+    def s_r_query_change(self, item_name):
+        if not item_name:
+            self.s_r_reset_query_fields()
+            return
+        item = self.queries.get(unicode(item_name), None)
+        if item is None:
+            self.s_r_reset_query_fields()
+            return
+
+        def set_index(attr, txt):
+            try:
+                attr.setCurrentIndex(attr.findText(txt))
+            except:
+                attr.setCurrentIndex(0)
+
+        set_index(self.search_mode, item['search_mode'])
+        set_index(self.search_field, item['search_field'])
+        self.s_r_template.setText(item['s_r_template'])
+        self.s_r_template_changed() #simulate gain/loss of focus
+        self.search_for.setText(item['search_for'])
+        self.case_sensitive.setChecked(item['case_sensitive'])
+        self.replace_with.setText(item['replace_with'])
+        set_index(self.replace_func, item['replace_func'])
+        set_index(self.destination_field, item['destination_field'])
+        set_index(self.replace_mode, item['replace_mode'])
+        self.comma_separated.setChecked(item['comma_separated'])
+        self.results_count.setValue(int(item['results_count']))
+        self.starting_from.setValue(int(item['starting_from']))
+        self.multiple_separator.setText(item['multiple_separator'])
+
+    def s_r_reset_query_fields(self):
+        # Don't reset the search mode. The user will probably want to use it
+        # as it was
+        self.search_field.setCurrentIndex(0)
+        self.s_r_template.setText("")
+        self.search_for.setText("")
+        self.case_sensitive.setChecked(False)
+        self.replace_with.setText("")
+        self.replace_func.setCurrentIndex(0)
+        self.destination_field.setCurrentIndex(0)
+        self.replace_mode.setCurrentIndex(0)
+        self.comma_separated.setChecked(True)
+        self.results_count.setValue(999)
+        self.starting_from.setValue(1)
+        self.multiple_separator.setText(" ::: ")
+

src/calibre/gui2/dialogs/metadata_bulk.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>850</width>
+    <width>962</width>
-    <height>650</height>
+    <height>645</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -44,8 +44,8 @@
        <rect>
         <x>0</x>
         <y>0</y>
-        <width>842</width>
+        <width>954</width>
-        <height>589</height>
+        <height>584</height>
        </rect>
       </property>
       <layout class="QVBoxLayout" name="verticalLayout_2">
@@ -574,7 +574,7 @@ Future conversion of these books will use the default settings.</string>
            <property name="sizeConstraint">
             <enum>QLayout::SetMinimumSize</enum>
            </property>
-           <item row="1" column="0" colspan="3">
+           <item row="0" column="0" colspan="4">
             <widget class="QLabel" name="s_r_heading">
              <property name="wordWrap">
               <bool>true</bool>
@@ -584,14 +584,91 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="2" column="0">
+           <item row="1" column="0">
             <widget class="QLabel" name="filler">
              <property name="text">
               <string/>
              </property>
             </widget>
            </item>
+           <item row="2" column="0" colspan="3">
+            <widget class="Line" name="line">
+             <property name="orientation">
+              <enum>Qt::Horizontal</enum>
+             </property>
+            </widget>
+           </item>
            <item row="3" column="0">
+            <widget class="QLabel" name="xlabel_22">
+             <property name="text">
+              <string>Load searc&amp;h/replace:</string>
+             </property>
+             <property name="buddy">
+              <cstring>search_field</cstring>
+             </property>
+            </widget>
+           </item>
+           <item row="3" column="1">
+            <widget class="QComboBox" name="query_field">
+             <property name="toolTip">
+              <string>Select saved search/replace to load.</string>
+             </property>
+            </widget>
+           </item>
+           <item row="3" column="2">
+            <layout class="QHBoxLayout" name="horizontalLayout_6">
+             <item>
+              <spacer name="horizontalSpacer_3">
+               <property name="orientation">
+                <enum>Qt::Horizontal</enum>
+               </property>
+               <property name="sizeType">
+                <enum>QSizePolicy::Fixed</enum>
+               </property>
+               <property name="sizeHint" stdset="0">
+                <size>
+                 <width>20</width>
+                 <height>20</height>
+                </size>
+               </property>
+              </spacer>
+             </item>
+             <item>
+              <widget class="QPushButton" name="save_button">
+               <property name="toolTip">
+                <string>Save current search/replace</string>
+               </property>
+               <property name="text">
+                <string>Sa&amp;ve</string>
+               </property>
+              </widget>
+             </item>
+             <item>
+              <widget class="QPushButton" name="remove_button">
+               <property name="toolTip">
+                <string>Delete saved search/replace</string>
+               </property>
+               <property name="text">
+                <string>Delete</string>
+               </property>
+              </widget>
+             </item>
+             <item>
+              <spacer name="horizontalSpacer_2">
+               <property name="orientation">
+                <enum>Qt::Horizontal</enum>
+               </property>
+               <property name="sizeHint" stdset="0">
+                <size>
+                 <width>40</width>
+                 <height>20</height>
+                </size>
+               </property>
+              </spacer>
+             </item>
+            </layout>
+           </item>
+           <item row="4" column="0">
             <widget class="QLabel" name="xlabel_21">
              <property name="text">
               <string>Search &amp;field:</string>
@@ -601,14 +678,14 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="3" column="1">
+           <item row="4" column="1">
             <widget class="QComboBox" name="search_field">
              <property name="toolTip">
               <string>The name of the field that you want to search</string>
              </property>
             </widget>
            </item>
-           <item row="3" column="2">
+           <item row="4" column="2">
             <layout class="QHBoxLayout" name="HLayout_3">
              <item>
               <widget class="QLabel" name="xlabel_24">
@@ -642,7 +719,7 @@ Future conversion of these books will use the default settings.</string>
              </item>
             </layout>
            </item>
-           <item row="4" column="0">
+           <item row="5" column="0">
             <widget class="QLabel" name="template_label">
              <property name="text">
               <string>Te&amp;mplate:</string>
@@ -652,7 +729,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="4" column="1">
+           <item row="5" column="1">
             <widget class="HistoryLineEdit" name="s_r_template">
              <property name="sizePolicy">
               <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
@@ -665,7 +742,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="5" column="0">
+           <item row="6" column="0">
             <widget class="QLabel" name="xlabel_2">
              <property name="text">
               <string>&amp;Search for:</string>
@@ -675,7 +752,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="5" column="1">
+           <item row="6" column="1">
             <widget class="HistoryLineEdit" name="search_for">
              <property name="sizePolicy">
               <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
@@ -688,7 +765,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="5" column="2">
+           <item row="6" column="2">
             <widget class="QCheckBox" name="case_sensitive">
              <property name="toolTip">
               <string>Check this box if the search string must match exactly upper and lower case. Uncheck it if case is to be ignored</string>
@@ -701,7 +778,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="6" column="0">
+           <item row="7" column="0">
             <widget class="QLabel" name="xlabel_4">
              <property name="text">
               <string>&amp;Replace with:</string>
@@ -711,14 +788,14 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="6" column="1">
+           <item row="7" column="1">
             <widget class="HistoryLineEdit" name="replace_with">
              <property name="toolTip">
               <string>The replacement text. The matched search text will be replaced with this string</string>
              </property>
             </widget>
            </item>
-           <item row="6" column="2">
+           <item row="7" column="2">
             <layout class="QHBoxLayout" name="verticalLayout">
              <item>
               <widget class="QLabel" name="label_41">
@@ -753,7 +830,7 @@ field is processed. In regular expression mode, only the matched text is process
              </item>
             </layout>
            </item>
-           <item row="7" column="0">
+           <item row="8" column="0">
             <widget class="QLabel" name="destination_field_label">
              <property name="text">
               <string>&amp;Destination field:</string>
@@ -763,7 +840,7 @@ field is processed. In regular expression mode, only the matched text is process
              </property>
             </widget>
            </item>
-           <item row="7" column="1">
+           <item row="8" column="1">
             <widget class="QComboBox" name="destination_field">
              <property name="toolTip">
               <string>The field that the text will be put into after all replacements.
@@ -771,7 +848,7 @@ If blank, the source field is used if the field is modifiable</string>
              </property>
             </widget>
            </item>
-           <item row="7" column="2">
+           <item row="8" column="2">
             <layout class="QHBoxLayout" name="verticalLayout">
              <item>
               <widget class="QLabel" name="replace_mode_label">
@@ -820,7 +897,7 @@ not multiple and the destination field is multiple</string>
              </item>
             </layout>
            </item>
-           <item row="8" column="1" colspan="2">
+           <item row="9" column="1" colspan="2">
             <layout class="QHBoxLayout" name="horizontalLayout_21">
              <item>
               <spacer name="HSpacer_347">
@@ -906,7 +983,7 @@ not multiple and the destination field is multiple</string>
              </item>
             </layout>
            </item>
-           <item row="9" column="0" colspan="4">
+           <item row="10" column="0" colspan="4">
             <widget class="QScrollArea" name="scrollArea11">
              <property name="frameShape">
               <enum>QFrame::NoFrame</enum>
@@ -919,8 +996,8 @@ not multiple and the destination field is multiple</string>
                <rect>
                 <x>0</x>
                 <y>0</y>
-                <width>197</width>
+                <width>938</width>
-                <height>60</height>
+                <height>268</height>
                </rect>
               </property>
               <layout class="QGridLayout" name="testgrid">
@@ -1030,6 +1107,9 @@ not multiple and the destination field is multiple</string>
   <tabstop>series_numbering_restarts</tabstop>
   <tabstop>series_start_number</tabstop>
   <tabstop>button_box</tabstop>
+  <tabstop>query_field</tabstop>
+  <tabstop>save_button</tabstop>
+  <tabstop>remove_button</tabstop>
   <tabstop>search_field</tabstop>
   <tabstop>search_mode</tabstop>
   <tabstop>s_r_template</tabstop>
@@ -1045,6 +1125,23 @@ not multiple and the destination field is multiple</string>
   <tabstop>multiple_separator</tabstop>
   <tabstop>test_text</tabstop>
   <tabstop>test_result</tabstop>
+  <tabstop>scrollArea</tabstop>
+  <tabstop>central_widget</tabstop>
+  <tabstop>swap_title_and_author</tabstop>
+  <tabstop>clear_series</tabstop>
+  <tabstop>adddate</tabstop>
+  <tabstop>clear_adddate_button</tabstop>
+  <tabstop>apply_adddate</tabstop>
+  <tabstop>pubdate</tabstop>
+  <tabstop>clear_pubdate_button</tabstop>
+  <tabstop>apply_pubdate</tabstop>
+  <tabstop>remove_format</tabstop>
+  <tabstop>change_title_to_title_case</tabstop>
+  <tabstop>remove_conversion_settings</tabstop>
+  <tabstop>cover_generate</tabstop>
+  <tabstop>cover_remove</tabstop>
+  <tabstop>cover_from_fmt</tabstop>
+  <tabstop>scrollArea11</tabstop>
  </tabstops>
  <resources>
   <include location="../../../../resources/images.qrc"/>

src/calibre/gui2/dialogs/scheduler.py

@@ -250,22 +250,27 @@ class Scheduler(QObject):
 
         self.timer = QTimer(self)
         self.timer.start(int(self.INTERVAL * 60 * 1000))
-        self.oldest_timer = QTimer()
-        self.connect(self.oldest_timer, SIGNAL('timeout()'), self.oldest_check)
         self.connect(self.timer, SIGNAL('timeout()'), self.check)
         self.oldest = gconf['oldest_news']
-        self.oldest_timer.start(int(60 * 60 * 1000))
         QTimer.singleShot(5 * 1000, self.oldest_check)
         self.database_changed = self.recipe_model.database_changed
 
     def oldest_check(self):
         if self.oldest > 0:
             delta = timedelta(days=self.oldest)
-            ids = self.recipe_model.db.tags_older_than(_('News'), delta)
+            try:
+                ids = self.recipe_model.db.tags_older_than(_('News'), delta)
+            except:
+                # Should never happen
+                ids = []
+                import traceback
+                traceback.print_exc()
             if ids:
                 ids = list(ids)
                 if ids:
                     self.delete_old_news.emit(ids)
+        QTimer.singleShot(60 * 60 * 1000, self.oldest_check)
+
 
     def show_dialog(self, *args):
         self.lock.lock()

src/calibre/gui2/dialogs/tag_categories.py

@@ -2,14 +2,14 @@ __license__   = 'GPL v3'
 
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
-
 from PyQt4.QtCore import SIGNAL, Qt
 from PyQt4.QtGui import QDialog, QIcon, QListWidgetItem
 
 from calibre.gui2.dialogs.tag_categories_ui import Ui_TagCategories
 from calibre.gui2.dialogs.confirm_delete import confirm
+from calibre.gui2 import error_dialog
 from calibre.constants import islinux
-from calibre.utils.icu import sort_key
+from calibre.utils.icu import sort_key, strcmp
 
 class Item:
     def __init__(self, name, label, index, icon, exists):
@@ -102,12 +102,13 @@ class TagCategories(QDialog, Ui_TagCategories):
             self.category_filter_box.addItem(v)
         self.current_cat_name = None
 
-        self.connect(self.apply_button,   SIGNAL('clicked()'), self.apply_tags)
+        self.apply_button.clicked.connect(self.apply_button_clicked)
-        self.connect(self.unapply_button, SIGNAL('clicked()'), self.unapply_tags)
+        self.unapply_button.clicked.connect(self.unapply_button_clicked)
-        self.connect(self.add_category_button, SIGNAL('clicked()'), self.add_category)
+        self.add_category_button.clicked.connect(self.add_category)
-        self.connect(self.category_box, SIGNAL('currentIndexChanged(int)'), self.select_category)
+        self.rename_category_button.clicked.connect(self.rename_category)
-        self.connect(self.category_filter_box, SIGNAL('currentIndexChanged(int)'), self.display_filtered_categories)
+        self.category_box.currentIndexChanged[int].connect(self.select_category)
-        self.connect(self.delete_category_button, SIGNAL('clicked()'), self.del_category)
+        self.category_filter_box.currentIndexChanged[int].connect(self.display_filtered_categories)
+        self.delete_category_button.clicked.connect(self.del_category)
         if islinux:
             self.available_items_box.itemDoubleClicked.connect(self.apply_tags)
         else:
@@ -119,6 +120,9 @@ class TagCategories(QDialog, Ui_TagCategories):
             l = self.category_box.findText(on_category)
             if l >= 0:
                 self.category_box.setCurrentIndex(l)
+        if self.current_cat_name is None:
+            self.category_box.setCurrentIndex(0)
+            self.select_category(0)
 
     def make_list_widget(self, item):
         n = item.name if item.exists else item.name + _(' (not on any book)')
@@ -137,6 +141,9 @@ class TagCategories(QDialog, Ui_TagCategories):
         for index in self.applied_items:
             self.applied_items_box.addItem(self.make_list_widget(self.all_items[index]))
 
+    def apply_button_clicked(self):
+        self.apply_tags(node=None)
+
     def apply_tags(self, node=None):
         if self.current_cat_name is None:
             return
@@ -148,6 +155,9 @@ class TagCategories(QDialog, Ui_TagCategories):
         self.applied_items.sort(key=lambda x:sort_key(self.all_items[x].name))
         self.display_filtered_categories(None)
 
+    def unapply_button_clicked(self):
+        self.unapply_tags(node=None)
+
     def unapply_tags(self, node=None):
         nodes = self.applied_items_box.selectedItems() if node is None else [node]
         for node in nodes:
@@ -160,15 +170,40 @@ class TagCategories(QDialog, Ui_TagCategories):
         cat_name = unicode(self.input_box.text()).strip()
         if cat_name == '':
             return False
+        for c in self.categories:
+            if strcmp(c, cat_name) == 0:
+                error_dialog(self, _('Name already used'),
+                        _('That name is already used, perhaps with different case.')).exec_()
+                return False
         if cat_name not in self.categories:
             self.category_box.clear()
             self.current_cat_name = cat_name
             self.categories[cat_name] = []
             self.applied_items = []
             self.populate_category_list()
-            self.category_box.setCurrentIndex(self.category_box.findText(cat_name))
+        self.input_box.clear()
-        else:
+        self.category_box.setCurrentIndex(self.category_box.findText(cat_name))
-            self.select_category(self.category_box.findText(cat_name))
+        return True
+
+    def rename_category(self):
+        self.save_category()
+        cat_name = unicode(self.input_box.text()).strip()
+        if cat_name == '':
+            return False
+        if not self.current_cat_name:
+            return False
+        for c in self.categories:
+            if strcmp(c, cat_name) == 0:
+                error_dialog(self, _('Name already used'),
+                        _('That name is already used, perhaps with different case.')).exec_()
+                return False
+        # The order below is important because of signals
+        self.categories[cat_name] = self.categories[self.current_cat_name]
+        del self.categories[self.current_cat_name]
+        self.current_cat_name = None
+        self.populate_category_list()
+        self.input_box.clear()
+        self.category_box.setCurrentIndex(self.category_box.findText(cat_name))
         return True
 
     def del_category(self):
@@ -196,7 +231,6 @@ class TagCategories(QDialog, Ui_TagCategories):
 
     def accept(self):
         self.save_category()
-        self.db.prefs['user_categories'] = self.categories
         QDialog.accept(self)
 
     def save_category(self):
@@ -208,5 +242,7 @@ class TagCategories(QDialog, Ui_TagCategories):
             self.categories[self.current_cat_name] = l
 
     def populate_category_list(self):
-        for n in sorted(self.categories.keys(), key=sort_key):
+        self.category_box.blockSignals(True)
-            self.category_box.addItem(n)
+        self.category_box.clear()
+        self.category_box.addItems(sorted(self.categories.keys(), key=sort_key))
+        self.category_box.blockSignals(False)

src/calibre/gui2/dialogs/tag_categories.ui

@@ -18,7 +18,139 @@
     <normaloff>:/images/chapters.png</normaloff>:/images/chapters.png</iconset>
   </property>
   <layout class="QGridLayout">
+   <item row="0" column="0">
+    <layout class="QHBoxLayout">
+     <item>
+      <widget class="QLabel" name="label_3">
+       <property name="sizePolicy">
+        <sizepolicy hsizetype="Fixed" vsizetype="Preferred">
+         <horstretch>0</horstretch>
+         <verstretch>0</verstretch>
+        </sizepolicy>
+       </property>
+       <property name="minimumSize">
+        <size>
+         <width>100</width>
+         <height>0</height>
+        </size>
+       </property>
+       <property name="text">
+        <string>Category name: </string>
+       </property>
+       <property name="alignment">
+        <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter</set>
+       </property>
+       <property name="buddy">
+        <cstring>category_box</cstring>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QComboBox" name="category_box">
+       <property name="sizePolicy">
+        <sizepolicy hsizetype="Fixed" vsizetype="Fixed">
+         <horstretch>160</horstretch>
+         <verstretch>0</verstretch>
+        </sizepolicy>
+       </property>
+       <property name="minimumSize">
+        <size>
+         <width>145</width>
+         <height>0</height>
+        </size>
+       </property>
+       <property name="toolTip">
+        <string>Select a category to edit</string>
+       </property>
+       <property name="editable">
+        <bool>false</bool>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
+   <item row="0" column="1">
+    <widget class="QToolButton" name="delete_category_button">
+     <property name="toolTip">
+      <string>Delete this selected tag category</string>
+     </property>
+     <property name="text">
+      <string>...</string>
+     </property>
+     <property name="icon">
+      <iconset>
+       <normaloff>:/images/minus.png</normaloff>:/images/minus.png</iconset>
+     </property>
+    </widget>
+   </item>
+   <item row="0" column="2">
+    <layout class="QHBoxLayout">
+     <item>
+      <widget class="QLineEdit" name="input_box">
+       <property name="sizePolicy">
+        <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
+         <horstretch>60</horstretch>
+         <verstretch>0</verstretch>
+        </sizepolicy>
+       </property>
+       <property name="toolTip">
+        <string>Enter a category name, then use the add button or the rename button</string>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QToolButton" name="add_category_button">
+       <property name="toolTip">
+        <string>Add a new category</string>
+       </property>
+       <property name="text">
+        <string>...</string>
+       </property>
+       <property name="icon">
+        <iconset>
+         <normaloff>:/images/plus.png</normaloff>:/images/plus.png
+        </iconset>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
+   <item row="0" column="3">
+    <widget class="QToolButton" name="rename_category_button">
+     <property name="toolTip">
+      <string>Rename the current category to the what is in the box</string>
+     </property>
+     <property name="text">
+      <string>...</string>
+     </property>
+     <property name="icon">
+      <iconset>
+       <normaloff>:/images/edit-undo.png</normaloff>:/images/edit-undo.png</iconset>
+     </property>
+    </widget>
+   </item>
    <item row="1" column="0">
+    <layout class="QHBoxLayout">
+     <item>
+      <widget class="QLabel" name="label_5">
+       <property name="text">
+        <string>Category filter: </string>
+       </property>
+       <property name="alignment">
+        <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter</set>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QComboBox" name="category_filter_box">
+       <property name="toolTip">
+        <string>Select the content kind of the new category</string>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
+   <item row="2" column="0">
     <layout class="QVBoxLayout">
      <item>
       <layout class="QHBoxLayout">
@@ -66,7 +198,7 @@
      </item>
     </layout>
    </item>
-   <item row="1" column="1">
+   <item row="2" column="1">
     <layout class="QVBoxLayout">
      <item>
       <spacer>
@@ -110,7 +242,7 @@
      </item>
     </layout>
    </item>
-   <item row="1" column="2">
+   <item row="2" column="2">
     <layout class="QVBoxLayout">
      <item>
       <layout class="QHBoxLayout">
@@ -151,7 +283,7 @@
      </item>
     </layout>
    </item>
-   <item row="1" column="3">
+   <item row="2" column="3">
     <layout class="QVBoxLayout">
      <item>
       <spacer>
@@ -195,7 +327,7 @@
      </item>
     </layout>
    </item>
-   <item row="3" column="0" colspan="4">
+   <item row="4" column="0" colspan="4">
     <widget class="QDialogButtonBox" name="buttonBox">
      <property name="orientation">
       <enum>Qt::Horizontal</enum>
@@ -208,141 +340,6 @@
      </property>
     </widget>
    </item>
-   <item row="0" column="0" colspan="4">
-    <layout class="QGridLayout" name="gridLayout">
-     <item row="0" column="0">
-      <widget class="QLabel" name="label_3">
-       <property name="sizePolicy">
-        <sizepolicy hsizetype="Fixed" vsizetype="Preferred">
-         <horstretch>0</horstretch>
-         <verstretch>0</verstretch>
-        </sizepolicy>
-       </property>
-       <property name="minimumSize">
-        <size>
-         <width>100</width>
-         <height>0</height>
-        </size>
-       </property>
-       <property name="text">
-        <string>Category name: </string>
-       </property>
-       <property name="alignment">
-        <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter</set>
-       </property>
-       <property name="buddy">
-        <cstring>category_box</cstring>
-       </property>
-      </widget>
-     </item>
-     <item row="0" column="1">
-      <widget class="QComboBox" name="category_box">
-       <property name="sizePolicy">
-        <sizepolicy hsizetype="Fixed" vsizetype="Fixed">
-         <horstretch>160</horstretch>
-         <verstretch>0</verstretch>
-        </sizepolicy>
-       </property>
-       <property name="minimumSize">
-        <size>
-         <width>145</width>
-         <height>0</height>
-        </size>
-       </property>
-       <property name="toolTip">
-        <string>Select a category to edit</string>
-       </property>
-       <property name="editable">
-        <bool>false</bool>
-       </property>
-      </widget>
-     </item>
-     <item row="0" column="2">
-      <widget class="QToolButton" name="delete_category_button">
-       <property name="toolTip">
-        <string>Delete this selected tag category</string>
-       </property>
-       <property name="text">
-        <string>...</string>
-       </property>
-       <property name="icon">
-        <iconset>
-         <normaloff>:/images/minus.png</normaloff>:/images/minus.png</iconset>
-       </property>
-      </widget>
-     </item>
-     <item row="0" column="3">
-      <spacer>
-       <property name="orientation">
-        <enum>Qt::Horizontal</enum>
-       </property>
-       <property name="sizeHint" stdset="0">
-        <size>
-         <width>40</width>
-         <height>20</height>
-        </size>
-       </property>
-      </spacer>
-     </item>
-     <item row="0" column="4">
-      <widget class="QLineEdit" name="input_box">
-       <property name="sizePolicy">
-        <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
-         <horstretch>60</horstretch>
-         <verstretch>0</verstretch>
-        </sizepolicy>
-       </property>
-       <property name="toolTip">
-        <string>Enter a new category name. Select the kind before adding it.</string>
-       </property>
-      </widget>
-     </item>
-     <item row="0" column="5">
-      <widget class="QToolButton" name="add_category_button">
-       <property name="toolTip">
-        <string>Add the new category</string>
-       </property>
-       <property name="text">
-        <string>...</string>
-       </property>
-       <property name="icon">
-        <iconset>
-         <normaloff>:/images/plus.png</normaloff>:/images/plus.png</iconset>
-       </property>
-      </widget>
-     </item>
-     <item row="1" column="5">
-      <spacer name="horizontalSpacer">
-       <property name="orientation">
-        <enum>Qt::Horizontal</enum>
-       </property>
-       <property name="sizeHint" stdset="0">
-        <size>
-         <width>20</width>
-         <height>20</height>
-        </size>
-       </property>
-      </spacer>
-     </item>
-     <item row="1" column="0">
-      <widget class="QLabel" name="label_5">
-       <property name="text">
-        <string>Category filter: </string>
-       </property>
-       <property name="alignment">
-        <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter</set>
-       </property>
-      </widget>
-     </item>
-     <item row="1" column="1">
-      <widget class="QComboBox" name="category_filter_box">
-       <property name="toolTip">
-        <string>Select the content kind of the new category</string>
-       </property>
-      </widget>
-     </item>
-    </layout>
-   </item>
   </layout>
  </widget>
  <resources>

src/calibre/gui2/dialogs/tag_editor.py

@@ -16,7 +16,7 @@ class TagEditor(QDialog, Ui_TagEditor):
         self.setupUi(self)
 
         self.db = db
-        self.index = db.row(id_)
+        self.index = db.row(id_) if id_ is not None else None
         if self.index is not None:
             tags = self.db.tags(self.index)
         else:

src/calibre/gui2/filename_pattern.ui

@@ -43,7 +43,17 @@ p, li { white-space: pre-wrap; }
      </property>
      <layout class="QVBoxLayout">
       <item>
-       <widget class="QLineEdit" name="re"/>
+       <widget class="QComboBox" name="re">
+        <property name="editable">
+         <bool>true</bool>
+        </property>
+        <property name="maxCount">
+         <number>10</number>
+        </property>
+        <property name="insertPolicy">
+         <enum>QComboBox::InsertAtTop</enum>
+        </property>
+       </widget>
       </item>
      </layout>
     </widget>
@@ -94,8 +104,8 @@ p, li { white-space: pre-wrap; }
             <rect>
              <x>0</x>
              <y>0</y>
-             <width>301</width>
+             <width>277</width>
-             <height>234</height>
+             <height>276</height>
             </rect>
            </property>
            <layout class="QGridLayout" name="gridLayout_2">

src/calibre/gui2/tag_view.py

@@ -98,6 +98,7 @@ class TagsView(QTreeView): # {{{
             self.collapse_model = 'disable'
         else:
             self.collapse_model = gprefs['tags_browser_partition_method']
+        self.search_icon = QIcon(I('search.png'))
 
     def set_pane_is_visible(self, to_what):
         pv = self.pane_is_visible
@@ -186,7 +187,7 @@ class TagsView(QTreeView): # {{{
             self.clear()
 
     def context_menu_handler(self, action=None, category=None,
-                             key=None, index=None):
+                             key=None, index=None, negate=None):
         if not action:
             return
         try:
@@ -199,12 +200,20 @@ class TagsView(QTreeView): # {{{
             if action == 'manage_categories':
                 self.user_category_edit.emit(category)
                 return
+            if action == 'search':
+                self.tags_marked.emit(('not ' if negate else '') +
+                                      category + ':"=' + key + '"')
+                return
+            if action == 'search_category':
+                self.tags_marked.emit(category + ':' + str(not negate))
+                return
             if action == 'manage_searches':
                 self.saved_search_edit.emit(category)
                 return
             if action == 'edit_author_sort':
                 self.author_sort_edit.emit(self, index)
                 return
+
             if action == 'hide':
                 self.hidden_categories.add(category)
             elif action == 'show':
@@ -245,19 +254,36 @@ class TagsView(QTreeView): # {{{
                 if key not in self.db.field_metadata:
                     return True
 
-                # If the user right-clicked on an editable item, then offer
+                # Did the user click on a leaf node?
-                # the possibility of renaming that item
+                if tag_name:
-                if tag_name and \
+                    # If the user right-clicked on an editable item, then offer
-                        (key in ['authors', 'tags', 'series', 'publisher', 'search'] or \
+                    # the possibility of renaming that item.
-                        self.db.field_metadata[key]['is_custom'] and \
+                    if key in ['authors', 'tags', 'series', 'publisher', 'search'] or \
-                        self.db.field_metadata[key]['datatype'] != 'rating'):
+                            (self.db.field_metadata[key]['is_custom'] and \
-                    self.context_menu.addAction(_('Rename \'%s\'')%tag_name,
+                             self.db.field_metadata[key]['datatype'] != 'rating'):
-                            partial(self.context_menu_handler, action='edit_item',
+                        # Add the 'rename' items
-                                    category=tag_item, index=index))
+                        self.context_menu.addAction(_('Rename %s')%tag_name,
-                    if key == 'authors':
+                                partial(self.context_menu_handler, action='edit_item',
-                        self.context_menu.addAction(_('Edit sort for \'%s\'')%tag_name,
+                                        category=tag_item, index=index))
-                                partial(self.context_menu_handler,
+                        if key == 'authors':
-                                        action='edit_author_sort', index=tag_id))
+                            self.context_menu.addAction(_('Edit sort for %s')%tag_name,
+                                    partial(self.context_menu_handler,
+                                            action='edit_author_sort', index=tag_id))
+                    # Add the search for value items
+                    n = tag_name
+                    c = category
+                    if self.db.field_metadata[key]['datatype'] == 'rating':
+                        n = str(len(tag_name))
+                    elif self.db.field_metadata[key]['kind'] in ['user', 'search']:
+                        c = tag_item.tag.category
+                    self.context_menu.addAction(self.search_icon,
+                            _('Search for %s')%tag_name,
+                            partial(self.context_menu_handler, action='search',
+                                    category=c, key=n, negate=False))
+                    self.context_menu.addAction(self.search_icon,
+                            _('Search for everything but %s')%tag_name,
+                            partial(self.context_menu_handler, action='search',
+                                    category=c, key=n, negate=True))
                     self.context_menu.addSeparator()
                 # Hide/Show/Restore categories
                 self.context_menu.addAction(_('Hide category %s') % category,
@@ -268,6 +294,16 @@ class TagsView(QTreeView): # {{{
                         m.addAction(col,
                             partial(self.context_menu_handler, action='show', category=col))
 
+                # search by category
+                if key != 'search':
+                    self.context_menu.addAction(self.search_icon,
+                            _('Search for books in category %s')%category,
+                            partial(self.context_menu_handler, action='search_category',
+                                    category=key, negate=False))
+                    self.context_menu.addAction(self.search_icon,
+                            _('Search for books not in category %s')%category,
+                            partial(self.context_menu_handler, action='search_category',
+                                    category=key, negate=True))
                 # Offer specific editors for tags/series/publishers/saved searches
                 self.context_menu.addSeparator()
                 if key in ['tags', 'publisher', 'series'] or \
@@ -540,10 +576,7 @@ class TagsModel(QAbstractItemModel): # {{{
         for i, r in enumerate(self.row_map):
             if self.hidden_categories and self.categories[i] in self.hidden_categories:
                 continue
-            if self.db.field_metadata[r]['kind'] != 'user':
+            tt = _(u'The lookup/search name is "{0}"').format(r)
-                tt = _('The lookup/search name is "{0}"').format(r)
-            else:
-                tt = ''
             TagTreeItem(parent=self.root_item,
                     data=self.categories[i],
                     category_icon=self.category_icon_map[r],
@@ -691,7 +724,11 @@ class TagsModel(QAbstractItemModel): # {{{
         for user_cat in sorted(self.db.prefs.get('user_categories', {}).keys(),
                                key=sort_key):
             cat_name = '@' + user_cat # add the '@' to avoid name collision
-            tb_cats.add_user_category(label=cat_name, name=user_cat)
+            try:
+                tb_cats.add_user_category(label=cat_name, name=user_cat)
+            except ValueError:
+                import traceback
+                traceback.print_exc()
         if len(saved_searches().names()):
             tb_cats.add_search_category(label='search', name=_('Searches'))
 
@@ -1147,9 +1184,14 @@ class TagBrowserMixin(object): # {{{
                 self.do_user_categories_edit())
 
     def do_user_categories_edit(self, on_category=None):
-        d = TagCategories(self, self.library_view.model().db, on_category)
+        db = self.library_view.model().db
-        d.exec_()
+        d = TagCategories(self, db, on_category)
-        if d.result() == d.Accepted:
+        if d.exec_() == d.Accepted:
+            db.prefs.set('user_categories', d.categories)
+            db.field_metadata.remove_user_categories()
+            for k in d.categories:
+                db.field_metadata.add_user_category('@' + k, k)
+            db.data.sqp_change_locations(db.field_metadata.get_search_terms())
             self.tags_view.set_new_model()
             self.tags_view.recount()
 

src/calibre/gui2/ui.py

@@ -638,8 +638,6 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         except KeyboardInterrupt:
             pass
         time.sleep(2)
-        if mb is not None:
-            mb.flush()
         self.hide_windows()
         return True
 

src/calibre/gui2/widgets.py

@@ -16,7 +16,6 @@ from PyQt4.Qt import QIcon, QFont, QLabel, QListWidget, QAction, \
                         QTimer, QRect
 
 from calibre.gui2 import NONE, error_dialog, pixmap_to_data, gprefs
-from calibre.constants import isosx
 from calibre.gui2.filename_pattern_ui import Ui_Form
 from calibre import fit_image
 from calibre.ebooks import BOOK_EXTENSIONS
@@ -67,17 +66,31 @@ class FilenamePattern(QWidget, Ui_Form):
         self.setupUi(self)
 
         self.connect(self.test_button, SIGNAL('clicked()'), self.do_test)
-        self.connect(self.re, SIGNAL('returnPressed()'), self.do_test)
+        self.connect(self.re.lineEdit(), SIGNAL('returnPressed()'), self.do_test)
-        self.initialize()
+        self.re.lineEdit().textChanged.connect(lambda x: self.changed_signal.emit())
-        self.re.textChanged.connect(lambda x: self.changed_signal.emit())
 
     def initialize(self, defaults=False):
+        # Get all itmes in the combobox. If we are resting
+        # to defaults we don't want to lose what the user
+        # has added.
+        val_hist = [unicode(self.re.lineEdit().text())] + [unicode(self.re.itemText(i)) for i in xrange(self.re.count())]
+        self.re.clear()
+        
         if defaults:
             val = prefs.defaults['filename_pattern']
         else:
             val = prefs['filename_pattern']
-        self.re.setText(val)
+        self.re.lineEdit().setText(val)
-
+        
+        val_hist += gprefs.get('filename_pattern_history', ['(?P<title>.+)', '(?P<author>[^_-]+) -?\s*(?P<series>[^_0-9-]*)(?P<series_index>[0-9]*)\s*-\s*(?P<title>[^_].+) ?'])
+        if val in val_hist:
+            del val_hist[val_hist.index(val)]
+        val_hist.insert(0, val)
+        for v in val_hist:
+            # Ensure we don't have duplicate items.
+            if v and self.re.findText(v) == -1:
+                self.re.addItem(v)
+        self.re.setCurrentIndex(0)
 
     def do_test(self):
         try:
@@ -110,12 +123,21 @@ class FilenamePattern(QWidget, Ui_Form):
 
 
     def pattern(self):
-        pat = unicode(self.re.text())
+        pat = unicode(self.re.lineEdit().text())
         return re.compile(pat)
 
     def commit(self):
         pat = self.pattern().pattern
         prefs['filename_pattern'] = pat
+        
+        history = []
+        history_pats = [unicode(self.re.lineEdit().text())] + [unicode(self.re.itemText(i)) for i in xrange(self.re.count())]
+        for p in history_pats[:14]:
+            # Ensure we don't have duplicate items.
+            if p and p not in history:
+                history.append(p)
+        gprefs['filename_pattern_history'] = history 
+        
         return pat
 
 
@@ -304,8 +326,9 @@ class FontFamilyModel(QAbstractListModel):
             return NONE
         if role == Qt.DisplayRole:
             return QVariant(family)
-        if not isosx and role == Qt.FontRole:
+        if False and role == Qt.FontRole:
-            # Causes a Qt crash with some fonts on OS X
+            # Causes a Qt crash with some fonts
+            # so disabled.
             return QVariant(QFont(family))
         return NONE
 

src/calibre/library/caches.py

@@ -10,7 +10,6 @@ import re, itertools, time, traceback
 from itertools import repeat
 from datetime import timedelta
 from threading import Thread
-from Queue import Empty
 
 from calibre.utils.config import tweaks
 from calibre.utils.date import parse_date, now, UNDEFINED_DATE
@@ -38,44 +37,47 @@ class MetadataBackup(Thread): # {{{
         self.get_metadata_for_dump = FunctionDispatcher(db.get_metadata_for_dump)
         self.clear_dirtied = FunctionDispatcher(db.clear_dirtied)
         self.set_dirtied = FunctionDispatcher(db.dirtied)
-        self.in_limbo = None
 
     def stop(self):
         self.keep_running = False
+
+    def break_cycles(self):
         # Break cycles so that this object doesn't hold references to db
         self.do_write = self.get_metadata_for_dump = self.clear_dirtied = \
             self.set_dirtied = self.db = None
 
     def run(self):
         while self.keep_running:
-            self.in_limbo = None
             try:
-                time.sleep(0.5) # Limit to two per second
+                time.sleep(2) # Limit to two per second
-                id_ = self.db.dirtied_queue.get(True, 1.45)
+                (id_, sequence) = self.db.get_a_dirtied_book()
-            except Empty:
+                if id_ is None:
-                continue
+                    continue
+                # print 'writer thread', id_, sequence
             except:
                 # Happens during interpreter shutdown
                 break
+            if not self.keep_running:
+                break
 
             try:
-                path, mi = self.get_metadata_for_dump(id_)
+                path, mi, sequence = self.get_metadata_for_dump(id_)
             except:
                 prints('Failed to get backup metadata for id:', id_, 'once')
                 traceback.print_exc()
                 time.sleep(2)
                 try:
-                    path, mi = self.get_metadata_for_dump(id_)
+                    path, mi, sequence = self.get_metadata_for_dump(id_)
                 except:
                     prints('Failed to get backup metadata for id:', id_, 'again, giving up')
                     traceback.print_exc()
                     continue
 
-            # at this point the dirty indication is off
-
             if mi is None:
+                self.clear_dirtied(id_, sequence)
                 continue
-            self.in_limbo = id_
+            if not self.keep_running:
+                break
 
             # Give the GUI thread a chance to do something. Python threads don't
             # have priorities, so this thread would naturally keep the processor
@@ -84,11 +86,13 @@ class MetadataBackup(Thread): # {{{
             try:
                 raw = metadata_to_opf(mi)
             except:
-                self.set_dirtied([id_])
                 prints('Failed to convert to opf for id:', id_)
                 traceback.print_exc()
                 continue
 
+            if not self.keep_running:
+                break
+
             time.sleep(0.1) # Give the GUI thread a chance to do something
             try:
                 self.do_write(path, raw)
@@ -98,19 +102,12 @@ class MetadataBackup(Thread): # {{{
                 try:
                     self.do_write(path, raw)
                 except:
-                    self.set_dirtied([id_])
                     prints('Failed to write backup metadata for id:', id_,
                             'again, giving up')
                     continue
-        self.in_limbo = None
 
-    def flush(self):
+            self.clear_dirtied(id_, sequence)
-        'Used during shutdown to ensure that a dirtied book is not missed'
+        self.break_cycles()
-        if self.in_limbo is not None:
-            try:
-                self.db.dirtied([self.in_limbo])
-            except:
-                traceback.print_exc()
 
     def write(self, path, raw):
         with lopen(path, 'wb') as f:
@@ -135,7 +132,7 @@ def _match(query, value, matchkind):
             pass
     return False
 
-class CacheRow(list):
+class CacheRow(list): # {{{
 
     def __init__(self, db, composites, val):
         self.db = db
@@ -166,14 +163,16 @@ class CacheRow(list):
     def __getslice__(self, i, j):
         return self.__getitem__(slice(i, j))
 
+# }}}
 
 class ResultCache(SearchQueryParser): # {{{
 
     '''
     Stores sorted and filtered metadata in memory.
     '''
-    def __init__(self, FIELD_MAP, field_metadata):
+    def __init__(self, FIELD_MAP, field_metadata, db_prefs=None):
         self.FIELD_MAP = FIELD_MAP
+        self.db_prefs = db_prefs
         self.composites = {}
         for key in field_metadata:
             if field_metadata[key]['datatype'] == 'composite':
@@ -183,15 +182,15 @@ class ResultCache(SearchQueryParser): # {{{
         self.first_sort = True
         self.search_restriction = ''
         self.field_metadata = field_metadata
-        self.all_search_locations = field_metadata.get_search_terms()
+        all_search_locations = field_metadata.get_search_terms()
-        SearchQueryParser.__init__(self, self.all_search_locations, optimize=True)
+        SearchQueryParser.__init__(self, all_search_locations, optimize=True)
         self.build_date_relop_dict()
         self.build_numeric_relop_dict()
 
     def break_cycles(self):
         self._data = self.field_metadata = self.FIELD_MAP = \
             self.numeric_search_relops = self.date_search_relops = \
-            self.all_search_locations = None
+            self.db_prefs = None
 
 
     def __getitem__(self, row):
@@ -405,6 +404,22 @@ class ResultCache(SearchQueryParser): # {{{
                 matches.add(item[0])
         return matches
 
+    def get_user_category_matches(self, location, query, candidates):
+        res = set([])
+        if self.db_prefs is None:
+            return  res
+        user_cats = self.db_prefs.get('user_categories', [])
+        if location not in user_cats:
+            return res
+        c = set(candidates)
+        for (item, category, ign) in user_cats[location]:
+            s = self.get_matches(category, '=' + item, candidates=c)
+            c -= s
+            res |= s
+        if query == 'false':
+            return candidates - res
+        return res
+
     def get_matches(self, location, query, allow_recursion=True, candidates=None):
         matches = set([])
         if candidates is None:
@@ -415,7 +430,7 @@ class ResultCache(SearchQueryParser): # {{{
         if query and query.strip():
             # get metadata key associated with the search term. Eliminates
             # dealing with plurals and other aliases
-            location = self.field_metadata.search_term_to_field_key(location.lower().strip())
+            location = self.field_metadata.search_term_to_field_key(icu_lower(location.strip()))
             if isinstance(location, list):
                 if allow_recursion:
                     for loc in location:
@@ -443,6 +458,10 @@ class ResultCache(SearchQueryParser): # {{{
                     return self.get_numeric_matches(location, query[1:],
                                                     candidates, val_func=vf)
 
+            # check for user categories
+            if len(location) >= 2 and location.startswith('@'):
+                return self.get_user_category_matches(location[1:], query.lower(),
+                                                      candidates)
             # everything else, or 'all' matches
             matchkind = CONTAINS_MATCH
             if (len(query) > 1):
@@ -468,6 +487,8 @@ class ResultCache(SearchQueryParser): # {{{
             for x in range(len(self.FIELD_MAP)):
                 col_datatype.append('')
             for x in self.field_metadata:
+                if x.startswith('@'):
+                    continue
                 if len(self.field_metadata[x]['search_terms']):
                     db_col[x] = self.field_metadata[x]['rec_index']
                     if self.field_metadata[x]['datatype'] not in \

src/calibre/library/catalog.py

@@ -1820,6 +1820,9 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
                 self.booksByTitle_noSeriesPrefix = nspt
 
             # Loop through the books by title
+            # Generate one divRunningTag per initial letter for the purposes of
+            # minimizing widows and orphans on readers that can handle large
+            # <divs> styled as inline-block
             title_list = self.booksByTitle
             if not self.useSeriesPrefixInTitlesSection:
                 title_list = self.booksByTitle_noSeriesPrefix
@@ -1832,7 +1835,7 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
                         divTag.insert(dtc, divRunningTag)
                         dtc += 1
                     divRunningTag = Tag(soup, 'div')
-                    divRunningTag['style'] = 'display:inline-block;width:100%'
+                    divRunningTag['class'] = "logical_group"
                     drtc = 0
                     current_letter = self.letter_or_symbol(book['title_sort'][0])
                     pIndexTag = Tag(soup, "p")
@@ -1954,6 +1957,8 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
             drtc = 0
 
             # Loop through booksByAuthor
+            # Each author/books group goes in an openingTag div (first) or
+            # a runningTag div (subsequent)
             book_count = 0
             current_author = ''
             current_letter = ''
@@ -1977,7 +1982,7 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
                     current_letter = self.letter_or_symbol(book['author_sort'][0].upper())
                     author_count = 0
                     divOpeningTag = Tag(soup, 'div')
-                    divOpeningTag['style'] = 'display:inline-block;width:100%'
+                    divOpeningTag['class'] = "logical_group"
                     dotc = 0
                     pIndexTag = Tag(soup, "p")
                     pIndexTag['class'] = "letter_index"
@@ -2001,7 +2006,7 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
 
                         # Create a divRunningTag for the rest of the authors in this letter
                         divRunningTag = Tag(soup, 'div')
-                        divRunningTag['style'] = 'display:inline-block;width:100%'
+                        divRunningTag['class'] = "logical_group"
                         drtc = 0
 
                     non_series_books = 0

src/calibre/library/database2.py

@@ -7,9 +7,9 @@ __docformat__ = 'restructuredtext en'
 The database used to store ebook metadata
 '''
 import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, json
+import threading, random
 from itertools import repeat
 from math import ceil
-from Queue import Queue
 
 from PyQt4.QtGui import QImage
 
@@ -117,7 +117,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
     def __init__(self, library_path, row_factory=False, default_prefs=None,
             read_only=False):
         self.field_metadata = FieldMetadata()
-        self.dirtied_queue = Queue()
+        # Create the lock to be used to guard access to the metadata writer
+        # queues. This must be an RLock, not a Lock
+        self.dirtied_lock = threading.RLock()
         if not os.path.exists(library_path):
             os.makedirs(library_path)
         self.listeners = set([])
@@ -186,6 +188,29 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         migrate_preference('saved_searches', {})
         set_saved_searches(self, 'saved_searches')
 
+        # Rename any user categories with names that differ only in case
+        user_cats = self.prefs.get('user_categories', [])
+        catmap = {}
+        for uc in user_cats:
+            ucl = icu_lower(uc)
+            if ucl not in catmap:
+                catmap[ucl] = []
+            catmap[ucl].append(uc)
+        cats_changed = False
+        for uc in catmap:
+            if len(catmap[uc]) > 1:
+                prints('found user category case overlap', catmap[uc])
+                cat = catmap[uc][0]
+                suffix = 1
+                while icu_lower((cat + unicode(suffix))) in catmap:
+                    suffix += 1
+                prints('Renaming user category %s to %s'%(cat, cat+unicode(suffix)))
+                user_cats[cat + unicode(suffix)] = user_cats[cat]
+                del user_cats[cat]
+                cats_changed = True
+        if cats_changed:
+            self.prefs.set('user_categories', user_cats)
+
         load_user_template_functions(self.prefs.get('user_template_functions', []))
 
         self.conn.executescript('''
@@ -332,7 +357,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 traceback.print_exc()
 
         self.book_on_device_func = None
-        self.data    = ResultCache(self.FIELD_MAP, self.field_metadata)
+        self.data    = ResultCache(self.FIELD_MAP, self.field_metadata, db_prefs=self.prefs)
         self.search  = self.data.search
         self.search_getting_ids  = self.data.search_getting_ids
         self.refresh = functools.partial(self.data.refresh, self)
@@ -353,9 +378,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 loc=self.FIELD_MAP['sort']))
 
         d = self.conn.get('SELECT book FROM metadata_dirtied', all=True)
-        for x in d:
+        with self.dirtied_lock:
-            self.dirtied_queue.put(x[0])
+            self.dirtied_sequence = 0
-        self.dirtied_cache = set([x[0] for x in d])
+            self.dirtied_cache = {}
+            for x in d:
+                self.dirtied_cache[x[0]] = self.dirtied_sequence
+                self.dirtied_sequence += 1
 
         self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self)
         self.refresh()
@@ -582,21 +610,27 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
     def metadata_for_field(self, key):
         return self.field_metadata[key]
 
-    def clear_dirtied(self, book_ids):
+    def clear_dirtied(self, book_id, sequence):
         '''
         Clear the dirtied indicator for the books. This is used when fetching
         metadata, creating an OPF, and writing a file are separated into steps.
         The last step is clearing the indicator
         '''
-        for book_id in book_ids:
+        with self.dirtied_lock:
-            self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?',
+            dc_sequence = self.dirtied_cache.get(book_id, None)
-                    (book_id,))
+#            print 'clear_dirty: check book', book_id, dc_sequence
-            # if a later exception prevents the commit, then the dirtied
+            if dc_sequence is None or sequence is None or dc_sequence == sequence:
-            # table will still have the book. No big deal, because the OPF
+#                print 'needs to be cleaned'
-            # is there and correct. We will simply do it again on next
+                self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?',
-            # start
+                        (book_id,))
-            self.dirtied_cache.discard(book_id)
+                self.conn.commit()
-        self.conn.commit()
+                try:
+                    del self.dirtied_cache[book_id]
+                except:
+                    pass
+            elif dc_sequence is not None:
+#                print 'book needs to be done again'
+                pass
 
     def dump_metadata(self, book_ids=None, remove_from_dirtied=True,
             commit=True):
@@ -609,38 +643,59 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         for book_id in book_ids:
             if not self.data.has_id(book_id):
                 continue
-            path, mi = self.get_metadata_for_dump(book_id,
+            path, mi, sequence = self.get_metadata_for_dump(book_id)
-                                        remove_from_dirtied=remove_from_dirtied)
             if path is None:
                 continue
             try:
                 raw = metadata_to_opf(mi)
                 with lopen(path, 'wb') as f:
                     f.write(raw)
+                if remove_from_dirtied:
+                    self.clear_dirtied(book_id, sequence)
             except:
-                # Something went wrong. Put the book back on the dirty list
+                pass
-                self.dirtied([book_id])
         if commit:
             self.conn.commit()
 
     def dirtied(self, book_ids, commit=True):
-        for book in frozenset(book_ids) - self.dirtied_cache:
+        changed = False
-            try:
+        for book in book_ids:
-                self.conn.execute(
+            with self.dirtied_lock:
-                    'INSERT INTO metadata_dirtied (book) VALUES (?)',
+#                print 'dirtied: check id', book
-                        (book,))
+                if book in self.dirtied_cache:
-                self.dirtied_queue.put(book)
+                    self.dirtied_cache[book] = self.dirtied_sequence
-            except IntegrityError:
+                    self.dirtied_sequence += 1
-                # Already in table
+                    continue
-                pass
+#                print 'book not already dirty'
-            # If the commit doesn't happen, then our cache will be wrong. This
+                try:
-            # could lead to a problem because we won't put the book back into
+                    self.conn.execute(
-            # the dirtied table. We deal with this by writing the dirty cache
+                        'INSERT INTO metadata_dirtied (book) VALUES (?)',
-            # back to the table on GUI exit. Not perfect, but probably OK
+                            (book,))
-            self.dirtied_cache.add(book)
+                    changed = True
-        if commit:
+                except IntegrityError:
+                    # Already in table
+                    pass
+                self.dirtied_cache[book] = self.dirtied_sequence
+                self.dirtied_sequence += 1
+        # If the commit doesn't happen, then the DB table will be wrong. This
+        # could lead to a problem because on restart, we won't put the book back
+        # into the dirtied_cache. We deal with this by writing the dirtied_cache
+        # back to the table on GUI exit. Not perfect, but probably OK
+        if commit and changed:
             self.conn.commit()
 
+    def get_a_dirtied_book(self):
+        with self.dirtied_lock:
+            l = len(self.dirtied_cache)
+            if l > 0:
+                # The random stuff is here to prevent a single book from
+                # blocking progress if its metadata cannot be written for some
+                # reason.
+                id_ = self.dirtied_cache.keys()[random.randint(0, l-1)]
+                sequence = self.dirtied_cache[id_]
+                return (id_, sequence)
+            return (None, None)
+
     def dirty_queue_length(self):
         return len(self.dirtied_cache)
 
@@ -653,12 +708,19 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         is no problem with setting a dirty indication for a book that isn't in
         fact dirty. Just wastes a few cycles.
         '''
-        book_ids = list(self.dirtied_cache)
+        with self.dirtied_lock:
-        self.dirtied_cache = set()
+            book_ids = list(self.dirtied_cache.keys())
-        self.dirtied(book_ids)
+            self.dirtied_cache = {}
+            self.dirtied(book_ids)
 
-    def get_metadata_for_dump(self, idx, remove_from_dirtied=True):
+    def get_metadata_for_dump(self, idx):
         path, mi = (None, None)
+        # get the current sequence number for this book to pass back to the
+        # backup thread. This will avoid double calls in the case where the
+        # thread has not done the work between the put and the get_metadata
+        with self.dirtied_lock:
+            sequence = self.dirtied_cache.get(idx, None)
+#        print 'get_md_for_dump', idx, sequence
         try:
             # While a book is being created, the path is empty. Don't bother to
             # try to write the opf, because it will go to the wrong folder.
@@ -673,16 +735,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             # This almost certainly means that the book has been deleted while
             # the backup operation sat in the queue.
             pass
-
+        return (path, mi, sequence)
-        try:
-            # clear the dirtied indicator. The user must put it back if
-            # something goes wrong with writing the OPF
-            if remove_from_dirtied:
-                self.clear_dirtied([idx])
-        except:
-            # No real problem. We will just do it again.
-            pass
-        return (path, mi)
 
     def get_metadata(self, idx, index_is_id=False, get_cover=False):
         '''
@@ -1376,10 +1429,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
     def tags_older_than(self, tag, delta):
         tag = tag.lower().strip()
         now = nowf()
+        tindex = self.FIELD_MAP['timestamp']
+        gindex = self.FIELD_MAP['tags']
         for r in self.data._data:
             if r is not None:
-                if (now - r[self.FIELD_MAP['timestamp']]) > delta:
+                if (now - r[tindex]) > delta:
-                    tags = r[self.FIELD_MAP['tags']]
+                    tags = r[gindex]
                     if tags and tag in [x.strip() for x in
                             tags.lower().split(',')]:
                         yield r[self.FIELD_MAP['id']]

src/calibre/library/field_metadata.py

@@ -474,6 +474,18 @@ class FieldMetadata(dict):
         for key in list(self._tb_cats.keys()):
             val = self._tb_cats[key]
             if val['is_category'] and val['kind'] in ('user', 'search'):
+                for k in self._tb_cats[key]['search_terms']:
+                    if k in self._search_term_map:
+                        del self._search_term_map[k]
+                del self._tb_cats[key]
+
+    def remove_user_categories(self):
+        for key in list(self._tb_cats.keys()):
+            val = self._tb_cats[key]
+            if val['is_category'] and val['kind']  == 'user':
+                for k in self._tb_cats[key]['search_terms']:
+                    if k in self._search_term_map:
+                        del self._search_term_map[k]
                 del self._tb_cats[key]
 
     def cc_series_index_column_for(self, key):
@@ -482,11 +494,15 @@ class FieldMetadata(dict):
     def add_user_category(self, label, name):
         if label in self._tb_cats:
             raise ValueError('Duplicate user field [%s]'%(label))
-        self._tb_cats[label] = {'table':None,        'column':None,
+        st = [label]
-                                'datatype':None,     'is_multiple':None,
+        if icu_lower(label) != label:
-                                'kind':'user',       'name':name,
+            st.append(icu_lower(label))
-                                'search_terms':[],    'is_custom':False,
+        self._tb_cats[label] = {'table':None,          'column':None,
+                                'datatype':None,       'is_multiple':None,
+                                'kind':'user',         'name':name,
+                                'search_terms':st,     'is_custom':False,
                                 'is_category':True}
+        self._add_search_terms_to_map(label, st)
 
     def add_search_category(self, label, name):
         if label in self._tb_cats:
@@ -518,7 +534,6 @@ class FieldMetadata(dict):
     def _add_search_terms_to_map(self, key, terms):
         if terms is not None:
             for t in terms:
-                t = t.lower()
                 if t in self._search_term_map:
                     raise ValueError('Attempt to add duplicate search term "%s"'%t)
                 self._search_term_map[t] = key

src/calibre/library/prefs.py

@@ -9,6 +9,7 @@ import json
 
 from calibre.constants import preferred_encoding
 from calibre.utils.config import to_json, from_json
+from calibre import prints
 
 class DBPrefs(dict):
 
@@ -17,7 +18,11 @@ class DBPrefs(dict):
         self.db = db
         self.defaults = {}
         for key, val in self.db.conn.get('SELECT key,val FROM preferences'):
-            val = self.raw_to_object(val)
+            try:
+                val = self.raw_to_object(val)
+            except:
+                prints('Failed to read value for:', key, 'from db')
+                continue
             dict.__setitem__(self, key, val)
 
     def raw_to_object(self, raw):

src/calibre/manual/faq.rst

@@ -310,7 +310,9 @@ What formats does |app| read metadata from?
 
 Where are the book files stored?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-When you first run |app|, it will ask you for a folder in which to store your books. Whenever you add a book to |app|, it will copy the book into that folder. Books in the folder are nicely arranged into sub-folders by Author and Title. Metadata about the books is stored in the file ``metadata.db`` (which is a sqlite database).
+When you first run |app|, it will ask you for a folder in which to store your books. Whenever you add a book to |app|, it will copy the book into that folder. Books in the folder are nicely arranged into sub-folders by Author and Title. Note that the contents of this folder are automatically managed by |app|, **do not** add any files/folders manually to this folder, as they may be automatically deleted. If you want to add a file associated to a particular book, use the top right area of :guilabel:`Edit metadata` dialog to do so. Then, |app| will automatically put that file into the correct folder and move it around when the title/author changes.
+
+Metadata about the books is stored in the file ``metadata.db`` at the top level of the library folder This file is is a sqlite database. When backing up your library make sure you copy the entire folder and all its sub-folders. 
 
 Why doesn't |app| let me store books in my own directory structure?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

src/calibre/utils/localization.py

@@ -105,6 +105,7 @@ _extra_lang_codes = {
         'en_TH' : _('English (Thailand)'),
         'en_CY' : _('English (Cyprus)'),
         'en_PK' : _('English (Pakistan)'),
+        'en_HR' : _('English (Croatia)'),
         'en_IL' : _('English (Israel)'),
         'en_SG' : _('English (Singapore)'),
         'en_YE' : _('English (Yemen)'),

src/calibre/utils/search_query_parser.py

@@ -119,6 +119,12 @@ class SearchQueryParser(object):
         return failed
 
     def __init__(self, locations, test=False, optimize=False):
+        self.sqp_initialize(locations, test=test, optimize=optimize)
+
+    def sqp_change_locations(self, locations):
+        self.sqp_initialize(locations, optimize=self.optimize)
+
+    def sqp_initialize(self, locations, test=False, optimize=False):
         self._tests_failed = False
         self.optimize = optimize
         # Define a token