[Sync] Sync with trunk. Revision: 7887

Changelog.yaml

@@ -4,12 +4,137 @@
 # for important features/bug fixes.
 # Also, each release can have new and improved recipes.
 
+#- version: ?.?.?
+#  date: 2011-??-??
+#
+#  new features:
+#    - title: 
+#  
+#  bug fixes:
+#    - title: 
+#
+#  improved recipes:
+#    - 
+#
+#  new recipes:
+#    - title: 
+
+
+- version: 0.7.43
+  date: 2011-01-28
+
+  new features:
+    - title: "Ask for confirmation when stopping running jobs"
+      tickets: [3101]
+
+    - title: "Combine the database integrity check and library check into a single menu item. Also nicer implementation of the db integrity check."
+
+    - title: "BiBTeX Catalog: Add option to include file paths in the catalog."
+      tickets: [8589]
+
+    - title: "Create 'generic' output profiles and generic devices in the welcome wizard"
+
+    - title: "Bulk metadata edit: Custom column widgets all have an apply checkbox next to them."
+
+    - title: "Only use LibraryThing to download metadata if the user provides a library thing username and password. Since LT doesn't like web scraping"
+
+    - title: "Allow renaming of user categories in the manage categories dialog. Also allow searching for books in a category from the Tag Browser by right clicking ona a category"
+
+    - title: "Folder device plugin: Add option to disable the use of sub folders"
+
+    - title: "Allow saving/loading of search and replace expressions in the bulk metadata edit dialog."
+
+    - title: "Remeber previously used regular expression in the add books preferences dialog"
+
+    - title: "Search and replace wizard: Cache the previously used input document."
+
+    - title: "Pressing Esc clears the current search in the main book list"
+
+    - title: "Preselct right formats when using send specific format to device"
+      tickets: [7834]
+
+    - title: "Regex wizard gets find next and previous match buttons"
+      tickets: [4486]
+  
+  bug fixes:
+    - title: "Do not allow customization of user interface plugins until calibre is restarted"
+      tickets: [8621]
+
+    - title: "EPUB Output: When using preserve cover aspect ratio, use the actual image sizes in the SVG template as otherwise ADE doesn't fully preserve the aspect ratio"
+
+    - title: "Fix completion on a word with a trailing space causing the first letter to be duplicated in the edit metadata dialog"
+
+    - title: "PML Input: PML x and Xn tags don't indent properly in TOC. Also handle invalid T markup and retain soft scene breaks"
+      tickets: [6194, 8565]
+
+    - title: "TXT Input: Retain whitespace at the beginning of lines. Don't preserve spaces in heuristic processing. Detect and retain soft scene breaks."
+
+    - title: "Fix Adding empty book - cover browser doesn't update"
+      tickets: [8557]
+
+    - title: "When generating author sort string ignore trailing Inc."
+      tickets: [8539]
+
+    - title: "When converting HTML/ZIP files do not leave temporary files that are only deleted on application shutdown."
+      tickets: [8597]
+
+    - title: "Don't crash if the prefs stored in the db are corrupted"
+
+    - title: "Catalog generation: Do not use inline-block CSS as apparently Adobe Digital Editions cannot handle it."
+      tickets: [8566]
+
+    - title: "Fix extra spaces being inserted into TOC title when reading TOC from OPF guide element."
+      tickets: [8569]
+
+    - title: "Remember window size for bulk metadata edit and catalog generation dialogs"
+      tickets: [8525]
+
+    - title: "Heuristics, italicize common cases: Enhance pattern matching to match punctuation after pattern."
+
+    - title: "Fix regression in converting HTML files that have ASCII-encoded non unicode characters inside their <style> tags. Apparently Word generates these."
+      tickets: [8494]
+
+  improved recipes:
+    - Calgary Herald
+    - The Economist
+    - New Yorker
+    - Heise
+    - HNA
+    - ZDNet
+    - NRC Handelsblad
+
+  new recipes:
+    - title: "SPIN Magazine"
+      author: Quistopher
+
+    - title: "Caps n Babes"
+      author: skyhawker
+
+    - title: "Leduc"
+      author: Brian Hahn
+
+    - title: "David Bravo's Blog, La Nueva Espana, 20 Minutos and La Tribuna de Talavera"
+      author: Luis Hernandez
+
+    - title: "Sinfest"
+      author: nadid
+
+    - title: "Various Czech news sources"
+      author: FunThomas
+
+    - title: "tportal.h"
+      author: Darko Miletic
+
+    - title: "Everett Herald"
+      author: "77jag5"
+
+    - title: "Roger Ebert"
+      author: Shane Erstad
+
 - version: 0.7.42
   date: 2011-01-21
 
   new features:
-    - title: "0.7.42 is a re-release of 0.7.41, because conversion to MOBI was broken in 0.7.41"
-
     - title: "Conversions: Replace the remove header/footer options with a more geenric search replace option, that allows you to not only remove but also replace text"
 
     - title: "Conversion: The preprocess html option has now become a new 'Heuristic Processing' option which allows you to control exactly which heuristics are used"

resources/calibre-portable.bat

@@ -6,25 +6,37 @@ REM  - Calibre Library Files
 REM  - Calibre Config Files
 REM  - Calibre Metadata database
 REM  - Calibre Source files
+REM  - Calibre Temp Files
 REM By setting the paths correctly it can be used to run:
 REM  - A "portable calibre" off a USB stick.
 REM  - A network installation with local metadata database
 REM    (for performance) and books stored on a network share 
+REM  - A local installation using customised settings
 REM
-REM If trying to run off a USB stick then the following 
+REM If trying to run off a USB stick then the folder structure
-REM folder structure is recommended:
+REM shown below is recommended (relative to the location of 
+REM this batch file).  This can structure can also be used
+REM when running of a local hard disk if you want to get the
+REM level of control this batch file provides.
 REM  - Calibre2			Location of program files
 REM  - CalibreConfig		Location of Configuration files
 REM  - CalibreLibrary		Location of Books and metadata
+REM  - CalibreSource 		Location of Calibre Source files (Optional)
+REM
+REM This batch file is designed so that if you create the recommended
+REM folder structure then it can be used 'as is' without modification.
 
 
 REM -------------------------------------
 REM Set up Calibre Config folder
+REM
+REM This is where user specific settings
+REM are stored.
 REM -------------------------------------
 
 IF EXIST CalibreConfig (
 	SET CALIBRE_CONFIG_DIRECTORY=%cd%\CalibreConfig
-	ECHO CONFIG=%cd%\CalibreConfig
+	ECHO CONFIG FILES:    %cd%\CalibreConfig
 )
 
 
@@ -35,21 +47,18 @@ REM Location where Book files are located
 REM Either set explicit path, or if running from a USB stick
 REM a relative path can be used to avoid need to know the
 REM drive letter of the USB stick.
-
+REM
 REM Comment out any of the following that are not to be used
+REM (although leaving them in does not really matter)
 REM --------------------------------------------------------------
 
 IF EXIST U:\eBooks\CalibreLibrary (
 	SET CALIBRE_LIBRARY_DIRECTORY=U:\eBOOKS\CalibreLibrary
-	ECHO LIBRARY=U:\eBOOKS\CalibreLibrary
+	ECHO LIBRARY FILES:   U:\eBOOKS\CalibreLibrary
 )
 IF EXIST CalibreLibrary (
 	SET CALIBRE_LIBRARY_DIRECTORY=%cd%\CalibreLibrary
-	ECHO LIBRARY=%cd%\CalibreLibrary
+	ECHO LIBRARY FILES:   %cd%\CalibreLibrary
-)
-IF EXIST CalibreBooks (
-	SET CALIBRE_LIBRARY_DIRECTORY=%cd%\CalibreBooks
-	ECHO LIBRARY=%cd%\CalibreBooks
 )
 
 
@@ -60,7 +69,7 @@ REM Location where the metadata.db file is located.  If not set
 REM the same location as Books files will be assumed.  This.
 REM options is used to get better performance when the Library is
 REM on a (slow) network drive.  Putting the metadata.db file 
-REM locally makes gives a big performance improvement.
+REM locally then makes gives a big performance improvement.
 REM
 REM NOTE.  If you use this option, then the ability to switch
 REM        libraries within Calibre will be disabled.  Therefore
@@ -68,19 +77,10 @@ REM        you do not want to set it if the metadata.db file
 REM        is at the same location as the book files.
 REM --------------------------------------------------------------
 
-IF EXIST CalibreBooks (
+IF EXIST %cd%\CalibreMetadata\metadata.db (
-	IF NOT "%CALIBRE_LIBRARY_DIRECTORY%" == "%cd%\CalibreBooks" (
-		SET SET CALIBRE_OVERRIDE_DATABASE_PATH=%cd%\CalibreBooks\metadata.db
-		ECHO DATABASE=%cd%\CalibreBooks\metadata.db
-		ECHO '
-		ECHO ***CAUTION*** Library Switching will be disabled 
-		ECHO '
-	)
-)
-IF EXIST CalibreMetadata (
 	IF NOT "%CALIBRE_LIBRARY_DIRECTORY%" == "%cd%\CalibreMetadata" (
 		SET CALIBRE_OVERRIDE_DATABASE_PATH=%cd%\CalibreMetadata\metadata.db
-		ECHO DATABASE=%cd%\CalibreMetadata\metadata.db
+		ECHO DATABASE:        %cd%\CalibreMetadata\metadata.db
 		ECHO '
 		ECHO ***CAUTION*** Library Switching will be disabled 
 		ECHO '
@@ -96,37 +96,60 @@ REM When running from source the GUI will have a '*' after the version.
 REM number that is displayed at the bottom of the Calibre main screen.
 REM --------------------------------------------------------------
 
-IF EXIST Calibre\src (
+IF EXIST CalibreSource\src (
-	SET CALIBRE_DEVELOP_FROM=%cd%\Calibre\src
+	SET CALIBRE_DEVELOP_FROM=%cd%\CalibreSource\src
-	ECHO SOURCE=%cd%\Calibre\src
+	ECHO SOURCE FILES:    %cd%\CalibreSource\src
-)
-IF EXIST D:\Calibre\Calibre\src (
-	SET CALIBRE_DEVELOP_FROM=D:\Calibre\Calibre\src
-	ECHO SOURCE=D:\Calibre\Calibre\src
 )
 
+
 REM --------------------------------------------------------------
 REM Specify Location of calibre binaries (optional)
 REM
 REM To avoid needing Calibre to be set in the search path, ensure
 REM that Calibre Program Files is current directory when starting.
 REM The following test falls back to using search path .
-REM This folder can be populated by cpying the Calibre2 folder from
+REM This folder can be populated by copying the Calibre2 folder from
-REM an existing isntallation or by isntalling direct to here.
+REM an existing installation or by installing direct to here.
 REM --------------------------------------------------------------
 
-IF EXIST Calibre2 (
+IF EXIST %cd%\Calibre2 (
-	Calibre2 CD Calibre2
+	CD %cd%\Calibre2
-	ECHO PROGRAMS=%cd%
+	ECHO PROGRAM FILES:   %cd%
 )
 
+
+REM --------------------------------------------------------------
+REM Location of Calibre Temporary files  (optional)
+REM
+REM Calibre creates a lot of temproary files while running
+REM In theory these are removed when Calibre finishes, but
+REM in practise files can be left behind (particularily if
+REM any errors occur.  Using this option allows some
+REM explicit clean-up of these files.
+REM If not set Calibre uses the normal system TEMP location
+REM --------------------------------------------------------------
+
+SET CALIBRE_TEMP_DIR=%TEMP%\CALIBRE_TEMP
+ECHO TEMPORARY FILES: %CALIBRE_TEMP_DIR%
+
+IF NOT "%CALIBRE_TEMP_DIR%" == "" (
+	IF EXIST "%CALIBRE_TEMP_DIR%" RMDIR /s /q "%CALIBRE_TEMP_DIR%"
+	MKDIR "%CALIBRE_TEMP_DIR%"
+	REM set the following for any components that do
+	REM not obey the CALIBRE_TEMP_DIR setting
+	SET TMP=%CALIBRE_TEMP_DIR%
+	SET TEMP=%CALIBRE_TEMP_DIR%
+)
+
+
 REM ----------------------------------------------------------
 REM  The following gives a chance to check the settings before
 REM  starting Calibre.  It can be commented out if not wanted.
 REM ----------------------------------------------------------
 
-echo "Press CTRL-C if you do not want to continue"
+ECHO '
-pause
+ECHO "Press CTRL-C if you do not want to continue"
+PAUSE
 
 
 REM --------------------------------------------------------
@@ -141,5 +164,7 @@ REM If used without /WAIT opotion launches Calibre and contines batch file.
 REM Use with /WAIT to wait until Calibre completes to run a task on exit
 REM --------------------------------------------------------
 
-echo "Starting up Calibre"
+ECHO "Starting up Calibre"
+ECHO OFF
+ECHO %cd%
 START /belownormal Calibre --with-library "%CALIBRE_LIBRARY_DIRECTORY%"

resources/catalog/stylesheet.css

@@ -62,6 +62,18 @@ div.description {
 	text-indent: 1em;
 	}
 
+/*
+* 	Attempt to minimize widows and orphans by logically grouping chunks
+* 	Recommend enabling for iPad
+*   Some reports of problems with Sony ereaders, presumably ADE engines
+*/
+/*
+div.logical_group {
+	display:inline-block;
+	width:100%;
+	}
+*/
+
 p.date_index {
 	font-size:x-large;
 	text-align:center;

resources/recipes/20_minutos.recipe

@@ -0,0 +1,70 @@
+# -*- coding: utf-8
+__license__   = 'GPL v3'
+__author__    = 'Luis Hernandez'
+__copyright__ = 'Luis Hernandez<tolyluis@gmail.com>'
+description   = 'Periódico gratuito en español - v0.8 - 27 Jan 2011'
+
+'''
+www.20minutos.es
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1294946868(BasicNewsRecipe):
+
+    title          = u'20 Minutos'
+    publisher      = u'Grupo 20 Minutos'
+
+    __author__            = 'Luis Hernández'
+    description           = 'Periódico gratuito en español'
+    cover_url     = 'http://estaticos.20minutos.es/mmedia/especiales/corporativo/css/img/logotipos_grupo20minutos.gif'
+
+    oldest_article = 5
+    max_articles_per_feed = 100
+
+    remove_javascript = True
+    no_stylesheets        = True
+    use_embedded_content  = False
+
+    encoding              = 'ISO-8859-1'
+    language              = 'es'
+    timefmt        = '[%a, %d %b, %Y]'
+
+    keep_only_tags     = [
+                                   dict(name='div', attrs={'id':['content','vinetas',]})
+                                  ,dict(name='div', attrs={'class':['boxed','description','lead','article-content','cuerpo estirar']})
+                                  ,dict(name='span', attrs={'class':['photo-bar']})
+                                  ,dict(name='ul', attrs={'class':['article-author']})
+                                ]
+
+    remove_tags_before = dict(name='ul' , attrs={'class':['servicios-sub']})
+    remove_tags_after  = dict(name='div' , attrs={'class':['related-news','col']})
+
+    remove_tags = [
+                     dict(name='ol', attrs={'class':['navigation',]})
+                    ,dict(name='span', attrs={'class':['action']})
+                    ,dict(name='div', attrs={'class':['twitter comments-list hidden','related-news','col','photo-gallery','calendario','article-comment','postto estirar','otras_vinetas estirar','kment','user-actions']})
+                    ,dict(name='div', attrs={'id':['twitter-destacados','eco-tabs','inner','vineta_calendario','vinetistas clearfix','otras_vinetas estirar','MIN1','main','SUP1','INT']})
+                    ,dict(name='ul', attrs={'class':['article-user-actions','stripped-list']})
+                    ,dict(name='ul', attrs={'id':['site-links']})
+                    ,dict(name='li', attrs={'class':['puntuacion','enviar','compartir']})
+                       ]
+
+    feeds = [
+              (u'Portada'              , u'http://www.20minutos.es/rss/')
+             ,(u'Nacional'             , u'http://www.20minutos.es/rss/nacional/')
+             ,(u'Internacional'       , u'http://www.20minutos.es/rss/internacional/')
+             ,(u'Economia'           , u'http://www.20minutos.es/rss/economia/')
+             ,(u'Deportes'            , u'http://www.20minutos.es/rss/deportes/')
+             ,(u'Tecnologia'          , u'http://www.20minutos.es/rss/tecnologia/')
+             ,(u'Gente - TV'         , u'http://www.20minutos.es/rss/gente-television/')
+             ,(u'Motor'                 , u'http://www.20minutos.es/rss/motor/')
+             ,(u'Salud'                 , u'http://www.20minutos.es/rss/belleza-y-salud/')
+             ,(u'Viajes'                , u'http://www.20minutos.es/rss/viajes/')
+             ,(u'Vivienda'             , u'http://www.20minutos.es/rss/vivienda/')
+             ,(u'Empleo'              , u'http://www.20minutos.es/rss/empleo/')
+             ,(u'Cine'                  , u'http://www.20minutos.es/rss/cine/')
+             ,(u'Musica'               , u'http://www.20minutos.es/rss/musica/')
+             ,(u'Vinetas'              , u'http://www.20minutos.es/rss/vinetas/')
+             ,(u'Comunidad20'     , u'http://www.20minutos.es/rss/zona20/')
+            ]

resources/recipes/abc.recipe

@@ -0,0 +1,43 @@
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class ABCRecipe(BasicNewsRecipe):
+  title          = u'ABC Linuxu'
+  oldest_article = 5
+  max_articles_per_feed = 3#5
+  __author__ = 'Funthomas'
+  language = 'cs'
+
+  feeds = [
+    #(u'Blogy', u'http://www.abclinuxu.cz/auto/blogDigest.rss'),
+    (u'Články', u'http://www.abclinuxu.cz/auto/abc.rss'),
+    (u'Zprávičky','http://www.abclinuxu.cz/auto/zpravicky.rss')
+  ]
+
+  remove_javascript = True
+  no_stylesheets    = True
+  remove_attributes = ['width','height']
+
+  remove_tags_before = dict(name='h1')
+  remove_tags = [
+    dict(attrs={'class':['meta-vypis','page_tools','cl_perex']}),
+    dict(attrs={'class':['cl_nadpis-link','komix-nav']})
+  ]
+
+  remove_tags_after = [
+    dict(name='div',attrs={'class':['cl_perex','komix-nav']}),
+    dict(attrs={'class':['meta-vypis','page_tools']}),
+    dict(name='',attrs={'':''}),
+  ]
+
+
+  preprocess_regexps = [
+    (re.compile(r'</div>.*<p class="perex">', re.DOTALL),lambda match: '</div><p class="perex">')
+  ]
+  def print_version(self, url):
+    return url + '?varianta=print&noDiz'
+
+  extra_css = '''
+            h1 {font-size:130%; font-weight:bold}
+            h3 {font-size:111%; font-weight:bold}
+        '''

resources/recipes/bbc_es.recipe

@@ -0,0 +1,53 @@
+__license__   = 'GPL v3'
+__author__    = 'Luis Hernandez'
+__copyright__ = 'Luis Hernandez<tolyluis@gmail.com>'
+__version__     = 'v1.0'
+__date__        = '29 January 2011'
+
+'''
+http://www.bbc.co.uk/mundo/
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1294946868(BasicNewsRecipe):
+
+    title          = u'BBC Mundo'
+    publisher      = u'BBC'
+
+    __author__            = 'Luis Hernandez'
+    description           = 'BBC World for spanish readers'
+
+    cover_url     = 'http://1.bp.blogspot.com/_NHiOjk_uZwU/TEYy7IJAdAI/AAAAAAAABP8/coAE-pJ7_5E/s1600/bbcmundo_h.png'
+    oldest_article = 2
+    max_articles_per_feed = 100
+
+    remove_javascript = True
+    no_stylesheets        = True
+    use_embedded_content  = False
+
+    language              = 'es'
+    remove_empty_feeds    = True
+    encoding               = 'UTF-8'
+    timefmt        = '[%a, %d %b, %Y]'
+
+    remove_tags_before = dict(name='div' , attrs={'class':['g-group']})
+    remove_tags_after = dict(name='div' , attrs={'class':[' g-w8']})
+
+    remove_tags = [
+                             dict(name='ul', attrs={'class':['document-tools blq-clearfix','blq-clearfix']})
+                            ,dict(name='div', attrs={'class':['box bx-quote-bubble','socialmedia-links','list li-carousel','list li-plain rolling-news','list li-plain','box bx-livestats','li-tab content','list li-relatedlinks','list li-relatedinternetlinks']})
+                         ]
+
+    feeds = [
+                  (u'Portada'               , u'http://www.bbc.co.uk/mundo/index.xml')
+                 ,(u'Ultimas Noticias'   , u'http://www.bbc.co.uk/mundo/ultimas_noticias/index.xml')
+                 ,(u'Internacional'       , u'http://www.bbc.co.uk/mundo/temas/internacional/index.xml')
+                 ,(u'Economia'            , u'http://www.bbc.co.uk/mundo/temas/economia/index.xml')
+                 ,(u'America Latina'     , u'http://www.bbc.co.uk/mundo/temas/america_latina/index.xml')
+                 ,(u'Ciencia'                , u'http://www.bbc.co.uk/mundo/temas/ciencia/index.xml')
+                 ,(u'Salud'                  , u'http://www.bbc.co.uk/mundo/temas/salud/index.xml')
+                 ,(u'Tecnologia'           , u'http://www.bbc.co.uk/mundo/temas/tecnologia/index.xml')
+                 ,(u'Cultura'                , u'http://www.bbc.co.uk/mundo/temas/cultura/index.xml')
+             ]
+

resources/recipes/calgary_herald.recipe

@@ -44,6 +44,7 @@ class CanWestPaper(BasicNewsRecipe):
 
     language = 'en_CA'
     __author__ = 'Nick Redding'
+    encoding = 'latin1'
     no_stylesheets = True
     timefmt = ' [%b %d]'
     extra_css = '''
@@ -97,6 +98,8 @@ class CanWestPaper(BasicNewsRecipe):
                 atag = h1tag.find('a',href=True)
                 if not atag:
                     continue
+                url = atag['href']
+                if not url.startswith('http:'):
                     url = self.url_prefix+'/news/todays-paper/'+atag['href']
                 #self.log("Section %s" % key)
                 #self.log("url %s" % url)

resources/recipes/capes_n_babes.recipe

@@ -0,0 +1,11 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class CapesnBabesRecipe(BasicNewsRecipe):
+    title          = u'Capes n Babes'
+    language       = 'en'
+    description = 'The Capes n Babes comic Blog'
+    __author__     = 'skyhawker'
+    oldest_article = 31
+    max_articles_per_feed = 100
+    use_embedded_content = True
+    feeds          = [(u'Capes & Babes', u'feed://www.capesnbabes.com/feed/')]

resources/recipes/dailytportal.recipe

@@ -0,0 +1,66 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011, Darko Miletic <darko.miletic at gmail.com>'
+'''
+daily.tportal.hr
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Pagina12(BasicNewsRecipe):
+    title                 = 'Daily tportal.h'
+    __author__            = 'Darko Miletic'
+    description           = 'News from Croatia'
+    publisher             = 'tportal.hr'
+    category              = 'news, politics, Croatia'
+    oldest_article        = 2
+    max_articles_per_feed = 200
+    no_stylesheets        = True
+    encoding              = 'utf-8'
+    use_embedded_content  = False
+    language              = 'en_HR'
+    remove_empty_feeds    = True
+    publication_type      = 'newsportal'
+    extra_css             = """
+                               body{font-family: Verdana,sans-serif }
+                               img{margin-bottom: 0.4em; display:block}
+                               h1,h2{color: #2D648A; font-family: Georgia,serif}
+                               .artAbstract{font-size: 1.2em; font-family: Georgia,serif}
+                            """
+
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
+
+    remove_tags = [
+                     dict(name=['meta','link','embed','object','iframe','base'])
+                    ,dict(name='div', attrs={'class':'artInfo'})
+                  ]
+    remove_attributes=['lang']
+
+    keep_only_tags=dict(attrs={'class':'articleDetails'})
+
+    feeds = [(u'News', u'http://daily.tportal.hr/rss/dailynaslovnicarss.xml')]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll('a'):
+            limg = item.find('img')
+            if item.string is not None:
+               str = item.string
+               item.replaceWith(str)
+            else:
+               if limg:
+                  item.name = 'div'
+                  item.attrs = []
+               else:
+                   str = self.tag_to_string(item)
+                   item.replaceWith(str)
+        for item in soup.findAll('img'):
+            if not item.has_key('alt'):
+               item['alt'] = 'image'
+        return soup
+

resources/recipes/dbb.recipe

@@ -0,0 +1,45 @@
+# -*- coding: utf-8
+
+__license__   = 'GPL v3'
+__author__    = 'Luis Hernandez'
+__copyright__ = 'Luis Hernandez<tolyluis@gmail.com>'
+
+'''
+http://www.filmica.com/david_bravo/
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1294946868(BasicNewsRecipe):
+
+    title             = u'Blog de David Bravo'
+    publisher      = u'Filmica'
+
+    __author__  = 'Luis Hernández'
+    description   = 'blog sobre leyes, p2p y copyright'
+    cover_url     = 'http://www.elpais.es/edigitales/image.php?foto=par/portada/1551.jpg'
+
+    oldest_article = 365
+    max_articles_per_feed = 100
+
+    remove_javascript = True
+    no_stylesheets        = True
+    use_embedded_content  = False
+
+    encoding              = 'ISO-8859-1'
+    language              = 'es'
+    timefmt        = '[%a, %d %b, %Y]'
+
+    keep_only_tags     = [
+                                    dict(name='div', attrs={'class':['blog','date','blogbody','comments-head','comments-body']})
+                                   ,dict(name='span', attrs={'class':['comments-post']})
+                                ]
+
+    remove_tags_before = dict(name='div' , attrs={'id':['bitacoras']})
+    remove_tags_after  = dict(name='div' , attrs={'id':['comments-body']})
+
+    extra_css             = ' p{text-align: justify; font-size: 100%} body{ text-align: left; font-family: serif; font-size: 100% } h2{ font-family: sans-serif; font-size:75%; font-weight: 800; text-align: justify } h3{ font-family: sans-serif; font-size:150%; font-weight: 600; text-align: left } img{margin-bottom: 0.4em} '
+
+
+
+    feeds          = [(u'Blog', u'http://www.filmica.com/david_bravo/index.rdf')]

resources/recipes/economist.recipe

@@ -22,8 +22,11 @@ class Economist(BasicNewsRecipe):
 
     oldest_article = 7.0
     cover_url = 'http://www.economist.com/images/covers/currentcoverus_large.jpg'
-    remove_tags = [dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']),
+    remove_tags = [
-            dict(attrs={'class':['dblClkTrk', 'ec-article-info']})]
+            dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']),
+            dict(attrs={'class':['dblClkTrk', 'ec-article-info']}),
+            {'class': lambda x: x and 'share-links-header' in x},
+    ]
     keep_only_tags = [dict(id='ec-article-body')]
     needs_subscription = False
     no_stylesheets = True

resources/recipes/economist_free.recipe

@@ -16,8 +16,11 @@ class Economist(BasicNewsRecipe):
 
     oldest_article = 7.0
     cover_url = 'http://www.economist.com/images/covers/currentcoverus_large.jpg'
-    remove_tags = [dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']),
+    remove_tags = [
-            dict(attrs={'class':['dblClkTrk', 'ec-article-info']})]
+            dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']),
+            dict(attrs={'class':['dblClkTrk', 'ec-article-info']}),
+            {'class': lambda x: x and 'share-links-header' in x},
+    ]
     keep_only_tags = [dict(id='ec-article-body')]
     no_stylesheets = True
     preprocess_regexps = [(re.compile('</html>.*', re.DOTALL),

resources/recipes/endgadget_ja.recipe

@@ -18,3 +18,6 @@ class EndgadgetJapan(BasicNewsRecipe):
     language = 'ja'
     encoding = 'utf-8'
     feeds          = [(u'engadget', u'http://japanese.engadget.com/rss.xml')]
+
+    remove_tags_before = dict(name="div", attrs={'id':"content_wrap"})
+    remove_tags_after = dict(name='h3', attrs={'id':'addcomments'})

resources/recipes/everett_herald.recipe

@@ -0,0 +1,36 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1295088390(BasicNewsRecipe):
+     title          = u'Everett Herald'
+     language = 'en'
+     __author__ = '77ja65'
+     oldest_article = 4
+     max_articles_per_feed = 50
+     no_stylesheets = True
+     masthead_url          = 'http://heraldnet.com/images/hnet/jQueryComponents/jQueryNavigation/heraldnet_logo.png'
+     extra_css      = '.headline {font-size: x-large;} \n .fact { padding-top: 10pt  }'
+
+     feeds          = [(u'Local News',
+ u'http://heraldnet.com/section/RSS02&mime=xml'),
+         (u'Sports', u'http://heraldnet.com/section/RSS04&mime=xml'),
+         (u'Entertainment',
+ u'http://heraldnet.com/section/RSS07&mime=xml'),
+         (u'Life', u'http://heraldnet.com/section/RSS03&mime=xml'),
+         (u'Breaking News',
+ u'http://heraldnet.com/section/RSS34&mime=xml'),
+         (u'Seahawks', u'http://heraldnet.com/section/RSS22&mime=xml'),
+         (u'HeraldNet', u'http://heraldnet.com/section/RSS01&mime=xml'),
+         (u'Inside Everett',
+ u'http://heraldnet.com/section/RSS26&mime=xml')
+         ]
+
+     def print_version(self, url):
+       return url + "&template=PrinterFriendly"
+
+     extra_css = '''
+                     h1{font-family:Arial,Helvetica,sans-serif; font-
+ weight:bold;font-size:large;}
+                     h2{font-family:Arial,Helvetica,sans-serif; font-
+ weight:normal;font-size:small;}
+                 '''
+

resources/recipes/explosm.recipe

@@ -0,0 +1,54 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+import re
+
+class Explosm(BasicNewsRecipe):
+    title              = u'Explosm Rotated'
+    __author__        = 'Andromeda Rabbit'
+    description      = 'Explosm'
+    language            = 'en'
+    use_embedded_content = False
+    no_stylesheets    = True
+    oldest_article    = 24
+    remove_javascript   = True
+    remove_empty_feeds  = True
+    max_articles_per_feed = 10
+
+    feeds = [
+             (u'Explosm Feed', u'http://feeds.feedburner.com/Explosm')
+             ]
+
+    #match_regexps = [r'http://www.explosm.net/comics/.*']
+
+    keep_only_tags   = [dict(name='img', attrs={'alt':'Cyanide and Happiness, a daily webcomic'})]
+    remove_tags = [dict(name='div'), dict(name='span'), dict(name='table'), dict(name='br'), dict(name='nobr'), dict(name='a'), dict(name='b')]
+
+    extra_css = '''
+                    h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
+                    h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
+                    p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
+                    body{font-family:Helvetica,Arial,sans-serif;font-size:small;}'''
+
+    def get_cover_url(self):
+        return 'http://cdn.shopify.com/s/files/1/0059/1872/products/cyanidetitle_large.jpg?1295846286'
+
+    def parse_feeds(self):
+        feeds = BasicNewsRecipe.parse_feeds(self)
+
+        for curfeed in feeds:
+            delList = []
+            for a,curarticle in enumerate(curfeed.articles):
+                if re.search(r'http://www.explosm.net/comics', curarticle.url) == None:
+                    delList.append(curarticle)
+            if len(delList)>0:
+                for d in delList:
+                    index = curfeed.articles.index(d)
+                    curfeed.articles[index:index+1] = []
+
+        return feeds
+
+    def skip_ad_pages(self, soup):
+        # Skip ad pages served before actual article
+        skip_tag = soup.find(name='img', attrs={'alt':'Cyanide and Happiness, a daily webcomic'})
+        if skip_tag is None:
+            return soup
+        return None

resources/recipes/heise.recipe

@@ -52,6 +52,7 @@ class heiseDe(BasicNewsRecipe):
 		   dict(id='navi_login'),
 		   dict(id='navigation'),
 		   dict(id='breadcrumb'),
+		   dict(id='adblockerwarnung'),
 		   dict(id=''),
 		   dict(id='sitemap'),
 		   dict(id='bannerzone'),
@@ -67,3 +68,4 @@ class heiseDe(BasicNewsRecipe):
 
 
 
+

resources/recipes/hna.recipe

@@ -21,7 +21,7 @@ class hnaDe(BasicNewsRecipe):
     max_articles_per_feed = 40
     no_stylesheets = True
     remove_javascript = True
-    encoding = 'iso-8859-1'
+    encoding = 'utf-8'
 
     remove_tags = [dict(id='topnav'),
 		   dict(id='nav_main'),
@@ -60,3 +60,4 @@ class hnaDe(BasicNewsRecipe):
     feeds =  [ ('hna_soehre', 'http://feeds2.feedburner.com/hna/soehre'),
 	       ('hna_kassel', 'http://feeds2.feedburner.com/hna/kassel') ]
 
+

resources/recipes/ibm_smarter_planet.recipe

@@ -1,17 +1,18 @@
+
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class AdvancedUserRecipe1293122276(BasicNewsRecipe):
-    title          = u'Smarter Planet | Tumblr for eReaders'
+    title          = u'Smarter Planet | Tumblr'
     __author__ = 'Jack Mason'
     author  = 'IBM Global Business Services'
     publisher             = 'IBM'
     language = 'en'
     category              = 'news, technology, IT, internet of things, analytics'
-    oldest_article = 7
+    oldest_article = 14
     max_articles_per_feed = 30
     no_stylesheets = True
     use_embedded_content = False
-    masthead_url          = 'http://30.media.tumblr.com/tumblr_l70dow9UmU1qzs4rbo1_r3_250.jpg'
+    masthead_url          = 'http://www.hellercd.com/wp-content/uploads/2010/09/hero.jpg'
     remove_tags_before = dict(id='item')
     remove_tags_after  = dict(id='item')
     remove_tags = [dict(attrs={'class':['sidebar', 'about', 'footer', 'description,' 'disqus', 'nav', 'notes', 'disqus_thread']}),
@@ -21,4 +22,3 @@ class AdvancedUserRecipe1293122276(BasicNewsRecipe):
 
 
     feeds          = [(u'Smarter Planet Tumblr', u'http://smarterplanet.tumblr.com/mobile/rss')]
-

resources/recipes/idnes.recipe

@@ -0,0 +1,54 @@
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class iHeuteRecipe(BasicNewsRecipe):
+    __author__  = 'FunThomas'
+    title = u'iDnes.cz'
+    publisher = u'MAFRA a.s.'
+    description = 'iDNES.cz Zprávy, Technet, Komiksy a další'
+    oldest_article = 3
+    max_articles_per_feed = 2
+
+    feeds = [
+            (u'Zprávy', u'http://servis.idnes.cz/rss.asp?c=zpravodaj'),
+            (u'Sport', u'http://servis.idnes.cz/rss.asp?c=sport'),
+            (u'Technet', u'http://servis.idnes.cz/rss.asp?c=technet'),
+            (u'Mobil', u'http://servis.idnes.cz/rss.asp?c=mobil'),
+            (u'Ekonomika', u'http://servis.idnes.cz/rss.asp?c=ekonomikah'),
+            #(u'Kultura', u'http://servis.idnes.cz/rss.asp?c=kultura'),
+            (u'Cestování', u'http://servis.idnes.cz/rss.asp?c=iglobe'),
+            #(u'Kavárna', u'http://servis.idnes.cz/rss.asp?r=kavarna'),
+            (u'Komixy', u'http://servis.idnes.cz/rss.asp?c=komiksy')
+            ]
+
+
+    encoding = 'cp1250'
+    language = 'cs'
+    cover_url = 'http://g.idnes.cz/u/loga-n4/idnes.gif'
+    remove_javascript = True
+    no_stylesheets = True
+
+    remove_attributes = ['width','height']
+    remove_tags = [dict(name='div',   attrs={'id':['zooming']}),
+                   dict(name='div',   attrs={'class':['related','mapa-wrapper']}),
+                   dict(name='table', attrs={'id':['opener-img','portal']}),
+                   dict(name='table', attrs={'class':['video-16ku9']})]
+    remove_tags_after  = [dict(name='div',attrs={'id':['related','related2']})]
+
+    keep_only_tags = [dict(name='div', attrs={'class':['art-full adwords-text','dil-day']})
+                      ,dict(name='table',attrs={'class':['kemel-box']})]
+
+    def print_version(self, url):
+        print_url = url
+        split_url = url.split("?")
+        if (split_url[0].rfind('dilbert.asp')  != -1):      #dilbert komix
+            print_url = print_url.replace('.htm','.gif&tisk=1')
+            print_url = print_url.replace('.asp','.aspx')
+        elif (split_url[0].rfind('kemel.asp')  == -1):      #not Kemel komix
+            print_url = 'http://zpravy.idnes.cz/tiskni.asp?' +  split_url[1]
+        #kemel                  kemel print page doesn't work
+        return print_url
+
+    extra_css = '''
+                  h1 {font-size:125%; font-weight:bold}
+                  h3 {font-size:110%; font-weight:bold}
+                '''

resources/recipes/la_nueva.recipe

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+__license__   = 'GPL v3'
+__author__    = 'Luis Hernandez'
+__copyright__ = 'Luis Hernandez<tolyluis@gmail.com>'
+description   = 'Diario independiente de Asturias - v1.0 - 27 Jan 2011'
+
+'''
+www.lne.es
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1294946868(BasicNewsRecipe):
+
+    title          = u'La Nueva España'
+    publisher      = u'Editorial Prensa Iberica'
+
+    __author__            = 'Luis Hernandez'
+    description           = 'Diario independiente de Asturias'
+    cover_url     = 'http://estaticos00.lne.es//elementosWeb/mediaweb/images/iconos/logo2.jpg'
+
+    oldest_article = 3
+    max_articles_per_feed = 100
+
+    remove_javascript = True
+    no_stylesheets        = True
+    use_embedded_content  = False
+
+    encoding              = 'ISO-8859-1'
+    language              = 'es'
+    timefmt        = '[%a, %d %b, %Y]'
+
+    keep_only_tags = [
+                                dict(name='div', attrs={'class':['noticia_titular','subtitulo','noticiadd2','noticia_texto']})
+                               ,dict(name='div', attrs={'id':['noticia_texto']})
+                             ]
+
+    extra_css             = ' p{text-align: justify; font-size: 100%} body{ text-align: left; font-family: serif; font-size: 100% } h1{ font-family: sans-serif; font-size:150%; font-weight: 600; text-align: justify; } h2{ font-family: sans-serif; font-size:120%; font-weight: 500; text-align: justify } '
+
+
+    remove_tags_before = dict(name='div' , attrs={'class':['contenedor']})
+    remove_tags_after = dict(name='div' , attrs={'class':['fin_noticia']})
+
+    remove_tags = [
+                             dict(name='div', attrs={'class':['epigrafe','antetitulo','bloqueclear','bloqueclear_video','cuadro_multimedia','cintillo2','editor_documentos','noticiadd','noticiadd3','noticiainterior','fin_noticia']})
+                            ,dict(name='div', attrs={'id':['evotos']})
+                         ]
+
+    feeds = [
+                   (u'Al minuto'            , u'http://www.lne.es/elementosInt/rss/AlMinuto')
+                  ,(u'General'              , u'http://www.lne.es/elementosInt/rss/55')
+                  ,(u'Nacional'             , u'http://www.lne.es/elementosInt/rss/43')
+                  ,(u'Internacional'      , u'http://www.lne.es/elementosInt/rss/44')
+                  ,(u'Economia'           , u'http://www.lne.es/elementosInt/rss/45')
+                  ,(u'Deportes'            , u'http://www.lne.es/elementosInt/rss/47')
+                  ,(u'Campeones'        , u'http://www.lne.es/elementosInt/rss/65')
+                  ,(u'Sociedad'            , u'http://www.lne.es/elementosInt/rss/46')
+                  ,(u'Sucesos'             , u'http://www.lne.es/elementosInt/rss/48')
+                  ,(u'Galeria'               , u'http://www.lne.es/elementosInt/rss/51')
+                  ,(u'Cultura'               , u'http://www.lne.es/elementosInt/rss/66')
+                  ,(u'Motor'                 , u'http://www.lne.es/elementosInt/rss/62')
+                  ,(u'Opinion'              , u'http://www.lne.es/elementosInt/rss/52')
+                  ,(u'Asturias'              , u'http://www.lne.es/elementosInt/rss/42')
+                  ,(u'Oviedo'               , u'http://www.lne.es/elementosInt/rss/31')
+                  ,(u'Gijon'                  , u'http://www.lne.es/elementosInt/rss/35')
+                  ,(u'Aviles'                 , u'http://www.lne.es/elementosInt/rss/36')
+                  ,(u'Nalon'                 , u'http://www.lne.es/elementosInt/rss/37')
+                  ,(u'Cuencas'             , u'http://www.lne.es/elementosInt/rss/38')
+                  ,(u'Caudal'               , u'http://www.lne.es/elementosInt/rss/39')
+                  ,(u'Oriente'              , u'http://www.lne.es/elementosInt/rss/40')
+                  ,(u'Occidente'          , u'http://www.lne.es/elementosInt/rss/41')
+                  ,(u'Mar y Campo'     , u'http://www.lne.es/elementosInt/rss/63')
+                  ,(u'Ultima'               , u'http://www.lne.es/elementosInt/rss/50')
+]

resources/recipes/la_tribuna.recipe

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+__license__   = 'GPL v3'
+__author__    = 'Luis Hernandez'
+__copyright__ = 'Luis Hernandez<tolyluis@gmail.com>'
+description   = 'Diario local de Talavera de la Reina - v1.2 - 27 Jan 2011'
+
+'''
+http://www.latribunadetalavera.es/
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1294946868(BasicNewsRecipe):
+
+    title             = u'La Tribuna de Talavera'
+    publisher      = u'Grupo PROMECAL'
+
+    __author__  = 'Luis Hernández'
+    description   = 'Diario local de Talavera de la Reina'
+    cover_url     = 'http://www.latribunadetalavera.es/entorno/mancheta.gif'
+
+    oldest_article = 5
+    max_articles_per_feed = 50
+
+    remove_javascript = True
+    no_stylesheets        = True
+    use_embedded_content  = False
+
+    encoding              = 'utf-8'
+    language              = 'es'
+    timefmt        = '[%a, %d %b, %Y]'
+
+    keep_only_tags     = [
+                                   dict(name='div', attrs={'id':['articulo']})
+                                  ,dict(name='div', attrs={'class':['foto']})
+                                  ,dict(name='p', attrs={'id':['texto']})
+                                ]
+
+    remove_tags_before = dict(name='div' , attrs={'class':['comparte']})
+    remove_tags_after  = dict(name='div' , attrs={'id':['relacionadas']})
+
+    extra_css             = ' p{text-align: justify; font-size: 100%} body{ text-align: left; font-family: serif; font-size: 100% } h1{ font-family: sans-serif; font-size:150%; font-weight: 700; text-align: justify; } h2{ font-family: sans-serif; font-size:120%; font-weight: 600; text-align: justify } h3{ font-family: sans-serif; font-size:60%; font-weight: 600; text-align: left } h4{ font-family: sans-serif; font-size:80%; font-weight: 600; text-align: left } h5{ font-family: sans-serif; font-size:70%; font-weight: 600; text-align: left }img{margin-bottom: 0.4em} '
+
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
+    feeds          = [(u'Portada', u'http://www.latribunadetalavera.es/rss.html')]

resources/recipes/ledevoir.recipe

@@ -9,6 +9,8 @@ __description__   = 'Canadian Paper '
 http://www.ledevoir.com/
 '''
 
+import re
+
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class ledevoir(BasicNewsRecipe):
@@ -32,6 +34,8 @@ class ledevoir(BasicNewsRecipe):
     remove_javascript     = True
     no_stylesheets        = True
 
+    preprocess_regexps = [(re.compile(r'(title|alt)=".*?>.*?"', re.DOTALL), lambda m: '')]
+
     keep_only_tags  = [
                         dict(name='div', attrs={'id':'article'}),
                         dict(name='ul', attrs={'id':'ariane'})

resources/recipes/leduc.recipe

@@ -0,0 +1,40 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1292550626(BasicNewsRecipe):
+   title = 'Leduc - Wetaskiwin Pipestone Flyer'
+   __author__ = 'Brian Hahn'
+   description = 'News from Alberta, Canada'
+   oldest_article = 56
+   max_articles_per_feed = 100
+   no_stylesheets = True
+   #delay = 1
+   use_embedded_content = False
+   publisher = 'Pipestone Publishing'
+   category = 'News, Alberta, Canada'
+   language = 'en_CA'
+   encoding = 'iso-8859-1'
+   cover_url = 'http://www.pipestoneflyer.ca/images/calibre-cover.jpg'
+   remove_tags_before = dict(id='ContentPanel')
+   remove_tags_after  = dict(id='ContentPanel')
+   remove_tags = [dict(name='div', attrs={'id':'StoryNav'}),dict(name='div', attrs={'id':'BottomAds'}),dict(name='div', attrs={'id':'MoreStoryLinks'})]
+   extra_css = 'img { margin:5px }'
+   feeds = [
+('Feature', 'http://www.pipestoneflyer.ca/Feature.rss'),
+('Editors Desk', 'http://www.pipestoneflyer.ca/Editor%27s%20Desk.rss'),
+('Letters', 'http://www.pipestoneflyer.ca/Letters.rss'),
+('A Loco Viewpoint', 'http://www.pipestoneflyer.ca/A%20Loco%20Viewpoint.rss'),
+('Lifes Doorway', 'http://www.pipestoneflyer.ca/Life%27s%20Doorway.rss'),
+('From the Otherside', 'http://www.pipestoneflyer.ca/From%20the%20Otherside.rss'),
+('Opinion', 'http://www.pipestoneflyer.ca/Opinion.rss'),
+('Community', 'http://www.pipestoneflyer.ca/Community.rss'),
+('Sports', 'http://www.pipestoneflyer.ca/Sports.rss'),
+('Chambers', 'http://www.pipestoneflyer.ca/Chambers.rss'),
+('Government', 'http://www.pipestoneflyer.ca/Government.rss'),
+('Environment', 'http://www.pipestoneflyer.ca/Environment.rss'),
+('Health', 'http://www.pipestoneflyer.ca/Health.rss'),
+('Funnies', 'http://www.pipestoneflyer.ca/Funnies.rss'),
+('Faith', 'http://www.pipestoneflyer.ca/Faith.rss'),
+('News and Views', 'http://www.pipestoneflyer.ca/News%20and%20Views.rss'),
+('Obituaries', 'http://www.pipestoneflyer.ca/Obituaries.rss'),
+('Police Blotter', 'http://www.pipestoneflyer.ca/Police%20Blotter.rss'),
+]

resources/recipes/new_yorker.recipe

@@ -1,5 +1,5 @@
 __license__   = 'GPL v3'
-__copyright__ = '2008-2010, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2011, Darko Miletic <darko.miletic at gmail.com>'
 '''
 newyorker.com
 '''
@@ -54,10 +54,10 @@ class NewYorker(BasicNewsRecipe):
                         ,dict(attrs={'id':['show-header','show-footer'] })
                      ]
     remove_attributes = ['lang']
-    feeds             = [(u'The New Yorker', u'http://feeds.newyorker.com/services/rss/feeds/everything.xml')]
+    feeds             = [(u'The New Yorker', u'http://www.newyorker.com/services/rss/feeds/everything.xml')]
 
     def print_version(self, url):
-        return url + '?printable=true'
+        return 'http://www.newyorker.com' + url + '?printable=true'
 
     def image_url_processor(self, baseurl, url):
         return url.strip()

resources/recipes/nytimes_sub.recipe

@@ -1,6 +1,5 @@
 #!/usr/bin/env  python
 # -*- coding: utf-8 -*-
-
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 '''
@@ -28,6 +27,10 @@ class NYTimes(BasicNewsRecipe):
     # previous paid versions of the new york times to best sent to the back issues folder on the kindle
     replaceKindleVersion = False
 
+    # download higher resolution images than the small thumbnails typically included in the article
+    # the down side of having large beautiful images is the file size is much larger, on the order of 7MB per paper
+    useHighResImages = True
+
     # includeSections: List of sections to include. If empty, all sections found will be included.
     # Otherwise, only the sections named will be included. For example,
     #
@@ -90,7 +93,6 @@ class NYTimes(BasicNewsRecipe):
                     (u'Sunday Magazine',u'magazine'),
                     (u'Week in Review',u'weekinreview')]
 
-
     if headlinesOnly:
         title='New York Times Headlines'
         description = 'Headlines from the New York Times'
@@ -127,7 +129,7 @@ class NYTimes(BasicNewsRecipe):
 
     earliest_date = date.today() - timedelta(days=oldest_article)
 
-    __author__  = 'GRiker/Kovid Goyal/Nick Redding'
+    __author__  = 'GRiker/Kovid Goyal/Nick Redding/Ben Collier'
     language = 'en'
     requires_version = (0, 7, 5)
 
@@ -149,7 +151,7 @@ class NYTimes(BasicNewsRecipe):
                             'dottedLine',
                             'entry-meta',
                             'entry-response module',
-                            'icon enlargeThis',
+                            #'icon enlargeThis', #removed to provide option for high res images
                             'leftNavTabs',
                             'metaFootnote',
                             'module box nav',
@@ -163,7 +165,23 @@ class NYTimes(BasicNewsRecipe):
                             'entry-tags', #added for DealBook
                             'footer promos clearfix', #added for DealBook
                             'footer links clearfix', #added for DealBook
-                            'inlineImage module', #added for DealBook
+                            'tabsContainer', #added for other blog downloads
+                            'column lastColumn', #added for other blog downloads
+                            'pageHeaderWithLabel', #added for other gadgetwise downloads
+                            'column two', #added for other blog downloads
+                            'column two last', #added for other blog downloads
+                            'column three', #added for other blog downloads
+                            'column three last', #added for other blog downloads
+                            'column four',#added for other blog downloads
+                            'column four last',#added for other blog downloads
+                            'column last', #added for other blog downloads
+                            'timestamp published', #added for other blog downloads
+                            'entry entry-related',
+                            'subNavigation tabContent active', #caucus blog navigation
+                            'columnGroup doubleRule',
+                            'mediaOverlay slideshow',
+                            'headlinesOnly multiline flush',
+                            'wideThumb',
                             re.compile('^subNavigation'),
                             re.compile('^leaderboard'),
                             re.compile('^module'),
@@ -254,7 +272,7 @@ class NYTimes(BasicNewsRecipe):
     def exclude_url(self,url):
         if not url.startswith("http"):
             return True
-        if not url.endswith(".html") and 'dealbook.nytimes.com' not in url: #added for DealBook
+        if not url.endswith(".html") and 'dealbook.nytimes.com' not in url and 'blogs.nytimes.com' not in url: #added for DealBook
             return True
         if 'nytimes.com' not in url:
             return True
@@ -592,19 +610,84 @@ class NYTimes(BasicNewsRecipe):
                     self.log("Skipping article dated %s" % date_str)
                     return None
 
-        kicker_tag = soup.find(attrs={'class':'kicker'})
+        #all articles are from today, no need to print the date on every page
-        if kicker_tag: # remove Op_Ed author head shots
+        try:
-            tagline = self.tag_to_string(kicker_tag)
+            if not self.webEdition:
-            if tagline=='Op-Ed Columnist':
+                date_tag = soup.find(True,attrs={'class': ['dateline','date']})
-                img_div = soup.find('div','inlineImage module')
+                if date_tag:
-                if img_div:
+                    date_tag.extract()
-                    img_div.extract()
+        except:
+            self.log("Error removing the published date")
 
+        if self.useHighResImages:
+            try:
+                #open up all the "Enlarge this Image" pop-ups and download the full resolution jpegs
+                enlargeThisList = soup.findAll('div',{'class':'icon enlargeThis'})
+                if enlargeThisList:
+                    for popupref in enlargeThisList:
+                        popupreflink = popupref.find('a')
+                        if popupreflink:
+                            reflinkstring = str(popupreflink['href'])
+                            refstart = reflinkstring.find("javascript:pop_me_up2('") + len("javascript:pop_me_up2('")
+                            refend = reflinkstring.find(".html", refstart) + len(".html")
+                            reflinkstring = reflinkstring[refstart:refend]
+
+                            popuppage = self.browser.open(reflinkstring)
+                            popuphtml = popuppage.read()
+                            popuppage.close()
+                            if popuphtml:
+                                st = time.localtime()
+                                year = str(st.tm_year)
+                                month = "%.2d" % st.tm_mon
+                                day = "%.2d" % st.tm_mday
+                                imgstartpos = popuphtml.find('http://graphics8.nytimes.com/images/' + year + '/' +  month +'/' + day +'/') + len('http://graphics8.nytimes.com/images/' + year + '/' +  month +'/' + day +'/')
+                                highResImageLink = 'http://graphics8.nytimes.com/images/' + year + '/' +  month +'/' + day +'/' + popuphtml[imgstartpos:popuphtml.find('.jpg',imgstartpos)+4]
+                                popupSoup = BeautifulSoup(popuphtml)
+                                highResTag = popupSoup.find('img', {'src':highResImageLink})
+                                if highResTag:
+                                    try:
+                                        newWidth = highResTag['width']
+                                        newHeight = highResTag['height']
+                                        imageTag = popupref.parent.find("img")
+                                    except:
+                                        self.log("Error: finding width and height of img")
+                                    popupref.extract()
+                                    if imageTag:
+                                        try:
+                                            imageTag['src'] = highResImageLink
+                                            imageTag['width'] = newWidth
+                                            imageTag['height'] = newHeight
+                                        except:
+                                            self.log("Error setting the src width and height parameters")
+            except Exception:
+                self.log("Error pulling high resolution images")
+
+            try:
+                #remove "Related content" bar
+                runAroundsFound = soup.findAll('div',{'class':['articleInline runaroundLeft','articleInline doubleRule runaroundLeft','articleInline runaroundLeft firstArticleInline']})
+                if runAroundsFound:
+                    for runAround in runAroundsFound:
+                        #find all section headers
+                        hlines = runAround.findAll(True ,{'class':['sectionHeader','sectionHeader flushBottom']})
+                        if hlines:
+                            for hline in hlines:
+                                hline.extract()
+            except:
+                self.log("Error removing related content bar")
+
+
+            try:
+                #in case pulling images failed, delete the enlarge this text
+                enlargeThisList = soup.findAll('div',{'class':'icon enlargeThis'})
+                if enlargeThisList:
+                    for popupref in enlargeThisList:
+                        popupref.extract()
+            except:
+                self.log("Error removing Enlarge this text")
 
         return self.strip_anchors(soup)
 
     def postprocess_html(self,soup, True):
-
         try:
                 if self.one_picture_per_article:
                         # Remove all images after first
@@ -766,6 +849,8 @@ class NYTimes(BasicNewsRecipe):
         try:
             if len(article.text_summary.strip()) == 0:
                 articlebodies = soup.findAll('div',attrs={'class':'articleBody'})
+                if not articlebodies: #added to account for blog formats
+                    articlebodies = soup.findAll('div', attrs={'class':'entry-content'}) #added to account for blog formats
                 if articlebodies:
                     for articlebody in articlebodies:
                         if articlebody:
@@ -774,13 +859,14 @@ class NYTimes(BasicNewsRecipe):
                                 refparagraph = self.massageNCXText(self.tag_to_string(p,use_alt=False)).strip()
                                 #account for blank paragraphs and short paragraphs by appending them to longer ones
                                 if len(refparagraph) > 0:
-                                    if len(refparagraph) > 70: #approximately one line of text
+                                    if len(refparagraph) > 140: #approximately two lines of text
                                         article.summary = article.text_summary = shortparagraph + refparagraph
                                         return
                                     else:
                                         shortparagraph = refparagraph + " "
                                         if shortparagraph.strip().find(" ") == -1 and not shortparagraph.strip().endswith(":"):
                                             shortparagraph = shortparagraph + "- "
+
         except:
             self.log("Error creating article descriptions")
             return

resources/recipes/onionavclub.recipe

@@ -26,7 +26,7 @@ class BBC(BasicNewsRecipe):
     extra_css      = '.headline {font-size: x-large;} \n .fact { padding-top: 10pt  }'
 
     feeds          = [
-                      ('Interviews', 'http://www.avclub.com/feed/interview/'),
+                      ('TV', 'http://www.avclub.com/feed/tv/'),
                       ('AV Club Daily', 'http://www.avclub.com/feed/daily'),
                       ('Film', 'http://www.avclub.com/feed/film/'),
                       ('Music', 'http://www.avclub.com/feed/music/'),

resources/recipes/roger_ebert.recipe

@@ -0,0 +1,120 @@
+import re
+import urllib2
+from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import BeautifulSoup, SoupStrainer
+
+class Ebert(BasicNewsRecipe):
+    title                 = 'Roger Ebert'
+    __author__            = 'Shane Erstad'
+    description           = 'Roger Ebert Movie Reviews'
+    publisher             = 'Chicago Sun Times'
+    category              = 'movies'
+    oldest_article        = 8
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    encoding              = 'utf-8'
+    masthead_url          = 'http://rogerebert.suntimes.com/graphics/global/roger.jpg'
+    language              = 'en'
+    remove_empty_feeds    = False
+    PREFIX                  = 'http://rogerebert.suntimes.com'
+    patternReviews                = r'<span class="*?movietitle"*?>(.*?)</span>.*?<div class="*?headline"*?>(.*?)</div>(.*?)</div>'
+    patternCommentary       = r'<div class="*?headline"*?>.*?(<a href="/apps/pbcs.dll/article\?AID=.*?COMMENTARY.*?" id="ltred">.*?</a>).*?<div class="blurb clear">(.*?)</div>'
+    patternPeople           = r'<div class="*?headline"*?>.*?(<a href="/apps/pbcs.dll/article\?AID=.*?PEOPLE.*?" id="ltred">.*?</a>).*?<div class="blurb clear">(.*?)</div>'
+    patternGlossary           = r'<div class="*?headline"*?>.*?(<a href="/apps/pbcs.dll/article\?AID=.*?GLOSSARY.*?" id="ltred">.*?</a>).*?<div class="blurb clear">(.*?)</div>'
+
+
+
+    conversion_options = {
+                          'comment'          : description
+                        , 'tags'             : category
+                        , 'publisher'        : publisher
+                        , 'language'         : language
+                        , 'linearize_tables' : True
+                        }
+
+
+    feeds          = [
+                        (u'Reviews'   , u'http://rogerebert.suntimes.com/apps/pbcs.dll/section?category=reviews' )
+                        ,(u'Commentary'   , u'http://rogerebert.suntimes.com/apps/pbcs.dll/section?category=COMMENTARY')
+                        ,(u'Great Movies'   , u'http://rogerebert.suntimes.com/apps/pbcs.dll/section?category=REVIEWS08')
+                        ,(u'People'   , u'http://rogerebert.suntimes.com/apps/pbcs.dll/section?category=PEOPLE')
+                        ,(u'Glossary'   , u'http://rogerebert.suntimes.com/apps/pbcs.dll/section?category=GLOSSARY')
+
+                     ]
+
+    preprocess_regexps = [
+        (re.compile(r'<font.*?>.*?This is a printer friendly.*?</font>.*?<hr>', re.DOTALL|re.IGNORECASE),
+            lambda m: '')
+    ]
+
+
+
+    def print_version(self, url):
+        return url + '&template=printart'
+
+    def parse_index(self):
+        totalfeeds = []
+        lfeeds = self.get_feeds()
+        for feedobj in lfeeds:
+            feedtitle, feedurl = feedobj
+            self.log('\tFeedurl: ', feedurl)
+            self.report_progress(0, _('Fetching feed')+' %s...'%(feedtitle if feedtitle else feedurl))
+            articles = []
+            page = urllib2.urlopen(feedurl).read()
+
+            if feedtitle == 'Reviews' or feedtitle == 'Great Movies':
+                    pattern = self.patternReviews
+            elif feedtitle == 'Commentary':
+                    pattern = self.patternCommentary
+            elif feedtitle == 'People':
+                    pattern = self.patternPeople
+            elif feedtitle == 'Glossary':
+                    pattern = self.patternGlossary
+
+
+            regex = re.compile(pattern, re.IGNORECASE|re.DOTALL)
+
+            for match in regex.finditer(page):
+                if feedtitle == 'Reviews' or feedtitle == 'Great Movies':
+                    movietitle = match.group(1)
+                    thislink = match.group(2)
+                    description = match.group(3)
+                elif feedtitle == 'Commentary' or feedtitle == 'People' or feedtitle == 'Glossary':
+                    thislink = match.group(1)
+                    description = match.group(2)
+
+                self.log(thislink)
+
+                for link in BeautifulSoup(thislink, parseOnlyThese=SoupStrainer('a')):
+                    thisurl = self.PREFIX + link['href']
+                    thislinktext = self.tag_to_string(link)
+
+                    if feedtitle == 'Reviews' or feedtitle == 'Great Movies':
+                        thistitle = movietitle
+                    elif feedtitle == 'Commentary' or feedtitle == 'People' or feedtitle == 'Glossary':
+                        thistitle = thislinktext
+
+                    if thistitle == '':
+                        thistitle = 'Ebert Journal Post'
+
+                    """
+                    pattern2 = r'AID=\/(.*?)\/'
+                    reg2 = re.compile(pattern2, re.IGNORECASE|re.DOTALL)
+                    match2 = reg2.search(thisurl)
+                    date = match2.group(1)
+                    c = time.strptime(match2.group(1),"%Y%m%d")
+                    date=time.strftime("%a, %b %d, %Y", c)
+                    self.log(date)
+                    """
+
+                    articles.append({
+                                      'title'      :thistitle
+                                     ,'date'       :''
+                                     ,'url'        :thisurl
+                                     ,'description':description
+                                    })
+            totalfeeds.append((feedtitle, articles))
+
+        return totalfeeds
+

resources/recipes/root.recipe

@@ -0,0 +1,39 @@
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1289939440(BasicNewsRecipe):
+  __author__  = 'FunThomas'
+  title       = u'Root.cz'
+  description = u'Zprávičky a články z Root.cz'
+  publisher   = u'Internet Info, s.r.o'
+  oldest_article = 2               #max stari clanku ve dnech
+  max_articles_per_feed = 50       #max pocet clanku na feed
+
+  feeds = [
+    (u'Články', u'http://www.root.cz/rss/clanky/'),
+    (u'Zprávičky', u'http://www.root.cz/rss/zpravicky/')
+  ]
+
+  publication_type = u'magazine'
+  language = u'cs'
+  no_stylesheets = True
+  remove_javascript = True
+  cover_url = u'http://i.iinfo.cz/urs/logo-root-bila-oranzova-cerna-111089527143118.gif'
+
+  remove_attributes = ['width','height','href'] #,'href'
+  keep_only_tags = [
+    dict(name='h1'),
+    dict(name='a',attrs={'class':'author'}),
+    dict(name='p', attrs={'class':'intro'}),
+    dict(name='div',attrs={'class':'urs'})
+  ]
+
+  preprocess_regexps = [
+    (re.compile(u'<p class="perex[^"]*">[^<]*<img[^>]*>', re.DOTALL),lambda match: '<p class="intro">'),
+    (re.compile(u'<h3><a name="tucnak">Tričko tučňák.*</body>', re.DOTALL),lambda match: '<!--deleted-->')
+  ]
+
+  extra_css = '''
+                h1 {font-size:130%; font-weight:bold}
+                h3 {font-size:111%; font-weight:bold}
+              '''

resources/recipes/sinfest.recipe

@@ -0,0 +1,33 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Nadid <nadid.skywalker at gmail.com>'
+'''
+http://www.sinfest.net
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class SinfestBig(BasicNewsRecipe):
+    title                  = 'Sinfest'
+    __author__             = 'nadid'
+    description            = 'Sinfest'
+    reverse_article_order = False
+    oldest_article         = 5
+    max_articles_per_feed  = 100
+    no_stylesheets         = True
+    use_embedded_content   = True
+    encoding               = 'utf-8'
+    publisher              = 'Tatsuya Ishida/Museworks'
+    category               = 'comic'
+    language               = 'en'
+
+    conversion_options = {
+                             'comments'        : description
+                            ,'tags'            : category
+                            ,'language'        : language
+                            ,'publisher'       : publisher
+                         }
+
+    feeds = [(u'SinFest', u'http://henrik.nyh.se/scrapers/sinfest.rss' )]
+    def get_article_url(self, article):
+        return article.get('link')
+

resources/recipes/spin_magazine.recipe

@@ -0,0 +1,15 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1296179411(BasicNewsRecipe):
+    title          = u'SPIN Magzine'
+    __author__     = 'Quistopher'
+    language = 'en'
+    oldest_article = 7
+    max_articles_per_feed = 100
+
+    feeds          = [
+			(u'Daily Noise Blog | SPIN.com', u'http://www.spin.com/blog/feed'),
+			(u'It Happened Last Night | SPIN.com', u'http://www.spin.com/it-happened-last-night/feed'),
+			(u'Album Reviews | SPIN.com', u'http://www.spin.com/album-reviews/feed')
+
+		     ]

resources/recipes/vijesti.recipe

@@ -1,6 +1,6 @@
 
 __license__   = 'GPL v3'
-__copyright__ = '2009-2010, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2009-2011, Darko Miletic <darko.miletic at gmail.com>'
 
 '''
 vijesti.me
@@ -18,12 +18,16 @@ class Vijesti(BasicNewsRecipe):
     oldest_article        = 2
     max_articles_per_feed = 150
     no_stylesheets        = True
-    encoding              = 'cp1250'
+    encoding              = 'utf8'
     use_embedded_content  = False
     language              = 'sr'
     publication_type      = 'newspaper'
-    masthead_url          = 'http://www.vijesti.me/img/logo.gif'    
+    extra_css             = """
-    extra_css             = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
+                                @font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} 
+                                @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} 
+                                body{font-family: Georgia,"Times New Roman",Times,serif1,serif} 
+                                .articledescription,.article,.chapter{font-family: sans1, sans-serif} 
+                            """
 
     conversion_options = {
                           'comment'          : description
@@ -34,11 +38,11 @@ class Vijesti(BasicNewsRecipe):
 
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
 
-    keep_only_tags = [dict(name='div', attrs={'id':'mainnews'})]
+    keep_only_tags = [dict(name='div', attrs={'id':['article_intro_text','article_text']})]
 
     remove_tags = [dict(name=['object','link','embed','form'])]
 
-    feeds = [(u'Sve vijesti', u'http://www.vijesti.me/rss.php' )]
+    feeds = [(u'Sve vijesti', u'http://www.vijesti.me/rss/' )]
 
     def preprocess_html(self, soup):
         return self.adeify_images(soup)

resources/recipes/zdnet.recipe

@@ -27,12 +27,34 @@ class cdnet(BasicNewsRecipe):
 		   dict(id='header'),
 		   dict(id='search'),
 		   dict(id='nav'),
+		   dict(id='blog-author-info'),
+		   dict(id='post-tags'),
+		   dict(id='bio-naraine'),
+		   dict(id='bio-kennedy'),
+		   dict(id='author-short-disclosure-kennedy'),
 		   dict(id=''),
 		   dict(name='div', attrs={'class':'banner'}),
+		   dict(name='div', attrs={'class':'int'}),
+		   dict(name='div', attrs={'class':'talkback clear space-2'}),
+		   dict(name='div', attrs={'class':'content-1 clear'}),
+		   dict(name='div', attrs={'class':'space-2'}),
+		   dict(name='div', attrs={'class':'space-3'}),
+		   dict(name='div', attrs={'class':'thumb-2 left'}),
+		   dict(name='div', attrs={'class':'hotspot'}),
+		   dict(name='div', attrs={'class':'hed hed-1 space-1'}),
+		   dict(name='div', attrs={'class':'view-1 clear content-3 space-2'}),
+		   dict(name='div', attrs={'class':'hed hed-1 space-1'}),
+		   dict(name='div', attrs={'class':'hed hed-1'}),
+		   dict(name='div', attrs={'class':'post-header'}),
+		   dict(name='div', attrs={'class':'lvl-nav clear'}),
+		   dict(name='div', attrs={'class':'t-share-overlay overlay-pop contain-overlay-4'}),
 		   dict(name='p', attrs={'class':'tags'}),
+		   dict(name='span', attrs={'class':'follow'}),
+		   dict(name='span', attrs={'class':'int'}),
+		   dict(name='h4', attrs={'class':'h s-4'}),
 		   dict(name='a', attrs={'href':'http://www.twitter.com/ryanaraine'}),
 		   dict(name='div', attrs={'class':'special1'})]
-    remove_tags_after = [dict(name='div', attrs={'class':'bloggerDesc clear'})]
+    remove_tags_after = [dict(name='div', attrs={'class':'clear'})]
 
     feeds =  [ ('zdnet', 'http://feeds.feedburner.com/zdnet/security') ]
 
@@ -43,3 +65,4 @@ class cdnet(BasicNewsRecipe):
         return soup
 
 
+

setup/installer/linux/freeze2.py

@@ -360,6 +360,9 @@ class LinuxFreeze(Command):
             def main():
                 try:
                     sys.argv[0] = sys.calibre_basename
+                    dfv = os.environ.get('CALIBRE_DEVELOP_FROM', None)
+                    if dfv and os.path.exists(dfv):
+                        sys.path.insert(0, os.path.abspath(dfv))
                     set_default_encoding()
                     set_helper()
                     set_qt_plugin_path()

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.7.42'
+__version__   = '0.7.43'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 
 import re

src/calibre/customize/profiles.py

@@ -495,6 +495,22 @@ class SonyReader900Output(SonyReaderOutput):
     screen_size               = (600, 999)
     comic_screen_size = screen_size
 
+class GenericEink(SonyReaderOutput):
+
+    name = 'Generic e-ink'
+    short_name = 'generic_eink'
+    description = _('Suitable for use with any e-ink device')
+    epub_periodical_format = None
+
+class GenericEinkLarge(GenericEink):
+
+    name = 'Generic e-ink large'
+    short_name = 'generic_eink_large'
+    description = _('Suitable for use with any large screen e-ink device')
+
+    screen_size               = (600, 999)
+    comic_screen_size = screen_size
+
 class JetBook5Output(OutputProfile):
 
     name        = 'JetBook 5-inch'
@@ -567,6 +583,7 @@ class CybookG3Output(OutputProfile):
 
     # Screen size is a best guess
     screen_size               = (600, 800)
+    comic_screen_size         = (600, 757)
     dpi                       = 168.451
     fbase                     = 16
     fsizes                    = [12, 12, 14, 16, 18, 20, 22, 24]
@@ -719,6 +736,6 @@ output_profiles = [OutputProfile, SonyReaderOutput, SonyReader300Output,
         iPadOutput, KoboReaderOutput, TabletOutput, SamsungGalaxy,
         SonyReaderLandscapeOutput, KindleDXOutput, IlliadOutput,
         IRexDR1000Output, IRexDR800Output, JetBook5Output, NookOutput,
-        BambookOutput, NookColorOutput]
+        BambookOutput, NookColorOutput, GenericEink, GenericEinkLarge]
 
 output_profiles.sort(cmp=lambda x,y:cmp(x.name.lower(), y.name.lower()))

src/calibre/devices/android/driver.py

@@ -19,7 +19,8 @@ class ANDROID(USBMS):
 
     VENDOR_ID   = {
             # HTC
-            0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226], 0x0c01 : [0x100, 0x0227], 0x0ff9
+            0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226], 0x0c01 : [0x100,
+                0x0227, 0x0226], 0x0ff9
                 : [0x0100, 0x0227, 0x0226], 0x0c87: [0x0100, 0x0227, 0x0226],
                 0xc92 : [0x100], 0xc97: [0x226], 0xc99 : [0x0100]},
 
@@ -54,7 +55,7 @@ class ANDROID(USBMS):
             0x1004 : { 0x61cc : [0x100] },
 
             # Archos
-            0x0e79 : { 0x1419: [0x0216], 0x1420 : [0x0216]},
+            0x0e79 : { 0x1419: [0x0216], 0x1420 : [0x0216], 0x1422 : [0x0216]},
 
             }
     EBOOK_DIR_MAIN = ['eBooks/import', 'wordplayer/calibretransfer', 'Books']
@@ -70,7 +71,7 @@ class ANDROID(USBMS):
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID',
             'SCH-I500_CARD', 'SPH-D700_CARD', 'MB810', 'GT-P1000', 'DESIRE',
-            'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT']
+            'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT']

src/calibre/devices/eb600/driver.py

@@ -21,6 +21,7 @@ from calibre.devices.usbms.driver import USBMS
 class EB600(USBMS):
 
     name           = 'Netronix EB600 Device Interface'
+    gui_name       = 'Netronix EB600'
     description    = _('Communicate with the EB600 eBook reader.')
     author         = 'Kovid Goyal'
     supported_platforms = ['windows', 'osx', 'linux']

src/calibre/devices/folder_device/driver.py

@@ -22,7 +22,7 @@ class FOLDER_DEVICE_FOR_CONFIG(USBMS):
     PRODUCT_ID  = [0xffff]
     BCD         = [0xffff]
     DEVICE_PLUGBOARD_NAME = 'FOLDER_DEVICE'
-
+    SUPPORTS_SUB_DIRS = True
 
 class FOLDER_DEVICE(USBMS):
     type = _('Device Interface')

src/calibre/devices/hanvon/driver.py

@@ -24,7 +24,7 @@ class N516(USBMS):
     supported_platforms = ['windows', 'osx', 'linux']
 
     # Ordered list of supported formats
-    FORMATS     = ['epub', 'prc', 'html', 'pdf', 'txt']
+    FORMATS     = ['epub', 'prc', 'mobi', 'html', 'pdf', 'txt']
 
     VENDOR_ID   = [0x0525]
     PRODUCT_ID  = [0xa4a5]

src/calibre/ebooks/conversion/plumber.py

@@ -576,10 +576,12 @@ OptionRecommendation(name='sr3_replace',
         if not input_fmt:
             raise ValueError('Input file must have an extension')
         input_fmt = input_fmt[1:].lower()
+        self.archive_input_tdir = None
         if input_fmt in ('zip', 'rar', 'oebzip'):
             self.log('Processing archive...')
-            tdir = PersistentTemporaryDirectory('_plumber')
+            tdir = PersistentTemporaryDirectory('_plumber_archive')
             self.input, input_fmt = self.unarchive(self.input, tdir)
+            self.archive_input_tdir = tdir
         if os.access(self.input, os.R_OK):
             nfp = run_plugins_on_preprocess(self.input, input_fmt)
             if nfp != self.input:

src/calibre/ebooks/conversion/utils.py

@@ -25,13 +25,15 @@ class HeuristicProcessor(object):
         self.chapters_with_title = 0
         self.blanks_deleted = False
         self.linereg = re.compile('(?<=<p).*?(?=</p>)', re.IGNORECASE|re.DOTALL)
-        self.blankreg = re.compile(r'\s*(?P<openline><p(?!\sid=\"softbreak\")[^>]*>)\s*(?P<closeline></p>)', re.IGNORECASE)
+        self.blankreg = re.compile(r'\s*(?P<openline><p(?!\sclass=\"softbreak\")[^>]*>)\s*(?P<closeline></p>)', re.IGNORECASE)
+        self.softbreak = re.compile(r'\s*(?P<openline><p(?=\sclass=\"softbreak\")[^>]*>)\s*(?P<closeline></p>)', re.IGNORECASE)
         self.multi_blank = re.compile(r'(\s*<p[^>]*>\s*</p>){2,}', re.IGNORECASE)
 
     def is_pdftohtml(self, src):
         return '<!-- created by calibre\'s pdftohtml -->' in src[:1000]
 
     def chapter_head(self, match):
+        from calibre.utils.html2text import html2text
         chap = match.group('chap')
         title = match.group('title')
         if not title:
@@ -40,10 +42,12 @@ class HeuristicProcessor(object):
                     " chapters. - " + unicode(chap))
             return '<h2>'+chap+'</h2>\n'
         else:
+            txt_chap = html2text(chap)
+            txt_title = html2text(title)
             self.html_preprocess_sections = self.html_preprocess_sections + 1
             self.log.debug("marked " + unicode(self.html_preprocess_sections) +
                     " chapters & titles. - " + unicode(chap) + ", " + unicode(title))
-            return '<h2>'+chap+'</h2>\n<h3>'+title+'</h3>\n'
+            return '<h2 title="'+txt_chap+', '+txt_title+'">'+chap+'</h2>\n<h3 class="sigilNotInTOC">'+title+'</h3>\n'
 
     def chapter_break(self, match):
         chap = match.group('section')
@@ -137,21 +141,21 @@ class HeuristicProcessor(object):
         ]
 
         ITALICIZE_STYLE_PATS = [
-            r'(?msu)(?<=\s)_(?P<words>\S[^_]{0,40}?\S)?_(?=\s)',
+            r'(?msu)(?<=\s)_(?P<words>\S[^_]{0,40}?\S)?_(?=[\s\.,\!\?])',
-            r'(?msu)(?<=\s)/(?P<words>\S[^/]{0,40}?\S)?/(?=\s)',
+            r'(?msu)(?<=\s)/(?P<words>\S[^/]{0,40}?\S)?/(?=[\s\.,\!\?])',
-            r'(?msu)(?<=\s)~~(?P<words>\S[^~]{0,40}?\S)?~~(?=\s)',
+            r'(?msu)(?<=\s)~~(?P<words>\S[^~]{0,40}?\S)?~~(?=[\s\.,\!\?])',
-            r'(?msu)(?<=\s)\*(?P<words>\S[^\*]{0,40}?\S)?\*(?=\s)',
+            r'(?msu)(?<=\s)\*(?P<words>\S[^\*]{0,40}?\S)?\*(?=[\s\.,\!\?])',
-            r'(?msu)(?<=\s)~(?P<words>\S[^~]{0,40}?\S)?~(?=\s)',
+            r'(?msu)(?<=\s)~(?P<words>\S[^~]{0,40}?\S)?~(?=[\s\.,\!\?])',
-            r'(?msu)(?<=\s)_/(?P<words>\S[^/_]{0,40}?\S)?/_(?=\s)',
+            r'(?msu)(?<=\s)_/(?P<words>\S[^/_]{0,40}?\S)?/_(?=[\s\.,\!\?])',
-            r'(?msu)(?<=\s)_\*(?P<words>\S[^\*_]{0,40}?\S)?\*_(?=\s)',
+            r'(?msu)(?<=\s)_\*(?P<words>\S[^\*_]{0,40}?\S)?\*_(?=[\s\.,\!\?])',
-            r'(?msu)(?<=\s)\*/(?P<words>\S[^/\*]{0,40}?\S)?/\*(?=\s)',
+            r'(?msu)(?<=\s)\*/(?P<words>\S[^/\*]{0,40}?\S)?/\*(?=[\s\.,\!\?])',
-            r'(?msu)(?<=\s)_\*/(?P<words>\S[^\*_]{0,40}?\S)?/\*_(?=\s)',
+            r'(?msu)(?<=\s)_\*/(?P<words>\S[^\*_]{0,40}?\S)?/\*_(?=[\s\.,\!\?])',
-            r'(?msu)(?<=\s)/:(?P<words>\S[^:/]{0,40}?\S)?:/(?=\s)',
+            r'(?msu)(?<=\s)/:(?P<words>\S[^:/]{0,40}?\S)?:/(?=[\s\.,\!\?])',
-            r'(?msu)(?<=\s)\|:(?P<words>\S[^:\|]{0,40}?\S)?:\|(?=\s)',
+            r'(?msu)(?<=\s)\|:(?P<words>\S[^:\|]{0,40}?\S)?:\|(?=[\s\.,\!\?])',
         ]
 
         for word in ITALICIZE_WORDS:
-            html = html.replace(word, '<i>%s</i>' % word)
+            html = re.sub(r'(?<=\s|>)' + re.escape(word) + r'(?=\s|<)', '<i>%s</i>' % word, html)
 
         for pat in ITALICIZE_STYLE_PATS:
             html = re.sub(pat, lambda mo: '<i>%s</i>' % mo.group('words'), html)
@@ -203,8 +207,8 @@ class HeuristicProcessor(object):
             blank_lines = ""
         opt_title_open = "("
         opt_title_close = ")?"
-        n_lookahead_open = "\s+(?!"
+        n_lookahead_open = "(?!\s*"
-        n_lookahead_close = ")"
+        n_lookahead_close = ")\s*"
 
         default_title = r"(<[ibu][^>]*>)?\s{0,3}(?!Chapter)([\w\:\'’\"-]+\s{0,3}){1,5}?(</[ibu][^>]*>)?(?=<)"
         simple_title = r"(<[ibu][^>]*>)?\s{0,3}(?!(Chapter|\s+<)).{0,65}?(</[ibu][^>]*>)?(?=<)"
@@ -215,7 +219,7 @@ class HeuristicProcessor(object):
             [r"[^'\"]?(Introduction|Synopsis|Acknowledgements|Epilogue|CHAPTER|Kapitel|Volume\b|Prologue|Book\b|Part\b|Dedication|Preface)\s*([\d\w-]+\:?\'?\s*){0,5}", True, True, True, False, "Searching for common section headings", 'common'],
             [r"[^'\"]?(CHAPTER|Kapitel)\s*([\dA-Z\-\'\"\?!#,]+\s*){0,7}\s*", True, True, True, False, "Searching for most common chapter headings", 'chapter'],  # Highest frequency headings which include titles
             [r"<b[^>]*>\s*(<span[^>]*>)?\s*(?!([*#•=]+\s*)+)(\s*(?=[\d.\w#\-*\s]+<)([\d.\w#-*]+\s*){1,5}\s*)(?!\.)(</span>)?\s*</b>", True, True, True, False, "Searching for emphasized lines", 'emphasized'], # Emphasized lines
-            [r"[^'\"]?(\d+(\.|:))\s*([\dA-Z\-\'\"#,]+\s*){0,7}\s*", True, True, True, False, "Searching for numeric chapter headings", 'numeric'],  # Numeric Chapters
+            [r"[^'\"]?(\d+(\.|:))\s*([\w\-\'\"#,]+\s*){0,7}\s*", True, True, True, False, "Searching for numeric chapter headings", 'numeric'],  # Numeric Chapters
             [r"([A-Z]\s+){3,}\s*([\d\w-]+\s*){0,3}\s*", True, True, True, False, "Searching for letter spaced headings", 'letter_spaced'],  # Spaced Lettering
             [r"[^'\"]?(\d+\.?\s+([\d\w-]+\:?\'?-?\s?){0,5})\s*", True, True, True, False, "Searching for numeric chapters with titles", 'numeric_title'], # Numeric Titles
             [r"[^'\"]?(\d+)\s*([\dA-Z\-\'\"\?!#,]+\s*){0,7}\s*", True, True, True, False, "Searching for simple numeric headings", 'plain_number'],  # Numeric Chapters, no dot or colon
@@ -275,7 +279,7 @@ class HeuristicProcessor(object):
                         self.log.debug(unicode(type_name)+" had "+unicode(hits)+" hits - "+unicode(self.chapters_no_title)+" chapters with no title, "+unicode(self.chapters_with_title)+" chapters with titles, "+unicode(float(self.chapters_with_title) / float(hits))+" percent. ")
                         if type_name == 'common':
                             analysis_result.append([chapter_type, n_lookahead_req, strict_title, ignorecase, title_req, log_message, type_name])
-                        elif self.min_chapters <= hits < max_chapters:
+                        elif self.min_chapters <= hits < max_chapters or self.min_chapters < 3 > hits:
                             analysis_result.append([chapter_type, n_lookahead_req, strict_title, ignorecase, title_req, log_message, type_name])
                             break
                 else:
@@ -367,6 +371,8 @@ class HeuristicProcessor(object):
         html = re.sub(ur'\s*<o:p>\s*</o:p>', ' ', html)
         # Delete microsoft 'smart' tags
         html = re.sub('(?i)</?st1:\w+>', '', html)
+        # Delete self closing paragraph tags
+        html = re.sub('<p\s?/>', '', html)
         # Get rid of empty span, bold, font, em, & italics tags
         html = re.sub(r"\s*<span[^>]*>\s*(<span[^>]*>\s*</span>){0,2}\s*</span>\s*", " ", html)
         html = re.sub(r"\s*<(font|[ibu]|em)[^>]*>\s*(<(font|[ibu]|em)[^>]*>\s*</(font|[ibu]|em)>\s*){0,2}\s*</(font|[ibu]|em)>", " ", html)
@@ -467,7 +473,7 @@ class HeuristicProcessor(object):
         if blanks_between_paragraphs and getattr(self.extra_opts, 'delete_blank_paragraphs', False):
             self.log.debug("deleting blank lines")
             self.blanks_deleted = True
-            html = self.multi_blank.sub('\n<p id="softbreak" style="margin-top:1.5em; margin-bottom:1.5em"> </p>', html)
+            html = self.multi_blank.sub('\n<p class="softbreak" style="margin-top:1.5em; margin-bottom:1.5em"> </p>', html)
             html = self.blankreg.sub('', html)
 
         # Determine line ending type
@@ -522,11 +528,11 @@ class HeuristicProcessor(object):
             # Center separator lines
             html = re.sub(u'<(?P<outer>p|div)[^>]*>\s*(<(?P<inner1>font|span|[ibu])[^>]*>)?\s*(<(?P<inner2>font|span|[ibu])[^>]*>)?\s*(<(?P<inner3>font|span|[ibu])[^>]*>)?\s*(?P<break>([*#•=✦]+\s*)+)\s*(</(?P=inner3)>)?\s*(</(?P=inner2)>)?\s*(</(?P=inner1)>)?\s*</(?P=outer)>', '<p style="text-align:center; margin-top:1.25em; margin-bottom:1.25em">' + '\g<break>' + '</p>', html)
             if not self.blanks_deleted:
-                html = self.multi_blank.sub('\n<p id="softbreak" style="margin-top:1.5em; margin-bottom:1.5em"> </p>', html)
+                html = self.multi_blank.sub('\n<p class="softbreak" style="margin-top:1.5em; margin-bottom:1.5em"> </p>', html)
-            html = re.sub('<p\s+id="softbreak"[^>]*>\s*</p>', '<div id="softbreak" style="margin-left: 45%; margin-right: 45%; margin-top:1.5em; margin-bottom:1.5em"><hr style="height: 3px; background:#505050" /></div>', html)
+            html = re.sub('<p\s+class="softbreak"[^>]*>\s*</p>', '<div id="softbreak" style="margin-left: 45%; margin-right: 45%; margin-top:1.5em; margin-bottom:1.5em"><hr style="height: 3px; background:#505050" /></div>', html)
 
         if self.deleted_nbsps:
             # put back non-breaking spaces in empty paragraphs to preserve original formatting
             html = self.blankreg.sub('\n'+r'\g<openline>'+u'\u00a0'+r'\g<closeline>', html)
-
+            html = self.softbreak.sub('\n'+r'\g<openline>'+u'\u00a0'+r'\g<closeline>', html)
         return html

src/calibre/ebooks/fb2/fb2ml.py

@@ -99,7 +99,10 @@ class FB2MLizer(object):
         metadata['appname'] = __appname__
         metadata['version'] = __version__
         metadata['date'] = '%i.%i.%i' % (datetime.now().day, datetime.now().month, datetime.now().year)
-        metadata['lang'] = u''.join(self.oeb_book.metadata.lang) if self.oeb_book.metadata.lang else 'en'
+        if self.oeb_book.metadata.language:
+            metadata['lang'] = self.oeb_book.metadata.language[0].value
+        else:
+            metadata['lang'] = u'en'
         metadata['id'] = None
         metadata['cover'] = self.get_cover()
 

src/calibre/ebooks/lit/input.py

@@ -39,12 +39,15 @@ class LITInput(InputFormatPlugin):
                     pre = body[0]
                     from calibre.ebooks.txt.processor import convert_basic, preserve_spaces, \
                         separate_paragraphs_single_line
+                    from calibre.ebooks.chardet import xml_to_unicode
                     from lxml import etree
                     import copy
                     html = separate_paragraphs_single_line(pre.text)
                     html = preserve_spaces(html)
                     html = convert_basic(html).replace('<html>',
                             '<html xmlns="%s">'%XHTML_NS)
+                    html = xml_to_unicode(html, strip_encoding_pats=True,
+                            resolve_entities=True)[0]
                     root = etree.fromstring(html)
                     body = XPath('//h:body')(root)
                     pre.tag = XHTML('div')

src/calibre/ebooks/metadata/__init__.py

@@ -36,6 +36,7 @@ def author_to_author_sort(author):
         return author
     author = _bracket_pat.sub('', author).strip()
     tokens = author.split()
+    if tokens and tokens[-1] not in ('Inc.', 'Inc'):
         tokens = tokens[-1:] + tokens[:-1]
         if len(tokens) > 1 and method != 'nocomma':
             tokens[0] += ','

src/calibre/ebooks/metadata/covers.py

@@ -121,6 +121,7 @@ class LibraryThingCovers(CoverDownload): # {{{
     LIBRARYTHING = 'http://www.librarything.com/isbn/'
 
     def get_cover_url(self, isbn, br, timeout=5.):
+
         try:
             src = br.open_novisit('http://www.librarything.com/isbn/'+isbn,
                     timeout=timeout).read().decode('utf-8', 'replace')
@@ -129,6 +130,8 @@ class LibraryThingCovers(CoverDownload): # {{{
                 err = Exception(_('LibraryThing.com timed out. Try again later.'))
             raise err
         else:
+            if '/wiki/index.php/HelpThing:Verify' in src:
+                raise Exception('LibraryThing is blocking calibre.')
             s = BeautifulSoup(src)
             url = s.find('td', attrs={'class':'left'})
             if url is None:
@@ -142,9 +145,12 @@ class LibraryThingCovers(CoverDownload): # {{{
             return url
 
     def has_cover(self, mi, ans, timeout=5.):
-        if not mi.isbn:
+        if not mi.isbn or not self.site_customization:
             return False
-        br = browser()
+        from calibre.ebooks.metadata.library_thing import get_browser, login
+        br = get_browser()
+        un, _, pw = self.site_customization.partition(':')
+        login(br, un, pw)
         try:
             self.get_cover_url(mi.isbn, br, timeout=timeout)
             self.debug('cover for', mi.isbn, 'found')
@@ -153,9 +159,12 @@ class LibraryThingCovers(CoverDownload): # {{{
             self.debug(e)
 
     def get_covers(self, mi, result_queue, abort, timeout=5.):
-        if not mi.isbn:
+        if not mi.isbn or not self.site_customization:
             return
-        br = browser()
+        from calibre.ebooks.metadata.library_thing import get_browser, login
+        br = get_browser()
+        un, _, pw = self.site_customization.partition(':')
+        login(br, un, pw)
         try:
             url = self.get_cover_url(mi.isbn, br, timeout=timeout)
             cover_data = br.open_novisit(url).read()
@@ -164,6 +173,11 @@ class LibraryThingCovers(CoverDownload): # {{{
             result_queue.put((False, self.exception_to_string(e),
                 traceback.format_exc(), self.name))
 
+    def customization_help(self, gui=False):
+        ans = _('To use librarything.com you must sign up for a %sfree account%s '
+                'and enter your username and password separated by a : below.')
+        return '<p>'+ans%('<a href="http://www.librarything.com">', '</a>')
+
 # }}}
 
 def check_for_cover(mi, timeout=5.): # {{{

src/calibre/ebooks/metadata/fetch.py

@@ -251,19 +251,26 @@ class LibraryThing(MetadataSource): # {{{
 
     name = 'LibraryThing'
     metadata_type = 'social'
-    description = _('Downloads series/tags/rating information from librarything.com')
+    description = _('Downloads series/covers/rating information from librarything.com')
 
     def fetch(self):
-        if not self.isbn:
+        if not self.isbn or not self.site_customization:
             return
         from calibre.ebooks.metadata.library_thing import get_social_metadata
+        un, _, pw = self.site_customization.partition(':')
         try:
             self.results = get_social_metadata(self.title, self.book_author,
-                    self.publisher, self.isbn)
+                    self.publisher, self.isbn, username=un, password=pw)
         except Exception, e:
             self.exception = e
             self.tb = traceback.format_exc()
 
+    @property
+    def string_customization_help(self):
+        ans = _('To use librarything.com you must sign up for a %sfree account%s '
+                'and enter your username and password separated by a : below.')
+        return '<p>'+ans%('<a href="http://www.librarything.com">', '</a>')
+
     # }}}
 
 
@@ -411,7 +418,7 @@ def search(title=None, author=None, publisher=None, isbn=None, isbndb_key=None,
                     r.pubdate = pubdate
 
     def fix_case(x):
-        if x and x.isupper():
+        if x:
             x = titlecase(x)
         return x
 

src/calibre/ebooks/metadata/library_thing.py

@@ -4,14 +4,13 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Fetch cover from LibraryThing.com based on ISBN number.
 '''
 
-import sys, socket, os, re, random
+import sys, re, random
 
 from lxml import html
 import mechanize
 
 from calibre import browser, prints
 from calibre.utils.config import OptionParser
-from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.ebooks.chardet import strip_encoding_declarations
 
 OPENLIBRARY = 'http://covers.openlibrary.org/b/isbn/%s-L.jpg?default=false'
@@ -28,6 +27,12 @@ def get_ua():
     ]
     return choices[random.randint(0, len(choices)-1)]
 
+_lt_br = None
+def get_browser():
+    global _lt_br
+    if _lt_br is None:
+        _lt_br = browser(user_agent=get_ua())
+    return _lt_br.clone_browser()
 
 class HeadRequest(mechanize.Request):
 
@@ -35,7 +40,7 @@ class HeadRequest(mechanize.Request):
         return 'HEAD'
 
 def check_for_cover(isbn, timeout=5.):
-    br = browser(user_agent=get_ua())
+    br = get_browser()
     br.set_handle_redirect(False)
     try:
         br.open_novisit(HeadRequest(OPENLIBRARY%isbn), timeout=timeout)
@@ -54,46 +59,16 @@ class ISBNNotFound(LibraryThingError):
 class ServerBusy(LibraryThingError):
     pass
 
-def login(br, username, password, force=True):
+def login(br, username, password):
-    br.open('http://www.librarything.com')
+    raw = br.open('http://www.librarything.com').read()
+    if '>Sign out' in raw:
+        return
     br.select_form('signup')
     br['formusername'] = username
     br['formpassword'] = password
-    br.submit()
+    raw = br.submit().read()
-
+    if '>Sign out' not in raw:
-
+        raise ValueError('Failed to login as %r:%r'%(username, password))
-def cover_from_isbn(isbn, timeout=5., username=None, password=None):
-    src = None
-    br = browser(user_agent=get_ua())
-    try:
-        return br.open(OPENLIBRARY%isbn, timeout=timeout).read(), 'jpg'
-    except:
-        pass # Cover not found
-    if username and password:
-        try:
-            login(br, username, password, force=False)
-        except:
-            pass
-    try:
-        src = br.open_novisit('http://www.librarything.com/isbn/'+isbn,
-                timeout=timeout).read().decode('utf-8', 'replace')
-    except Exception, err:
-        if isinstance(getattr(err, 'args', [None])[0], socket.timeout):
-            err = LibraryThingError(_('LibraryThing.com timed out. Try again later.'))
-        raise err
-    else:
-        s = BeautifulSoup(src)
-        url = s.find('td', attrs={'class':'left'})
-        if url is None:
-            if s.find('div', attrs={'class':'highloadwarning'}) is not None:
-                raise ServerBusy(_('Could not fetch cover as server is experiencing high load. Please try again later.'))
-            raise ISBNNotFound('ISBN: '+isbn+_(' not found.'))
-        url = url.find('img')
-        if url is None:
-            raise LibraryThingError(_('LibraryThing.com server error. Try again later.'))
-        url = re.sub(r'_S[XY]\d+', '', url['src'])
-        cover_data = br.open_novisit(url).read()
-        return cover_data, url.rpartition('.')[-1]
 
 def option_parser():
     parser = OptionParser(usage=\
@@ -113,15 +88,16 @@ def get_social_metadata(title, authors, publisher, isbn, username=None,
     from calibre.ebooks.metadata import MetaInformation
     mi = MetaInformation(title, authors)
     if isbn:
-        br = browser(user_agent=get_ua())
+        br = get_browser()
-        if username and password:
         try:
-                login(br, username, password, force=False)
+            login(br, username, password)
-            except:
-                pass
 
             raw = br.open_novisit('http://www.librarything.com/isbn/'
                         +isbn).read()
+        except:
+            return mi
+        if '/wiki/index.php/HelpThing:Verify' in raw:
+            raise Exception('LibraryThing is blocking calibre.')
         if not raw:
             return mi
         raw = raw.decode('utf-8', 'replace')
@@ -172,15 +148,46 @@ def main(args=sys.argv):
         parser.print_help()
         return 1
     isbn = args[1]
-    mi = get_social_metadata('', [], '', isbn)
+    from calibre.customize.ui import metadata_sources, cover_sources
+    lt = None
+    for x in metadata_sources('social'):
+        if x.name == 'LibraryThing':
+            lt = x
+            break
+    lt('', '', '', isbn, True)
+    lt.join()
+    if lt.exception:
+        print lt.tb
+        return 1
+    mi = lt.results
     prints(mi)
-    cover_data, ext = cover_from_isbn(isbn, username=opts.username,
+    mi.isbn = isbn
-            password=opts.password)
+
-    if not ext:
+    lt = None
-        ext = 'jpg'
+    for x in cover_sources():
-    oname = os.path.abspath(isbn+'.'+ext)
+        if x.name == 'librarything.com covers':
-    open(oname, 'w').write(cover_data)
+            lt = x
-    print 'Cover saved to file', oname
+            break
+
+    from threading import Event
+    from Queue import Queue
+    ev = Event()
+    lt.has_cover(mi, ev)
+    hc = ev.is_set()
+    print 'Has cover:', hc
+    if hc:
+        abort = Event()
+        temp = Queue()
+        lt.get_covers(mi, temp, abort)
+
+        cover = temp.get_nowait()
+        if cover[0]:
+            open(isbn + '.jpg', 'wb').write(cover[1])
+            print 'Cover saved to:', isbn+'.jpg'
+        else:
+            print 'Cover download failed'
+            print cover[2]
+
     return 0
 
 if __name__ == '__main__':

src/calibre/ebooks/metadata/sources/__init__.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+

src/calibre/ebooks/mobi/reader.py

@@ -488,7 +488,7 @@ class MobiReader(object):
 
 
     def remove_random_bytes(self, html):
-        return re.sub('\x14|\x15|\x19|\x1c|\x1d|\xef|\x12|\x13|\xec|\x08',
+        return re.sub('\x14|\x15|\x19|\x1c|\x1d|\xef|\x12|\x13|\xec|\x08|\x01|\x02|\x03|\x04|\x05|\x06|\x07',
                     '', html)
 
     def ensure_unit(self, raw, unit='px'):

src/calibre/ebooks/oeb/base.py

@@ -221,7 +221,10 @@ def rewrite_links(root, link_repl_func, resolve_base_href=False):
                         el.text):
             stylesheet = parseString(el.text)
             replaceUrls(stylesheet, link_repl_func)
-            el.text = '\n'+stylesheet.cssText + '\n'
+            repl = stylesheet.cssText
+            if isbytestring(repl):
+                repl = repl.decode('utf-8')
+            el.text = '\n'+ repl + '\n'
 
         if 'style' in el.attrib:
             text = el.attrib['style']
@@ -234,8 +237,11 @@ def rewrite_links(root, link_repl_func, resolve_base_href=False):
                             set_property(item)
                     elif v.CSS_PRIMITIVE_VALUE == v.cssValueType:
                         set_property(v)
-                el.attrib['style'] = stext.cssText.replace('\n', ' ').replace('\r',
+                repl = stext.cssText.replace('\n', ' ').replace('\r',
                         ' ')
+                if isbytestring(repl):
+                    repl = repl.decode('utf-8')
+                el.attrib['style'] = repl
 
 
 

src/calibre/ebooks/oeb/reader.py

@@ -17,7 +17,7 @@ from lxml import etree
 import cssutils
 
 from calibre.ebooks.oeb.base import OPF1_NS, OPF2_NS, OPF2_NSMAP, DC11_NS, \
-    DC_NSES, OPF
+    DC_NSES, OPF, xml2text
 from calibre.ebooks.oeb.base import OEB_DOCS, OEB_STYLES, OEB_IMAGES, \
     PAGE_MAP_MIME, JPEG_MIME, NCX_MIME, SVG_MIME
 from calibre.ebooks.oeb.base import XMLDECL_RE, COLLAPSE_RE, \
@@ -423,7 +423,7 @@ class OEBReader(object):
             path, frag = urldefrag(href)
             if path not in self.oeb.manifest.hrefs:
                 continue
-            title = ' '.join(xpath(anchor, './/text()'))
+            title = xml2text(anchor)
             title = COLLAPSE_RE.sub(' ', title.strip())
             if href not in titles:
                 order.append(href)

src/calibre/ebooks/oeb/transforms/cover.py

@@ -141,8 +141,8 @@ class CoverManager(object):
             if width is None or height is None:
                 self.log.warning('Failed to read cover dimensions')
                 width, height = 600, 800
-            if self.preserve_aspect_ratio:
+            #if self.preserve_aspect_ratio:
-                width, height = 600, 800
+            #    width, height = 600, 800
             self.svg_template = self.svg_template.replace('__viewbox__',
                     '0 0 %d %d'%(width, height))
             self.svg_template = self.svg_template.replace('__width__',

src/calibre/ebooks/pdf/output.py

@@ -47,7 +47,7 @@ class PDFOutput(OutputFormatPlugin):
                     OptionRecommendation(name='preserve_cover_aspect_ratio',
                         recommended_value=False,
                         help=_('Preserve the aspect ratio of the cover, instead'
-                            ' of stretching it to fill the ull first page of the'
+                            ' of stretching it to fill the full first page of the'
                             ' generated pdf.')
                         ),
                  ])

src/calibre/ebooks/pml/pmlconverter.py

@@ -143,7 +143,9 @@ class PML_HTMLizer(object):
 
     def __init__(self):
         self.state = {}
-        self.toc = TOC()
+        # toc consists of a tuple
+        # (level, (href, id, text))
+        self.toc = []
         self.file_name = ''
 
     def prepare_pml(self, pml):
@@ -494,19 +496,20 @@ class PML_HTMLizer(object):
         output = []
 
         self.state = {}
-        self.toc = TOC()
+        self.toc = []
         self.file_name = file_name
 
         indent_state = {'t': False, 'T': False}
         adv_indent_val = ''
+        # Keep track of the number of empty lines
+        # between paragraphs. When we reach a set number
+        # we assume it's a soft scene break.
+        empty_count = 0
 
         for s in self.STATES:
             self.state[s] = [False, ''];
 
         for line in pml.splitlines():
-            if not line:
-                continue
-
             parsed = []
             empty = True
             basic_indent = indent_state['t']
@@ -541,6 +544,7 @@ class PML_HTMLizer(object):
                         # inside of ="" so we don't have do special processing
                         # for C.
                         t = ''
+                        level = 0
                         if c in 'XC':
                             level = line.read(1)
                         id = 'pml_toc-%s' % len(self.toc)
@@ -552,7 +556,7 @@ class PML_HTMLizer(object):
                         if not value or value == '':
                             text = t
                         else:
-                            self.toc.add_item(os.path.basename(self.file_name), id, value)
+                            self.toc.append((level, (os.path.basename(self.file_name), id, value)))
                             text = '%s<span id="%s"></span>' % (t, id)
                     elif c == 'm':
                         empty = False
@@ -575,10 +579,15 @@ class PML_HTMLizer(object):
                         if indent_state[c]:
                             basic_indent = True
                     elif c == 'T':
-                        indent_state[c] = not indent_state[c]
+                        # Ensure we only store the value on the first T set for the line.
-                        if indent_state[c]:
+                        if not indent_state['T']:
                             adv_indent = True
                             adv_indent_val = self.code_value(line)
+                        else:
+                            # We detected a T previously on this line.
+                            # Don't replace the first detected value.
+                            self.code_value(line)
+                        indent_state['T'] = True
                     elif c == '-':
                         empty = False
                         text = '&shy;'
@@ -592,7 +601,12 @@ class PML_HTMLizer(object):
                 parsed.append(text)
                 c = line.read(1)
 
-            if not empty:
+            if empty:
+                empty_count += 1
+                if empty_count == 3:
+                    output.append('<p>&nbsp;</p>')
+            else:
+                empty_count = 0
                 text = self.end_line()
                 parsed.append(text)
                 
@@ -613,7 +627,72 @@ class PML_HTMLizer(object):
         return output
 
     def get_toc(self):
-        return self.toc
+        '''
+        Toc can have up to 5 levels, 0 - 4 inclusive.
+        
+        This function will add items to their appropriate
+        depth in the TOC tree. If the specified depth is
+        invalid (item would not have a valid parent) add
+        it to the next valid level above the specified
+        level.
+        '''
+        # Base toc object all items will be added to.
+        n_toc = TOC()
+        # Used to track nodes in the toc so we can add
+        # sub items to the appropriate place in tree.
+        t_l0 = None
+        t_l1 = None
+        t_l2 = None
+        t_l3 = None
+
+        for level, (href, id, text) in self.toc:
+            if level == u'0':
+                t_l0 = n_toc.add_item(href, id, text)
+                t_l1 = None
+                t_l2 = None
+                t_l3 = None
+            elif level == u'1':
+                if t_l0 == None:
+                    t_l0 = n_toc
+                t_l1 = t_l0.add_item(href, id, text)
+                t_l2 = None
+                t_l3 = None
+            elif level == u'2':
+                if t_l1 == None:
+                    if t_l0 == None:
+                        t_l1 = n_toc
+                    else:
+                        t_l1 = t_l0
+                t_l2 = t_l1.add_item(href, id, text)
+                t_l3 = None
+            elif level == u'3':
+                if t_l2 == None:
+                    if t_l1 == None:
+                        if t_l0 == None:
+                            t_l2 = n_toc
+                        else:
+                            t_l2 = t_l0
+                    else:
+                        t_l2 = t_l1
+                t_l3 = t_l2.add_item(href, id, text)
+            # Level 4.
+            # Anything above 4 is invalid but we will count
+            # it as level 4.
+            else:
+                if t_l3 == None:
+                    if t_l2 == None:
+                        if t_l1 == None:
+                            if t_l0 == None:
+                                t_l3 = n_toc
+                            else:
+                                t_l3 = t_l0
+                        else:
+                            t_l3 = t_l1
+                    else:
+                        t_l3 = t_l2
+                t_l3.add_item(href, id, text)
+
+        return n_toc
 
 
 def pml_to_html(pml):

src/calibre/ebooks/rtf/rtfml.py

@@ -24,14 +24,15 @@ from calibre.utils.magick.draw import save_cover_data_to, identify_data
 TAGS = {
     'b': '\\b',
     'del': '\\deleted',
-    'h1': '\\b \\par \\pard \\hyphpar',
+    'h1': '\\s1 \\afs32',
-    'h2': '\\b \\par \\pard \\hyphpar',
+    'h2': '\\s2 \\afs28',
-    'h3': '\\b \\par \\pard \\hyphpar',
+    'h3': '\\s3 \\afs28',
-    'h4': '\\b \\par \\pard \\hyphpar',
+    'h4': '\\s4 \\afs23',
-    'h5': '\\b \\par \\pard \\hyphpar',
+    'h5': '\\s5 \\afs23',
-    'h6': '\\b \\par \\pard \\hyphpar',
+    'h6': '\\s6 \\afs21',
-    'li': '\\par \\pard \\hyphpar \t',
+    'i': '\\i',
-    'p': '\\par \\pard \\hyphpar \t',
+    'li': '\t',
+    'p': '\t',
     'sub': '\\sub',
     'sup': '\\super',
     'u': '\\ul',
@@ -39,15 +40,9 @@ TAGS = {
 
 SINGLE_TAGS = {
     'br': '\n{\\line }\n',
-    'div': '\n{\\line }\n',
-}
-
-SINGLE_TAGS_END = {
-    'div': '\n{\\line }\n',
 }
 
 STYLES = [
-    ('display', {'block': '\\par \\pard \\hyphpar'}),
     ('font-weight', {'bold': '\\b', 'bolder': '\\b'}),
     ('font-style', {'italic': '\\i'}),
     ('text-align', {'center': '\\qc', 'left': '\\ql', 'right': '\\qr'}),
@@ -55,6 +50,7 @@ STYLES = [
 ]
 
 BLOCK_TAGS = [
+    'div',
     'p',
     'h1',
     'h2',
@@ -112,14 +108,16 @@ class RTFMLizer(object):
                 stylizer = Stylizer(item.data, item.href, self.oeb_book,
                         self.opts, self.opts.output_profile)
                 output += self.dump_text(item.data.find(XHTML('body')), stylizer)
-                output += '{\\page } '
+                output += '{\\page }'
         for item in self.oeb_book.spine:
             self.log.debug('Converting %s to RTF markup...' % item.href)
             content = unicode(etree.tostring(item.data, encoding=unicode))
             content = self.remove_newlines(content)
+            content = self.remove_tabs(content)
             content = etree.fromstring(content)
             stylizer = Stylizer(content, item.href, self.oeb_book, self.opts, self.opts.output_profile)
             output += self.dump_text(content.find(XHTML('body')), stylizer)
+            output += '{\\page }'
         output += self.footer()
         output = self.insert_images(output)
         output = self.clean_text(output)
@@ -134,8 +132,23 @@ class RTFMLizer(object):
 
         return text
 
+    def remove_tabs(self, text):
+        self.log.debug('\Replace tabs with space for processing...')
+        text = text.replace('\t', ' ')
+
+        return text
+
     def header(self):
-        return u'{\\rtf1{\\info{\\title %s}{\\author %s}}\\ansi\\ansicpg1252\\deff0\\deflang1033' % (self.oeb_book.metadata.title[0].value, authors_to_string([x.value for x in self.oeb_book.metadata.creator]))
+        header = u'{\\rtf1{\\info{\\title %s}{\\author %s}}\\ansi\\ansicpg1252\\deff0\\deflang1033\n' % (self.oeb_book.metadata.title[0].value, authors_to_string([x.value for x in self.oeb_book.metadata.creator]))
+        return header + \
+            '{\\fonttbl{\\f0\\froman\\fprq2\\fcharset128 Times New Roman;}{\\f1\\froman\\fprq2\\fcharset128 Times New Roman;}{\\f2\\fswiss\\fprq2\\fcharset128 Arial;}{\\f3\\fnil\\fprq2\\fcharset128 Arial;}{\\f4\\fnil\\fprq2\\fcharset128 MS Mincho;}{\\f5\\fnil\\fprq2\\fcharset128 Tahoma;}{\\f6\\fnil\\fprq0\\fcharset128 Tahoma;}}\n' \
+            '{\\stylesheet{\\ql \\li0\\ri0\\nowidctlpar\\wrapdefault\\faauto\\rin0\\lin0\\itap0 \\rtlch\\fcs1 \\af25\\afs24\\alang1033 \\ltrch\\fcs0 \\fs24\\lang1033\\langfe255\\cgrid\\langnp1033\\langfenp255 \\snext0 Normal;}\n' \
+            '{\\s1\\ql \\li0\\ri0\\sb240\\sa120\\keepn\\nowidctlpar\\wrapdefault\\faauto\\outlinelevel0\\rin0\\lin0\\itap0 \\rtlch\\fcs1 \\ab\\af0\\afs32\\alang1033 \\ltrch\\fcs0 \\b\\fs32\\lang1033\\langfe255\\loch\\f1\\hich\\af1\\dbch\\af26\\cgrid\\langnp1033\\langfenp255 \\sbasedon15 \\snext16 \\slink21 heading 1;}\n' \
+            '{\\s2\\ql \\li0\\ri0\\sb240\\sa120\\keepn\\nowidctlpar\\wrapdefault\\faauto\\outlinelevel1\\rin0\\lin0\\itap0 \\rtlch\\fcs1 \\ab\\ai\\af0\\afs28\\alang1033 \\ltrch\\fcs0 \\b\\i\\fs28\\lang1033\\langfe255\\loch\\f1\\hich\\af1\\dbch\\af26\\cgrid\\langnp1033\\langfenp255 \\sbasedon15 \\snext16 \\slink22 heading 2;}\n' \
+            '{\\s3\\ql \\li0\\ri0\\sb240\\sa120\\keepn\\nowidctlpar\\wrapdefault\\faauto\\outlinelevel2\\rin0\\lin0\\itap0 \\rtlch\\fcs1 \\ab\\af0\\afs28\\alang1033 \\ltrch\\fcs0 \\b\\fs28\\lang1033\\langfe255\\loch\\f1\\hich\\af1\\dbch\\af26\\cgrid\\langnp1033\\langfenp255 \\sbasedon15 \\snext16 \\slink23 heading 3;}\n' \
+            '{\\s4\\ql \\li0\\ri0\\sb240\\sa120\\keepn\\nowidctlpar\\wrapdefault\\faauto\\outlinelevel3\\rin0\\lin0\\itap0 \\rtlch\\fcs1 \\ab\\ai\\af0\\afs23\\alang1033 \\ltrch\\fcs0\\b\\i\\fs23\\lang1033\\langfe255\\loch\\f1\\hich\\af1\\dbch\\af26\\cgrid\\langnp1033\\langfenp255 \\sbasedon15 \\snext16 \\slink24 heading 4;}\n' \
+            '{\\s5\\ql \\li0\\ri0\\sb240\\sa120\\keepn\\nowidctlpar\\wrapdefault\\faauto\\outlinelevel4\\rin0\\lin0\\itap0 \\rtlch\\fcs1 \\ab\\af0\\afs23\\alang1033 \\ltrch\\fcs0 \\b\\fs23\\lang1033\\langfe255\\loch\\f1\\hich\\af1\\dbch\\af26\\cgrid\\langnp1033\\langfenp255 \\sbasedon15 \\snext16 \\slink25 heading 5;}\n' \
+            '{\\s6\\ql \\li0\\ri0\\sb240\\sa120\\keepn\\nowidctlpar\\wrapdefault\\faauto\\outlinelevel5\\rin0\\lin0\\itap0 \\rtlch\\fcs1 \\ab\\af0\\afs21\\alang1033 \\ltrch\\fcs0 \\b\\fs21\\lang1033\\langfe255\\loch\\f1\\hich\\af1\\dbch\\af26\\cgrid\\langnp1033\\langfenp255 \\sbasedon15 \\snext16 \\slink26 heading 6;}}\n'
 
     def footer(self):
         return ' }'
@@ -170,19 +183,16 @@ class RTFMLizer(object):
         return (hex_string, width, height)
 
     def clean_text(self, text):
-        # Remove excess spaces at beginning and end of lines
-        text = re.sub('(?m)^[ ]+', '', text)
-        text = re.sub('(?m)[ ]+$', '', text)
-
         # Remove excessive newlines
-        #text = re.sub('%s{1,1}' % os.linesep, '%s%s' % (os.linesep, os.linesep), text)
         text = re.sub('%s{3,}' % os.linesep, '%s%s' % (os.linesep, os.linesep), text)
 
         # Remove excessive spaces
         text = re.sub('[ ]{2,}', ' ', text)
+        text = re.sub('\t{2,}', '\t', text)
+        text = re.sub('\t ', '\t', text)
 
+        # Remove excessive line breaks
         text = re.sub(r'(\{\\line \}\s*){3,}', r'{\\line }{\\line }', text)
-        #text = re.compile(r'(\{\\line \}\s*)+(?P<brackets>}*)\s*\{\\par').sub(lambda mo: r'%s{\\par' % mo.group('brackets'), text)
 
         # Remove non-breaking spaces
         text = text.replace(u'\xa0', ' ')
@@ -222,7 +232,7 @@ class RTFMLizer(object):
             block_start = ''
             block_end = ''
             if 'block' not in tag_stack:
-                block_start = '{\\par \\pard \\hyphpar '
+                block_start = '{\\par\\pard\\hyphpar '
                 block_end = '}'
             text += '%s SPECIAL_IMAGE-%s-REPLACE_ME %s' % (block_start, src, block_end)
 
@@ -245,7 +255,7 @@ class RTFMLizer(object):
                 tag_stack.append(style_tag)
 
         # Proccess tags that contain text.
-        if hasattr(elem, 'text') and elem.text != None and elem.text.strip() != '':
+        if hasattr(elem, 'text') and elem.text:
             text += txt2rtf(elem.text)
 
         for item in elem:
@@ -254,16 +264,15 @@ class RTFMLizer(object):
         for i in range(0, tag_count):
             end_tag =  tag_stack.pop()
             if end_tag != 'block':
+                if tag in BLOCK_TAGS:
+                    text += u'\\par\\pard\\plain\\hyphpar}'
+                else:
                     text += u'}'
 
-        single_tag_end = SINGLE_TAGS_END.get(tag, None)
+        if hasattr(elem, 'tail') and elem.tail:
-        if single_tag_end:
-            text += single_tag_end
-
-        if hasattr(elem, 'tail') and elem.tail != None and elem.tail.strip() != '':
             if 'block' in tag_stack:
                 text += '%s' % txt2rtf(elem.tail)
             else:
-                text += '{\\par \\pard \\hyphpar %s}' % txt2rtf(elem.tail)
+                text += '{\\par\\pard\\hyphpar %s}' % txt2rtf(elem.tail)
 
         return text

src/calibre/ebooks/txt/input.py

@@ -83,7 +83,6 @@ class TXTInput(InputFormatPlugin):
             setattr(options, 'markup_chapter_headings', True)
             setattr(options, 'italicize_common_cases', True)
             setattr(options, 'fix_indents', True)
-            setattr(options, 'preserve_spaces', True)
             setattr(options, 'delete_blank_paragraphs', True)
             setattr(options, 'format_scene_breaks', True)
             setattr(options, 'dehyphenate', True)

src/calibre/ebooks/txt/output.py

@@ -8,7 +8,6 @@ import os
 
 from calibre.customize.conversion import OutputFormatPlugin, \
     OptionRecommendation
-from calibre.ebooks.txt.markdownml import MarkdownMLizer
 from calibre.ebooks.txt.txtml import TXTMLizer
 from calibre.ebooks.txt.newlines import TxtNewlines, specified_newlines
 
@@ -44,24 +43,32 @@ class TXTOutput(OutputFormatPlugin):
             recommended_value=False, level=OptionRecommendation.LOW,
             help=_('Force splitting on the max-line-length value when no space '
             'is present. Also allows max-line-length to be below the minimum')),
-        OptionRecommendation(name='markdown_format',
+        OptionRecommendation(name='txt_output_formatting',
-            recommended_value=False, level=OptionRecommendation.LOW,
+             recommended_value='plain',
-            help=_('Produce Markdown formatted text.')),
+             choices=['plain', 'markdown', 'textile'],
+             help=_('Formatting used within the document.\n'
+                    '* plain: Produce plain text.\n'
+                    '* markdown: Produce Markdown formatted text.\n'
+                    '* textile: Produce Textile formatted text.')),
         OptionRecommendation(name='keep_links',
             recommended_value=False, level=OptionRecommendation.LOW,
             help=_('Do not remove links within the document. This is only ' \
-            'useful when paired with the markdown-format option because' \
+            'useful when paired with a txt-output-formatting option that '
-            ' links are always removed with plain text output.')),
+            'is not none because links are always removed with plain text output.')),
         OptionRecommendation(name='keep_image_references',
             recommended_value=False, level=OptionRecommendation.LOW,
             help=_('Do not remove image references within the document. This is only ' \
-            'useful when paired with the markdown-format option because' \
+            'useful when paired with a txt-output-formatting option that '
-            ' image references are always removed with plain text output.')),
+            'is not none because links are always removed with plain text output.')),
      ])
 
     def convert(self, oeb_book, output_path, input_plugin, opts, log):
-        if opts.markdown_format:
+        if opts.txt_output_formatting.lower() == 'markdown':
+            from calibre.ebooks.txt.markdownml import MarkdownMLizer
             writer = MarkdownMLizer(log)
+        elif opts.txt_output_formatting.lower() == 'textile':
+            from calibre.ebooks.txt.textileml import TextileMLizer
+            writer = TextileMLizer(log)
         else:
             writer = TXTMLizer(log)
 

src/calibre/ebooks/txt/processor.py

@@ -20,9 +20,13 @@ HTML_TEMPLATE = u'<html><head><meta http-equiv="Content-Type" content="text/html
 def clean_txt(txt):
     if isbytestring(txt):
         txt = txt.decode('utf-8', 'replace')
-    # Strip whitespace from the beginning and end of the line. Also replace
+    # Strip whitespace from the end of the line. Also replace
     # all line breaks with \n.
-    txt = '\n'.join([line.strip() for line in txt.splitlines()])
+    txt = '\n'.join([line.rstrip() for line in txt.splitlines()])
+    
+    # Replace whitespace at the beginning of the list with &nbsp;
+    txt = re.sub('(?m)(?P<space>[ ]+)', lambda mo: '&nbsp;' * mo.groups('space').count(' '), txt)
+    txt = re.sub('(?m)(?P<space>[\t]+)', lambda mo: '&nbsp;' * 4 * mo.groups('space').count('\t'), txt)
 
     # Condense redundant spaces
     txt = re.sub('[ ]{2,}', ' ', txt)
@@ -31,7 +35,7 @@ def clean_txt(txt):
     txt = re.sub('^\s+(?=.)', '', txt)
     txt = re.sub('(?<=.)\s+$', '', txt)
     # Remove excessive line breaks.
-    txt = re.sub('\n{3,}', '\n\n', txt)
+    txt = re.sub('\n{5,}', '\n\n\n\n', txt)
     #remove ASCII invalid chars : 0 to 8 and 11-14 to 24
     txt = clean_ascii_chars(txt)
 
@@ -59,10 +63,16 @@ def convert_basic(txt, title='', epub_split_size_kb=0):
     txt = split_txt(txt, epub_split_size_kb)
 
     lines = []
+    blank_count = 0
     # Split into paragraphs based on having a blank line between text.
-    for line in txt.split('\n\n'):
+    for line in txt.split('\n'):
         if line.strip():
+            blank_count = 0
             lines.append(u'<p>%s</p>' % prepare_string_for_xml(line.replace('\n', ' ')))
+        else:
+            blank_count += 1
+            if blank_count == 2:
+                lines.append(u'<p>&nbsp;</p>')
 
     return HTML_TEMPLATE % (title, u'\n'.join(lines))
 
@@ -85,7 +95,7 @@ def normalize_line_endings(txt):
     return txt
 
 def separate_paragraphs_single_line(txt):
-    txt = re.sub(u'(?<=.)\n(?=.)', '\n\n', txt)
+    txt = txt.replace('\n', '\n\n')
     return txt
 
 def separate_paragraphs_print_formatted(txt):
@@ -93,7 +103,7 @@ def separate_paragraphs_print_formatted(txt):
     return txt
 
 def preserve_spaces(txt):
-    txt = txt.replace(' ', '&nbsp;')
+    txt = re.sub('(?P<space>[ ]{2,})', lambda mo: ' ' + ('&nbsp;' * (len(mo.group('space')) - 1)), txt)
     txt = txt.replace('\t', '&nbsp;&nbsp;&nbsp;&nbsp;')
     return txt
 

src/calibre/ebooks/txt/textileml.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+'''
+Transform OEB content into Textile formatted plain text
+'''
+
+import re
+
+from lxml import etree
+
+from calibre.ebooks.oeb.base import XHTML
+from calibre.utils.html2textile import html2textile
+
+class TextileMLizer(object):
+
+    def __init__(self, log):
+        self.log = log
+
+    def extract_content(self, oeb_book, opts):
+        self.log.info('Converting XHTML to Textile formatted TXT...')
+        self.oeb_book = oeb_book
+        self.opts = opts
+
+        return self.mlize_spine()
+
+    def mlize_spine(self):
+        output = [u'']
+
+        for item in self.oeb_book.spine:
+            self.log.debug('Converting %s to Textile formatted TXT...' % item.href)
+
+            html = unicode(etree.tostring(item.data.find(XHTML('body')), encoding=unicode))
+
+            if not self.opts.keep_links:
+                html = re.sub(r'<\s*a[^>]*>', '', html)
+                html = re.sub(r'<\s*/\s*a\s*>', '', html)
+            if not self.opts.keep_image_references:
+                html = re.sub(r'<\s*img[^>]*>', '', html)
+                html = re.sub(r'<\s*img\s*>', '', html)
+
+            text = html2textile(html)
+
+            # Ensure the section ends with at least two new line characters.
+            # This is to prevent the last paragraph from a section being
+            # combined into the fist paragraph of the next.
+            end_chars = text[-4:]
+            # Convert all newlines to \n
+            end_chars = end_chars.replace('\r\n', '\n')
+            end_chars = end_chars.replace('\r', '\n')
+            end_chars = end_chars[-2:]
+            if not end_chars[1] == '\n':
+                text += '\n\n'
+            if end_chars[1] == '\n' and not end_chars[0] == '\n':
+                text += '\n'
+
+            output += text
+
+        output = u''.join(output)
+
+        return output

src/calibre/ebooks/txt/txtml.py

@@ -222,6 +222,8 @@ class TXTMLizer(object):
         # Scene breaks.
         if tag == 'hr':
             text.append('\n\n* * *\n\n')
+        elif style['margin-top']:
+            text.append('\n\n' + '\n' * round(style['margin-top']))
 
         # Process tags that contain text.
         if hasattr(elem, 'text') and elem.text:

src/calibre/gui2/__init__.py

@@ -8,12 +8,12 @@ from urllib import unquote
 from PyQt4.Qt import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, \
                          QByteArray, QTranslator, QCoreApplication, QThread, \
                          QEvent, QTimer, pyqtSignal, QDate, QDesktopServices, \
-                         QFileDialog, QMessageBox, QPixmap, QFileIconProvider, \
+                         QFileDialog, QFileIconProvider, \
-                         QIcon, QApplication, QDialog, QPushButton, QUrl, QFont
+                         QIcon, QApplication, QDialog, QUrl, QFont
 
 ORG_NAME = 'KovidsBrain'
 APP_UID  = 'libprs500'
-from calibre.constants import islinux, iswindows, isosx, isfreebsd, isfrozen
+from calibre.constants import islinux, iswindows, isfreebsd, isfrozen
 from calibre.utils.config import Config, ConfigProxy, dynamic, JSONConfig
 from calibre.utils.localization import set_qt_translator
 from calibre.ebooks.metadata.meta import get_metadata, metadata_from_formats
@@ -120,6 +120,8 @@ def _config():
         help='Search history for the LRF viewer')
     c.add_opt('scheduler_search_history', default=[],
         help='Search history for the recipe scheduler')
+    c.add_opt('plugin_search_history', default=[],
+        help='Search history for the recipe scheduler')
     c.add_opt('worker_limit', default=6,
             help=_('Maximum number of waiting worker processes'))
     c.add_opt('get_social_metadata', default=True,
@@ -138,6 +140,7 @@ def _config():
             help=_('Show the average rating per item indication in the tag browser'))
     c.add_opt('disable_animations', default=False,
             help=_('Disable UI animations'))
+    c.add_opt
     return ConfigProxy(c)
 
 config = _config()
@@ -178,104 +181,36 @@ def is_widescreen():
 def extension(path):
     return os.path.splitext(path)[1][1:].lower()
 
-class CopyButton(QPushButton):
-
-    ACTION_KEYS = [Qt.Key_Enter, Qt.Key_Return, Qt.Key_Space]
-
-    def copied(self):
-        self.emit(SIGNAL('copy()'))
-        self.setDisabled(True)
-        self.setText(_('Copied'))
-
-
-    def keyPressEvent(self, ev):
-        try:
-            if ev.key() in self.ACTION_KEYS:
-                self.copied()
-                return
-        except:
-            pass
-        QPushButton.keyPressEvent(self, ev)
-
-
-    def keyReleaseEvent(self, ev):
-        try:
-            if ev.key() in self.ACTION_KEYS:
-                return
-        except:
-            pass
-        QPushButton.keyReleaseEvent(self, ev)
-
-    def mouseReleaseEvent(self, ev):
-        ev.accept()
-        self.copied()
-
-class MessageBox(QMessageBox):
-
-    def __init__(self, type_, title, msg, buttons, parent, det_msg=''):
-        QMessageBox.__init__(self, type_, title, msg, buttons, parent)
-        self.title = title
-        self.msg = msg
-        self.det_msg = det_msg
-        self.setDetailedText(det_msg)
-        # Cannot set keyboard shortcut as the event is not easy to filter
-        self.cb = CopyButton(_('Copy') if isosx else _('Copy to Clipboard'))
-        self.connect(self.cb, SIGNAL('copy()'), self.copy_to_clipboard)
-        self.addButton(self.cb, QMessageBox.ActionRole)
-        default_button = self.button(self.Ok)
-        if default_button is None:
-            default_button = self.button(self.Yes)
-        if default_button is not None:
-            self.setDefaultButton(default_button)
-
-    def copy_to_clipboard(self):
-        QApplication.clipboard().setText('%s: %s\n\n%s' %
-                (self.title, self.msg, self.det_msg))
-
-
 
 def warning_dialog(parent, title, msg, det_msg='', show=False,
         show_copy_button=True):
-    d = MessageBox(QMessageBox.Warning, 'WARNING: '+title, msg, QMessageBox.Ok,
+    from calibre.gui2.dialogs.message_box import MessageBox
-                    parent, det_msg)
+    d = MessageBox(MessageBox.WARNING, 'WARNING: '+title, msg, det_msg, parent=parent,
-    d.setEscapeButton(QMessageBox.Ok)
+            show_copy_button=show_copy_button)
-    d.setIconPixmap(QPixmap(I('dialog_warning.png')))
-    if not show_copy_button:
-        d.cb.setVisible(False)
     if show:
         return d.exec_()
     return d
 
 def error_dialog(parent, title, msg, det_msg='', show=False,
         show_copy_button=True):
-    d = MessageBox(QMessageBox.Critical, 'ERROR: '+title, msg, QMessageBox.Ok,
+    from calibre.gui2.dialogs.message_box import MessageBox
-                    parent, det_msg)
+    d = MessageBox(MessageBox.ERROR, 'ERROR: '+title, msg, det_msg, parent=parent,
-    d.setIconPixmap(QPixmap(I('dialog_error.png')))
+                    show_copy_button=show_copy_button)
-    d.setEscapeButton(QMessageBox.Ok)
-    if not show_copy_button:
-        d.cb.setVisible(False)
     if show:
         return d.exec_()
     return d
 
-def question_dialog(parent, title, msg, det_msg='', show_copy_button=True,
+def question_dialog(parent, title, msg, det_msg='', show_copy_button=False):
-        buttons=QMessageBox.Yes|QMessageBox.No, yes_button=QMessageBox.Yes):
+    from calibre.gui2.dialogs.message_box import MessageBox
-    d = MessageBox(QMessageBox.Question, title, msg, buttons,
+    d = MessageBox(MessageBox.QUESTION, title, msg, det_msg, parent=parent,
-                    parent, det_msg)
+                    show_copy_button=show_copy_button)
-    d.setIconPixmap(QPixmap(I('dialog_question.png')))
+    return d.exec_() == d.Accepted
-    d.setEscapeButton(QMessageBox.No)
-    if not show_copy_button:
-        d.cb.setVisible(False)
-
-    return d.exec_() == yes_button
 
 def info_dialog(parent, title, msg, det_msg='', show=False,
         show_copy_button=True):
-    d = MessageBox(QMessageBox.Information, title, msg, QMessageBox.Ok,
+    from calibre.gui2.dialogs.message_box import MessageBox
-                    parent, det_msg)
+    d = MessageBox(MessageBox.INFO, title, msg, det_msg, parent=parent,
-    d.setIconPixmap(QPixmap(I('dialog_information.png')))
+                    show_copy_button=show_copy_button)
-    if not show_copy_button:
-        d.cb.setVisible(False)
 
     if show:
         return d.exec_()
@@ -550,6 +485,14 @@ def choose_dir(window, name, title, default_dir='~'):
     if dir:
         return dir[0]
 
+def choose_osx_app(window, name, title, default_dir='/Applications'):
+    fd = FileDialog(title=title, parent=window, name=name, mode=QFileDialog.ExistingFile,
+            default_dir=default_dir)
+    app = fd.get_files()
+    fd.setParent(None)
+    if app:
+        return app
+
 def choose_files(window, name, title,
                  filters=[], all_files=True, select_only_single_file=False):
     '''

src/calibre/gui2/actions/add.py

@@ -100,6 +100,9 @@ class AddAction(InterfaceAction):
                 mi = MetaInformation(_('Unknown'), dlg.selected_authors)
                 self.gui.library_view.model().db.import_book(mi, [])
             self.gui.library_view.model().books_added(num)
+            if hasattr(self.gui, 'db_images'):
+                self.gui.db_images.reset()
+            self.gui.tags_view.recount()
 
     def add_isbns(self, books, add_tags=[]):
         from calibre.ebooks.metadata import MetaInformation

src/calibre/gui2/actions/annotate.py

@@ -9,7 +9,7 @@ import os, datetime
 
 from PyQt4.Qt import pyqtSignal, QModelIndex, QThread, Qt
 
-from calibre.gui2 import error_dialog, gprefs
+from calibre.gui2 import error_dialog
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag, NavigableString
 from calibre import strftime
 from calibre.gui2.actions import InterfaceAction
@@ -165,10 +165,12 @@ class FetchAnnotationsAction(InterfaceAction):
                 ka_soup.insert(0,divTag)
                 return ka_soup
 
+            '''
             def mark_book_as_read(self,id):
                 read_tag = gprefs.get('catalog_epub_mobi_read_tag')
                 if read_tag:
                     self.db.set_tags(id, [read_tag], append=True)
+            '''
 
             def canceled(self):
                 self.pd.hide()
@@ -201,10 +203,12 @@ class FetchAnnotationsAction(InterfaceAction):
                         # Update library comments
                         self.db.set_comment(id, mi.comments)
 
+                        '''
                         # Update 'read' tag except for Catalogs/Clippings
                         if bm.value.percent_read >= self.FINISHED_READING_PCT_THRESHOLD:
                             if not set(mi.tags).intersection(ignore_tags):
                                 self.mark_book_as_read(id)
+                        '''
 
                         # Add bookmark file to id
                         self.db.add_format_with_hooks(id, bm.value.bookmark_extension,

src/calibre/gui2/actions/catalog.py

@@ -17,7 +17,7 @@ from calibre.gui2.actions import InterfaceAction
 class GenerateCatalogAction(InterfaceAction):
 
     name = 'Generate Catalog'
-    action_spec = (_('Create catalog of books in your calibre library'), None, None, None)
+    action_spec = (_('Create a catalog of the books in your calibre library'), None, None, None)
     dont_add_to = frozenset(['toolbar-device', 'context-menu-device'])
 
     def generate_catalog(self):

src/calibre/gui2/actions/choose_library.py

@@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
 import os, shutil
 from functools import partial
 
-from PyQt4.Qt import QMenu, Qt, QInputDialog, QThread, pyqtSignal, QProgressDialog
+from PyQt4.Qt import QMenu, Qt, QInputDialog
 
 from calibre import isbytestring
 from calibre.constants import filesystem_encoding
@@ -16,7 +16,6 @@ from calibre.utils.config import prefs
 from calibre.gui2 import gprefs, warning_dialog, Dispatcher, error_dialog, \
     question_dialog, info_dialog
 from calibre.gui2.actions import InterfaceAction
-from calibre.gui2.dialogs.check_library import CheckLibraryDialog
 
 class LibraryUsageStats(object): # {{{
 
@@ -76,76 +75,6 @@ class LibraryUsageStats(object): # {{{
         self.write_stats()
 # }}}
 
-# Check Integrity {{{
-
-class VacThread(QThread):
-
-    check_done = pyqtSignal(object, object)
-    callback   = pyqtSignal(object, object)
-
-    def __init__(self, parent, db):
-        QThread.__init__(self, parent)
-        self.db = db
-        self._parent = parent
-
-    def run(self):
-        err = bad = None
-        try:
-            bad = self.db.check_integrity(self.callbackf)
-        except:
-            import traceback
-            err = traceback.format_exc()
-        self.check_done.emit(bad, err)
-
-    def callbackf(self, progress, msg):
-        self.callback.emit(progress, msg)
-
-
-class CheckIntegrity(QProgressDialog):
-
-    def __init__(self, db, parent=None):
-        QProgressDialog.__init__(self, parent)
-        self.db = db
-        self.setCancelButton(None)
-        self.setMinimum(0)
-        self.setMaximum(100)
-        self.setWindowTitle(_('Checking database integrity'))
-        self.setAutoReset(False)
-        self.setValue(0)
-
-        self.vthread = VacThread(self, db)
-        self.vthread.check_done.connect(self.check_done,
-                type=Qt.QueuedConnection)
-        self.vthread.callback.connect(self.callback, type=Qt.QueuedConnection)
-        self.vthread.start()
-
-    def callback(self, progress, msg):
-        self.setLabelText(msg)
-        self.setValue(int(100*progress))
-
-    def check_done(self, bad, err):
-        if err:
-            error_dialog(self, _('Error'),
-                    _('Failed to check database integrity'),
-                    det_msg=err, show=True)
-        elif bad:
-            titles = [self.db.title(x, index_is_id=True) for x in bad]
-            det_msg = '\n'.join(titles)
-            warning_dialog(self, _('Some inconsistencies found'),
-                    _('The following books had formats or covers listed in the '
-                        'database that are not actually available. '
-                        'The entries for the formats/covers have been removed. '
-                        'You should check them manually. This can '
-                        'happen if you manipulate the files in the '
-                        'library folder directly.'), det_msg=det_msg, show=True)
-        else:
-            info_dialog(self, _('No errors found'),
-                    _('The integrity check completed with no uncorrectable errors found.'),
-                    show=True)
-        self.reset()
-
-# }}}
-
 class ChooseLibraryAction(InterfaceAction):
 
     name = 'Choose Library'
@@ -209,14 +138,12 @@ class ChooseLibraryAction(InterfaceAction):
                                       None, None), attr='action_check_library')
         ac.triggered.connect(self.check_library, type=Qt.QueuedConnection)
         self.maintenance_menu.addAction(ac)
-        ac = self.create_action(spec=(_('Check database integrity'), 'lt.png',
+        ac = self.create_action(spec=(_('Restore database'), 'lt.png',
-                                      None, None), attr='action_check_database')
+                                      None, None),
-        ac.triggered.connect(self.check_database, type=Qt.QueuedConnection)
+                                      attr='action_restore_database')
-        self.maintenance_menu.addAction(ac)
-        ac = self.create_action(spec=(_('Recover database'), 'lt.png',
-                                    None, None), attr='action_restore_database')
         ac.triggered.connect(self.restore_database, type=Qt.QueuedConnection)
         self.maintenance_menu.addAction(ac)
+
         self.choose_menu.addMenu(self.maintenance_menu)
 
     def pick_random(self, *args):
@@ -343,31 +270,52 @@ class ChooseLibraryAction(InterfaceAction):
         db.dirtied(list(db.data.iterallids()))
         info_dialog(self.gui, _('Backup metadata'),
             _('Metadata will be backed up while calibre is running, at the '
-              'rate of approximately 1 book per second.'), show=True)
+              'rate of approximately 1 book every three seconds.'), show=True)
-
-    def check_library(self):
-        db = self.gui.library_view.model().db
-        d = CheckLibraryDialog(self.gui.parent(), db)
-        d.exec_()
-
-    def check_database(self, *args):
-        m = self.gui.library_view.model()
-        m.stop_metadata_backup()
-        try:
-            d = CheckIntegrity(m.db, self.gui)
-            d.exec_()
-        finally:
-            m.start_metadata_backup()
 
     def restore_database(self):
-        info_dialog(self.gui, _('Recover database'), '<p>'+
+        from calibre.gui2.dialogs.restore_library import restore_database
-            _(
+        m = self.gui.library_view.model()
-              'This command rebuilds your calibre database from the information '
+        m.stop_metadata_backup()
-              'stored by calibre in the OPF files.<p>'
+        db = m.db
-              'This function is not currently available in the GUI. You can '
+        db.prefs.disable_setting = True
-              'recover your database using the \'calibredb restore_database\' '
+        if restore_database(db, self.gui):
-              'command line function.'
+            self.gui.library_moved(db.library_path, call_close=False)
-              ), show=True)
+
+    def check_library(self):
+        from calibre.gui2.dialogs.check_library import CheckLibraryDialog, DBCheck
+        self.gui.library_view.save_state()
+        m = self.gui.library_view.model()
+        m.stop_metadata_backup()
+        db = m.db
+        db.prefs.disable_setting = True
+
+        d = DBCheck(self.gui, db)
+        d.start()
+        try:
+            d.conn.close()
+        except:
+            pass
+        d.break_cycles()
+        self.gui.library_moved(db.library_path, call_close=not
+                d.closed_orig_conn)
+        if d.rejected:
+            return
+        if d.error is None:
+            if not question_dialog(self.gui, _('Success'),
+                    _('Found no errors in your calibre library database.'
+                        ' Do you want calibre to check if the files in your '
+                        ' library match the information in the database?')):
+                return
+        else:
+            return error_dialog(self.gui, _('Failed'),
+                    _('Database integrity check failed, click Show details'
+                        ' for details.'), show=True, det_msg=d.error[1])
+
+        d = CheckLibraryDialog(self.gui, m.db)
+        if not d.do_exec():
+            info_dialog(self.gui, _('No problems found'),
+                    _('The files in your library match the information '
+                      'in the database.'), show=True)
 
     def switch_requested(self, location):
         if not self.change_library_allowed():
@@ -385,13 +333,27 @@ class ChooseLibraryAction(InterfaceAction):
 
         prefs['library_path'] = loc
         #from calibre.utils.mem import memory
-        #import weakref, gc
+        #import weakref
-        #ref = weakref.ref(self.gui.library_view.model().db)
+        #from PyQt4.Qt import QTimer
-        #before = memory()/1024**2
+        #self.dbref = weakref.ref(self.gui.library_view.model().db)
+        #self.before_mem = memory()/1024**2
         self.gui.library_moved(loc)
-        #print gc.get_referrers(ref)[0]
+        #QTimer.singleShot(5000, self.debug_leak)
-        #for i in xrange(3): gc.collect()
+
-        #print 'leaked:', memory()/1024**2 - before
+    def debug_leak(self):
+        import gc
+        from calibre.utils.mem import memory
+        ref = self.dbref
+        for i in xrange(3): gc.collect()
+        if ref() is not None:
+            print 'DB object alive:', ref()
+            for r in gc.get_referrers(ref())[:10]:
+                print r
+                print
+        print 'before:', self.before_mem
+        print 'after:', memory()/1024**2
+        self.dbref = self.before_mem = None
+
 
     def qs_requested(self, idx, *args):
         self.switch_requested(self.qs_locations[idx])

src/calibre/gui2/actions/convert.py

@@ -31,7 +31,7 @@ class ConvertAction(InterfaceAction):
                 partial(self.convert_ebook, False, bulk=True))
         cm.addSeparator()
         ac = cm.addAction(
-                _('Create catalog of books in your calibre library'))
+                _('Create a catalog of the books in your calibre library'))
         ac.triggered.connect(self.gui.iactions['Generate Catalog'].generate_catalog)
         self.qaction.setMenu(cm)
         self.qaction.triggered.connect(self.convert_ebook)

src/calibre/gui2/catalog/catalog_bibtex.py

@@ -19,7 +19,9 @@ class PluginWidget(QWidget, Ui_Form):
                      ('bib_entry', 0), #mixed
                      ('bibfile_enc', 0), #utf-8
                      ('bibfile_enctag', 0), #strict
-                     ('impcit', True) ]
+                     ('impcit', True),
+                     ('addfiles', False),
+                     ]
 
     sync_enabled = False
     formats = set(['bib'])
@@ -49,7 +51,7 @@ class PluginWidget(QWidget, Ui_Form):
             opt_value = gprefs.get(self.name + '_' + opt[0], opt[1])
             if opt[0] in ['bibfile_enc', 'bibfile_enctag', 'bib_entry']:
                 getattr(self, opt[0]).setCurrentIndex(opt_value)
-            elif opt[0] == 'impcit' :
+            elif opt[0] in ['impcit', 'addfiles'] :
                 getattr(self, opt[0]).setChecked(opt_value)
             else:
                 getattr(self, opt[0]).setText(opt_value)
@@ -76,7 +78,7 @@ class PluginWidget(QWidget, Ui_Form):
         for opt in self.OPTION_FIELDS:
             if opt[0] in ['bibfile_enc', 'bibfile_enctag', 'bib_entry']:
                 opt_value = getattr(self,opt[0]).currentIndex()
-            elif opt[0] == 'impcit' :
+            elif opt[0] in ['impcit', 'addfiles'] :
                 opt_value = getattr(self, opt[0]).isChecked()
             else :
                 opt_value = unicode(getattr(self, opt[0]).text())

src/calibre/gui2/catalog/catalog_bibtex.ui

@@ -47,7 +47,7 @@
      </item>
     </widget>
    </item>
-   <item row="1" column="1" rowspan="12">
+   <item row="1" column="1" rowspan="11">
     <widget class="QListWidget" name="db_fields">
      <property name="sizePolicy">
       <sizepolicy hsizetype="Preferred" vsizetype="Expanding">
@@ -141,6 +141,13 @@
     </widget>
    </item>
    <item row="8" column="0">
+    <widget class="QCheckBox" name="addfiles">
+     <property name="text">
+      <string>Add files path with formats?</string>
+     </property>
+    </widget>
+   </item>
+   <item row="9" column="0">
     <widget class="QLabel" name="label_3">
      <property name="text">
       <string>Expression to form the BibTeX citation tag:</string>

src/calibre/gui2/catalog/catalog_epub_mobi.py

@@ -335,7 +335,7 @@ class PluginWidget(QWidget,Ui_Form):
         '''
 
         return
-
+        '''
         if new_state == 0:
             # unchecked
             self.merge_source_field.setEnabled(False)
@@ -348,6 +348,7 @@ class PluginWidget(QWidget,Ui_Form):
             self.merge_before.setEnabled(True)
             self.merge_after.setEnabled(True)
             self.include_hr.setEnabled(True)
+        '''
 
     def header_note_source_field_changed(self,new_index):
         '''

src/calibre/gui2/convert/bulk.py

@@ -4,6 +4,8 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
+import shutil
+
 from PyQt4.Qt import QString, SIGNAL
 
 from calibre.gui2.convert.single import Config, sort_formats_by_preference, \
@@ -108,6 +110,11 @@ class BulkConfig(Config):
         idx = oidx if -1 < oidx < self._groups_model.rowCount() else 0
         self.groups.setCurrentIndex(self._groups_model.index(idx))
         self.stack.setCurrentIndex(idx)
+        try:
+            shutil.rmtree(self.plumber.archive_input_tdir, ignore_errors=True)
+        except:
+            pass
+
 
     def setup_output_formats(self, db, preferred_output_format):
         if preferred_output_format:

src/calibre/gui2/convert/look_and_feel.ui

@@ -43,7 +43,7 @@
       <double>0.000000000000000</double>
      </property>
      <property name="maximum">
-      <double>30.000000000000000</double>
+      <double>50.000000000000000</double>
      </property>
      <property name="singleStep">
       <double>1.000000000000000</double>

src/calibre/gui2/convert/regex_builder.py

@@ -6,7 +6,7 @@ __docformat__ = 'restructuredtext en'
 
 import re
 
-from PyQt4.QtCore import SIGNAL, Qt
+from PyQt4.QtCore import SIGNAL, Qt, pyqtSignal
 from PyQt4.QtGui import QDialog, QWidget, QDialogButtonBox, \
                         QBrush, QTextCursor, QTextEdit
 
@@ -19,8 +19,8 @@ from calibre.gui2.dialogs.choose_format import ChooseFormatDialog
 
 class RegexBuilder(QDialog, Ui_RegexBuilder):
 
-    def __init__(self, db, book_id, regex, *args):
+    def __init__(self, db, book_id, regex, doc=None, parent=None):
-        QDialog.__init__(self, *args)
+        QDialog.__init__(self, parent)
         self.setupUi(self)
 
         self.regex.setText(regex)
@@ -28,9 +28,13 @@ class RegexBuilder(QDialog, Ui_RegexBuilder):
 
         if not db or not book_id:
             self.button_box.addButton(QDialogButtonBox.Open)
-        elif not self.select_format(db, book_id):
+        elif not doc and not self.select_format(db, book_id):
             self.cancelled = True
             return
+
+        if doc:
+            self.preview.setPlainText(doc)
+
         self.cancelled = False
         self.connect(self.button_box, SIGNAL('clicked(QAbstractButton*)'), self.button_clicked)
         self.connect(self.regex, SIGNAL('textChanged(QString)'), self.regex_valid)
@@ -153,24 +157,41 @@ class RegexBuilder(QDialog, Ui_RegexBuilder):
         if button == self.button_box.button(QDialogButtonBox.Ok):
             self.accept()
 
+    def doc(self):
+        return unicode(self.preview.toPlainText())
+
 class RegexEdit(QWidget, Ui_Edit):
 
+    doc_update = pyqtSignal(unicode)
+
     def __init__(self, parent=None):
         QWidget.__init__(self, parent)
         self.setupUi(self)
 
         self.book_id = None
         self.db = None
+        self.doc_cache = None
 
         self.connect(self.button, SIGNAL('clicked()'), self.builder)
 
     def builder(self):
-        bld = RegexBuilder(self.db, self.book_id, self.edit.text(), self)
+        bld = RegexBuilder(self.db, self.book_id, self.edit.text(), self.doc_cache, self)
         if bld.cancelled:
             return
+        if not self.doc_cache:
+            self.doc_cache = bld.doc()
+            self.doc_update.emit(self.doc_cache)
         if bld.exec_() == bld.Accepted:
             self.edit.setText(bld.regex.text())
 
+    def doc(self):
+        return self.doc_cache
+
+    def setObjectName(self, *args):
+        QWidget.setObjectName(self, *args)
+        if hasattr(self, 'edit'):
+            self.edit.initialize('regex_edit_'+unicode(self.objectName()))
+
     def set_msg(self, msg):
         self.msg.setText(msg)
 
@@ -180,8 +201,11 @@ class RegexEdit(QWidget, Ui_Edit):
     def set_db(self, db):
         self.db = db
 
+    def set_doc(self, doc):
+        self.doc_cache = doc
+
     def break_cycles(self):
-        self.db = None
+        self.db = self.doc_cache = None
 
     @property
     def text(self):

src/calibre/gui2/convert/search_and_replace.py

@@ -35,13 +35,32 @@ class SearchAndReplaceWidget(Widget, Ui_Form):
         self.opt_sr3_search.set_book_id(book_id)
         self.opt_sr3_search.set_db(db)
 
+        self.opt_sr1_search.doc_update.connect(self.update_doc)
+        self.opt_sr2_search.doc_update.connect(self.update_doc)
+        self.opt_sr3_search.doc_update.connect(self.update_doc)
+
     def break_cycles(self):
         Widget.break_cycles(self)
 
+        def d(x):
+            try:
+                x.disconnect()
+            except:
+                pass
+
+        d(self.opt_sr1_search)
+        d(self.opt_sr2_search)
+        d(self.opt_sr3_search)
+
         self.opt_sr1_search.break_cycles()
         self.opt_sr2_search.break_cycles()
         self.opt_sr3_search.break_cycles()
 
+    def update_doc(self, doc):
+        self.opt_sr1_search.set_doc(doc)
+        self.opt_sr2_search.set_doc(doc)
+        self.opt_sr3_search.set_doc(doc)
+
     def pre_commit_check(self):
         for x in ('sr1_search', 'sr2_search', 'sr3_search'):
             x = getattr(self, 'opt_'+x)

src/calibre/gui2/convert/single.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import sys, cPickle
+import sys, cPickle, shutil
 
 from PyQt4.Qt import QString, SIGNAL, QAbstractListModel, Qt, QVariant, QFont
 
@@ -224,6 +224,10 @@ class Config(ResizableDialog, Ui_Dialog):
         idx = oidx if -1 < oidx < self._groups_model.rowCount() else 0
         self.groups.setCurrentIndex(self._groups_model.index(idx))
         self.stack.setCurrentIndex(idx)
+        try:
+            shutil.rmtree(self.plumber.archive_input_tdir, ignore_errors=True)
+        except:
+            pass
 
 
     def setup_input_output_formats(self, db, book_id, preferred_input_format,

src/calibre/gui2/convert/txt_output.py

@@ -4,7 +4,6 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
-from PyQt4.Qt import Qt
 
 from calibre.gui2.convert.txt_output_ui import Ui_Form
 from calibre.gui2.convert import Widget
@@ -21,26 +20,14 @@ class PluginWidget(Widget, Ui_Form):
     def __init__(self, parent, get_option, get_help, db=None, book_id=None):
         Widget.__init__(self, parent,
         ['newline', 'max_line_length', 'force_max_line_length',
-        'inline_toc', 'markdown_format', 'keep_links', 'keep_image_references',
+        'inline_toc', 'txt_output_formatting', 'keep_links', 'keep_image_references',
         'txt_output_encoding'])
         self.db, self.book_id = db, book_id
         for x in get_option('newline').option.choices:
             self.opt_newline.addItem(x)
+        for x in get_option('txt_output_formatting').option.choices:
+            self.opt_txt_output_formatting.addItem(x)
         self.initialize_options(get_option, get_help, db, book_id)
 
-        self.opt_markdown_format.stateChanged.connect(self.enable_markdown_format)
-        self.enable_markdown_format(self.opt_markdown_format.checkState())
-
     def break_cycles(self):
         Widget.break_cycles(self)
-
-        try:
-            self.opt_markdown_format.stateChanged.disconnect()
-        except:
-            pass
-
-    def enable_markdown_format(self, state):
-        state = state == Qt.Checked
-        self.opt_keep_links.setEnabled(state)
-        self.opt_keep_image_references.setEnabled(state)
-

src/calibre/gui2/convert/txt_output.ui

@@ -6,15 +6,38 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>477</width>
+    <width>392</width>
-    <height>300</height>
+    <height>346</height>
    </rect>
   </property>
   <property name="windowTitle">
    <string>Form</string>
   </property>
+  <layout class="QVBoxLayout" name="verticalLayout_2">
+   <item>
+    <widget class="QGroupBox" name="groupBox">
+     <property name="title">
+      <string>General</string>
+     </property>
      <layout class="QGridLayout" name="gridLayout">
       <item row="0" column="0">
+       <widget class="QLabel" name="label_3">
+        <property name="text">
+         <string>Output &amp;Encoding:</string>
+        </property>
+        <property name="buddy">
+         <cstring>opt_txt_output_encoding</cstring>
+        </property>
+       </widget>
+      </item>
+      <item row="0" column="1">
+       <widget class="EncodingComboBox" name="opt_txt_output_encoding">
+        <property name="editable">
+         <bool>true</bool>
+        </property>
+       </widget>
+      </item>
+      <item row="1" column="0">
        <widget class="QLabel" name="label">
         <property name="text">
          <string>&amp;Line ending style:</string>
@@ -24,32 +47,31 @@
         </property>
        </widget>
       </item>
-   <item row="0" column="1">
+      <item row="1" column="1">
        <widget class="QComboBox" name="opt_newline"/>
       </item>
-   <item row="8" column="0">
+      <item row="2" column="0">
-    <spacer name="verticalSpacer">
+       <widget class="QLabel" name="label_4">
-     <property name="orientation">
-      <enum>Qt::Vertical</enum>
-     </property>
-     <property name="sizeHint" stdset="0">
-      <size>
-       <width>20</width>
-       <height>246</height>
-      </size>
-     </property>
-    </spacer>
-   </item>
-   <item row="4" column="0" colspan="2">
-    <widget class="QCheckBox" name="opt_inline_toc">
         <property name="text">
-      <string>&amp;Inline TOC</string>
+         <string>&amp;Formatting:</string>
+        </property>
+        <property name="buddy">
+         <cstring>opt_txt_output_formatting</cstring>
         </property>
        </widget>
       </item>
-   <item row="1" column="1">
+      <item row="2" column="1">
-    <widget class="QSpinBox" name="opt_max_line_length"/>
+       <widget class="QComboBox" name="opt_txt_output_formatting"/>
       </item>
+     </layout>
+    </widget>
+   </item>
+   <item>
+    <widget class="QGroupBox" name="groupBox_2">
+     <property name="title">
+      <string>Plain</string>
+     </property>
+     <layout class="QGridLayout" name="gridLayout_2">
       <item row="1" column="0">
        <widget class="QLabel" name="label_2">
         <property name="text">
@@ -60,46 +82,47 @@
         </property>
        </widget>
       </item>
-   <item row="3" column="0" colspan="2">
+      <item row="1" column="1">
+       <widget class="QSpinBox" name="opt_max_line_length"/>
+      </item>
+      <item row="2" column="0" colspan="2">
        <widget class="QCheckBox" name="opt_force_max_line_length">
         <property name="text">
          <string>Force maximum line length</string>
         </property>
        </widget>
       </item>
-   <item row="5" column="0">
+      <item row="0" column="0">
-    <widget class="QCheckBox" name="opt_markdown_format">
+       <widget class="QCheckBox" name="opt_inline_toc">
         <property name="text">
-      <string>Apply Markdown formatting to text</string>
+         <string>&amp;Inline TOC</string>
         </property>
        </widget>
       </item>
-   <item row="6" column="0">
+     </layout>
+    </widget>
+   </item>
+   <item>
+    <widget class="QGroupBox" name="groupBox_3">
+     <property name="title">
+      <string>Markdown, Textile</string>
+     </property>
+     <layout class="QVBoxLayout" name="verticalLayout">
+      <item>
        <widget class="QCheckBox" name="opt_keep_links">
         <property name="text">
          <string>Do not remove links (&lt;a&gt; tags) before processing</string>
         </property>
        </widget>
       </item>
-   <item row="7" column="0">
+      <item>
        <widget class="QCheckBox" name="opt_keep_image_references">
         <property name="text">
          <string>Do not remove image references before processing</string>
         </property>
        </widget>
       </item>
-   <item row="2" column="0">
+     </layout>
-    <widget class="QLabel" name="label_3">
-     <property name="text">
-      <string>Output Encoding:</string>
-     </property>
-    </widget>
-   </item>
-   <item row="2" column="1">
-    <widget class="EncodingComboBox" name="opt_txt_output_encoding">
-     <property name="editable">
-      <bool>true</bool>
-     </property>
     </widget>
    </item>
   </layout>

src/calibre/gui2/custom_column_widgets.py

@@ -151,12 +151,27 @@ class DateEdit(QDateEdit):
     def set_to_today(self):
         self.setDate(now())
 
+    def set_to_clear(self):
+        self.setDate(UNDEFINED_QDATE)
+
 class DateTime(Base):
 
     def setup_ui(self, parent):
         cm = self.col_metadata
-        self.widgets = [QLabel('&'+cm['name']+':', parent), DateEdit(parent),
+        self.widgets = [QLabel('&'+cm['name']+':', parent), DateEdit(parent)]
-            QLabel(''), QPushButton(_('Set \'%s\' to today')%cm['name'], parent)]
+        self.widgets.append(QLabel(''))
+        w = QWidget(parent)
+        self.widgets.append(w)
+        l = QHBoxLayout()
+        l.setContentsMargins(0, 0, 0, 0)
+        w.setLayout(l)
+        l.addStretch(1)
+        self.today_button = QPushButton(_('Set \'%s\' to today')%cm['name'], parent)
+        l.addWidget(self.today_button)
+        self.clear_button = QPushButton(_('Clear \'%s\'')%cm['name'], parent)
+        l.addWidget(self.clear_button)
+        l.addStretch(2)
+
         w = self.widgets[1]
         format = cm['display'].get('date_format','')
         if not format:
@@ -165,7 +180,8 @@ class DateTime(Base):
         w.setCalendarPopup(True)
         w.setMinimumDate(UNDEFINED_QDATE)
         w.setSpecialValueText(_('Undefined'))
-        self.widgets[3].clicked.connect(w.set_to_today)
+        self.today_button.clicked.connect(w.set_to_today)
+        self.clear_button.clicked.connect(w.set_to_clear)
 
     def setter(self, val):
         if val is None:
@@ -470,11 +486,48 @@ class BulkBase(Base):
         self.setter(val)
 
     def commit(self, book_ids, notify=False):
+        if not self.a_c_checkbox.isChecked():
+            return
         val = self.gui_val
         val = self.normalize_ui_val(val)
         if val != self.initial_val:
             self.db.set_custom_bulk(book_ids, val, num=self.col_id, notify=notify)
 
+    def make_widgets(self, parent, main_widget_class, extra_label_text=''):
+        w = QWidget(parent)
+        self.widgets = [QLabel('&'+self.col_metadata['name']+':', w), w]
+        l = QHBoxLayout()
+        l.setContentsMargins(0, 0, 0, 0)
+        w.setLayout(l)
+        self.main_widget = main_widget_class(w)
+        l.addWidget(self.main_widget)
+        l.setStretchFactor(self.main_widget, 10)
+        self.a_c_checkbox = QCheckBox( _('Apply changes'), w)
+        l.addWidget(self.a_c_checkbox)
+        self.ignore_change_signals = True
+
+        # connect to the various changed signals so we can auto-update the
+        # apply changes checkbox
+        if hasattr(self.main_widget, 'editTextChanged'):
+            # editable combobox widgets
+            self.main_widget.editTextChanged.connect(self.a_c_checkbox_changed)
+        if hasattr(self.main_widget, 'textChanged'):
+            # lineEdit widgets
+            self.main_widget.textChanged.connect(self.a_c_checkbox_changed)
+        if hasattr(self.main_widget, 'currentIndexChanged'):
+            # combobox widgets
+            self.main_widget.currentIndexChanged[int].connect(self.a_c_checkbox_changed)
+        if hasattr(self.main_widget, 'valueChanged'):
+            # spinbox widgets
+            self.main_widget.valueChanged.connect(self.a_c_checkbox_changed)
+        if hasattr(self.main_widget, 'dateChanged'):
+            # dateEdit widgets
+            self.main_widget.dateChanged.connect(self.a_c_checkbox_changed)
+
+    def a_c_checkbox_changed(self):
+        if not self.ignore_change_signals:
+            self.a_c_checkbox.setChecked(True)
+
 class BulkBool(BulkBase, Bool):
 
     def get_initial_value(self, book_ids):
@@ -484,58 +537,144 @@ class BulkBool(BulkBase, Bool):
             if tweaks['bool_custom_columns_are_tristate'] == 'no' and val is None:
                 val = False
             if value is not None and value != val:
-                return 'nochange'
+                return None
             value = val
         return value
 
     def setup_ui(self, parent):
-        self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent),
+        self.make_widgets(parent, QComboBox)
-                QComboBox(parent)]
+        items = [_('Yes'), _('No'), _('Undefined')]
-        w = self.widgets[1]
+        icons = [I('ok.png'), I('list_remove.png'), I('blank.png')]
-        items = [_('Yes'), _('No'), _('Undefined'), _('Do not change')]
+        self.main_widget.blockSignals(True)
-        icons = [I('ok.png'), I('list_remove.png'), I('blank.png'), I('blank.png')]
         for icon, text in zip(icons, items):
-            w.addItem(QIcon(icon), text)
+            self.main_widget.addItem(QIcon(icon), text)
+        self.main_widget.blockSignals(False)
 
     def getter(self):
-        val = self.widgets[1].currentIndex()
+        val = self.main_widget.currentIndex()
-        return {3: 'nochange', 2: None, 1: False, 0: True}[val]
+        return {2: None, 1: False, 0: True}[val]
 
     def setter(self, val):
-        val = {'nochange': 3, None: 2, False: 1, True: 0}[val]
+        val = {None: 2, False: 1, True: 0}[val]
-        self.widgets[1].setCurrentIndex(val)
+        self.main_widget.setCurrentIndex(val)
+        self.ignore_change_signals = False
 
     def commit(self, book_ids, notify=False):
+        if not self.a_c_checkbox.isChecked():
+            return
         val = self.gui_val
         val = self.normalize_ui_val(val)
-        if val != self.initial_val and val != 'nochange':
         if tweaks['bool_custom_columns_are_tristate'] == 'no' and val is None:
             val = False
         self.db.set_custom_bulk(book_ids, val, num=self.col_id, notify=notify)
 
-class BulkInt(BulkBase, Int):
+class BulkInt(BulkBase):
-    pass
 
-class BulkFloat(BulkBase, Float):
+    def setup_ui(self, parent):
-    pass
+        self.make_widgets(parent, QSpinBox)
+        self.main_widget.setRange(-100, sys.maxint)
+        self.main_widget.setSpecialValueText(_('Undefined'))
+        self.main_widget.setSingleStep(1)
 
-class BulkRating(BulkBase, Rating):
+    def setter(self, val):
-    pass
+        if val is None:
+            val = self.main_widget.minimum()
+        else:
+            val = int(val)
+        self.main_widget.setValue(val)
+        self.ignore_change_signals = False
 
-class BulkDateTime(BulkBase, DateTime):
+    def getter(self):
-    pass
+        val = self.main_widget.value()
+        if val == self.main_widget.minimum():
+            val = None
+        return val
+
+class BulkFloat(BulkInt):
+
+    def setup_ui(self, parent):
+        self.make_widgets(parent, QDoubleSpinBox)
+        self.main_widget.setRange(-100., float(sys.maxint))
+        self.main_widget.setDecimals(2)
+        self.main_widget.setSpecialValueText(_('Undefined'))
+        self.main_widget.setSingleStep(1)
+
+class BulkRating(BulkBase):
+
+    def setup_ui(self, parent):
+        self.make_widgets(parent, QSpinBox)
+        self.main_widget.setRange(0, 5)
+        self.main_widget.setSuffix(' '+_('star(s)'))
+        self.main_widget.setSpecialValueText(_('Unrated'))
+        self.main_widget.setSingleStep(1)
+
+    def setter(self, val):
+        if val is None:
+            val = 0
+        self.main_widget.setValue(int(round(val/2.)))
+        self.ignore_change_signals = False
+
+    def getter(self):
+        val = self.main_widget.value()
+        if val == 0:
+            val = None
+        else:
+            val *= 2
+        return val
+
+class BulkDateTime(BulkBase):
+
+    def setup_ui(self, parent):
+        cm = self.col_metadata
+        self.make_widgets(parent, DateEdit)
+        self.widgets.append(QLabel(''))
+        w = QWidget(parent)
+        self.widgets.append(w)
+        l = QHBoxLayout()
+        l.setContentsMargins(0, 0, 0, 0)
+        w.setLayout(l)
+        l.addStretch(1)
+        self.today_button = QPushButton(_('Set \'%s\' to today')%cm['name'], parent)
+        l.addWidget(self.today_button)
+        self.clear_button = QPushButton(_('Clear \'%s\'')%cm['name'], parent)
+        l.addWidget(self.clear_button)
+        l.addStretch(2)
+
+        w = self.main_widget
+        format = cm['display'].get('date_format','')
+        if not format:
+            format = 'dd MMM yyyy'
+        w.setDisplayFormat(format)
+        w.setCalendarPopup(True)
+        w.setMinimumDate(UNDEFINED_QDATE)
+        w.setSpecialValueText(_('Undefined'))
+        self.today_button.clicked.connect(w.set_to_today)
+        self.clear_button.clicked.connect(w.set_to_clear)
+
+    def setter(self, val):
+        if val is None:
+            val = self.main_widget.minimumDate()
+        else:
+            val = QDate(val.year, val.month, val.day)
+        self.main_widget.setDate(val)
+        self.ignore_change_signals = False
+
+    def getter(self):
+        val = self.main_widget.date()
+        if val == UNDEFINED_QDATE:
+            val = None
+        else:
+            val = qt_to_dt(val)
+        return val
 
 class BulkSeries(BulkBase):
 
     def setup_ui(self, parent):
+        self.make_widgets(parent, EnComboBox)
         values = self.all_values = list(self.db.all_custom(num=self.col_id))
         values.sort(key=sort_key)
-        w = EnComboBox(parent)
+        self.main_widget.setSizeAdjustPolicy(self.main_widget.AdjustToMinimumContentsLengthWithIcon)
-        w.setSizeAdjustPolicy(w.AdjustToMinimumContentsLengthWithIcon)
+        self.main_widget.setMinimumContentsLength(25)
-        w.setMinimumContentsLength(25)
-        self.name_widget = w
-        self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), w]
-
         self.widgets.append(QLabel('', parent))
         w = QWidget(parent)
         layout = QHBoxLayout(w)
@@ -555,15 +694,24 @@ class BulkSeries(BulkBase):
         layout.addWidget(self.series_start_number)
         layout.addItem(QSpacerItem(20, 10, QSizePolicy.Expanding, QSizePolicy.Minimum))
         self.widgets.append(w)
+        self.idx_widget.stateChanged.connect(self.check_changed_checkbox)
+        self.force_number.stateChanged.connect(self.check_changed_checkbox)
+        self.series_start_number.valueChanged.connect(self.check_changed_checkbox)
+        self.remove_series.stateChanged.connect(self.check_changed_checkbox)
+        self.ignore_change_signals = False
+
+    def check_changed_checkbox(self):
+        self.a_c_checkbox.setChecked(True)
 
     def initialize(self, book_id):
         self.idx_widget.setChecked(False)
         for c in self.all_values:
-            self.name_widget.addItem(c)
+            self.main_widget.addItem(c)
-        self.name_widget.setEditText('')
+        self.main_widget.setEditText('')
+        self.a_c_checkbox.setChecked(False)
 
     def getter(self):
-        n = unicode(self.name_widget.currentText()).strip()
+        n = unicode(self.main_widget.currentText()).strip()
         i = self.idx_widget.checkState()
         f = self.force_number.checkState()
         s = self.series_start_number.value()
@@ -571,6 +719,8 @@ class BulkSeries(BulkBase):
         return n, i, f, s, r
 
     def commit(self, book_ids, notify=False):
+        if not self.a_c_checkbox.isChecked():
+            return
         val, update_indices, force_start, at_value, clear = self.gui_val
         val = None if clear else self.normalize_ui_val(val)
         if clear or val != '':
@@ -598,9 +748,9 @@ class BulkEnumeration(BulkBase, Enumeration):
 
     def get_initial_value(self, book_ids):
         value = None
-        ret_value = None
+        first = True
         dialog_shown = False
-        for i,book_id in enumerate(book_ids):
+        for book_id in book_ids:
             val = self.db.get_custom(book_id, num=self.col_id, index_is_id=True)
             if val and val not in self.col_metadata['display']['enum_values']:
                 if not dialog_shown:
@@ -610,44 +760,32 @@ class BulkEnumeration(BulkBase, Enumeration):
                                                     self.col_metadata['name']),
                             show=True, show_copy_button=False)
                     dialog_shown = True
-                ret_value = ' nochange '
+            if first:
-            elif (value is not None and value != val) or (val and i != 0):
-                ret_value = ' nochange '
                 value = val
-        if ret_value is None:
+                first = False
+            elif value != val:
+                value = None
+        if not value:
+            self.ignore_change_signals = False
         return value
-        return ret_value
 
     def setup_ui(self, parent):
-        self.parent = parent
+        self.make_widgets(parent, QComboBox)
-        self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent),
-                QComboBox(parent)]
-        w = self.widgets[1]
         vals = self.col_metadata['display']['enum_values']
-        w.addItem('Do Not Change')
+        self.main_widget.blockSignals(True)
-        w.addItem('')
+        self.main_widget.addItem('')
-        for v in vals:
+        self.main_widget.addItems(vals)
-            w.addItem(v)
+        self.main_widget.blockSignals(False)
 
     def getter(self):
-        if self.widgets[1].currentIndex() == 0:
+        return unicode(self.main_widget.currentText())
-            return ' nochange '
-        return unicode(self.widgets[1].currentText())
 
     def setter(self, val):
-        if val == ' nochange ':
-            self.widgets[1].setCurrentIndex(0)
-        else:
         if val is None:
-                self.widgets[1].setCurrentIndex(1)
+            self.main_widget.setCurrentIndex(0)
         else:
-                self.widgets[1].setCurrentIndex(self.widgets[1].findText(val))
+            self.main_widget.setCurrentIndex(self.main_widget.findText(val))
-
+        self.ignore_change_signals = False
-    def commit(self, book_ids, notify=False):
-        val = self.gui_val
-        val = self.normalize_ui_val(val)
-        if val != self.initial_val and val != ' nochange ':
-            self.db.set_custom_bulk(book_ids, val, num=self.col_id, notify=notify)
 
 class RemoveTags(QWidget):
 
@@ -658,11 +796,10 @@ class RemoveTags(QWidget):
         layout.setContentsMargins(0, 0, 0, 0)
 
         self.tags_box = CompleteLineEdit(parent, values)
-        layout.addWidget(self.tags_box, stretch = 1)
+        layout.addWidget(self.tags_box, stretch=3)
-        # self.tags_box.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Preferred)
-
         self.checkbox = QCheckBox(_('Remove all tags'), parent)
         layout.addWidget(self.checkbox)
+        layout.addStretch(1)
         self.setLayout(layout)
         self.connect(self.checkbox, SIGNAL('stateChanged(int)'), self.box_touched)
 
@@ -679,39 +816,45 @@ class BulkText(BulkBase):
         values = self.all_values = list(self.db.all_custom(num=self.col_id))
         values.sort(key=sort_key)
         if self.col_metadata['is_multiple']:
-            w = CompleteLineEdit(parent, values)
+            self.make_widgets(parent, CompleteLineEdit,
-            w.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Preferred)
+                              extra_label_text=_('tags to add'))
-            self.widgets = [QLabel('&'+self.col_metadata['name']+': ' +
+            self.main_widget.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Preferred)
-                                   _('tags to add'), parent), w]
+            self.adding_widget = self.main_widget
-            self.adding_widget = w
 
             w = RemoveTags(parent, values)
             self.widgets.append(QLabel('&'+self.col_metadata['name']+': ' +
                                        _('tags to remove'), parent))
             self.widgets.append(w)
             self.removing_widget = w
+            w.tags_box.textChanged.connect(self.a_c_checkbox_changed)
+            w.checkbox.stateChanged.connect(self.a_c_checkbox_changed)
         else:
-            w = EnComboBox(parent)
+            self.make_widgets(parent, EnComboBox)
-            w.setSizeAdjustPolicy(w.AdjustToMinimumContentsLengthWithIcon)
+            self.main_widget.setSizeAdjustPolicy(
-            w.setMinimumContentsLength(25)
+                        self.main_widget.AdjustToMinimumContentsLengthWithIcon)
-            self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), w]
+            self.main_widget.setMinimumContentsLength(25)
+        self.ignore_change_signals = False
 
     def initialize(self, book_ids):
         if self.col_metadata['is_multiple']:
-            self.widgets[1].update_items_cache(self.all_values)
+            self.main_widget.update_items_cache(self.all_values)
         else:
             val = self.get_initial_value(book_ids)
             self.initial_val = val = self.normalize_db_val(val)
             idx = None
+            self.main_widget.blockSignals(True)
             for i, c in enumerate(self.all_values):
                 if c == val:
                     idx = i
-                self.widgets[1].addItem(c)
+                self.main_widget.addItem(c)
-            self.widgets[1].setEditText('')
+            self.main_widget.setEditText('')
             if idx is not None:
-                self.widgets[1].setCurrentIndex(idx)
+                self.main_widget.setCurrentIndex(idx)
+            self.main_widget.blockSignals(False)
 
     def commit(self, book_ids, notify=False):
+        if not self.a_c_checkbox.isChecked():
+            return
         if self.col_metadata['is_multiple']:
             remove_all, adding, rtext = self.gui_val
             remove = set()
@@ -740,7 +883,7 @@ class BulkText(BulkBase):
                     unicode(self.adding_widget.text()), \
                     unicode(self.removing_widget.tags_box.text())
 
-        val = unicode(self.widgets[1].currentText()).strip()
+        val = unicode(self.main_widget.currentText()).strip()
         if not val:
             val = None
         return val

src/calibre/gui2/device.py

@@ -7,13 +7,13 @@ import os, traceback, Queue, time, cStringIO, re, sys
 from threading import Thread
 
 from PyQt4.Qt import QMenu, QAction, QActionGroup, QIcon, SIGNAL, \
-                     Qt, pyqtSignal, QDialog, QMessageBox
+                     Qt, pyqtSignal, QDialog
 
 from calibre.customize.ui import available_input_formats, available_output_formats, \
     device_plugins
 from calibre.devices.interface import DevicePlugin
 from calibre.devices.errors import UserFeedback, OpenFeedback
-from calibre.gui2.dialogs.choose_format import ChooseFormatDialog
+from calibre.gui2.dialogs.choose_format_device import ChooseFormatDeviceDialog
 from calibre.utils.ipc.job import BaseJob
 from calibre.devices.scanner import DeviceScanner
 from calibre.gui2 import config, error_dialog, Dispatcher, dynamic, \
@@ -609,10 +609,8 @@ class DeviceMixin(object): # {{{
         autos = u'\n'.join(map(unicode, map(force_unicode, autos)))
         return self.ask_a_yes_no_question(
                 _('No suitable formats'), msg,
-                buttons=QMessageBox.Yes|QMessageBox.Cancel,
                 ans_when_user_unavailable=True,
-                det_msg=autos,
+                det_msg=autos
-                show_copy_button=False
         )
 
     def set_default_thumbnail(self, height):
@@ -689,7 +687,7 @@ class DeviceMixin(object): # {{{
         except:
             pass
         if not self.device_error_dialog.isVisible():
-            self.device_error_dialog.setDetailedText(job.details)
+            self.device_error_dialog.set_details(job.details)
             self.device_error_dialog.show()
 
     # Device connected {{{
@@ -826,8 +824,24 @@ class DeviceMixin(object): # {{{
 
         fmt = None
         if specific:
-            d = ChooseFormatDialog(self, _('Choose format to send to device'),
+            formats = []
-                                self.device_manager.device.settings().format_map)
+            aval_out_formats = available_output_formats()
+            format_count = {}
+            for row in rows:
+                fmts = self.library_view.model().db.formats(row.row())
+                if fmts:
+                    for f in fmts.split(','):
+                        f = f.lower()
+                        if format_count.has_key(f):
+                            format_count[f] += 1
+                        else:
+                            format_count[f] = 1
+            for f in self.device_manager.device.settings().format_map:
+                if f in format_count.keys():
+                    formats.append((f, _('%i of %i Books' % (format_count[f], len(rows))), True if f in aval_out_formats else False))
+                elif f in aval_out_formats:
+                    formats.append((f, _('0 of %i Books' % len(rows)), True))
+            d = ChooseFormatDeviceDialog(self, _('Choose format to send to device'), formats)
             if d.exec_() != QDialog.Accepted:
                 return
             if d.format():

src/calibre/gui2/dialogs/catalog.py

@@ -8,15 +8,12 @@ __docformat__ = 'restructuredtext en'
 
 import os, sys
 
-from PyQt4 import QtGui
-from PyQt4.Qt import QDialog, SIGNAL
-
 from calibre.customize.ui import config
 from calibre.gui2.dialogs.catalog_ui import Ui_Dialog
-from calibre.gui2 import dynamic
+from calibre.gui2 import dynamic, ResizableDialog
 from calibre.customize.ui import catalog_plugins
 
-class Catalog(QDialog, Ui_Dialog):
+class Catalog(ResizableDialog, Ui_Dialog):
     ''' Catalog Dialog builder'''
 
     def __init__(self, parent, dbspec, ids, db):
@@ -24,10 +21,8 @@ class Catalog(QDialog, Ui_Dialog):
         from calibre import prints as info
         from PyQt4.uic import compileUi
 
-        QDialog.__init__(self, parent)
+        ResizableDialog.__init__(self, parent)
 
-        # Run the dialog setup generated from catalog.ui
-        self.setupUi(self)
         self.dbspec, self.ids = dbspec, ids
 
         # Display the number of books we've been passed
@@ -120,11 +115,13 @@ class Catalog(QDialog, Ui_Dialog):
             self.sync.setChecked(dynamic.get('catalog_sync_to_device', True))
 
         self.format.currentIndexChanged.connect(self.show_plugin_tab)
-        self.connect(self.buttonBox.button(QtGui.QDialogButtonBox.Apply),
+        self.buttonBox.button(self.buttonBox.Apply).clicked.connect(self.apply)
-                     SIGNAL("clicked()"),
-                     self.apply)
         self.show_plugin_tab(None)
 
+        geom = dynamic.get('catalog_window_geom', None)
+        if geom is not None:
+            self.restoreGeometry(bytes(geom))
+
     def show_plugin_tab(self, idx):
         cf = unicode(self.format.currentText()).lower()
         while self.tabs.count() > 1:
@@ -157,8 +154,9 @@ class Catalog(QDialog, Ui_Dialog):
         dynamic.set('catalog_last_used_title', self.catalog_title)
         self.catalog_sync = bool(self.sync.isChecked())
         dynamic.set('catalog_sync_to_device', self.catalog_sync)
+        dynamic.set('catalog_window_geom', bytearray(self.saveGeometry()))
 
-    def apply(self):
+    def apply(self, *args):
         # Store current values without building catalog
         self.save_catalog_settings()
         if self.tabs.count() > 1:
@@ -166,4 +164,9 @@ class Catalog(QDialog, Ui_Dialog):
 
     def accept(self):
         self.save_catalog_settings()
-        return QDialog.accept(self)
+        return ResizableDialog.accept(self)
+
+    def reject(self):
+        dynamic.set('catalog_window_geom', bytearray(self.saveGeometry()))
+        ResizableDialog.reject(self)
+

src/calibre/gui2/dialogs/catalog.ui

@@ -14,7 +14,7 @@
    <string>Generate catalog</string>
   </property>
   <property name="windowIcon">
-   <iconset>
+   <iconset resource="../../../../resources/images.qrc">
     <normaloff>:/images/library.png</normaloff>:/images/library.png</iconset>
   </property>
   <layout class="QGridLayout" name="gridLayout">
@@ -31,7 +31,38 @@
      </property>
     </widget>
    </item>
+   <item row="2" column="1">
+    <widget class="QDialogButtonBox" name="buttonBox">
+     <property name="orientation">
+      <enum>Qt::Horizontal</enum>
+     </property>
+     <property name="standardButtons">
+      <set>QDialogButtonBox::Apply|QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
+     </property>
+    </widget>
+   </item>
    <item row="1" column="0" colspan="2">
+    <widget class="QScrollArea" name="scrollArea">
+     <property name="frameShape">
+      <enum>QFrame::NoFrame</enum>
+     </property>
+     <property name="widgetResizable">
+      <bool>true</bool>
+     </property>
+     <widget class="QWidget" name="scrollAreaWidgetContents">
+      <property name="geometry">
+       <rect>
+        <x>0</x>
+        <y>0</y>
+        <width>666</width>
+        <height>599</height>
+       </rect>
+      </property>
+      <layout class="QVBoxLayout" name="verticalLayout">
+       <property name="margin">
+        <number>0</number>
+       </property>
+       <item>
         <widget class="QTabWidget" name="tabs">
          <property name="sizePolicy">
           <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
@@ -82,13 +113,6 @@
            <item row="1" column="2">
             <widget class="QLineEdit" name="title"/>
            </item>
-       <item row="3" column="0">
-        <widget class="QCheckBox" name="sync">
-         <property name="text">
-          <string>&amp;Send catalog to device automatically</string>
-         </property>
-        </widget>
-       </item>
            <item row="2" column="1">
             <spacer name="verticalSpacer">
              <property name="orientation">
@@ -96,30 +120,31 @@
              </property>
              <property name="sizeHint" stdset="0">
               <size>
-           <width>20</width>
+               <width>0</width>
-           <height>299</height>
+               <height>0</height>
               </size>
              </property>
             </spacer>
            </item>
+           <item row="3" column="0">
+            <widget class="QCheckBox" name="sync">
+             <property name="text">
+              <string>&amp;Send catalog to device automatically</string>
+             </property>
+            </widget>
+           </item>
           </layout>
          </widget>
         </widget>
        </item>
-   <item row="2" column="1">
+      </layout>
-    <widget class="QDialogButtonBox" name="buttonBox">
+     </widget>
-     <property name="orientation">
-      <enum>Qt::Horizontal</enum>
-     </property>
-     <property name="standardButtons">
-      <set>QDialogButtonBox::Apply|QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
-     </property>
     </widget>
    </item>
   </layout>
  </widget>
  <resources>
-  <include location="../../../work/calibre/resources/images.qrc"/>
+  <include location="../../../../resources/images.qrc"/>
  </resources>
  <connections>
   <connection>

src/calibre/gui2/dialogs/check_library.py

@@ -3,16 +3,138 @@ __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __license__   = 'GPL v3'
 
-import os
+import os, shutil
 
 from PyQt4.Qt import QDialog, QVBoxLayout, QHBoxLayout, QTreeWidget, QLabel, \
             QPushButton, QDialogButtonBox, QApplication, QTreeWidgetItem, \
-            QLineEdit, Qt
+            QLineEdit, Qt, QProgressBar, QSize, QTimer
 
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.library.check_library import CheckLibrary, CHECKS
 from calibre.library.database2 import delete_file, delete_tree
-from calibre import prints
+from calibre import prints, as_unicode
+from calibre.ptempfile import PersistentTemporaryFile
+from calibre.library.sqlite import DBThread, OperationalError
+
+class DBCheck(QDialog):
+
+    def __init__(self, parent, db):
+        QDialog.__init__(self, parent)
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+        self.l1 = QLabel(_('Checking database integrity')+'...')
+        self.setWindowTitle(_('Checking database integrity'))
+        self.l.addWidget(self.l1)
+        self.pb = QProgressBar(self)
+        self.l.addWidget(self.pb)
+        self.pb.setMaximum(0)
+        self.pb.setMinimum(0)
+        self.msg = QLabel('')
+        self.l.addWidget(self.msg)
+        self.msg.setWordWrap(True)
+        self.bb = QDialogButtonBox(QDialogButtonBox.Cancel)
+        self.l.addWidget(self.bb)
+        self.bb.rejected.connect(self.reject)
+        self.resize(self.sizeHint() + QSize(100, 50))
+        self.error = None
+        self.db = db
+        self.closed_orig_conn = False
+
+    def start(self):
+        self.user_version = self.db.user_version
+        self.rejected = False
+        self.db.clean()
+        self.db.conn.close()
+        self.closed_orig_conn = True
+        t = DBThread(self.db.dbpath, False)
+        t.connect()
+        self.conn = t.conn
+        self.dump = self.conn.iterdump()
+        self.statements = []
+        self.count = 0
+        self.msg.setText(_('Dumping database to SQL'))
+        # Give the backup thread time to stop
+        QTimer.singleShot(2000, self.do_one_dump)
+        self.exec_()
+
+    def do_one_dump(self):
+        if self.rejected:
+            return
+        try:
+            try:
+                self.statements.append(self.dump.next())
+                self.count += 1
+            except StopIteration:
+                self.start_load()
+                return
+            QTimer.singleShot(0, self.do_one_dump)
+        except Exception, e:
+            import traceback
+            self.error = (as_unicode(e), traceback.format_exc())
+            self.reject()
+
+    def start_load(self):
+        try:
+            self.conn.close()
+            self.pb.setMaximum(self.count)
+            self.pb.setValue(0)
+            self.msg.setText(_('Loading database from SQL'))
+            self.db.conn.close()
+            self.ndbpath = PersistentTemporaryFile('.db')
+            self.ndbpath.close()
+            self.ndbpath = self.ndbpath.name
+            t = DBThread(self.ndbpath, False)
+            t.connect()
+            self.conn = t.conn
+            self.conn.execute('create temporary table temp_sequence(id INTEGER PRIMARY KEY AUTOINCREMENT)')
+            self.conn.commit()
+
+            QTimer.singleShot(0, self.do_one_load)
+        except Exception, e:
+            import traceback
+            self.error = (as_unicode(e), traceback.format_exc())
+            self.reject()
+
+
+    def do_one_load(self):
+        if self.rejected:
+            return
+        if self.count > 0:
+            try:
+                try:
+                    self.conn.execute(self.statements.pop(0))
+                except OperationalError:
+                    if self.count > 1:
+                        # The last statement in the dump could be an extra
+                        # commit, so ignore it.
+                        raise
+                self.pb.setValue(self.pb.value() + 1)
+                self.count -= 1
+                QTimer.singleShot(0, self.do_one_load)
+            except Exception, e:
+                import traceback
+                self.error = (as_unicode(e), traceback.format_exc())
+                self.reject()
+
+        else:
+            self.replace_db()
+
+    def replace_db(self):
+        self.conn.commit()
+        self.conn.execute('pragma user_version=%d'%int(self.user_version))
+        self.conn.commit()
+        self.conn.close()
+        shutil.copyfile(self.ndbpath, self.db.dbpath)
+        self.db = None
+        self.accept()
+
+    def break_cycles(self):
+        self.statements = self.unpickler = self.db = self.conn = None
+
+    def reject(self):
+        self.rejected = True
+        QDialog.reject(self)
+
 
 class Item(QTreeWidgetItem):
     pass
@@ -23,7 +145,7 @@ class CheckLibraryDialog(QDialog):
         QDialog.__init__(self, parent)
         self.db = db
 
-        self.setWindowTitle(_('Check Library'))
+        self.setWindowTitle(_('Check Library -- Problems Found'))
 
         self._layout = QVBoxLayout(self)
         self.setLayout(self._layout)
@@ -32,7 +154,7 @@ class CheckLibraryDialog(QDialog):
         self.log.itemChanged.connect(self.item_changed)
         self._layout.addWidget(self.log)
 
-        self.check_button = QPushButton(_('&Run the check'))
+        self.check_button = QPushButton(_('&Run the check again'))
         self.check_button.setDefault(False)
         self.check_button.clicked.connect(self.run_the_check)
         self.copy_button = QPushButton(_('Copy &to clipboard'))
@@ -80,8 +202,17 @@ class CheckLibraryDialog(QDialog):
         self.resize(750, 500)
         self.bbox.setEnabled(True)
 
+    def do_exec(self):
         self.run_the_check()
 
+        probs = 0
+        for c in self.problem_count:
+            probs += self.problem_count[c]
+        if probs == 0:
+            return False
+        self.exec_()
+        return True
+
     def accept(self):
         self.db.prefs['check_library_ignore_extensions'] = \
                                             unicode(self.ext_ignores.text())
@@ -103,7 +234,10 @@ class CheckLibraryDialog(QDialog):
             attr, h, checkable, fixable = check
             list = getattr(checker, attr, None)
             if list is None:
+                self.problem_count[attr] = 0
                 return
+            else:
+                self.problem_count[attr] = len(list)
 
             tl = Item()
             tl.setText(0, h)
@@ -134,6 +268,7 @@ class CheckLibraryDialog(QDialog):
         t.setHeaderLabels([_('Name'), _('Path from library')])
         self.all_items = []
         self.top_level_items = {}
+        self.problem_count = {}
         for check in CHECKS:
             builder(t, checker, check)
 

src/calibre/gui2/dialogs/choose_format_device.py

@@ -0,0 +1,53 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+
+from PyQt4.Qt import QDialog, QTreeWidgetItem, QIcon, SIGNAL
+
+from calibre.gui2 import file_icon_provider
+from calibre.gui2.dialogs.choose_format_device_ui import Ui_ChooseFormatDeviceDialog
+
+class ChooseFormatDeviceDialog(QDialog, Ui_ChooseFormatDeviceDialog):
+
+    def __init__(self, window, msg, formats):
+        '''
+        formats is a list of tuples: [(format, exists, convertible)].
+            format: Lower case format identifier. E.G. mobi
+            exists: String representing the number of books that
+                    exist in the format.
+            convertible: True if the format is a convertible format.
+        formats should be ordered in the device's preferred format ordering.
+        '''
+        QDialog.__init__(self, window)
+        Ui_ChooseFormatDeviceDialog.__init__(self)
+        self.setupUi(self)
+        self.connect(self.formats, SIGNAL('activated(QModelIndex)'),
+                self.activated_slot)
+
+        self.msg.setText(msg)
+        for i, (format, exists, convertible) in enumerate(formats):
+            t_item = QTreeWidgetItem()
+            t_item.setIcon(0, file_icon_provider().icon_from_ext(format.lower()))
+            t_item.setText(0, format.upper())
+            t_item.setText(1, exists)
+            if convertible:
+                t_item.setIcon(2, QIcon(I('ok.png')))
+            self.formats.addTopLevelItem(t_item)
+            if i == 0:
+                self.formats.setCurrentItem(t_item)
+                t_item.setSelected(True)
+        self.formats.resizeColumnToContents(2)
+        self.formats.resizeColumnToContents(1)
+        self.formats.resizeColumnToContents(0)
+        self.formats.header().resizeSection(0, self.formats.header().sectionSize(0) * 2)
+        self._format = None
+
+    def activated_slot(self, *args):
+        self.accept()
+
+    def format(self):
+        return self._format
+
+    def accept(self):
+        self._format = unicode(self.formats.currentItem().text(0))
+        return QDialog.accept(self)
+

src/calibre/gui2/dialogs/choose_format_device.ui

@@ -0,0 +1,111 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>ChooseFormatDeviceDialog</class>
+ <widget class="QDialog" name="ChooseFormatDeviceDialog">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>507</width>
+    <height>377</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Choose Format</string>
+  </property>
+  <property name="windowIcon">
+   <iconset resource="../../../../resources/images.qrc">
+    <normaloff>:/images/mimetypes/unknown.png</normaloff>:/images/mimetypes/unknown.png</iconset>
+  </property>
+  <layout class="QVBoxLayout">
+   <item>
+    <widget class="QLabel" name="msg">
+     <property name="text">
+      <string/>
+     </property>
+    </widget>
+   </item>
+   <item>
+    <widget class="QTreeWidget" name="formats">
+     <property name="alternatingRowColors">
+      <bool>true</bool>
+     </property>
+     <property name="iconSize">
+      <size>
+       <width>64</width>
+       <height>64</height>
+      </size>
+     </property>
+     <property name="allColumnsShowFocus">
+      <bool>true</bool>
+     </property>
+     <column>
+      <property name="text">
+       <string>Format</string>
+      </property>
+     </column>
+     <column>
+      <property name="text">
+       <string>Existing</string>
+      </property>
+      <property name="textAlignment">
+       <set>AlignLeft|AlignVCenter</set>
+      </property>
+     </column>
+     <column>
+      <property name="text">
+       <string>Convertible</string>
+      </property>
+     </column>
+    </widget>
+   </item>
+   <item>
+    <widget class="QDialogButtonBox" name="buttonBox">
+     <property name="orientation">
+      <enum>Qt::Horizontal</enum>
+     </property>
+     <property name="standardButtons">
+      <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
+     </property>
+    </widget>
+   </item>
+  </layout>
+ </widget>
+ <resources>
+  <include location="../../../../resources/images.qrc"/>
+ </resources>
+ <connections>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>accepted()</signal>
+   <receiver>ChooseFormatDeviceDialog</receiver>
+   <slot>accept()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>248</x>
+     <y>254</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>157</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>rejected()</signal>
+   <receiver>ChooseFormatDeviceDialog</receiver>
+   <slot>reject()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>316</x>
+     <y>260</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>286</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
+</ui>

src/calibre/gui2/dialogs/confirm_delete.py

@@ -21,10 +21,10 @@ class Dialog(QDialog, Ui_Dialog):
         self.again.stateChanged.connect(self.toggle)
         self.buttonBox.setFocus(Qt.OtherFocusReason)
 
-
     def toggle(self, *args):
         dynamic[_config_name(self.name)] = self.again.isChecked()
 
+
 def confirm(msg, name, parent=None, pixmap='dialog_warning.png'):
     if not dynamic.get(_config_name(name), True):
         return True

src/calibre/gui2/dialogs/message_box.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+from PyQt4.Qt import QDialog, QIcon, QApplication, QSize, QKeySequence, \
+    QAction, Qt
+
+from calibre.constants import __version__
+from calibre.gui2.dialogs.message_box_ui import Ui_Dialog
+
+class MessageBox(QDialog, Ui_Dialog):
+
+    ERROR = 0
+    WARNING = 1
+    INFO = 2
+    QUESTION = 3
+
+    def __init__(self, type_, title, msg, det_msg='', show_copy_button=True,
+            parent=None):
+        QDialog.__init__(self, parent)
+        icon = {
+                self.ERROR : 'error',
+                self.WARNING: 'warning',
+                self.INFO:    'information',
+                self.QUESTION: 'question',
+        }[type_]
+        icon = 'dialog_%s.png'%icon
+        self.icon = QIcon(I(icon))
+        self.setupUi(self)
+
+        self.setWindowTitle(title)
+        self.setWindowIcon(self.icon)
+        self.icon_label.setPixmap(self.icon.pixmap(128, 128))
+        self.msg.setText(msg)
+        self.det_msg.setPlainText(det_msg)
+        self.det_msg.setVisible(False)
+
+        if show_copy_button:
+            self.ctc_button = self.bb.addButton(_('&Copy to clipboard'),
+                    self.bb.ActionRole)
+            self.ctc_button.clicked.connect(self.copy_to_clipboard)
+
+
+        self.show_det_msg = _('Show &details')
+        self.hide_det_msg = _('Hide &details')
+        self.det_msg_toggle = self.bb.addButton(self.show_det_msg, self.bb.ActionRole)
+        self.det_msg_toggle.clicked.connect(self.toggle_det_msg)
+        self.det_msg_toggle.setToolTip(
+                _('Show detailed information about this error'))
+
+        self.copy_action = QAction(self)
+        self.addAction(self.copy_action)
+        self.copy_action.setShortcuts(QKeySequence.Copy)
+        self.copy_action.triggered.connect(self.copy_to_clipboard)
+
+        self.is_question = type_ == self.QUESTION
+        if self.is_question:
+            self.bb.setStandardButtons(self.bb.Yes|self.bb.No)
+            self.bb.button(self.bb.Yes).setDefault(True)
+        else:
+            self.bb.button(self.bb.Ok).setDefault(True)
+
+        if not det_msg:
+            self.det_msg_toggle.setVisible(False)
+
+        self.do_resize()
+
+
+    def toggle_det_msg(self, *args):
+        vis = unicode(self.det_msg_toggle.text()) == self.hide_det_msg
+        self.det_msg_toggle.setText(self.show_det_msg if vis else
+                self.hide_det_msg)
+        self.det_msg.setVisible(not vis)
+        self.do_resize()
+
+    def do_resize(self):
+        sz = self.sizeHint() + QSize(100, 0)
+        sz.setWidth(min(500, sz.width()))
+        sz.setHeight(min(500, sz.height()))
+        self.resize(sz)
+
+    def copy_to_clipboard(self, *args):
+        QApplication.clipboard().setText(
+                'calibre, version %s\n%s: %s\n\n%s' %
+                (__version__, unicode(self.windowTitle()),
+                    unicode(self.msg.text()),
+                    unicode(self.det_msg.toPlainText())))
+        self.ctc_button.setText(_('Copied'))
+
+    def showEvent(self, ev):
+        ret = QDialog.showEvent(self, ev)
+        if self.is_question:
+            try:
+                self.bb.button(self.bb.Yes).setFocus(Qt.OtherFocusReason)
+            except:
+                pass# Buttons were changed
+        else:
+            self.bb.button(self.bb.Ok).setFocus(Qt.OtherFocusReason)
+        return ret
+
+    def set_details(self, msg):
+        if not msg:
+            msg = ''
+        self.det_msg.setPlainText(msg)
+        self.det_msg_toggle.setText(self.show_det_msg)
+        self.det_msg_toggle.setVisible(bool(msg))
+        self.det_msg.setVisible(False)
+        self.do_resize()
+
+if __name__ == '__main__':
+    app = QApplication([])
+    from calibre.gui2 import question_dialog
+    print question_dialog(None, 'title', 'msg <a href="http://google.com">goog</a> ',
+            det_msg='det '*1000,
+            show_copy_button=True)

src/calibre/gui2/dialogs/message_box.ui

@@ -0,0 +1,105 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Dialog</class>
+ <widget class="QDialog" name="Dialog">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>497</width>
+    <height>235</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Dialog</string>
+  </property>
+  <layout class="QGridLayout" name="gridLayout">
+   <item row="0" column="0">
+    <widget class="QLabel" name="icon_label">
+     <property name="maximumSize">
+      <size>
+       <width>68</width>
+       <height>68</height>
+      </size>
+     </property>
+     <property name="text">
+      <string/>
+     </property>
+     <property name="pixmap">
+      <pixmap resource="../../../../resources/images.qrc">:/images/dialog_warning.png</pixmap>
+     </property>
+     <property name="scaledContents">
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
+   <item row="0" column="1">
+    <widget class="QLabel" name="msg">
+     <property name="text">
+      <string/>
+     </property>
+     <property name="wordWrap">
+      <bool>true</bool>
+     </property>
+     <property name="openExternalLinks">
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
+   <item row="1" column="0" colspan="2">
+    <widget class="QPlainTextEdit" name="det_msg">
+     <property name="readOnly">
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="0" colspan="2">
+    <widget class="QDialogButtonBox" name="bb">
+     <property name="orientation">
+      <enum>Qt::Horizontal</enum>
+     </property>
+     <property name="standardButtons">
+      <set>QDialogButtonBox::Ok</set>
+     </property>
+    </widget>
+   </item>
+  </layout>
+ </widget>
+ <resources>
+  <include location="../../../../resources/images.qrc"/>
+ </resources>
+ <connections>
+  <connection>
+   <sender>bb</sender>
+   <signal>accepted()</signal>
+   <receiver>Dialog</receiver>
+   <slot>accept()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>248</x>
+     <y>254</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>157</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>bb</sender>
+   <signal>rejected()</signal>
+   <receiver>Dialog</receiver>
+   <slot>reject()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>316</x>
+     <y>260</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>286</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
+</ui>

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -6,8 +6,8 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import re, os
 
 from PyQt4.Qt import Qt, QDialog, QGridLayout, QVBoxLayout, QFont, QLabel, \
-                     pyqtSignal, QDialogButtonBox
+                     pyqtSignal, QDialogButtonBox, QInputDialog, QLineEdit, \
-from PyQt4 import QtGui
+                     QDate
 
 from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
 from calibre.gui2.dialogs.tag_editor import TagEditor
@@ -15,9 +15,10 @@ from calibre.ebooks.metadata import string_to_authors, authors_to_string
 from calibre.ebooks.metadata.book.base import composite_formatter
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.gui2.custom_column_widgets import populate_metadata_page
-from calibre.gui2 import error_dialog, ResizableDialog, UNDEFINED_QDATE
+from calibre.gui2 import error_dialog, ResizableDialog, UNDEFINED_QDATE, \
+    gprefs, question_dialog
 from calibre.gui2.progress_indicator import ProgressIndicator
-from calibre.utils.config import dynamic
+from calibre.utils.config import dynamic, JSONConfig
 from calibre.utils.titlecase import titlecase
 from calibre.utils.icu import sort_key, capitalize
 from calibre.utils.config import prefs, tweaks
@@ -302,6 +303,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         self.pubdate.setSpecialValueText(_('Undefined'))
         self.clear_pubdate_button.clicked.connect(self.clear_pubdate)
         self.pubdate.dateChanged.connect(self.do_apply_pubdate)
+        self.adddate.setDate(QDate.currentDate())
         self.adddate.setMinimumDate(UNDEFINED_QDATE)
         self.adddate.setSpecialValueText(_('Undefined'))
         self.clear_adddate_button.clicked.connect(self.clear_adddate)
@@ -320,8 +322,15 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
             'This operation cannot be canceled or undone'))
         self.do_again = False
         self.central_widget.setCurrentIndex(tab)
+        geom = gprefs.get('bulk_metadata_window_geometry', None)
+        if geom is not None:
+            self.restoreGeometry(bytes(geom))
         self.exec_()
 
+    def save_state(self, *args):
+        gprefs['bulk_metadata_window_geometry'] = \
+            bytearray(self.saveGeometry())
+
     def do_apply_pubdate(self, *args):
         self.apply_pubdate.setChecked(True)
 
@@ -365,16 +374,16 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         offset = 10
         self.s_r_number_of_books = min(10, len(self.ids))
         for i in range(1,self.s_r_number_of_books+1):
-            w = QtGui.QLabel(self.tabWidgetPage3)
+            w = QLabel(self.tabWidgetPage3)
             w.setText(_('Book %d:')%i)
             self.testgrid.addWidget(w, i+offset, 0, 1, 1)
-            w = QtGui.QLineEdit(self.tabWidgetPage3)
+            w = QLineEdit(self.tabWidgetPage3)
             w.setReadOnly(True)
             name = 'book_%d_text'%i
             setattr(self, name, w)
             self.book_1_text.setObjectName(name)
             self.testgrid.addWidget(w, i+offset, 1, 1, 1)
-            w = QtGui.QLineEdit(self.tabWidgetPage3)
+            w = QLineEdit(self.tabWidgetPage3)
             w.setReadOnly(True)
             name = 'book_%d_result'%i
             setattr(self, name, w)
@@ -451,6 +460,15 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         self.results_count.valueChanged[int].connect(self.s_r_display_bounds_changed)
         self.starting_from.valueChanged[int].connect(self.s_r_display_bounds_changed)
 
+        self.save_button.clicked.connect(self.s_r_save_query)
+        self.remove_button.clicked.connect(self.s_r_remove_query)
+
+        self.queries = JSONConfig("search_replace_queries")
+        self.query_field.addItem("")
+        self.query_field.addItems(sorted([q for q in self.queries], key=sort_key))
+        self.query_field.currentIndexChanged[str].connect(self.s_r_query_change)
+        self.query_field.setCurrentIndex(0)
+
     def s_r_get_field(self, mi, field):
         if field:
             if field == '{template}':
@@ -780,7 +798,12 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
             self.series_start_number.setEnabled(False)
             self.series_start_number.setValue(1)
 
+    def reject(self):
+        self.save_state()
+        ResizableDialog.reject(self)
+
     def accept(self):
+        self.save_state()
         if len(self.ids) < 1:
             return QDialog.accept(self)
 
@@ -862,3 +885,112 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
     def series_changed(self, *args):
         self.write_series = True
 
+    def s_r_remove_query(self, *args):
+        if self.query_field.currentIndex() == 0:
+            return
+
+        if not question_dialog(self, _("Delete saved search/replace"),
+                _("The selected saved search/replace will be deleted. "
+                    "Are you sure?")):
+            return
+
+        item_id = self.query_field.currentIndex()
+        item_name = unicode(self.query_field.currentText())
+
+        self.query_field.blockSignals(True)
+        self.query_field.removeItem(item_id)
+        self.query_field.blockSignals(False)
+        self.query_field.setCurrentIndex(0)
+
+        if item_name in self.queries.keys():
+            del(self.queries[item_name])
+            self.queries.commit()
+
+    def s_r_save_query(self, *args):
+        name, ok =  QInputDialog.getText(self, _('Save search/replace'),
+                _('Search/replace name:'))
+        if not ok:
+            return
+
+        new = True
+        name = unicode(name)
+        if name in self.queries.keys():
+            if not question_dialog(self, _("Save search/replace"),
+                    _("That saved search/replace already exists and will be overwritten. "
+                        "Are you sure?")):
+                return
+            new = False
+
+        query = {}
+        query['name'] = name
+        query['search_field'] = unicode(self.search_field.currentText())
+        query['search_mode'] = unicode(self.search_mode.currentText())
+        query['s_r_template'] = unicode(self.s_r_template.text())
+        query['search_for'] = unicode(self.search_for.text())
+        query['case_sensitive'] = self.case_sensitive.isChecked()
+        query['replace_with'] = unicode(self.replace_with.text())
+        query['replace_func'] = unicode(self.replace_func.currentText())
+        query['destination_field'] = unicode(self.destination_field.currentText())
+        query['replace_mode'] = unicode(self.replace_mode.currentText())
+        query['comma_separated'] = self.comma_separated.isChecked()
+        query['results_count'] = self.results_count.value()
+        query['starting_from'] = self.starting_from.value()
+        query['multiple_separator'] = unicode(self.multiple_separator.text())
+
+        self.queries[name] = query
+        self.queries.commit()
+
+        if new:
+            self.query_field.blockSignals(True)
+            self.query_field.clear()
+            self.query_field.addItem('')
+            self.query_field.addItems(sorted([q for q in self.queries], key=sort_key))
+            self.query_field.blockSignals(False)
+        self.query_field.setCurrentIndex(self.query_field.findText(name))
+
+    def s_r_query_change(self, item_name):
+        if not item_name:
+            self.s_r_reset_query_fields()
+            return
+        item = self.queries.get(unicode(item_name), None)
+        if item is None:
+            self.s_r_reset_query_fields()
+            return
+
+        def set_index(attr, txt):
+            try:
+                attr.setCurrentIndex(attr.findText(txt))
+            except:
+                attr.setCurrentIndex(0)
+
+        set_index(self.search_mode, item['search_mode'])
+        set_index(self.search_field, item['search_field'])
+        self.s_r_template.setText(item['s_r_template'])
+        self.s_r_template_changed() #simulate gain/loss of focus
+        self.search_for.setText(item['search_for'])
+        self.case_sensitive.setChecked(item['case_sensitive'])
+        self.replace_with.setText(item['replace_with'])
+        set_index(self.replace_func, item['replace_func'])
+        set_index(self.destination_field, item['destination_field'])
+        set_index(self.replace_mode, item['replace_mode'])
+        self.comma_separated.setChecked(item['comma_separated'])
+        self.results_count.setValue(int(item['results_count']))
+        self.starting_from.setValue(int(item['starting_from']))
+        self.multiple_separator.setText(item['multiple_separator'])
+
+    def s_r_reset_query_fields(self):
+        # Don't reset the search mode. The user will probably want to use it
+        # as it was
+        self.search_field.setCurrentIndex(0)
+        self.s_r_template.setText("")
+        self.search_for.setText("")
+        self.case_sensitive.setChecked(False)
+        self.replace_with.setText("")
+        self.replace_func.setCurrentIndex(0)
+        self.destination_field.setCurrentIndex(0)
+        self.replace_mode.setCurrentIndex(0)
+        self.comma_separated.setChecked(True)
+        self.results_count.setValue(999)
+        self.starting_from.setValue(1)
+        self.multiple_separator.setText(" ::: ")
+

src/calibre/gui2/dialogs/metadata_bulk.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>850</width>
+    <width>962</width>
-    <height>650</height>
+    <height>645</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -44,8 +44,8 @@
        <rect>
         <x>0</x>
         <y>0</y>
-        <width>842</width>
+        <width>954</width>
-        <height>589</height>
+        <height>584</height>
        </rect>
       </property>
       <layout class="QVBoxLayout" name="verticalLayout_2">
@@ -574,7 +574,7 @@ Future conversion of these books will use the default settings.</string>
            <property name="sizeConstraint">
             <enum>QLayout::SetMinimumSize</enum>
            </property>
-           <item row="1" column="0" colspan="3">
+           <item row="0" column="0" colspan="4">
             <widget class="QLabel" name="s_r_heading">
              <property name="wordWrap">
               <bool>true</bool>
@@ -584,14 +584,91 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="2" column="0">
+           <item row="1" column="0">
             <widget class="QLabel" name="filler">
              <property name="text">
               <string/>
              </property>
             </widget>
            </item>
+           <item row="2" column="0" colspan="3">
+            <widget class="Line" name="line">
+             <property name="orientation">
+              <enum>Qt::Horizontal</enum>
+             </property>
+            </widget>
+           </item>
            <item row="3" column="0">
+            <widget class="QLabel" name="xlabel_22">
+             <property name="text">
+              <string>Load searc&amp;h/replace:</string>
+             </property>
+             <property name="buddy">
+              <cstring>search_field</cstring>
+             </property>
+            </widget>
+           </item>
+           <item row="3" column="1">
+            <widget class="QComboBox" name="query_field">
+             <property name="toolTip">
+              <string>Select saved search/replace to load.</string>
+             </property>
+            </widget>
+           </item>
+           <item row="3" column="2">
+            <layout class="QHBoxLayout" name="horizontalLayout_6">
+             <item>
+              <spacer name="horizontalSpacer_3">
+               <property name="orientation">
+                <enum>Qt::Horizontal</enum>
+               </property>
+               <property name="sizeType">
+                <enum>QSizePolicy::Fixed</enum>
+               </property>
+               <property name="sizeHint" stdset="0">
+                <size>
+                 <width>20</width>
+                 <height>20</height>
+                </size>
+               </property>
+              </spacer>
+             </item>
+             <item>
+              <widget class="QPushButton" name="save_button">
+               <property name="toolTip">
+                <string>Save current search/replace</string>
+               </property>
+               <property name="text">
+                <string>Sa&amp;ve</string>
+               </property>
+              </widget>
+             </item>
+             <item>
+              <widget class="QPushButton" name="remove_button">
+               <property name="toolTip">
+                <string>Delete saved search/replace</string>
+               </property>
+               <property name="text">
+                <string>Delete</string>
+               </property>
+              </widget>
+             </item>
+             <item>
+              <spacer name="horizontalSpacer_2">
+               <property name="orientation">
+                <enum>Qt::Horizontal</enum>
+               </property>
+               <property name="sizeHint" stdset="0">
+                <size>
+                 <width>40</width>
+                 <height>20</height>
+                </size>
+               </property>
+              </spacer>
+             </item>
+            </layout>
+           </item>
+           <item row="4" column="0">
             <widget class="QLabel" name="xlabel_21">
              <property name="text">
               <string>Search &amp;field:</string>
@@ -601,14 +678,14 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="3" column="1">
+           <item row="4" column="1">
             <widget class="QComboBox" name="search_field">
              <property name="toolTip">
               <string>The name of the field that you want to search</string>
              </property>
             </widget>
            </item>
-           <item row="3" column="2">
+           <item row="4" column="2">
             <layout class="QHBoxLayout" name="HLayout_3">
              <item>
               <widget class="QLabel" name="xlabel_24">
@@ -642,7 +719,7 @@ Future conversion of these books will use the default settings.</string>
              </item>
             </layout>
            </item>
-           <item row="4" column="0">
+           <item row="5" column="0">
             <widget class="QLabel" name="template_label">
              <property name="text">
               <string>Te&amp;mplate:</string>
@@ -652,7 +729,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="4" column="1">
+           <item row="5" column="1">
             <widget class="HistoryLineEdit" name="s_r_template">
              <property name="sizePolicy">
               <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
@@ -665,7 +742,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="5" column="0">
+           <item row="6" column="0">
             <widget class="QLabel" name="xlabel_2">
              <property name="text">
               <string>&amp;Search for:</string>
@@ -675,7 +752,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="5" column="1">
+           <item row="6" column="1">
             <widget class="HistoryLineEdit" name="search_for">
              <property name="sizePolicy">
               <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
@@ -688,7 +765,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="5" column="2">
+           <item row="6" column="2">
             <widget class="QCheckBox" name="case_sensitive">
              <property name="toolTip">
               <string>Check this box if the search string must match exactly upper and lower case. Uncheck it if case is to be ignored</string>
@@ -701,7 +778,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="6" column="0">
+           <item row="7" column="0">
             <widget class="QLabel" name="xlabel_4">
              <property name="text">
               <string>&amp;Replace with:</string>
@@ -711,14 +788,14 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="6" column="1">
+           <item row="7" column="1">
             <widget class="HistoryLineEdit" name="replace_with">
              <property name="toolTip">
               <string>The replacement text. The matched search text will be replaced with this string</string>
              </property>
             </widget>
            </item>
-           <item row="6" column="2">
+           <item row="7" column="2">
             <layout class="QHBoxLayout" name="verticalLayout">
              <item>
               <widget class="QLabel" name="label_41">
@@ -753,7 +830,7 @@ field is processed. In regular expression mode, only the matched text is process
              </item>
             </layout>
            </item>
-           <item row="7" column="0">
+           <item row="8" column="0">
             <widget class="QLabel" name="destination_field_label">
              <property name="text">
               <string>&amp;Destination field:</string>
@@ -763,7 +840,7 @@ field is processed. In regular expression mode, only the matched text is process
              </property>
             </widget>
            </item>
-           <item row="7" column="1">
+           <item row="8" column="1">
             <widget class="QComboBox" name="destination_field">
              <property name="toolTip">
               <string>The field that the text will be put into after all replacements.
@@ -771,7 +848,7 @@ If blank, the source field is used if the field is modifiable</string>
              </property>
             </widget>
            </item>
-           <item row="7" column="2">
+           <item row="8" column="2">
             <layout class="QHBoxLayout" name="verticalLayout">
              <item>
               <widget class="QLabel" name="replace_mode_label">
@@ -820,7 +897,7 @@ not multiple and the destination field is multiple</string>
              </item>
             </layout>
            </item>
-           <item row="8" column="1" colspan="2">
+           <item row="9" column="1" colspan="2">
             <layout class="QHBoxLayout" name="horizontalLayout_21">
              <item>
               <spacer name="HSpacer_347">
@@ -906,7 +983,7 @@ not multiple and the destination field is multiple</string>
              </item>
             </layout>
            </item>
-           <item row="9" column="0" colspan="4">
+           <item row="10" column="0" colspan="4">
             <widget class="QScrollArea" name="scrollArea11">
              <property name="frameShape">
               <enum>QFrame::NoFrame</enum>
@@ -919,8 +996,8 @@ not multiple and the destination field is multiple</string>
                <rect>
                 <x>0</x>
                 <y>0</y>
-                <width>197</width>
+                <width>938</width>
-                <height>60</height>
+                <height>268</height>
                </rect>
               </property>
               <layout class="QGridLayout" name="testgrid">
@@ -1030,6 +1107,9 @@ not multiple and the destination field is multiple</string>
   <tabstop>series_numbering_restarts</tabstop>
   <tabstop>series_start_number</tabstop>
   <tabstop>button_box</tabstop>
+  <tabstop>query_field</tabstop>
+  <tabstop>save_button</tabstop>
+  <tabstop>remove_button</tabstop>
   <tabstop>search_field</tabstop>
   <tabstop>search_mode</tabstop>
   <tabstop>s_r_template</tabstop>
@@ -1045,6 +1125,23 @@ not multiple and the destination field is multiple</string>
   <tabstop>multiple_separator</tabstop>
   <tabstop>test_text</tabstop>
   <tabstop>test_result</tabstop>
+  <tabstop>scrollArea</tabstop>
+  <tabstop>central_widget</tabstop>
+  <tabstop>swap_title_and_author</tabstop>
+  <tabstop>clear_series</tabstop>
+  <tabstop>adddate</tabstop>
+  <tabstop>clear_adddate_button</tabstop>
+  <tabstop>apply_adddate</tabstop>
+  <tabstop>pubdate</tabstop>
+  <tabstop>clear_pubdate_button</tabstop>
+  <tabstop>apply_pubdate</tabstop>
+  <tabstop>remove_format</tabstop>
+  <tabstop>change_title_to_title_case</tabstop>
+  <tabstop>remove_conversion_settings</tabstop>
+  <tabstop>cover_generate</tabstop>
+  <tabstop>cover_remove</tabstop>
+  <tabstop>cover_from_fmt</tabstop>
+  <tabstop>scrollArea11</tabstop>
  </tabstops>
  <resources>
   <include location="../../../../resources/images.qrc"/>

src/calibre/gui2/dialogs/metadata_single.py

@@ -11,7 +11,7 @@ from functools import partial
 from threading import Thread
 
 from PyQt4.Qt import SIGNAL, QObject, Qt, QTimer, QDate, \
-    QPixmap, QListWidgetItem, QDialog, pyqtSignal, QMessageBox, QIcon, \
+    QPixmap, QListWidgetItem, QDialog, pyqtSignal, QIcon, \
     QPushButton
 
 from calibre.gui2 import error_dialog, file_icon_provider, dynamic, \
@@ -208,6 +208,8 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
         from calibre.gui2 import config
         title = unicode(self.title.text()).strip()
         author = unicode(self.authors.text()).strip()
+        if author.endswith('&'):
+            author = author[:-1].strip()
         if not title or not author:
             return error_dialog(self, _('Specify title and author'),
                     _('You must specify a title and author before generating '
@@ -768,9 +770,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
             if question_dialog(self, _('Tags changed'),
                     _('You have changed the tags. In order to use the tags'
                        ' editor, you must either discard or apply these '
-                       'changes'), show_copy_button=False,
+                       'changes. Apply changes?'), show_copy_button=False):
-                    buttons=QMessageBox.Apply|QMessageBox.Discard,
-                    yes_button=QMessageBox.Apply):
                 self.apply_tags(commit=True, notify=True)
                 self.original_tags = unicode(self.tags.text())
             else:

src/calibre/gui2/dialogs/restore_library.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from PyQt4.Qt import QDialog, QLabel, QVBoxLayout, QDialogButtonBox, \
+        QProgressBar, QSize, QTimer, pyqtSignal, Qt
+
+from calibre.library.restore import Restore
+from calibre.gui2 import error_dialog, question_dialog, warning_dialog, \
+    info_dialog
+
+class DBRestore(QDialog):
+
+    update_signal = pyqtSignal(object, object)
+
+    def __init__(self, parent, library_path):
+        QDialog.__init__(self, parent)
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+        self.l1 = QLabel('<b>'+_('Restoring database from backups, do not'
+            ' interrupt, this will happen in three stages')+'...')
+        self.setWindowTitle(_('Restoring database'))
+        self.l.addWidget(self.l1)
+        self.pb = QProgressBar(self)
+        self.l.addWidget(self.pb)
+        self.pb.setMaximum(0)
+        self.pb.setMinimum(0)
+        self.msg = QLabel('')
+        self.l.addWidget(self.msg)
+        self.msg.setWordWrap(True)
+        self.bb = QDialogButtonBox(QDialogButtonBox.Cancel)
+        self.l.addWidget(self.bb)
+        self.bb.rejected.connect(self.reject)
+        self.resize(self.sizeHint() + QSize(100, 50))
+        self.error = None
+        self.rejected = False
+        self.library_path = library_path
+        self.update_signal.connect(self.do_update, type=Qt.QueuedConnection)
+
+        self.restorer = Restore(library_path, self)
+        self.restorer.daemon = True
+
+        # Give the metadata backup thread time to stop
+        QTimer.singleShot(2000, self.start)
+
+
+    def start(self):
+        self.restorer.start()
+        QTimer.singleShot(10, self.update)
+
+    def reject(self):
+        self.rejected = True
+        self.restorer.progress_callback = lambda x, y: x
+        QDialog.rejecet(self)
+
+    def update(self):
+        if self.restorer.is_alive():
+            QTimer.singleShot(10, self.update)
+        else:
+            self.restorer.progress_callback = lambda x, y: x
+            self.accept()
+
+    def __call__(self, msg, step):
+        self.update_signal.emit(msg, step)
+
+    def do_update(self, msg, step):
+        if msg is None:
+            self.pb.setMaximum(step)
+        else:
+            self.msg.setText(msg)
+            self.pb.setValue(step)
+
+
+def restore_database(db, parent=None):
+    if not question_dialog(parent, _('Are you sure?'), '<p>'+
+            _('Your list of books, with all their metadata is '
+                'stored in a single file, called a database. '
+                'In addition, metadata for each individual '
+                'book is stored in that books\' folder, as '
+                'a backup.'
+                '<p>This operation will rebuild '
+                'the database from the individual book '
+                'metadata. This is useful if the '
+                'database has been corrupted and you get a '
+                'blank list of books. Note that restoring only '
+                'restores books, not any settings stored in the '
+                'database, or any custom recipes.'
+                '<p>Do you want to restore the database?')):
+        return False
+    db.conn.close()
+    d = DBRestore(parent, db.library_path)
+    d.exec_()
+    r = d.restorer
+    d.restorer = None
+    if d.rejected:
+        return True
+    if r.tb is not None:
+        error_dialog(parent, _('Failed'),
+        _('Restoring database failed, click Show details to see details'),
+        det_msg=r.tb, show=True)
+    else:
+        if r.errors_occurred:
+            warning_dialog(parent, _('Success'),
+                    _('Restoring the database succeeded with some warnings'
+                        ' click Show details to see the details.'),
+                    det_msg=r.report, show=True)
+        else:
+            info_dialog(parent, _('Success'),
+                    _('Restoring database was successful'), show=True,
+                    show_copy_button=False)
+    return True
+

src/calibre/gui2/dialogs/scheduler.py

@@ -250,22 +250,27 @@ class Scheduler(QObject):
 
         self.timer = QTimer(self)
         self.timer.start(int(self.INTERVAL * 60 * 1000))
-        self.oldest_timer = QTimer()
-        self.connect(self.oldest_timer, SIGNAL('timeout()'), self.oldest_check)
         self.connect(self.timer, SIGNAL('timeout()'), self.check)
         self.oldest = gconf['oldest_news']
-        self.oldest_timer.start(int(60 * 60 * 1000))
         QTimer.singleShot(5 * 1000, self.oldest_check)
         self.database_changed = self.recipe_model.database_changed
 
     def oldest_check(self):
         if self.oldest > 0:
             delta = timedelta(days=self.oldest)
-            ids = self.recipe_model.db.tags_older_than(_('News'), delta)
+            try:
+                ids = list(self.recipe_model.db.tags_older_than(_('News'),
+                    delta))
+            except:
+                # Should never happen
+                ids = []
+                import traceback
+                traceback.print_exc()
             if ids:
-                ids = list(ids)
                 if ids:
                     self.delete_old_news.emit(ids)
+        QTimer.singleShot(60 * 60 * 1000, self.oldest_check)
+
 
     def show_dialog(self, *args):
         self.lock.lock()

src/calibre/gui2/dialogs/tag_categories.py

@@ -2,14 +2,14 @@ __license__   = 'GPL v3'
 
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
-
 from PyQt4.QtCore import SIGNAL, Qt
 from PyQt4.QtGui import QDialog, QIcon, QListWidgetItem
 
 from calibre.gui2.dialogs.tag_categories_ui import Ui_TagCategories
 from calibre.gui2.dialogs.confirm_delete import confirm
+from calibre.gui2 import error_dialog
 from calibre.constants import islinux
-from calibre.utils.icu import sort_key
+from calibre.utils.icu import sort_key, strcmp
 
 class Item:
     def __init__(self, name, label, index, icon, exists):
@@ -102,12 +102,13 @@ class TagCategories(QDialog, Ui_TagCategories):
             self.category_filter_box.addItem(v)
         self.current_cat_name = None
 
-        self.connect(self.apply_button,   SIGNAL('clicked()'), self.apply_tags)
+        self.apply_button.clicked.connect(self.apply_button_clicked)
-        self.connect(self.unapply_button, SIGNAL('clicked()'), self.unapply_tags)
+        self.unapply_button.clicked.connect(self.unapply_button_clicked)
-        self.connect(self.add_category_button, SIGNAL('clicked()'), self.add_category)
+        self.add_category_button.clicked.connect(self.add_category)
-        self.connect(self.category_box, SIGNAL('currentIndexChanged(int)'), self.select_category)
+        self.rename_category_button.clicked.connect(self.rename_category)
-        self.connect(self.category_filter_box, SIGNAL('currentIndexChanged(int)'), self.display_filtered_categories)
+        self.category_box.currentIndexChanged[int].connect(self.select_category)
-        self.connect(self.delete_category_button, SIGNAL('clicked()'), self.del_category)
+        self.category_filter_box.currentIndexChanged[int].connect(self.display_filtered_categories)
+        self.delete_category_button.clicked.connect(self.del_category)
         if islinux:
             self.available_items_box.itemDoubleClicked.connect(self.apply_tags)
         else:
@@ -119,6 +120,9 @@ class TagCategories(QDialog, Ui_TagCategories):
             l = self.category_box.findText(on_category)
             if l >= 0:
                 self.category_box.setCurrentIndex(l)
+        if self.current_cat_name is None:
+            self.category_box.setCurrentIndex(0)
+            self.select_category(0)
 
     def make_list_widget(self, item):
         n = item.name if item.exists else item.name + _(' (not on any book)')
@@ -137,6 +141,9 @@ class TagCategories(QDialog, Ui_TagCategories):
         for index in self.applied_items:
             self.applied_items_box.addItem(self.make_list_widget(self.all_items[index]))
 
+    def apply_button_clicked(self):
+        self.apply_tags(node=None)
+
     def apply_tags(self, node=None):
         if self.current_cat_name is None:
             return
@@ -148,6 +155,9 @@ class TagCategories(QDialog, Ui_TagCategories):
         self.applied_items.sort(key=lambda x:sort_key(self.all_items[x].name))
         self.display_filtered_categories(None)
 
+    def unapply_button_clicked(self):
+        self.unapply_tags(node=None)
+
     def unapply_tags(self, node=None):
         nodes = self.applied_items_box.selectedItems() if node is None else [node]
         for node in nodes:
@@ -160,15 +170,40 @@ class TagCategories(QDialog, Ui_TagCategories):
         cat_name = unicode(self.input_box.text()).strip()
         if cat_name == '':
             return False
+        for c in self.categories:
+            if strcmp(c, cat_name) == 0:
+                error_dialog(self, _('Name already used'),
+                        _('That name is already used, perhaps with different case.')).exec_()
+                return False
         if cat_name not in self.categories:
             self.category_box.clear()
             self.current_cat_name = cat_name
             self.categories[cat_name] = []
             self.applied_items = []
             self.populate_category_list()
+        self.input_box.clear()
+        self.category_box.setCurrentIndex(self.category_box.findText(cat_name))
+        return True
+
+    def rename_category(self):
+        self.save_category()
+        cat_name = unicode(self.input_box.text()).strip()
+        if cat_name == '':
+            return False
+        if not self.current_cat_name:
+            return False
+        for c in self.categories:
+            if strcmp(c, cat_name) == 0:
+                error_dialog(self, _('Name already used'),
+                        _('That name is already used, perhaps with different case.')).exec_()
+                return False
+        # The order below is important because of signals
+        self.categories[cat_name] = self.categories[self.current_cat_name]
+        del self.categories[self.current_cat_name]
+        self.current_cat_name = None
+        self.populate_category_list()
+        self.input_box.clear()
         self.category_box.setCurrentIndex(self.category_box.findText(cat_name))
-        else:
-            self.select_category(self.category_box.findText(cat_name))
         return True
 
     def del_category(self):
@@ -196,7 +231,6 @@ class TagCategories(QDialog, Ui_TagCategories):
 
     def accept(self):
         self.save_category()
-        self.db.prefs['user_categories'] = self.categories
         QDialog.accept(self)
 
     def save_category(self):
@@ -208,5 +242,7 @@ class TagCategories(QDialog, Ui_TagCategories):
             self.categories[self.current_cat_name] = l
 
     def populate_category_list(self):
-        for n in sorted(self.categories.keys(), key=sort_key):
+        self.category_box.blockSignals(True)
-            self.category_box.addItem(n)
+        self.category_box.clear()
+        self.category_box.addItems(sorted(self.categories.keys(), key=sort_key))
+        self.category_box.blockSignals(False)

src/calibre/gui2/dialogs/tag_categories.ui

@@ -18,7 +18,139 @@
     <normaloff>:/images/chapters.png</normaloff>:/images/chapters.png</iconset>
   </property>
   <layout class="QGridLayout">
+   <item row="0" column="0">
+    <layout class="QHBoxLayout">
+     <item>
+      <widget class="QLabel" name="label_3">
+       <property name="sizePolicy">
+        <sizepolicy hsizetype="Fixed" vsizetype="Preferred">
+         <horstretch>0</horstretch>
+         <verstretch>0</verstretch>
+        </sizepolicy>
+       </property>
+       <property name="minimumSize">
+        <size>
+         <width>100</width>
+         <height>0</height>
+        </size>
+       </property>
+       <property name="text">
+        <string>Category name: </string>
+       </property>
+       <property name="alignment">
+        <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter</set>
+       </property>
+       <property name="buddy">
+        <cstring>category_box</cstring>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QComboBox" name="category_box">
+       <property name="sizePolicy">
+        <sizepolicy hsizetype="Fixed" vsizetype="Fixed">
+         <horstretch>160</horstretch>
+         <verstretch>0</verstretch>
+        </sizepolicy>
+       </property>
+       <property name="minimumSize">
+        <size>
+         <width>145</width>
+         <height>0</height>
+        </size>
+       </property>
+       <property name="toolTip">
+        <string>Select a category to edit</string>
+       </property>
+       <property name="editable">
+        <bool>false</bool>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
+   <item row="0" column="1">
+    <widget class="QToolButton" name="delete_category_button">
+     <property name="toolTip">
+      <string>Delete this selected tag category</string>
+     </property>
+     <property name="text">
+      <string>...</string>
+     </property>
+     <property name="icon">
+      <iconset>
+       <normaloff>:/images/minus.png</normaloff>:/images/minus.png</iconset>
+     </property>
+    </widget>
+   </item>
+   <item row="0" column="2">
+    <layout class="QHBoxLayout">
+     <item>
+      <widget class="QLineEdit" name="input_box">
+       <property name="sizePolicy">
+        <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
+         <horstretch>60</horstretch>
+         <verstretch>0</verstretch>
+        </sizepolicy>
+       </property>
+       <property name="toolTip">
+        <string>Enter a category name, then use the add button or the rename button</string>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QToolButton" name="add_category_button">
+       <property name="toolTip">
+        <string>Add a new category</string>
+       </property>
+       <property name="text">
+        <string>...</string>
+       </property>
+       <property name="icon">
+        <iconset>
+         <normaloff>:/images/plus.png</normaloff>:/images/plus.png
+        </iconset>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
+   <item row="0" column="3">
+    <widget class="QToolButton" name="rename_category_button">
+     <property name="toolTip">
+      <string>Rename the current category to the what is in the box</string>
+     </property>
+     <property name="text">
+      <string>...</string>
+     </property>
+     <property name="icon">
+      <iconset>
+       <normaloff>:/images/edit-undo.png</normaloff>:/images/edit-undo.png</iconset>
+     </property>
+    </widget>
+   </item>
    <item row="1" column="0">
+    <layout class="QHBoxLayout">
+     <item>
+      <widget class="QLabel" name="label_5">
+       <property name="text">
+        <string>Category filter: </string>
+       </property>
+       <property name="alignment">
+        <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter</set>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QComboBox" name="category_filter_box">
+       <property name="toolTip">
+        <string>Select the content kind of the new category</string>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
+   <item row="2" column="0">
     <layout class="QVBoxLayout">
      <item>
       <layout class="QHBoxLayout">
@@ -66,7 +198,7 @@
      </item>
     </layout>
    </item>
-   <item row="1" column="1">
+   <item row="2" column="1">
     <layout class="QVBoxLayout">
      <item>
       <spacer>
@@ -110,7 +242,7 @@
      </item>
     </layout>
    </item>
-   <item row="1" column="2">
+   <item row="2" column="2">
     <layout class="QVBoxLayout">
      <item>
       <layout class="QHBoxLayout">
@@ -151,7 +283,7 @@
      </item>
     </layout>
    </item>
-   <item row="1" column="3">
+   <item row="2" column="3">
     <layout class="QVBoxLayout">
      <item>
       <spacer>
@@ -195,7 +327,7 @@
      </item>
     </layout>
    </item>
-   <item row="3" column="0" colspan="4">
+   <item row="4" column="0" colspan="4">
     <widget class="QDialogButtonBox" name="buttonBox">
      <property name="orientation">
       <enum>Qt::Horizontal</enum>
@@ -208,141 +340,6 @@
      </property>
     </widget>
    </item>
-   <item row="0" column="0" colspan="4">
-    <layout class="QGridLayout" name="gridLayout">
-     <item row="0" column="0">
-      <widget class="QLabel" name="label_3">
-       <property name="sizePolicy">
-        <sizepolicy hsizetype="Fixed" vsizetype="Preferred">
-         <horstretch>0</horstretch>
-         <verstretch>0</verstretch>
-        </sizepolicy>
-       </property>
-       <property name="minimumSize">
-        <size>
-         <width>100</width>
-         <height>0</height>
-        </size>
-       </property>
-       <property name="text">
-        <string>Category name: </string>
-       </property>
-       <property name="alignment">
-        <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter</set>
-       </property>
-       <property name="buddy">
-        <cstring>category_box</cstring>
-       </property>
-      </widget>
-     </item>
-     <item row="0" column="1">
-      <widget class="QComboBox" name="category_box">
-       <property name="sizePolicy">
-        <sizepolicy hsizetype="Fixed" vsizetype="Fixed">
-         <horstretch>160</horstretch>
-         <verstretch>0</verstretch>
-        </sizepolicy>
-       </property>
-       <property name="minimumSize">
-        <size>
-         <width>145</width>
-         <height>0</height>
-        </size>
-       </property>
-       <property name="toolTip">
-        <string>Select a category to edit</string>
-       </property>
-       <property name="editable">
-        <bool>false</bool>
-       </property>
-      </widget>
-     </item>
-     <item row="0" column="2">
-      <widget class="QToolButton" name="delete_category_button">
-       <property name="toolTip">
-        <string>Delete this selected tag category</string>
-       </property>
-       <property name="text">
-        <string>...</string>
-       </property>
-       <property name="icon">
-        <iconset>
-         <normaloff>:/images/minus.png</normaloff>:/images/minus.png</iconset>
-       </property>
-      </widget>
-     </item>
-     <item row="0" column="3">
-      <spacer>
-       <property name="orientation">
-        <enum>Qt::Horizontal</enum>
-       </property>
-       <property name="sizeHint" stdset="0">
-        <size>
-         <width>40</width>
-         <height>20</height>
-        </size>
-       </property>
-      </spacer>
-     </item>
-     <item row="0" column="4">
-      <widget class="QLineEdit" name="input_box">
-       <property name="sizePolicy">
-        <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
-         <horstretch>60</horstretch>
-         <verstretch>0</verstretch>
-        </sizepolicy>
-       </property>
-       <property name="toolTip">
-        <string>Enter a new category name. Select the kind before adding it.</string>
-       </property>
-      </widget>
-     </item>
-     <item row="0" column="5">
-      <widget class="QToolButton" name="add_category_button">
-       <property name="toolTip">
-        <string>Add the new category</string>
-       </property>
-       <property name="text">
-        <string>...</string>
-       </property>
-       <property name="icon">
-        <iconset>
-         <normaloff>:/images/plus.png</normaloff>:/images/plus.png</iconset>
-       </property>
-      </widget>
-     </item>
-     <item row="1" column="5">
-      <spacer name="horizontalSpacer">
-       <property name="orientation">
-        <enum>Qt::Horizontal</enum>
-       </property>
-       <property name="sizeHint" stdset="0">
-        <size>
-         <width>20</width>
-         <height>20</height>
-        </size>
-       </property>
-      </spacer>
-     </item>
-     <item row="1" column="0">
-      <widget class="QLabel" name="label_5">
-       <property name="text">
-        <string>Category filter: </string>
-       </property>
-       <property name="alignment">
-        <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter</set>
-       </property>
-      </widget>
-     </item>
-     <item row="1" column="1">
-      <widget class="QComboBox" name="category_filter_box">
-       <property name="toolTip">
-        <string>Select the content kind of the new category</string>
-       </property>
-      </widget>
-     </item>
-    </layout>
-   </item>
   </layout>
  </widget>
  <resources>

src/calibre/gui2/dialogs/tag_editor.py

@@ -16,7 +16,7 @@ class TagEditor(QDialog, Ui_TagEditor):
         self.setupUi(self)
 
         self.db = db
-        self.index = db.row(id_)
+        self.index = db.row(id_) if id_ is not None else None
         if self.index is not None:
             tags = self.db.tags(self.index)
         else:

src/calibre/gui2/filename_pattern.ui

@@ -43,7 +43,17 @@ p, li { white-space: pre-wrap; }
      </property>
      <layout class="QVBoxLayout">
       <item>
-       <widget class="QLineEdit" name="re"/>
+       <widget class="QComboBox" name="re">
+        <property name="editable">
+         <bool>true</bool>
+        </property>
+        <property name="maxCount">
+         <number>10</number>
+        </property>
+        <property name="insertPolicy">
+         <enum>QComboBox::InsertAtTop</enum>
+        </property>
+       </widget>
       </item>
      </layout>
     </widget>
@@ -94,8 +104,8 @@ p, li { white-space: pre-wrap; }
             <rect>
              <x>0</x>
              <y>0</y>
-             <width>301</width>
+             <width>277</width>
-             <height>234</height>
+             <height>276</height>
             </rect>
            </property>
            <layout class="QGridLayout" name="gridLayout_2">

src/calibre/gui2/jobs.py

@@ -19,7 +19,7 @@ from PyQt4.Qt import QAbstractTableModel, QVariant, QModelIndex, Qt, \
 
 from calibre.utils.ipc.server import Server
 from calibre.utils.ipc.job import ParallelJob
-from calibre.gui2 import Dispatcher, error_dialog, NONE, config, gprefs
+from calibre.gui2 import Dispatcher, error_dialog, question_dialog, NONE, config, gprefs
 from calibre.gui2.device import DeviceJob
 from calibre.gui2.dialogs.jobs_ui import Ui_JobsDialog
 from calibre import __appname__
@@ -380,8 +380,8 @@ class JobsDialog(QDialog, Ui_JobsDialog):
         self.model = model
         self.setWindowModality(Qt.NonModal)
         self.setWindowTitle(__appname__ + _(' - Jobs'))
-        self.kill_button.clicked.connect(self.kill_job)
         self.details_button.clicked.connect(self.show_details)
+        self.kill_button.clicked.connect(self.kill_job)
         self.stop_all_jobs_button.clicked.connect(self.kill_all_jobs)
         self.pb_delegate = ProgressBarDelegate(self)
         self.jobs_view.setItemDelegateForColumn(2, self.pb_delegate)
@@ -415,18 +415,19 @@ class JobsDialog(QDialog, Ui_JobsDialog):
         d.exec_()
         d.timer.stop()
 
-    def kill_job(self, *args):
-        for index in self.jobs_view.selectedIndexes():
-            row = index.row()
-            self.model.kill_job(row, self)
-            return
-
     def show_details(self, *args):
         for index in self.jobs_view.selectedIndexes():
             self.show_job_details(index)
             return
 
+    def kill_job(self, *args):
+        if question_dialog(self, _('Are you sure?'), _('Do you really want to stop the selected job?')):
+            for index in self.jobs_view.selectedIndexes():
+                row = index.row()
+                self.model.kill_job(row, self)
+
     def kill_all_jobs(self, *args):
+        if question_dialog(self, _('Are you sure?'), _('Do you really want to stop all non-device jobs?')):
             self.model.kill_all_jobs()
 
     def closeEvent(self, e):

src/calibre/gui2/main.py

@@ -4,7 +4,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import sys, os, time, socket, traceback
 from functools import partial
 
-from PyQt4.Qt import QCoreApplication, QIcon, QMessageBox, QObject, QTimer, \
+from PyQt4.Qt import QCoreApplication, QIcon, QObject, QTimer, \
         QThread, pyqtSignal, Qt, QProgressDialog, QString, QPixmap, \
         QSplashScreen, QApplication
 
@@ -150,13 +150,13 @@ class GuiRunner(QObject):
         if DEBUG:
             prints('Starting up...')
 
-    def start_gui(self):
+    def start_gui(self, db):
         from calibre.gui2.ui import Main
         main = Main(self.opts, gui_debug=self.gui_debug)
         if self.splash_screen is not None:
             self.splash_screen.showMessage(_('Initializing user interface...'))
             self.splash_screen.finish(main)
-        main.initialize(self.library_path, self.db, self.listener, self.actions)
+        main.initialize(self.library_path, db, self.listener, self.actions)
         if DEBUG:
             prints('Started up in', time.time() - self.startup_time)
         add_filesystem_book = partial(main.iactions['Add Books'].add_filesystem_book, allow_device=False)
@@ -200,8 +200,7 @@ class GuiRunner(QObject):
                     det_msg=traceback.format_exc(), show=True)
                 self.initialization_failed()
 
-        self.db = db
+        self.start_gui(db)
-        self.start_gui()
 
     def initialize_db(self):
         db = None
@@ -320,9 +319,6 @@ def run_gui(opts, args, actions, listener, app, gui_debug=None):
 
 def cant_start(msg=_('If you are sure it is not running')+', ',
         what=None):
-    d = QMessageBox(QMessageBox.Critical, _('Cannot Start ')+__appname__,
-        '<p>'+(_('%s is already running.')%__appname__)+'</p>',
-        QMessageBox.Ok)
     base = '<p>%s</p><p>%s %s'
     where = __appname__ + ' '+_('may be running in the system tray, in the')+' '
     if isosx:
@@ -335,8 +331,10 @@ def cant_start(msg=_('If you are sure it is not running')+', ',
         else:
             what = _('try deleting the file')+': '+ADDRESS
 
-    d.setInformativeText(base%(where, msg, what))
+    info = base%(where, msg, what)
-    d.exec_()
+    error_dialog(None, _('Cannot Start ')+__appname__,
+        '<p>'+(_('%s is already running.')%__appname__)+'</p>'+info, show=True)
+
     raise SystemExit(1)
 
 def communicate(args):

src/calibre/gui2/main_window.py

@@ -87,7 +87,7 @@ class MainWindow(QMainWindow):
             fe = sio.getvalue()
             prints(fe, file=sys.stderr)
             msg = '<b>%s</b>:'%type.__name__ + unicode(str(value), 'utf8', 'replace')
-            error_dialog(self, _('ERROR: Unhandled exception'), msg, det_msg=fe,
+            error_dialog(self, _('Unhandled exception'), msg, det_msg=fe,
                     show=True)
         except BaseException:
             pass

src/calibre/gui2/metadata/basic_widgets.py

@@ -10,7 +10,7 @@ import textwrap, re, os
 from PyQt4.Qt import Qt, QDateEdit, QDate, \
     QIcon, QToolButton, QWidget, QLabel, QGridLayout, \
     QDoubleSpinBox, QListWidgetItem, QSize, QPixmap, \
-    QPushButton, QSpinBox, QMessageBox, QLineEdit
+    QPushButton, QSpinBox, QLineEdit
 
 from calibre.gui2.widgets import EnLineEdit, CompleteComboBox, \
         EnComboBox, FormatList, ImageView, CompleteLineEdit
@@ -848,9 +848,7 @@ class TagsEdit(CompleteLineEdit): # {{{
             if question_dialog(self, _('Tags changed'),
                     _('You have changed the tags. In order to use the tags'
                        ' editor, you must either discard or apply these '
-                       'changes'), show_copy_button=False,
+                       'changes. Apply changes?'), show_copy_button=False):
-                    buttons=QMessageBox.Apply|QMessageBox.Discard,
-                    yes_button=QMessageBox.Apply):
                 self.commit(db, id_)
                 db.commit()
                 self.original_val = self.current_val