Sync to trunk.

resources/calibre-portable.bat

@@ -6,25 +6,37 @@ REM  - Calibre Library Files
 REM  - Calibre Config Files
 REM  - Calibre Metadata database
 REM  - Calibre Source files
+REM  - Calibre Temp Files
 REM By setting the paths correctly it can be used to run:
 REM  - A "portable calibre" off a USB stick.
 REM  - A network installation with local metadata database
 REM    (for performance) and books stored on a network share 
+REM  - A local installation using customised settings
 REM
-REM If trying to run off a USB stick then the following 
+REM If trying to run off a USB stick then the folder structure
-REM folder structure is recommended:
+REM shown below is recommended (relative to the location of 
+REM this batch file).  This can structure can also be used
+REM when running of a local hard disk if you want to get the
+REM level of control this batch file provides.
 REM  - Calibre2			Location of program files
 REM  - CalibreConfig		Location of Configuration files
 REM  - CalibreLibrary		Location of Books and metadata
+REM  - CalibreSource 		Location of Calibre Source files (Optional)
+REM
+REM This batch file is designed so that if you create the recommended
+REM folder structure then it can be used 'as is' without modification.
 
 
 REM -------------------------------------
 REM Set up Calibre Config folder
+REM
+REM This is where user specific settings
+REM are stored.
 REM -------------------------------------
 
 IF EXIST CalibreConfig (
 	SET CALIBRE_CONFIG_DIRECTORY=%cd%\CalibreConfig
-	ECHO CONFIG=%cd%\CalibreConfig
+	ECHO CONFIG FILES:    %cd%\CalibreConfig
 )
 
 
@@ -35,21 +47,18 @@ REM Location where Book files are located
 REM Either set explicit path, or if running from a USB stick
 REM a relative path can be used to avoid need to know the
 REM drive letter of the USB stick.
-
+REM
 REM Comment out any of the following that are not to be used
+REM (although leaving them in does not really matter)
 REM --------------------------------------------------------------
 
 IF EXIST U:\eBooks\CalibreLibrary (
 	SET CALIBRE_LIBRARY_DIRECTORY=U:\eBOOKS\CalibreLibrary
-	ECHO LIBRARY=U:\eBOOKS\CalibreLibrary
+	ECHO LIBRARY FILES:   U:\eBOOKS\CalibreLibrary
 )
 IF EXIST CalibreLibrary (
 	SET CALIBRE_LIBRARY_DIRECTORY=%cd%\CalibreLibrary
-	ECHO LIBRARY=%cd%\CalibreLibrary
+	ECHO LIBRARY FILES:   %cd%\CalibreLibrary
-)
-IF EXIST CalibreBooks (
-	SET CALIBRE_LIBRARY_DIRECTORY=%cd%\CalibreBooks
-	ECHO LIBRARY=%cd%\CalibreBooks
 )
 
 
@@ -60,7 +69,7 @@ REM Location where the metadata.db file is located.  If not set
 REM the same location as Books files will be assumed.  This.
 REM options is used to get better performance when the Library is
 REM on a (slow) network drive.  Putting the metadata.db file 
-REM locally makes gives a big performance improvement.
+REM locally then makes gives a big performance improvement.
 REM
 REM NOTE.  If you use this option, then the ability to switch
 REM        libraries within Calibre will be disabled.  Therefore
@@ -68,19 +77,10 @@ REM        you do not want to set it if the metadata.db file
 REM        is at the same location as the book files.
 REM --------------------------------------------------------------
 
-IF EXIST CalibreBooks (
+IF EXIST %cd%\CalibreMetadata\metadata.db (
-	IF NOT "%CALIBRE_LIBRARY_DIRECTORY%" == "%cd%\CalibreBooks" (
-		SET SET CALIBRE_OVERRIDE_DATABASE_PATH=%cd%\CalibreBooks\metadata.db
-		ECHO DATABASE=%cd%\CalibreBooks\metadata.db
-		ECHO '
-		ECHO ***CAUTION*** Library Switching will be disabled 
-		ECHO '
-	)
-)
-IF EXIST CalibreMetadata (
 	IF NOT "%CALIBRE_LIBRARY_DIRECTORY%" == "%cd%\CalibreMetadata" (
 		SET CALIBRE_OVERRIDE_DATABASE_PATH=%cd%\CalibreMetadata\metadata.db
-		ECHO DATABASE=%cd%\CalibreMetadata\metadata.db
+		ECHO DATABASE:        %cd%\CalibreMetadata\metadata.db
 		ECHO '
 		ECHO ***CAUTION*** Library Switching will be disabled 
 		ECHO '
@@ -96,37 +96,60 @@ REM When running from source the GUI will have a '*' after the version.
 REM number that is displayed at the bottom of the Calibre main screen.
 REM --------------------------------------------------------------
 
-IF EXIST Calibre\src (
+IF EXIST CalibreSource\src (
-	SET CALIBRE_DEVELOP_FROM=%cd%\Calibre\src
+	SET CALIBRE_DEVELOP_FROM=%cd%\CalibreSource\src
-	ECHO SOURCE=%cd%\Calibre\src
+	ECHO SOURCE FILES:    %cd%\CalibreSource\src
-)
-IF EXIST D:\Calibre\Calibre\src (
-	SET CALIBRE_DEVELOP_FROM=D:\Calibre\Calibre\src
-	ECHO SOURCE=D:\Calibre\Calibre\src
 )
 
+
 REM --------------------------------------------------------------
 REM Specify Location of calibre binaries (optional)
 REM
 REM To avoid needing Calibre to be set in the search path, ensure
 REM that Calibre Program Files is current directory when starting.
 REM The following test falls back to using search path .
-REM This folder can be populated by cpying the Calibre2 folder from
+REM This folder can be populated by copying the Calibre2 folder from
-REM an existing isntallation or by isntalling direct to here.
+REM an existing installation or by installing direct to here.
 REM --------------------------------------------------------------
 
-IF EXIST Calibre2 (
+IF EXIST %cd%\Calibre2 (
-	Calibre2 CD Calibre2
+	CD %cd%\Calibre2
-	ECHO PROGRAMS=%cd%
+	ECHO PROGRAM FILES:   %cd%
 )
 
+
+REM --------------------------------------------------------------
+REM Location of Calibre Temporary files  (optional)
+REM
+REM Calibre creates a lot of temproary files while running
+REM In theory these are removed when Calibre finishes, but
+REM in practise files can be left behind (particularily if
+REM any errors occur.  Using this option allows some
+REM explicit clean-up of these files.
+REM If not set Calibre uses the normal system TEMP location
+REM --------------------------------------------------------------
+
+SET CALIBRE_TEMP_DIR=%TEMP%\CALIBRE_TEMP
+ECHO TEMPORARY FILES: %CALIBRE_TEMP_DIR%
+
+IF NOT "%CALIBRE_TEMP_DIR%" == "" (
+	IF EXIST "%CALIBRE_TEMP_DIR%" RMDIR /s /q "%CALIBRE_TEMP_DIR%"
+	MKDIR "%CALIBRE_TEMP_DIR%"
+	REM set the following for any components that do
+	REM not obey the CALIBRE_TEMP_DIR setting
+	SET TMP=%CALIBRE_TEMP_DIR%
+	SET TEMP=%CALIBRE_TEMP_DIR%
+)
+
+
 REM ----------------------------------------------------------
 REM  The following gives a chance to check the settings before
 REM  starting Calibre.  It can be commented out if not wanted.
 REM ----------------------------------------------------------
 
-echo "Press CTRL-C if you do not want to continue"
+ECHO '
-pause
+ECHO "Press CTRL-C if you do not want to continue"
+PAUSE
 
 
 REM --------------------------------------------------------
@@ -141,5 +164,7 @@ REM If used without /WAIT opotion launches Calibre and contines batch file.
 REM Use with /WAIT to wait until Calibre completes to run a task on exit
 REM --------------------------------------------------------
 
-echo "Starting up Calibre"
+ECHO "Starting up Calibre"
-START /belownormal Calibre --with-library "%CALIBRE_LIBRARY_DIRECTORY%"
+ECHO OFF
+ECHO %cd%
+START /belownormal Calibre --with-library "%CALIBRE_LIBRARY_DIRECTORY%"

resources/catalog/stylesheet.css

@@ -62,6 +62,18 @@ div.description {
 	text-indent: 1em;
 	}
 
+/*
+* 	Attempt to minimize widows and orphans by logically grouping chunks
+* 	Recommend enabling for iPad
+*   Some reports of problems with Sony ereaders, presumably ADE engines
+*/
+/*
+div.logical_group {
+	display:inline-block;
+	width:100%;
+	}
+*/
+
 p.date_index {
 	font-size:x-large;
 	text-align:center;

resources/recipes/20_minutos.recipe

@@ -0,0 +1,67 @@
+# -*- coding: utf-8
+__license__   = 'GPL v3'
+__author__    = 'Luis Hernandez'
+__copyright__ = 'Luis Hernandez<tolyluis@gmail.com>'
+description   = 'Periódico gratuito en español - v0.5 - 25 Jan 2011'
+
+'''
+www.20minutos.es
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1294946868(BasicNewsRecipe):
+
+    title          = u'20 Minutos'
+    publisher      = u'Grupo 20 Minutos'
+
+    __author__            = u'Luis Hernández'
+    description           = u'Periódico gratuito en español'
+    cover_url     = 'http://estaticos.20minutos.es/mmedia/especiales/corporativo/css/img/logotipos_grupo20minutos.gif'
+
+    oldest_article = 5
+    max_articles_per_feed = 100
+
+    remove_javascript = True
+    no_stylesheets        = True
+    use_embedded_content  = False
+
+    encoding              = 'ISO-8859-1'
+    language              = 'es'
+    timefmt        = '[%a, %d %b, %Y]'
+
+    keep_only_tags     = [dict(name='div', attrs={'id':['content']})
+                                  ,dict(name='div', attrs={'class':['boxed','description','lead','article-content']})
+                                  ,dict(name='span', attrs={'class':['photo-bar']})
+                                  ,dict(name='ul', attrs={'class':['article-author']})
+                                ]
+
+    remove_tags_before = dict(name='ul' , attrs={'class':['servicios-sub']})
+    remove_tags_after  = dict(name='div' , attrs={'class':['related-news','col']})
+
+    remove_tags = [
+                     dict(name='ol', attrs={'class':['navigation',]})
+                    ,dict(name='span', attrs={'class':['action']})
+                    ,dict(name='div', attrs={'class':['twitter comments-list hidden','related-news','col']})
+                    ,dict(name='div', attrs={'id':['twitter-destacados']})
+                    ,dict(name='ul', attrs={'class':['article-user-actions','stripped-list']})
+                                          ]
+
+    feeds = [
+              (u'Portada'              , u'http://www.20minutos.es/rss/')
+             ,(u'Nacional'             , u'http://www.20minutos.es/rss/nacional/')
+             ,(u'Internacional'       , u'http://www.20minutos.es/rss/internacional/')
+             ,(u'Economia'           , u'http://www.20minutos.es/rss/economia/')
+             ,(u'Deportes'            , u'http://www.20minutos.es/rss/deportes/')
+             ,(u'Tecnologia'          , u'http://www.20minutos.es/rss/tecnologia/')
+             ,(u'Gente - TV'         , u'http://www.20minutos.es/rss/gente-television/')
+             ,(u'Motor'                 , u'http://www.20minutos.es/rss/motor/')
+             ,(u'Salud'                 , u'http://www.20minutos.es/rss/belleza-y-salud/')
+             ,(u'Viajes'                , u'http://www.20minutos.es/rss/viajes/')
+             ,(u'Vivienda'             , u'http://www.20minutos.es/rss/vivienda/')
+             ,(u'Empleo'              , u'http://www.20minutos.es/rss/empleo/')
+             ,(u'Cine'                  , u'http://www.20minutos.es/rss/cine/')
+             ,(u'Musica'               , u'http://www.20minutos.es/rss/musica/')
+             ,(u'Comunidad20'     , u'http://www.20minutos.es/rss/zona20/')
+            ]
+

resources/recipes/abc.recipe

@@ -0,0 +1,43 @@
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class ABCRecipe(BasicNewsRecipe):
+  title          = u'ABC Linuxu'
+  oldest_article = 5
+  max_articles_per_feed = 3#5
+  __author__ = 'Funthomas'
+  language = 'cs'
+
+  feeds = [
+    #(u'Blogy', u'http://www.abclinuxu.cz/auto/blogDigest.rss'),
+    (u'Články', u'http://www.abclinuxu.cz/auto/abc.rss'),
+    (u'Zprávičky','http://www.abclinuxu.cz/auto/zpravicky.rss')
+  ]
+
+  remove_javascript = True
+  no_stylesheets    = True
+  remove_attributes = ['width','height']
+
+  remove_tags_before = dict(name='h1')
+  remove_tags = [
+    dict(attrs={'class':['meta-vypis','page_tools','cl_perex']}),
+    dict(attrs={'class':['cl_nadpis-link','komix-nav']})
+  ]
+
+  remove_tags_after = [
+    dict(name='div',attrs={'class':['cl_perex','komix-nav']}),
+    dict(attrs={'class':['meta-vypis','page_tools']}),
+    dict(name='',attrs={'':''}),
+  ]
+
+
+  preprocess_regexps = [
+    (re.compile(r'</div>.*<p class="perex">', re.DOTALL),lambda match: '</div><p class="perex">')
+  ]
+  def print_version(self, url):
+    return url + '?varianta=print&noDiz'
+
+  extra_css = '''
+            h1 {font-size:130%; font-weight:bold}
+            h3 {font-size:111%; font-weight:bold}
+        '''

resources/recipes/dailytportal.recipe

@@ -0,0 +1,66 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011, Darko Miletic <darko.miletic at gmail.com>'
+'''
+daily.tportal.hr
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Pagina12(BasicNewsRecipe):
+    title                 = 'Daily tportal.h'
+    __author__            = 'Darko Miletic'
+    description           = 'News from Croatia'
+    publisher             = 'tportal.hr'
+    category              = 'news, politics, Croatia'
+    oldest_article        = 2
+    max_articles_per_feed = 200
+    no_stylesheets        = True
+    encoding              = 'utf-8'
+    use_embedded_content  = False
+    language              = 'en_HR'
+    remove_empty_feeds    = True
+    publication_type      = 'newsportal'
+    extra_css             = """
+                               body{font-family: Verdana,sans-serif }
+                               img{margin-bottom: 0.4em; display:block}
+                               h1,h2{color: #2D648A; font-family: Georgia,serif}
+                               .artAbstract{font-size: 1.2em; font-family: Georgia,serif}
+                            """
+
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
+
+    remove_tags = [
+                     dict(name=['meta','link','embed','object','iframe','base'])
+                    ,dict(name='div', attrs={'class':'artInfo'})
+                  ]
+    remove_attributes=['lang']
+
+    keep_only_tags=dict(attrs={'class':'articleDetails'})
+
+    feeds = [(u'News', u'http://daily.tportal.hr/rss/dailynaslovnicarss.xml')]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll('a'):
+            limg = item.find('img')
+            if item.string is not None:
+               str = item.string
+               item.replaceWith(str)
+            else:
+               if limg:
+                  item.name = 'div'
+                  item.attrs = []
+               else:
+                   str = self.tag_to_string(item)
+                   item.replaceWith(str)
+        for item in soup.findAll('img'):
+            if not item.has_key('alt'):
+               item['alt'] = 'image'
+        return soup
+

resources/recipes/everett_herald.recipe

@@ -0,0 +1,36 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1295088390(BasicNewsRecipe):
+     title          = u'Everett Herald'
+     language = 'en'
+     __author__ = '77ja65'
+     oldest_article = 4
+     max_articles_per_feed = 50
+     no_stylesheets = True
+     masthead_url          = 'http://heraldnet.com/images/hnet/jQueryComponents/jQueryNavigation/heraldnet_logo.png'
+     extra_css      = '.headline {font-size: x-large;} \n .fact { padding-top: 10pt  }'
+
+     feeds          = [(u'Local News',
+ u'http://heraldnet.com/section/RSS02&mime=xml'),
+         (u'Sports', u'http://heraldnet.com/section/RSS04&mime=xml'),
+         (u'Entertainment',
+ u'http://heraldnet.com/section/RSS07&mime=xml'),
+         (u'Life', u'http://heraldnet.com/section/RSS03&mime=xml'),
+         (u'Breaking News',
+ u'http://heraldnet.com/section/RSS34&mime=xml'),
+         (u'Seahawks', u'http://heraldnet.com/section/RSS22&mime=xml'),
+         (u'HeraldNet', u'http://heraldnet.com/section/RSS01&mime=xml'),
+         (u'Inside Everett',
+ u'http://heraldnet.com/section/RSS26&mime=xml')
+         ]
+
+     def print_version(self, url):
+       return url + "&template=PrinterFriendly"
+
+     extra_css = '''
+                     h1{font-family:Arial,Helvetica,sans-serif; font-
+ weight:bold;font-size:large;}
+                     h2{font-family:Arial,Helvetica,sans-serif; font-
+ weight:normal;font-size:small;}
+                 '''
+

resources/recipes/heise.recipe

@@ -52,6 +52,7 @@ class heiseDe(BasicNewsRecipe):
 		   dict(id='navi_login'),
 		   dict(id='navigation'),
 		   dict(id='breadcrumb'),
+		   dict(id='adblockerwarnung'),
 		   dict(id=''),
 		   dict(id='sitemap'),
 		   dict(id='bannerzone'),
@@ -67,3 +68,4 @@ class heiseDe(BasicNewsRecipe):
 
 
 
+

resources/recipes/hna.recipe

@@ -21,7 +21,7 @@ class hnaDe(BasicNewsRecipe):
     max_articles_per_feed = 40
     no_stylesheets = True
     remove_javascript = True
-    encoding = 'iso-8859-1'
+    encoding = 'utf-8'
 
     remove_tags = [dict(id='topnav'),
 		   dict(id='nav_main'),
@@ -60,3 +60,4 @@ class hnaDe(BasicNewsRecipe):
     feeds =  [ ('hna_soehre', 'http://feeds2.feedburner.com/hna/soehre'),
 	       ('hna_kassel', 'http://feeds2.feedburner.com/hna/kassel') ]
 
+

resources/recipes/idnes.recipe

@@ -0,0 +1,54 @@
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class iHeuteRecipe(BasicNewsRecipe):
+    __author__  = 'FunThomas'
+    title = u'iDnes.cz'
+    publisher = u'MAFRA a.s.'
+    description = 'iDNES.cz Zprávy, Technet, Komiksy a další'
+    oldest_article = 3
+    max_articles_per_feed = 2
+
+    feeds = [
+            (u'Zprávy', u'http://servis.idnes.cz/rss.asp?c=zpravodaj'),
+            (u'Sport', u'http://servis.idnes.cz/rss.asp?c=sport'),
+            (u'Technet', u'http://servis.idnes.cz/rss.asp?c=technet'),
+            (u'Mobil', u'http://servis.idnes.cz/rss.asp?c=mobil'),
+            (u'Ekonomika', u'http://servis.idnes.cz/rss.asp?c=ekonomikah'),
+            #(u'Kultura', u'http://servis.idnes.cz/rss.asp?c=kultura'),
+            (u'Cestování', u'http://servis.idnes.cz/rss.asp?c=iglobe'),
+            #(u'Kavárna', u'http://servis.idnes.cz/rss.asp?r=kavarna'),
+            (u'Komixy', u'http://servis.idnes.cz/rss.asp?c=komiksy')
+            ]
+
+
+    encoding = 'cp1250'
+    language = 'cs'
+    cover_url = 'http://g.idnes.cz/u/loga-n4/idnes.gif'
+    remove_javascript = True
+    no_stylesheets = True
+
+    remove_attributes = ['width','height']
+    remove_tags = [dict(name='div',   attrs={'id':['zooming']}),
+                   dict(name='div',   attrs={'class':['related','mapa-wrapper']}),
+                   dict(name='table', attrs={'id':['opener-img','portal']}),
+                   dict(name='table', attrs={'class':['video-16ku9']})]
+    remove_tags_after  = [dict(name='div',attrs={'id':['related','related2']})]
+
+    keep_only_tags = [dict(name='div', attrs={'class':['art-full adwords-text','dil-day']})
+                      ,dict(name='table',attrs={'class':['kemel-box']})]
+
+    def print_version(self, url):
+        print_url = url
+        split_url = url.split("?")
+        if (split_url[0].rfind('dilbert.asp')  != -1):      #dilbert komix
+            print_url = print_url.replace('.htm','.gif&tisk=1')
+            print_url = print_url.replace('.asp','.aspx')
+        elif (split_url[0].rfind('kemel.asp')  == -1):      #not Kemel komix
+            print_url = 'http://zpravy.idnes.cz/tiskni.asp?' +  split_url[1]
+        #kemel                  kemel print page doesn't work
+        return print_url
+
+    extra_css = '''
+                  h1 {font-size:125%; font-weight:bold}
+                  h3 {font-size:110%; font-weight:bold}
+                '''

resources/recipes/la_tribuna.recipe

@@ -0,0 +1,29 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1294946868(BasicNewsRecipe):
+    title             = u'La Tribuna de Talavera'
+    __author__  = 'Luis Hernández'
+    description   = 'Diario de Talavera de la Reina'
+    cover_url     = 'http://www.latribunadetalavera.es/entorno/mancheta.gif'
+
+    oldest_article = 5
+    max_articles_per_feed = 50
+
+    remove_javascript = True
+    no_stylesheets        = True
+    use_embedded_content  = False
+
+    encoding              = 'utf-8'
+    language              = 'es'
+    timefmt        = '[%a, %d %b, %Y]'
+
+    keep_only_tags     = [dict(name='div', attrs={'id':['articulo']})
+                                  ,dict(name='div', attrs={'class':['foto']})
+                                  ,dict(name='p', attrs={'id':['texto']})
+                                ]
+
+    remove_tags_before = dict(name='div' , attrs={'class':['comparte']})
+    remove_tags_after  = dict(name='div' , attrs={'id':['relacionadas']})
+
+
+    feeds          = [(u'Portada', u'http://www.latribunadetalavera.es/rss.html')]

resources/recipes/new_yorker.recipe

@@ -1,5 +1,5 @@
 __license__   = 'GPL v3'
-__copyright__ = '2008-2010, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2011, Darko Miletic <darko.miletic at gmail.com>'
 '''
 newyorker.com
 '''
@@ -54,10 +54,10 @@ class NewYorker(BasicNewsRecipe):
                         ,dict(attrs={'id':['show-header','show-footer'] })
                      ]
     remove_attributes = ['lang']
-    feeds             = [(u'The New Yorker', u'http://feeds.newyorker.com/services/rss/feeds/everything.xml')]
+    feeds             = [(u'The New Yorker', u'http://www.newyorker.com/services/rss/feeds/everything.xml')]
 
     def print_version(self, url):
-        return url + '?printable=true'
+        return 'http://www.newyorker.com' + url + '?printable=true'
 
     def image_url_processor(self, baseurl, url):
         return url.strip()

resources/recipes/nytimes_sub.recipe

@@ -1,6 +1,5 @@
 #!/usr/bin/env  python
 # -*- coding: utf-8 -*-
-
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 '''
@@ -28,6 +27,10 @@ class NYTimes(BasicNewsRecipe):
     # previous paid versions of the new york times to best sent to the back issues folder on the kindle
     replaceKindleVersion = False
 
+    # download higher resolution images than the small thumbnails typically included in the article
+    # the down side of having large beautiful images is the file size is much larger, on the order of 7MB per paper
+    useHighResImages = True
+
     # includeSections: List of sections to include. If empty, all sections found will be included.
     # Otherwise, only the sections named will be included. For example,
     #
@@ -90,7 +93,6 @@ class NYTimes(BasicNewsRecipe):
                     (u'Sunday Magazine',u'magazine'),
                     (u'Week in Review',u'weekinreview')]
 
-
     if headlinesOnly:
         title='New York Times Headlines'
         description = 'Headlines from the New York Times'
@@ -127,7 +129,7 @@ class NYTimes(BasicNewsRecipe):
 
     earliest_date = date.today() - timedelta(days=oldest_article)
 
-    __author__  = 'GRiker/Kovid Goyal/Nick Redding'
+    __author__  = 'GRiker/Kovid Goyal/Nick Redding/Ben Collier'
     language = 'en'
     requires_version = (0, 7, 5)
 
@@ -149,7 +151,7 @@ class NYTimes(BasicNewsRecipe):
                             'dottedLine',
                             'entry-meta',
                             'entry-response module',
-                            'icon enlargeThis',
+                            #'icon enlargeThis', #removed to provide option for high res images
                             'leftNavTabs',
                             'metaFootnote',
                             'module box nav',
@@ -163,7 +165,23 @@ class NYTimes(BasicNewsRecipe):
                             'entry-tags', #added for DealBook
                             'footer promos clearfix', #added for DealBook
                             'footer links clearfix', #added for DealBook
-                            'inlineImage module', #added for DealBook
+                            'tabsContainer', #added for other blog downloads
+                            'column lastColumn', #added for other blog downloads
+                            'pageHeaderWithLabel', #added for other gadgetwise downloads
+                            'column two', #added for other blog downloads
+                            'column two last', #added for other blog downloads
+                            'column three', #added for other blog downloads
+                            'column three last', #added for other blog downloads
+                            'column four',#added for other blog downloads
+                            'column four last',#added for other blog downloads
+                            'column last', #added for other blog downloads
+                            'timestamp published', #added for other blog downloads
+                            'entry entry-related',
+                            'subNavigation tabContent active', #caucus blog navigation
+                            'columnGroup doubleRule',
+                            'mediaOverlay slideshow',
+                            'headlinesOnly multiline flush',
+                            'wideThumb',
                             re.compile('^subNavigation'),
                             re.compile('^leaderboard'),
                             re.compile('^module'),
@@ -254,7 +272,7 @@ class NYTimes(BasicNewsRecipe):
     def exclude_url(self,url):
         if not url.startswith("http"):
             return True
-        if not url.endswith(".html") and 'dealbook.nytimes.com' not in url: #added for DealBook
+        if not url.endswith(".html") and 'dealbook.nytimes.com' not in url and 'blogs.nytimes.com' not in url: #added for DealBook
             return True
         if 'nytimes.com' not in url:
             return True
@@ -592,19 +610,84 @@ class NYTimes(BasicNewsRecipe):
                     self.log("Skipping article dated %s" % date_str)
                     return None
 
-        kicker_tag = soup.find(attrs={'class':'kicker'})
+        #all articles are from today, no need to print the date on every page
-        if kicker_tag: # remove Op_Ed author head shots
+        try:
-            tagline = self.tag_to_string(kicker_tag)
+            if not self.webEdition:
-            if tagline=='Op-Ed Columnist':
+                date_tag = soup.find(True,attrs={'class': ['dateline','date']})
-                img_div = soup.find('div','inlineImage module')
+                if date_tag:
-                if img_div:
+                    date_tag.extract()
-                    img_div.extract()
+        except:
+            self.log("Error removing the published date")
 
+        if self.useHighResImages:
+            try:
+                #open up all the "Enlarge this Image" pop-ups and download the full resolution jpegs
+                enlargeThisList = soup.findAll('div',{'class':'icon enlargeThis'})
+                if enlargeThisList:
+                    for popupref in enlargeThisList:
+                        popupreflink = popupref.find('a')
+                        if popupreflink:
+                            reflinkstring = str(popupreflink['href'])
+                            refstart = reflinkstring.find("javascript:pop_me_up2('") + len("javascript:pop_me_up2('")
+                            refend = reflinkstring.find(".html", refstart) + len(".html")
+                            reflinkstring = reflinkstring[refstart:refend]
+
+                            popuppage = self.browser.open(reflinkstring)
+                            popuphtml = popuppage.read()
+                            popuppage.close()
+                            if popuphtml:
+                                st = time.localtime()
+                                year = str(st.tm_year)
+                                month = "%.2d" % st.tm_mon
+                                day = "%.2d" % st.tm_mday
+                                imgstartpos = popuphtml.find('http://graphics8.nytimes.com/images/' + year + '/' +  month +'/' + day +'/') + len('http://graphics8.nytimes.com/images/' + year + '/' +  month +'/' + day +'/')
+                                highResImageLink = 'http://graphics8.nytimes.com/images/' + year + '/' +  month +'/' + day +'/' + popuphtml[imgstartpos:popuphtml.find('.jpg',imgstartpos)+4]
+                                popupSoup = BeautifulSoup(popuphtml)
+                                highResTag = popupSoup.find('img', {'src':highResImageLink})
+                                if highResTag:
+                                    try:
+                                        newWidth = highResTag['width']
+                                        newHeight = highResTag['height']
+                                        imageTag = popupref.parent.find("img")
+                                    except:
+                                        self.log("Error: finding width and height of img")
+                                    popupref.extract()
+                                    if imageTag:
+                                        try:
+                                            imageTag['src'] = highResImageLink
+                                            imageTag['width'] = newWidth
+                                            imageTag['height'] = newHeight
+                                        except:
+                                            self.log("Error setting the src width and height parameters")
+            except Exception:
+                self.log("Error pulling high resolution images")
+
+            try:
+                #remove "Related content" bar
+                runAroundsFound = soup.findAll('div',{'class':['articleInline runaroundLeft','articleInline doubleRule runaroundLeft','articleInline runaroundLeft firstArticleInline']})
+                if runAroundsFound:
+                    for runAround in runAroundsFound:
+                        #find all section headers
+                        hlines = runAround.findAll(True ,{'class':['sectionHeader','sectionHeader flushBottom']})
+                        if hlines:
+                            for hline in hlines:
+                                hline.extract()
+            except:
+                self.log("Error removing related content bar")
+
+
+            try:
+                #in case pulling images failed, delete the enlarge this text
+                enlargeThisList = soup.findAll('div',{'class':'icon enlargeThis'})
+                if enlargeThisList:
+                    for popupref in enlargeThisList:
+                        popupref.extract()
+            except:
+                self.log("Error removing Enlarge this text")
 
         return self.strip_anchors(soup)
 
     def postprocess_html(self,soup, True):
-
         try:
                 if self.one_picture_per_article:
                         # Remove all images after first
@@ -766,6 +849,8 @@ class NYTimes(BasicNewsRecipe):
         try:
             if len(article.text_summary.strip()) == 0:
                 articlebodies = soup.findAll('div',attrs={'class':'articleBody'})
+                if not articlebodies: #added to account for blog formats
+                    articlebodies = soup.findAll('div', attrs={'class':'entry-content'}) #added to account for blog formats
                 if articlebodies:
                     for articlebody in articlebodies:
                         if articlebody:
@@ -774,13 +859,14 @@ class NYTimes(BasicNewsRecipe):
                                 refparagraph = self.massageNCXText(self.tag_to_string(p,use_alt=False)).strip()
                                 #account for blank paragraphs and short paragraphs by appending them to longer ones
                                 if len(refparagraph) > 0:
-                                    if len(refparagraph) > 70: #approximately one line of text
+                                    if len(refparagraph) > 140: #approximately two lines of text
                                         article.summary = article.text_summary = shortparagraph + refparagraph
                                         return
                                     else:
                                         shortparagraph = refparagraph + " "
                                         if shortparagraph.strip().find(" ") == -1 and not shortparagraph.strip().endswith(":"):
                                             shortparagraph = shortparagraph + "- "
+
         except:
             self.log("Error creating article descriptions")
             return

resources/recipes/roger_ebert.recipe

@@ -0,0 +1,120 @@
+import re
+import urllib2
+from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import BeautifulSoup, SoupStrainer
+
+class Ebert(BasicNewsRecipe):
+    title                 = 'Roger Ebert'
+    __author__            = 'Shane Erstad'
+    description           = 'Roger Ebert Movie Reviews'
+    publisher             = 'Chicago Sun Times'
+    category              = 'movies'
+    oldest_article        = 8
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    encoding              = 'utf-8'
+    masthead_url          = 'http://rogerebert.suntimes.com/graphics/global/roger.jpg'
+    language              = 'en'
+    remove_empty_feeds    = False
+    PREFIX                  = 'http://rogerebert.suntimes.com'
+    patternReviews                = r'<span class="*?movietitle"*?>(.*?)</span>.*?<div class="*?headline"*?>(.*?)</div>(.*?)</div>'
+    patternCommentary       = r'<div class="*?headline"*?>.*?(<a href="/apps/pbcs.dll/article\?AID=.*?COMMENTARY.*?" id="ltred">.*?</a>).*?<div class="blurb clear">(.*?)</div>'
+    patternPeople           = r'<div class="*?headline"*?>.*?(<a href="/apps/pbcs.dll/article\?AID=.*?PEOPLE.*?" id="ltred">.*?</a>).*?<div class="blurb clear">(.*?)</div>'
+    patternGlossary           = r'<div class="*?headline"*?>.*?(<a href="/apps/pbcs.dll/article\?AID=.*?GLOSSARY.*?" id="ltred">.*?</a>).*?<div class="blurb clear">(.*?)</div>'
+
+
+
+    conversion_options = {
+                          'comment'          : description
+                        , 'tags'             : category
+                        , 'publisher'        : publisher
+                        , 'language'         : language
+                        , 'linearize_tables' : True
+                        }
+
+
+    feeds          = [
+                        (u'Reviews'   , u'http://rogerebert.suntimes.com/apps/pbcs.dll/section?category=reviews' )
+                        ,(u'Commentary'   , u'http://rogerebert.suntimes.com/apps/pbcs.dll/section?category=COMMENTARY')
+                        ,(u'Great Movies'   , u'http://rogerebert.suntimes.com/apps/pbcs.dll/section?category=REVIEWS08')
+                        ,(u'People'   , u'http://rogerebert.suntimes.com/apps/pbcs.dll/section?category=PEOPLE')
+                        ,(u'Glossary'   , u'http://rogerebert.suntimes.com/apps/pbcs.dll/section?category=GLOSSARY')
+
+                     ]
+
+    preprocess_regexps = [
+        (re.compile(r'<font.*?>.*?This is a printer friendly.*?</font>.*?<hr>', re.DOTALL|re.IGNORECASE),
+            lambda m: '')
+    ]
+
+
+
+    def print_version(self, url):
+        return url + '&template=printart'
+
+    def parse_index(self):
+        totalfeeds = []
+        lfeeds = self.get_feeds()
+        for feedobj in lfeeds:
+            feedtitle, feedurl = feedobj
+            self.log('\tFeedurl: ', feedurl)
+            self.report_progress(0, _('Fetching feed')+' %s...'%(feedtitle if feedtitle else feedurl))
+            articles = []
+            page = urllib2.urlopen(feedurl).read()
+
+            if feedtitle == 'Reviews' or feedtitle == 'Great Movies':
+                    pattern = self.patternReviews
+            elif feedtitle == 'Commentary':
+                    pattern = self.patternCommentary
+            elif feedtitle == 'People':
+                    pattern = self.patternPeople
+            elif feedtitle == 'Glossary':
+                    pattern = self.patternGlossary
+
+
+            regex = re.compile(pattern, re.IGNORECASE|re.DOTALL)
+
+            for match in regex.finditer(page):
+                if feedtitle == 'Reviews' or feedtitle == 'Great Movies':
+                    movietitle = match.group(1)
+                    thislink = match.group(2)
+                    description = match.group(3)
+                elif feedtitle == 'Commentary' or feedtitle == 'People' or feedtitle == 'Glossary':
+                    thislink = match.group(1)
+                    description = match.group(2)
+
+                self.log(thislink)
+
+                for link in BeautifulSoup(thislink, parseOnlyThese=SoupStrainer('a')):
+                    thisurl = self.PREFIX + link['href']
+                    thislinktext = self.tag_to_string(link)
+
+                    if feedtitle == 'Reviews' or feedtitle == 'Great Movies':
+                        thistitle = movietitle
+                    elif feedtitle == 'Commentary' or feedtitle == 'People' or feedtitle == 'Glossary':
+                        thistitle = thislinktext
+
+                    if thistitle == '':
+                        thistitle = 'Ebert Journal Post'
+
+                    """
+                    pattern2 = r'AID=\/(.*?)\/'
+                    reg2 = re.compile(pattern2, re.IGNORECASE|re.DOTALL)
+                    match2 = reg2.search(thisurl)
+                    date = match2.group(1)
+                    c = time.strptime(match2.group(1),"%Y%m%d")
+                    date=time.strftime("%a, %b %d, %Y", c)
+                    self.log(date)
+                    """
+
+                    articles.append({
+                                      'title'      :thistitle
+                                     ,'date'       :''
+                                     ,'url'        :thisurl
+                                     ,'description':description
+                                    })
+            totalfeeds.append((feedtitle, articles))
+
+        return totalfeeds
+

resources/recipes/root.recipe

@@ -0,0 +1,39 @@
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1289939440(BasicNewsRecipe):
+  __author__  = 'FunThomas'
+  title       = u'Root.cz'
+  description = u'Zprávičky a články z Root.cz'
+  publisher   = u'Internet Info, s.r.o'
+  oldest_article = 2               #max stari clanku ve dnech
+  max_articles_per_feed = 50       #max pocet clanku na feed
+
+  feeds = [
+    (u'Články', u'http://www.root.cz/rss/clanky/'),
+    (u'Zprávičky', u'http://www.root.cz/rss/zpravicky/')
+  ]
+
+  publication_type = u'magazine'
+  language = u'cs'
+  no_stylesheets = True
+  remove_javascript = True
+  cover_url = u'http://i.iinfo.cz/urs/logo-root-bila-oranzova-cerna-111089527143118.gif'
+
+  remove_attributes = ['width','height','href'] #,'href'
+  keep_only_tags = [
+    dict(name='h1'),
+    dict(name='a',attrs={'class':'author'}),
+    dict(name='p', attrs={'class':'intro'}),
+    dict(name='div',attrs={'class':'urs'})
+  ]
+
+  preprocess_regexps = [
+    (re.compile(u'<p class="perex[^"]*">[^<]*<img[^>]*>', re.DOTALL),lambda match: '<p class="intro">'),
+    (re.compile(u'<h3><a name="tucnak">Tričko tučňák.*</body>', re.DOTALL),lambda match: '<!--deleted-->')
+  ]
+
+  extra_css = '''
+                h1 {font-size:130%; font-weight:bold}
+                h3 {font-size:111%; font-weight:bold}
+              '''

resources/recipes/sinfest.recipe

@@ -0,0 +1,33 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Nadid <nadid.skywalker at gmail.com>'
+'''
+http://www.sinfest.net
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class SinfestBig(BasicNewsRecipe):
+    title                  = 'Sinfest'
+    __author__             = 'nadid'
+    description            = 'Sinfest'
+    reverse_article_order = False
+    oldest_article         = 5
+    max_articles_per_feed  = 100
+    no_stylesheets         = True
+    use_embedded_content   = True
+    encoding               = 'utf-8'
+    publisher              = 'Tatsuya Ishida/Museworks'
+    category               = 'comic'
+    language               = 'en'
+
+    conversion_options = {
+                             'comments'        : description
+                            ,'tags'            : category
+                            ,'language'        : language
+                            ,'publisher'       : publisher
+                         }
+
+    feeds = [(u'SinFest', u'http://henrik.nyh.se/scrapers/sinfest.rss' )]
+    def get_article_url(self, article):
+        return article.get('link')
+

resources/recipes/zdnet.recipe

@@ -27,12 +27,34 @@ class cdnet(BasicNewsRecipe):
 		   dict(id='header'),
 		   dict(id='search'),
 		   dict(id='nav'),
+		   dict(id='blog-author-info'),
+		   dict(id='post-tags'),
+		   dict(id='bio-naraine'),
+		   dict(id='bio-kennedy'),
+		   dict(id='author-short-disclosure-kennedy'),
 		   dict(id=''),
 		   dict(name='div', attrs={'class':'banner'}),
+		   dict(name='div', attrs={'class':'int'}),
+		   dict(name='div', attrs={'class':'talkback clear space-2'}),
+		   dict(name='div', attrs={'class':'content-1 clear'}),
+		   dict(name='div', attrs={'class':'space-2'}),
+		   dict(name='div', attrs={'class':'space-3'}),
+		   dict(name='div', attrs={'class':'thumb-2 left'}),
+		   dict(name='div', attrs={'class':'hotspot'}),
+		   dict(name='div', attrs={'class':'hed hed-1 space-1'}),
+		   dict(name='div', attrs={'class':'view-1 clear content-3 space-2'}),
+		   dict(name='div', attrs={'class':'hed hed-1 space-1'}),
+		   dict(name='div', attrs={'class':'hed hed-1'}),
+		   dict(name='div', attrs={'class':'post-header'}),
+		   dict(name='div', attrs={'class':'lvl-nav clear'}),
+		   dict(name='div', attrs={'class':'t-share-overlay overlay-pop contain-overlay-4'}),
 		   dict(name='p', attrs={'class':'tags'}),
+		   dict(name='span', attrs={'class':'follow'}),
+		   dict(name='span', attrs={'class':'int'}),
+		   dict(name='h4', attrs={'class':'h s-4'}),
 		   dict(name='a', attrs={'href':'http://www.twitter.com/ryanaraine'}),
 		   dict(name='div', attrs={'class':'special1'})]
-    remove_tags_after = [dict(name='div', attrs={'class':'bloggerDesc clear'})]
+    remove_tags_after = [dict(name='div', attrs={'class':'clear'})]
 
     feeds =  [ ('zdnet', 'http://feeds.feedburner.com/zdnet/security') ]
 
@@ -43,3 +65,4 @@ class cdnet(BasicNewsRecipe):
         return soup
 
 
+

src/calibre/devices/android/driver.py

@@ -54,7 +54,7 @@ class ANDROID(USBMS):
             0x1004 : { 0x61cc : [0x100] },
 
             # Archos
-            0x0e79 : { 0x1419: [0x0216], 0x1420 : [0x0216]},
+            0x0e79 : { 0x1419: [0x0216], 0x1420 : [0x0216], 0x1422 : [0x0216]},
 
             }
     EBOOK_DIR_MAIN = ['eBooks/import', 'wordplayer/calibretransfer', 'Books']
@@ -70,7 +70,7 @@ class ANDROID(USBMS):
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID',
             'SCH-I500_CARD', 'SPH-D700_CARD', 'MB810', 'GT-P1000', 'DESIRE',
-            'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT']
+            'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT']

src/calibre/devices/folder_device/driver.py

@@ -22,7 +22,7 @@ class FOLDER_DEVICE_FOR_CONFIG(USBMS):
     PRODUCT_ID  = [0xffff]
     BCD         = [0xffff]
     DEVICE_PLUGBOARD_NAME = 'FOLDER_DEVICE'
-
+    SUPPORTS_SUB_DIRS = True
 
 class FOLDER_DEVICE(USBMS):
     type = _('Device Interface')

src/calibre/ebooks/conversion/utils.py

@@ -25,13 +25,15 @@ class HeuristicProcessor(object):
         self.chapters_with_title = 0
         self.blanks_deleted = False
         self.linereg = re.compile('(?<=<p).*?(?=</p>)', re.IGNORECASE|re.DOTALL)
-        self.blankreg = re.compile(r'\s*(?P<openline><p(?!\sid=\"softbreak\")[^>]*>)\s*(?P<closeline></p>)', re.IGNORECASE)
+        self.blankreg = re.compile(r'\s*(?P<openline><p(?!\sclass=\"softbreak\")[^>]*>)\s*(?P<closeline></p>)', re.IGNORECASE)
+        self.softbreak = re.compile(r'\s*(?P<openline><p(?=\sclass=\"softbreak\")[^>]*>)\s*(?P<closeline></p>)', re.IGNORECASE)
         self.multi_blank = re.compile(r'(\s*<p[^>]*>\s*</p>){2,}', re.IGNORECASE)
 
     def is_pdftohtml(self, src):
         return '<!-- created by calibre\'s pdftohtml -->' in src[:1000]
 
     def chapter_head(self, match):
+        from calibre.utils.html2text import html2text
         chap = match.group('chap')
         title = match.group('title')
         if not title:
@@ -40,10 +42,12 @@ class HeuristicProcessor(object):
                     " chapters. - " + unicode(chap))
             return '<h2>'+chap+'</h2>\n'
         else:
+            txt_chap = html2text(chap)
+            txt_title = html2text(title)
             self.html_preprocess_sections = self.html_preprocess_sections + 1
             self.log.debug("marked " + unicode(self.html_preprocess_sections) +
                     " chapters & titles. - " + unicode(chap) + ", " + unicode(title))
-            return '<h2>'+chap+'</h2>\n<h3>'+title+'</h3>\n'
+            return '<h2 title="'+txt_chap+', '+txt_title+'">'+chap+'</h2>\n<h3 class="sigilNotInTOC">'+title+'</h3>\n'
 
     def chapter_break(self, match):
         chap = match.group('section')
@@ -203,8 +207,8 @@ class HeuristicProcessor(object):
             blank_lines = ""
         opt_title_open = "("
         opt_title_close = ")?"
-        n_lookahead_open = "\s+(?!"
+        n_lookahead_open = "(?!\s*"
-        n_lookahead_close = ")"
+        n_lookahead_close = ")\s*"
 
         default_title = r"(<[ibu][^>]*>)?\s{0,3}(?!Chapter)([\w\:\'’\"-]+\s{0,3}){1,5}?(</[ibu][^>]*>)?(?=<)"
         simple_title = r"(<[ibu][^>]*>)?\s{0,3}(?!(Chapter|\s+<)).{0,65}?(</[ibu][^>]*>)?(?=<)"
@@ -215,7 +219,7 @@ class HeuristicProcessor(object):
             [r"[^'\"]?(Introduction|Synopsis|Acknowledgements|Epilogue|CHAPTER|Kapitel|Volume\b|Prologue|Book\b|Part\b|Dedication|Preface)\s*([\d\w-]+\:?\'?\s*){0,5}", True, True, True, False, "Searching for common section headings", 'common'],
             [r"[^'\"]?(CHAPTER|Kapitel)\s*([\dA-Z\-\'\"\?!#,]+\s*){0,7}\s*", True, True, True, False, "Searching for most common chapter headings", 'chapter'],  # Highest frequency headings which include titles
             [r"<b[^>]*>\s*(<span[^>]*>)?\s*(?!([*#•=]+\s*)+)(\s*(?=[\d.\w#\-*\s]+<)([\d.\w#-*]+\s*){1,5}\s*)(?!\.)(</span>)?\s*</b>", True, True, True, False, "Searching for emphasized lines", 'emphasized'], # Emphasized lines
-            [r"[^'\"]?(\d+(\.|:))\s*([\dA-Z\-\'\"#,]+\s*){0,7}\s*", True, True, True, False, "Searching for numeric chapter headings", 'numeric'],  # Numeric Chapters
+            [r"[^'\"]?(\d+(\.|:))\s*([\w\-\'\"#,]+\s*){0,7}\s*", True, True, True, False, "Searching for numeric chapter headings", 'numeric'],  # Numeric Chapters
             [r"([A-Z]\s+){3,}\s*([\d\w-]+\s*){0,3}\s*", True, True, True, False, "Searching for letter spaced headings", 'letter_spaced'],  # Spaced Lettering
             [r"[^'\"]?(\d+\.?\s+([\d\w-]+\:?\'?-?\s?){0,5})\s*", True, True, True, False, "Searching for numeric chapters with titles", 'numeric_title'], # Numeric Titles
             [r"[^'\"]?(\d+)\s*([\dA-Z\-\'\"\?!#,]+\s*){0,7}\s*", True, True, True, False, "Searching for simple numeric headings", 'plain_number'],  # Numeric Chapters, no dot or colon
@@ -275,7 +279,7 @@ class HeuristicProcessor(object):
                         self.log.debug(unicode(type_name)+" had "+unicode(hits)+" hits - "+unicode(self.chapters_no_title)+" chapters with no title, "+unicode(self.chapters_with_title)+" chapters with titles, "+unicode(float(self.chapters_with_title) / float(hits))+" percent. ")
                         if type_name == 'common':
                             analysis_result.append([chapter_type, n_lookahead_req, strict_title, ignorecase, title_req, log_message, type_name])
-                        elif self.min_chapters <= hits < max_chapters:
+                        elif self.min_chapters <= hits < max_chapters or self.min_chapters < 3 > hits:
                             analysis_result.append([chapter_type, n_lookahead_req, strict_title, ignorecase, title_req, log_message, type_name])
                             break
                 else:
@@ -367,6 +371,8 @@ class HeuristicProcessor(object):
         html = re.sub(ur'\s*<o:p>\s*</o:p>', ' ', html)
         # Delete microsoft 'smart' tags
         html = re.sub('(?i)</?st1:\w+>', '', html)
+        # Delete self closing paragraph tags
+        html = re.sub('<p\s?/>', '', html)
         # Get rid of empty span, bold, font, em, & italics tags
         html = re.sub(r"\s*<span[^>]*>\s*(<span[^>]*>\s*</span>){0,2}\s*</span>\s*", " ", html)
         html = re.sub(r"\s*<(font|[ibu]|em)[^>]*>\s*(<(font|[ibu]|em)[^>]*>\s*</(font|[ibu]|em)>\s*){0,2}\s*</(font|[ibu]|em)>", " ", html)
@@ -467,7 +473,7 @@ class HeuristicProcessor(object):
         if blanks_between_paragraphs and getattr(self.extra_opts, 'delete_blank_paragraphs', False):
             self.log.debug("deleting blank lines")
             self.blanks_deleted = True
-            html = self.multi_blank.sub('\n<p id="softbreak" style="margin-top:1.5em; margin-bottom:1.5em"> </p>', html)
+            html = self.multi_blank.sub('\n<p class="softbreak" style="margin-top:1.5em; margin-bottom:1.5em"> </p>', html)
             html = self.blankreg.sub('', html)
 
         # Determine line ending type
@@ -522,11 +528,11 @@ class HeuristicProcessor(object):
             # Center separator lines
             html = re.sub(u'<(?P<outer>p|div)[^>]*>\s*(<(?P<inner1>font|span|[ibu])[^>]*>)?\s*(<(?P<inner2>font|span|[ibu])[^>]*>)?\s*(<(?P<inner3>font|span|[ibu])[^>]*>)?\s*(?P<break>([*#•=✦]+\s*)+)\s*(</(?P=inner3)>)?\s*(</(?P=inner2)>)?\s*(</(?P=inner1)>)?\s*</(?P=outer)>', '<p style="text-align:center; margin-top:1.25em; margin-bottom:1.25em">' + '\g<break>' + '</p>', html)
             if not self.blanks_deleted:
-                html = self.multi_blank.sub('\n<p id="softbreak" style="margin-top:1.5em; margin-bottom:1.5em"> </p>', html)
+                html = self.multi_blank.sub('\n<p class="softbreak" style="margin-top:1.5em; margin-bottom:1.5em"> </p>', html)
-            html = re.sub('<p\s+id="softbreak"[^>]*>\s*</p>', '<div id="softbreak" style="margin-left: 45%; margin-right: 45%; margin-top:1.5em; margin-bottom:1.5em"><hr style="height: 3px; background:#505050" /></div>', html)
+            html = re.sub('<p\s+class="softbreak"[^>]*>\s*</p>', '<div id="softbreak" style="margin-left: 45%; margin-right: 45%; margin-top:1.5em; margin-bottom:1.5em"><hr style="height: 3px; background:#505050" /></div>', html)
 
         if self.deleted_nbsps:
             # put back non-breaking spaces in empty paragraphs to preserve original formatting
             html = self.blankreg.sub('\n'+r'\g<openline>'+u'\u00a0'+r'\g<closeline>', html)
-
+            html = self.softbreak.sub('\n'+r'\g<openline>'+u'\u00a0'+r'\g<closeline>', html)
         return html

src/calibre/ebooks/metadata/fetch.py

@@ -411,7 +411,7 @@ def search(title=None, author=None, publisher=None, isbn=None, isbndb_key=None,
                     r.pubdate = pubdate
 
     def fix_case(x):
-        if x and x.isupper():
+        if x:
             x = titlecase(x)
         return x
 

src/calibre/ebooks/metadata/sources/__init__.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+

src/calibre/ebooks/oeb/base.py

@@ -221,7 +221,10 @@ def rewrite_links(root, link_repl_func, resolve_base_href=False):
                         el.text):
             stylesheet = parseString(el.text)
             replaceUrls(stylesheet, link_repl_func)
-            el.text = '\n'+stylesheet.cssText + '\n'
+            repl = stylesheet.cssText
+            if isbytestring(repl):
+                repl = repl.decode('utf-8')
+            el.text = '\n'+ repl + '\n'
 
         if 'style' in el.attrib:
             text = el.attrib['style']
@@ -234,8 +237,11 @@ def rewrite_links(root, link_repl_func, resolve_base_href=False):
                             set_property(item)
                     elif v.CSS_PRIMITIVE_VALUE == v.cssValueType:
                         set_property(v)
-                el.attrib['style'] = stext.cssText.replace('\n', ' ').replace('\r',
+                repl = stext.cssText.replace('\n', ' ').replace('\r',
                         ' ')
+                if isbytestring(repl):
+                    repl = repl.decode('utf-8')
+                el.attrib['style'] = repl
 
 
 

src/calibre/ebooks/oeb/reader.py

@@ -17,7 +17,7 @@ from lxml import etree
 import cssutils
 
 from calibre.ebooks.oeb.base import OPF1_NS, OPF2_NS, OPF2_NSMAP, DC11_NS, \
-    DC_NSES, OPF
+    DC_NSES, OPF, xml2text
 from calibre.ebooks.oeb.base import OEB_DOCS, OEB_STYLES, OEB_IMAGES, \
     PAGE_MAP_MIME, JPEG_MIME, NCX_MIME, SVG_MIME
 from calibre.ebooks.oeb.base import XMLDECL_RE, COLLAPSE_RE, \
@@ -423,7 +423,7 @@ class OEBReader(object):
             path, frag = urldefrag(href)
             if path not in self.oeb.manifest.hrefs:
                 continue
-            title = ' '.join(xpath(anchor, './/text()'))
+            title = xml2text(anchor)
             title = COLLAPSE_RE.sub(' ', title.strip())
             if href not in titles:
                 order.append(href)

src/calibre/gui2/__init__.py

@@ -550,6 +550,14 @@ def choose_dir(window, name, title, default_dir='~'):
     if dir:
         return dir[0]
 
+def choose_osx_app(window, name, title, default_dir='/Applications'):
+    fd = FileDialog(title=title, parent=window, name=name, mode=QFileDialog.ExistingFile,
+            default_dir=default_dir)
+    app = fd.get_files()
+    fd.setParent(None)
+    if app:
+        return app
+
 def choose_files(window, name, title,
                  filters=[], all_files=True, select_only_single_file=False):
     '''

src/calibre/gui2/actions/annotate.py

@@ -9,7 +9,7 @@ import os, datetime
 
 from PyQt4.Qt import pyqtSignal, QModelIndex, QThread, Qt
 
-from calibre.gui2 import error_dialog, gprefs
+from calibre.gui2 import error_dialog
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag, NavigableString
 from calibre import strftime
 from calibre.gui2.actions import InterfaceAction
@@ -165,10 +165,12 @@ class FetchAnnotationsAction(InterfaceAction):
                 ka_soup.insert(0,divTag)
                 return ka_soup
 
+            '''
             def mark_book_as_read(self,id):
                 read_tag = gprefs.get('catalog_epub_mobi_read_tag')
                 if read_tag:
                     self.db.set_tags(id, [read_tag], append=True)
+            '''
 
             def canceled(self):
                 self.pd.hide()
@@ -201,10 +203,12 @@ class FetchAnnotationsAction(InterfaceAction):
                         # Update library comments
                         self.db.set_comment(id, mi.comments)
 
+                        '''
                         # Update 'read' tag except for Catalogs/Clippings
                         if bm.value.percent_read >= self.FINISHED_READING_PCT_THRESHOLD:
                             if not set(mi.tags).intersection(ignore_tags):
                                 self.mark_book_as_read(id)
+                        '''
 
                         # Add bookmark file to id
                         self.db.add_format_with_hooks(id, bm.value.bookmark_extension,

src/calibre/gui2/actions/choose_library.py

@@ -385,13 +385,27 @@ class ChooseLibraryAction(InterfaceAction):
 
         prefs['library_path'] = loc
         #from calibre.utils.mem import memory
-        #import weakref, gc
+        #import weakref
-        #ref = weakref.ref(self.gui.library_view.model().db)
+        #from PyQt4.Qt import QTimer
-        #before = memory()/1024**2
+        #self.dbref = weakref.ref(self.gui.library_view.model().db)
+        #self.before_mem = memory()/1024**2
         self.gui.library_moved(loc)
-        #print gc.get_referrers(ref)[0]
+        #QTimer.singleShot(5000, self.debug_leak)
-        #for i in xrange(3): gc.collect()
+
-        #print 'leaked:', memory()/1024**2 - before
+    def debug_leak(self):
+        import gc
+        from calibre.utils.mem import memory
+        ref = self.dbref
+        for i in xrange(3): gc.collect()
+        if ref() is not None:
+            print 'DB object alive:', ref()
+            for r in gc.get_referrers(ref())[:10]:
+                print r
+                print
+        print 'before:', self.before_mem
+        print 'after:', memory()/1024**2
+        self.dbref = self.before_mem = None
+
 
     def qs_requested(self, idx, *args):
         self.switch_requested(self.qs_locations[idx])

src/calibre/gui2/catalog/catalog_epub_mobi.py

@@ -335,7 +335,7 @@ class PluginWidget(QWidget,Ui_Form):
         '''
 
         return
-
+        '''
         if new_state == 0:
             # unchecked
             self.merge_source_field.setEnabled(False)
@@ -348,6 +348,7 @@ class PluginWidget(QWidget,Ui_Form):
             self.merge_before.setEnabled(True)
             self.merge_after.setEnabled(True)
             self.include_hr.setEnabled(True)
+        '''
 
     def header_note_source_field_changed(self,new_index):
         '''

src/calibre/gui2/convert/look_and_feel.ui

@@ -43,7 +43,7 @@
       <double>0.000000000000000</double>
      </property>
      <property name="maximum">
-      <double>30.000000000000000</double>
+      <double>50.000000000000000</double>
      </property>
      <property name="singleStep">
       <double>1.000000000000000</double>

src/calibre/gui2/convert/regex_builder.py

@@ -31,10 +31,10 @@ class RegexBuilder(QDialog, Ui_RegexBuilder):
         elif not doc and not self.select_format(db, book_id):
             self.cancelled = True
             return
-        
+
         if doc:
             self.preview.setPlainText(doc)
-            
+
         self.cancelled = False
         self.connect(self.button_box, SIGNAL('clicked(QAbstractButton*)'), self.button_clicked)
         self.connect(self.regex, SIGNAL('textChanged(QString)'), self.regex_valid)
@@ -156,7 +156,7 @@ class RegexBuilder(QDialog, Ui_RegexBuilder):
                 self.open_book(files[0])
         if button == self.button_box.button(QDialogButtonBox.Ok):
             self.accept()
-    
+
     def doc(self):
         return unicode(self.preview.toPlainText())
 
@@ -200,12 +200,12 @@ class RegexEdit(QWidget, Ui_Edit):
 
     def set_db(self, db):
         self.db = db
-        
+
     def set_doc(self, doc):
         self.doc_cache = doc
 
     def break_cycles(self):
-        self.db = None
+        self.db = self.doc_cache = None
 
     @property
     def text(self):

src/calibre/gui2/convert/search_and_replace.py

@@ -34,17 +34,23 @@ class SearchAndReplaceWidget(Widget, Ui_Form):
         self.opt_sr3_search.set_msg(_('&Search Regular Expression'))
         self.opt_sr3_search.set_book_id(book_id)
         self.opt_sr3_search.set_db(db)
-        
+
         self.opt_sr1_search.doc_update.connect(self.update_doc)
         self.opt_sr2_search.doc_update.connect(self.update_doc)
         self.opt_sr3_search.doc_update.connect(self.update_doc)
 
     def break_cycles(self):
         Widget.break_cycles(self)
-        
+
-        self.opt_sr1_search.doc_update.disconnect()
+        def d(x):
-        self.opt_sr2_search.doc_update.disconnect()
+            try:
-        self.opt_sr3_search.doc_update.disconnect()
+                x.disconnect()
+            except:
+                pass
+
+        d(self.opt_sr1_search)
+        d(self.opt_sr2_search)
+        d(self.opt_sr3_search)
 
         self.opt_sr1_search.break_cycles()
         self.opt_sr2_search.break_cycles()

src/calibre/gui2/device.py

@@ -830,12 +830,14 @@ class DeviceMixin(object): # {{{
             aval_out_formats = available_output_formats()
             format_count = {}
             for row in rows:
-                for f in self.library_view.model().db.formats(row.row()).split(','):
+                fmts = self.library_view.model().db.formats(row.row())
-                    f = f.lower()
+                if fmts:
-                    if format_count.has_key(f):
+                    for f in fmts.split(','):
-                        format_count[f] += 1
+                        f = f.lower()
-                    else:
+                        if format_count.has_key(f):
-                        format_count[f] = 1
+                            format_count[f] += 1
+                        else:
+                            format_count[f] = 1
             for f in self.device_manager.device.settings().format_map:
                 if f in format_count.keys():
                     formats.append((f, _('%i of %i Books' % (format_count[f], len(rows))), True if f in aval_out_formats else False))

src/calibre/gui2/dialogs/choose_format_device.ui

@@ -14,14 +14,14 @@
    <string>Choose Format</string>
   </property>
   <property name="windowIcon">
-   <iconset>
+   <iconset resource="../../../../resources/images.qrc">
     <normaloff>:/images/mimetypes/unknown.png</normaloff>:/images/mimetypes/unknown.png</iconset>
   </property>
   <layout class="QVBoxLayout">
    <item>
     <widget class="QLabel" name="msg">
      <property name="text">
-      <string>TextLabel</string>
+      <string/>
      </property>
     </widget>
    </item>

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -6,8 +6,8 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import re, os
 
 from PyQt4.Qt import Qt, QDialog, QGridLayout, QVBoxLayout, QFont, QLabel, \
-                     pyqtSignal, QDialogButtonBox
+                     pyqtSignal, QDialogButtonBox, QInputDialog, QLineEdit, \
-from PyQt4 import QtGui
+                     QMessageBox, QDate
 
 from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
 from calibre.gui2.dialogs.tag_editor import TagEditor
@@ -15,9 +15,9 @@ from calibre.ebooks.metadata import string_to_authors, authors_to_string
 from calibre.ebooks.metadata.book.base import composite_formatter
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.gui2.custom_column_widgets import populate_metadata_page
-from calibre.gui2 import error_dialog, ResizableDialog, UNDEFINED_QDATE
+from calibre.gui2 import error_dialog, ResizableDialog, UNDEFINED_QDATE, gprefs
 from calibre.gui2.progress_indicator import ProgressIndicator
-from calibre.utils.config import dynamic
+from calibre.utils.config import dynamic, JSONConfig
 from calibre.utils.titlecase import titlecase
 from calibre.utils.icu import sort_key, capitalize
 from calibre.utils.config import prefs, tweaks
@@ -302,6 +302,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         self.pubdate.setSpecialValueText(_('Undefined'))
         self.clear_pubdate_button.clicked.connect(self.clear_pubdate)
         self.pubdate.dateChanged.connect(self.do_apply_pubdate)
+        self.adddate.setDate(QDate.currentDate())
         self.adddate.setMinimumDate(UNDEFINED_QDATE)
         self.adddate.setSpecialValueText(_('Undefined'))
         self.clear_adddate_button.clicked.connect(self.clear_adddate)
@@ -320,8 +321,15 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
             'This operation cannot be canceled or undone'))
         self.do_again = False
         self.central_widget.setCurrentIndex(tab)
+        geom = gprefs.get('bulk_metadata_window_geometry', None)
+        if geom is not None:
+            self.restoreGeometry(bytes(geom))
         self.exec_()
 
+    def save_state(self, *args):
+        gprefs['bulk_metadata_window_geometry'] = \
+            bytearray(self.saveGeometry())
+
     def do_apply_pubdate(self, *args):
         self.apply_pubdate.setChecked(True)
 
@@ -365,16 +373,16 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         offset = 10
         self.s_r_number_of_books = min(10, len(self.ids))
         for i in range(1,self.s_r_number_of_books+1):
-            w = QtGui.QLabel(self.tabWidgetPage3)
+            w = QLabel(self.tabWidgetPage3)
             w.setText(_('Book %d:')%i)
             self.testgrid.addWidget(w, i+offset, 0, 1, 1)
-            w = QtGui.QLineEdit(self.tabWidgetPage3)
+            w = QLineEdit(self.tabWidgetPage3)
             w.setReadOnly(True)
             name = 'book_%d_text'%i
             setattr(self, name, w)
             self.book_1_text.setObjectName(name)
             self.testgrid.addWidget(w, i+offset, 1, 1, 1)
-            w = QtGui.QLineEdit(self.tabWidgetPage3)
+            w = QLineEdit(self.tabWidgetPage3)
             w.setReadOnly(True)
             name = 'book_%d_result'%i
             setattr(self, name, w)
@@ -451,6 +459,15 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         self.results_count.valueChanged[int].connect(self.s_r_display_bounds_changed)
         self.starting_from.valueChanged[int].connect(self.s_r_display_bounds_changed)
 
+        self.save_button.clicked.connect(self.s_r_save_query)
+        self.remove_button.clicked.connect(self.s_r_remove_query)
+
+        self.queries = JSONConfig("search_replace_queries")
+        self.query_field.addItem("")
+        self.query_field.addItems(sorted([q for q in self.queries], key=sort_key))
+        self.query_field.currentIndexChanged[str].connect(self.s_r_query_change)
+        self.query_field.setCurrentIndex(0)
+
     def s_r_get_field(self, mi, field):
         if field:
             if field == '{template}':
@@ -780,7 +797,12 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
             self.series_start_number.setEnabled(False)
             self.series_start_number.setValue(1)
 
+    def reject(self):
+        self.save_state()
+        ResizableDialog.reject(self)
+
     def accept(self):
+        self.save_state()
         if len(self.ids) < 1:
             return QDialog.accept(self)
 
@@ -862,3 +884,117 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
     def series_changed(self, *args):
         self.write_series = True
 
+    def s_r_remove_query(self, *args):
+        if self.query_field.currentIndex() == 0:
+            return
+
+        ret = QMessageBox.question(self, _("Delete saved search/replace"),
+                _("The selected saved search/replace will be deleted. "
+                  "Are you sure?"),
+                QMessageBox.Ok, QMessageBox.Cancel)
+
+        if ret == QMessageBox.Cancel:
+            return
+
+        item_id = self.query_field.currentIndex()
+        item_name = unicode(self.query_field.currentText())
+
+        self.query_field.blockSignals(True)
+        self.query_field.removeItem(item_id)
+        self.query_field.blockSignals(False)
+        self.query_field.setCurrentIndex(0)
+
+        if item_name in self.queries.keys():
+            del(self.queries[item_name])
+            self.queries.commit()
+
+    def s_r_save_query(self, *args):
+        name, ok =  QInputDialog.getText(self, _('Save search/replace'),
+                _('Search/replace name:'))
+        if not ok:
+            return
+
+        new = True
+        name = unicode(name)
+        if name in self.queries.keys():
+            ret = QMessageBox.question(self, _("Save search/replace"),
+                    _("That saved search/replace already exists and will be overwritten. "
+                      "Are you sure?"),
+                    QMessageBox.Ok, QMessageBox.Cancel)
+            if ret == QMessageBox.Cancel:
+                return
+            new = False
+
+        query = {}
+        query['name'] = name
+        query['search_field'] = unicode(self.search_field.currentText())
+        query['search_mode'] = unicode(self.search_mode.currentText())
+        query['s_r_template'] = unicode(self.s_r_template.text())
+        query['search_for'] = unicode(self.search_for.text())
+        query['case_sensitive'] = self.case_sensitive.isChecked()
+        query['replace_with'] = unicode(self.replace_with.text())
+        query['replace_func'] = unicode(self.replace_func.currentText())
+        query['destination_field'] = unicode(self.destination_field.currentText())
+        query['replace_mode'] = unicode(self.replace_mode.currentText())
+        query['comma_separated'] = self.comma_separated.isChecked()
+        query['results_count'] = self.results_count.value()
+        query['starting_from'] = self.starting_from.value()
+        query['multiple_separator'] = unicode(self.multiple_separator.text())
+
+        self.queries[name] = query
+        self.queries.commit()
+
+        if new:
+            self.query_field.blockSignals(True)
+            self.query_field.clear()
+            self.query_field.addItem('')
+            self.query_field.addItems(sorted([q for q in self.queries], key=sort_key))
+            self.query_field.blockSignals(False)
+        self.query_field.setCurrentIndex(self.query_field.findText(name))
+
+    def s_r_query_change(self, item_name):
+        if not item_name:
+            self.s_r_reset_query_fields()
+            return
+        item = self.queries.get(unicode(item_name), None)
+        if item is None:
+            self.s_r_reset_query_fields()
+            return
+
+        def set_index(attr, txt):
+            try:
+                attr.setCurrentIndex(attr.findText(txt))
+            except:
+                attr.setCurrentIndex(0)
+
+        set_index(self.search_mode, item['search_mode'])
+        set_index(self.search_field, item['search_field'])
+        self.s_r_template.setText(item['s_r_template'])
+        self.s_r_template_changed() #simulate gain/loss of focus
+        self.search_for.setText(item['search_for'])
+        self.case_sensitive.setChecked(item['case_sensitive'])
+        self.replace_with.setText(item['replace_with'])
+        set_index(self.replace_func, item['replace_func'])
+        set_index(self.destination_field, item['destination_field'])
+        set_index(self.replace_mode, item['replace_mode'])
+        self.comma_separated.setChecked(item['comma_separated'])
+        self.results_count.setValue(int(item['results_count']))
+        self.starting_from.setValue(int(item['starting_from']))
+        self.multiple_separator.setText(item['multiple_separator'])
+
+    def s_r_reset_query_fields(self):
+        # Don't reset the search mode. The user will probably want to use it
+        # as it was
+        self.search_field.setCurrentIndex(0)
+        self.s_r_template.setText("")
+        self.search_for.setText("")
+        self.case_sensitive.setChecked(False)
+        self.replace_with.setText("")
+        self.replace_func.setCurrentIndex(0)
+        self.destination_field.setCurrentIndex(0)
+        self.replace_mode.setCurrentIndex(0)
+        self.comma_separated.setChecked(True)
+        self.results_count.setValue(999)
+        self.starting_from.setValue(1)
+        self.multiple_separator.setText(" ::: ")
+

src/calibre/gui2/dialogs/metadata_bulk.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>850</width>
+    <width>962</width>
-    <height>650</height>
+    <height>727</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -44,8 +44,8 @@
        <rect>
         <x>0</x>
         <y>0</y>
-        <width>842</width>
+        <width>954</width>
-        <height>589</height>
+        <height>666</height>
        </rect>
       </property>
       <layout class="QVBoxLayout" name="verticalLayout_2">
@@ -574,7 +574,7 @@ Future conversion of these books will use the default settings.</string>
            <property name="sizeConstraint">
             <enum>QLayout::SetMinimumSize</enum>
            </property>
-           <item row="1" column="0" colspan="3">
+           <item row="0" column="0" colspan="4">
             <widget class="QLabel" name="s_r_heading">
              <property name="wordWrap">
               <bool>true</bool>
@@ -584,14 +584,91 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="2" column="0">
+           <item row="1" column="0">
             <widget class="QLabel" name="filler">
              <property name="text">
               <string/>
              </property>
             </widget>
            </item>
+           <item row="2" column="0" colspan="3">
+            <widget class="Line" name="line">
+             <property name="orientation">
+              <enum>Qt::Horizontal</enum>
+             </property>
+            </widget>
+           </item>
            <item row="3" column="0">
+            <widget class="QLabel" name="xlabel_22">
+             <property name="text">
+              <string>Load searc&amp;h/replace:</string>
+             </property>
+             <property name="buddy">
+              <cstring>search_field</cstring>
+             </property>
+            </widget>
+           </item>
+           <item row="3" column="1">
+            <widget class="QComboBox" name="query_field">
+             <property name="toolTip">
+              <string>Select saved search/replace to load.</string>
+             </property>
+            </widget>
+           </item>
+           <item row="3" column="2">
+            <layout class="QHBoxLayout" name="horizontalLayout_6">
+             <item>
+              <spacer name="horizontalSpacer_3">
+               <property name="orientation">
+                <enum>Qt::Horizontal</enum>
+               </property>
+               <property name="sizeType">
+                <enum>QSizePolicy::Fixed</enum>
+               </property>
+               <property name="sizeHint" stdset="0">
+                <size>
+                 <width>20</width>
+                 <height>20</height>
+                </size>
+               </property>
+              </spacer>
+             </item>
+             <item>
+              <widget class="QPushButton" name="save_button">
+               <property name="toolTip">
+                <string>Save current search/replace</string>
+               </property>
+               <property name="text">
+                <string>Sa&amp;ve</string>
+               </property>
+              </widget>
+             </item>
+             <item>
+              <widget class="QPushButton" name="remove_button">
+               <property name="toolTip">
+                <string>Delete saved search/replace</string>
+               </property>
+               <property name="text">
+                <string>Delete</string>
+               </property>
+              </widget>
+             </item>
+             <item>
+              <spacer name="horizontalSpacer_2">
+               <property name="orientation">
+                <enum>Qt::Horizontal</enum>
+               </property>
+               <property name="sizeHint" stdset="0">
+                <size>
+                 <width>40</width>
+                 <height>20</height>
+                </size>
+               </property>
+              </spacer>
+             </item>
+            </layout>
+           </item>
+           <item row="4" column="0">
             <widget class="QLabel" name="xlabel_21">
              <property name="text">
               <string>Search &amp;field:</string>
@@ -601,14 +678,14 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="3" column="1">
+           <item row="4" column="1">
             <widget class="QComboBox" name="search_field">
              <property name="toolTip">
               <string>The name of the field that you want to search</string>
              </property>
             </widget>
            </item>
-           <item row="3" column="2">
+           <item row="4" column="2">
             <layout class="QHBoxLayout" name="HLayout_3">
              <item>
               <widget class="QLabel" name="xlabel_24">
@@ -642,7 +719,7 @@ Future conversion of these books will use the default settings.</string>
              </item>
             </layout>
            </item>
-           <item row="4" column="0">
+           <item row="5" column="0">
             <widget class="QLabel" name="template_label">
              <property name="text">
               <string>Te&amp;mplate:</string>
@@ -652,7 +729,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="4" column="1">
+           <item row="5" column="1">
             <widget class="HistoryLineEdit" name="s_r_template">
              <property name="sizePolicy">
               <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
@@ -665,7 +742,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="5" column="0">
+           <item row="6" column="0">
             <widget class="QLabel" name="xlabel_2">
              <property name="text">
               <string>&amp;Search for:</string>
@@ -675,7 +752,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="5" column="1">
+           <item row="6" column="1">
             <widget class="HistoryLineEdit" name="search_for">
              <property name="sizePolicy">
               <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
@@ -688,7 +765,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="5" column="2">
+           <item row="6" column="2">
             <widget class="QCheckBox" name="case_sensitive">
              <property name="toolTip">
               <string>Check this box if the search string must match exactly upper and lower case. Uncheck it if case is to be ignored</string>
@@ -701,7 +778,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="6" column="0">
+           <item row="7" column="0">
             <widget class="QLabel" name="xlabel_4">
              <property name="text">
               <string>&amp;Replace with:</string>
@@ -711,14 +788,14 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="6" column="1">
+           <item row="7" column="1">
             <widget class="HistoryLineEdit" name="replace_with">
              <property name="toolTip">
               <string>The replacement text. The matched search text will be replaced with this string</string>
              </property>
             </widget>
            </item>
-           <item row="6" column="2">
+           <item row="7" column="2">
             <layout class="QHBoxLayout" name="verticalLayout">
              <item>
               <widget class="QLabel" name="label_41">
@@ -753,7 +830,7 @@ field is processed. In regular expression mode, only the matched text is process
              </item>
             </layout>
            </item>
-           <item row="7" column="0">
+           <item row="8" column="0">
             <widget class="QLabel" name="destination_field_label">
              <property name="text">
               <string>&amp;Destination field:</string>
@@ -763,7 +840,7 @@ field is processed. In regular expression mode, only the matched text is process
              </property>
             </widget>
            </item>
-           <item row="7" column="1">
+           <item row="8" column="1">
             <widget class="QComboBox" name="destination_field">
              <property name="toolTip">
               <string>The field that the text will be put into after all replacements.
@@ -771,7 +848,7 @@ If blank, the source field is used if the field is modifiable</string>
              </property>
             </widget>
            </item>
-           <item row="7" column="2">
+           <item row="8" column="2">
             <layout class="QHBoxLayout" name="verticalLayout">
              <item>
               <widget class="QLabel" name="replace_mode_label">
@@ -820,7 +897,7 @@ not multiple and the destination field is multiple</string>
              </item>
             </layout>
            </item>
-           <item row="8" column="1" colspan="2">
+           <item row="9" column="1" colspan="2">
             <layout class="QHBoxLayout" name="horizontalLayout_21">
              <item>
               <spacer name="HSpacer_347">
@@ -906,7 +983,7 @@ not multiple and the destination field is multiple</string>
              </item>
             </layout>
            </item>
-           <item row="9" column="0" colspan="4">
+           <item row="10" column="0" colspan="4">
             <widget class="QScrollArea" name="scrollArea11">
              <property name="frameShape">
               <enum>QFrame::NoFrame</enum>
@@ -1030,6 +1107,9 @@ not multiple and the destination field is multiple</string>
   <tabstop>series_numbering_restarts</tabstop>
   <tabstop>series_start_number</tabstop>
   <tabstop>button_box</tabstop>
+  <tabstop>query_field</tabstop>
+  <tabstop>save_button</tabstop>
+  <tabstop>remove_button</tabstop>
   <tabstop>search_field</tabstop>
   <tabstop>search_mode</tabstop>
   <tabstop>s_r_template</tabstop>
@@ -1045,6 +1125,23 @@ not multiple and the destination field is multiple</string>
   <tabstop>multiple_separator</tabstop>
   <tabstop>test_text</tabstop>
   <tabstop>test_result</tabstop>
+  <tabstop>scrollArea</tabstop>
+  <tabstop>central_widget</tabstop>
+  <tabstop>swap_title_and_author</tabstop>
+  <tabstop>clear_series</tabstop>
+  <tabstop>adddate</tabstop>
+  <tabstop>clear_adddate_button</tabstop>
+  <tabstop>apply_adddate</tabstop>
+  <tabstop>pubdate</tabstop>
+  <tabstop>clear_pubdate_button</tabstop>
+  <tabstop>apply_pubdate</tabstop>
+  <tabstop>remove_format</tabstop>
+  <tabstop>change_title_to_title_case</tabstop>
+  <tabstop>remove_conversion_settings</tabstop>
+  <tabstop>cover_generate</tabstop>
+  <tabstop>cover_remove</tabstop>
+  <tabstop>cover_from_fmt</tabstop>
+  <tabstop>scrollArea11</tabstop>
  </tabstops>
  <resources>
   <include location="../../../../resources/images.qrc"/>

src/calibre/gui2/dialogs/scheduler.py

@@ -250,22 +250,27 @@ class Scheduler(QObject):
 
         self.timer = QTimer(self)
         self.timer.start(int(self.INTERVAL * 60 * 1000))
-        self.oldest_timer = QTimer()
-        self.connect(self.oldest_timer, SIGNAL('timeout()'), self.oldest_check)
         self.connect(self.timer, SIGNAL('timeout()'), self.check)
         self.oldest = gconf['oldest_news']
-        self.oldest_timer.start(int(60 * 60 * 1000))
         QTimer.singleShot(5 * 1000, self.oldest_check)
         self.database_changed = self.recipe_model.database_changed
 
     def oldest_check(self):
         if self.oldest > 0:
             delta = timedelta(days=self.oldest)
-            ids = self.recipe_model.db.tags_older_than(_('News'), delta)
+            try:
+                ids = self.recipe_model.db.tags_older_than(_('News'), delta)
+            except:
+                # Should never happen
+                ids = []
+                import traceback
+                traceback.print_exc()
             if ids:
                 ids = list(ids)
                 if ids:
                     self.delete_old_news.emit(ids)
+        QTimer.singleShot(60 * 60 * 1000, self.oldest_check)
+
 
     def show_dialog(self, *args):
         self.lock.lock()

src/calibre/gui2/dialogs/tag_editor.py

@@ -16,7 +16,7 @@ class TagEditor(QDialog, Ui_TagEditor):
         self.setupUi(self)
 
         self.db = db
-        self.index = db.row(id_)
+        self.index = db.row(id_) if id_ is not None else None
         if self.index is not None:
             tags = self.db.tags(self.index)
         else:

src/calibre/gui2/main.py

@@ -150,13 +150,13 @@ class GuiRunner(QObject):
         if DEBUG:
             prints('Starting up...')
 
-    def start_gui(self):
+    def start_gui(self, db):
         from calibre.gui2.ui import Main
         main = Main(self.opts, gui_debug=self.gui_debug)
         if self.splash_screen is not None:
             self.splash_screen.showMessage(_('Initializing user interface...'))
             self.splash_screen.finish(main)
-        main.initialize(self.library_path, self.db, self.listener, self.actions)
+        main.initialize(self.library_path, db, self.listener, self.actions)
         if DEBUG:
             prints('Started up in', time.time() - self.startup_time)
         add_filesystem_book = partial(main.iactions['Add Books'].add_filesystem_book, allow_device=False)
@@ -200,8 +200,7 @@ class GuiRunner(QObject):
                     det_msg=traceback.format_exc(), show=True)
                 self.initialization_failed()
 
-        self.db = db
+        self.start_gui(db)
-        self.start_gui()
 
     def initialize_db(self):
         db = None

src/calibre/gui2/tag_view.py

@@ -98,6 +98,7 @@ class TagsView(QTreeView): # {{{
             self.collapse_model = 'disable'
         else:
             self.collapse_model = gprefs['tags_browser_partition_method']
+        self.search_icon = QIcon(I('search.png'))
 
     def set_pane_is_visible(self, to_what):
         pv = self.pane_is_visible
@@ -114,6 +115,9 @@ class TagsView(QTreeView): # {{{
 
     def set_database(self, db, tag_match, sort_by):
         self.hidden_categories = config['tag_browser_hidden_categories']
+        old = getattr(self, '_model', None)
+        if old is not None:
+            old.break_cycles()
         self._model = TagsModel(db, parent=self,
                                 hidden_categories=self.hidden_categories,
                                 search_restriction=None,
@@ -183,7 +187,7 @@ class TagsView(QTreeView): # {{{
             self.clear()
 
     def context_menu_handler(self, action=None, category=None,
-                             key=None, index=None):
+                             key=None, index=None, negate=None):
         if not action:
             return
         try:
@@ -196,12 +200,20 @@ class TagsView(QTreeView): # {{{
             if action == 'manage_categories':
                 self.user_category_edit.emit(category)
                 return
+            if action == 'search':
+                self.tags_marked.emit(('not ' if negate else '') +
+                                      category + ':"=' + key + '"')
+                return
+            if action == 'search_category':
+                self.tags_marked.emit(category + ':' + str(not negate))
+                return
             if action == 'manage_searches':
                 self.saved_search_edit.emit(category)
                 return
             if action == 'edit_author_sort':
                 self.author_sort_edit.emit(self, index)
                 return
+
             if action == 'hide':
                 self.hidden_categories.add(category)
             elif action == 'show':
@@ -242,19 +254,36 @@ class TagsView(QTreeView): # {{{
                 if key not in self.db.field_metadata:
                     return True
 
-                # If the user right-clicked on an editable item, then offer
+                # Did the user click on a leaf node?
-                # the possibility of renaming that item
+                if tag_name:
-                if tag_name and \
+                    # If the user right-clicked on an editable item, then offer
-                        (key in ['authors', 'tags', 'series', 'publisher', 'search'] or \
+                    # the possibility of renaming that item.
-                        self.db.field_metadata[key]['is_custom'] and \
+                    if key in ['authors', 'tags', 'series', 'publisher', 'search'] or \
-                        self.db.field_metadata[key]['datatype'] != 'rating'):
+                            (self.db.field_metadata[key]['is_custom'] and \
-                    self.context_menu.addAction(_('Rename \'%s\'')%tag_name,
+                             self.db.field_metadata[key]['datatype'] != 'rating'):
-                            partial(self.context_menu_handler, action='edit_item',
+                        # Add the 'rename' items
-                                    category=tag_item, index=index))
+                        self.context_menu.addAction(_('Rename %s')%tag_name,
-                    if key == 'authors':
+                                partial(self.context_menu_handler, action='edit_item',
-                        self.context_menu.addAction(_('Edit sort for \'%s\'')%tag_name,
+                                        category=tag_item, index=index))
-                                partial(self.context_menu_handler,
+                        if key == 'authors':
-                                        action='edit_author_sort', index=tag_id))
+                            self.context_menu.addAction(_('Edit sort for %s')%tag_name,
+                                    partial(self.context_menu_handler,
+                                            action='edit_author_sort', index=tag_id))
+                    # Add the search for value items
+                    n = tag_name
+                    c = category
+                    if self.db.field_metadata[key]['datatype'] == 'rating':
+                        n = str(len(tag_name))
+                    elif self.db.field_metadata[key]['kind'] in ['user', 'search']:
+                        c = tag_item.tag.category
+                    self.context_menu.addAction(self.search_icon,
+                            _('Search for %s')%tag_name,
+                            partial(self.context_menu_handler, action='search',
+                                    category=c, key=n, negate=False))
+                    self.context_menu.addAction(self.search_icon,
+                            _('Search for everything but %s')%tag_name,
+                            partial(self.context_menu_handler, action='search',
+                                    category=c, key=n, negate=True))
                     self.context_menu.addSeparator()
                 # Hide/Show/Restore categories
                 self.context_menu.addAction(_('Hide category %s') % category,
@@ -265,6 +294,16 @@ class TagsView(QTreeView): # {{{
                         m.addAction(col,
                             partial(self.context_menu_handler, action='show', category=col))
 
+                # search by category
+                if key != 'search':
+                    self.context_menu.addAction(self.search_icon,
+                            _('Search for books in category %s')%category,
+                            partial(self.context_menu_handler, action='search_category',
+                                    category=key, negate=False))
+                    self.context_menu.addAction(self.search_icon,
+                            _('Search for books not in category %s')%category,
+                            partial(self.context_menu_handler, action='search_category',
+                                    category=key, negate=True))
                 # Offer specific editors for tags/series/publishers/saved searches
                 self.context_menu.addSeparator()
                 if key in ['tags', 'publisher', 'series'] or \
@@ -371,6 +410,9 @@ class TagsView(QTreeView): # {{{
     # model. Reason: it is much easier than reconstructing the browser tree.
     def set_new_model(self, filter_categories_by=None):
         try:
+            old = getattr(self, '_model', None)
+            if old is not None:
+                old.break_cycles()
             self._model = TagsModel(self.db, parent=self,
                                     hidden_categories=self.hidden_categories,
                                     search_restriction=self.search_restriction,
@@ -509,8 +551,8 @@ class TagsModel(QAbstractItemModel): # {{{
         QAbstractItemModel.__init__(self, parent)
 
         # must do this here because 'QPixmap: Must construct a QApplication
-        # before a QPaintDevice'. The ':' in front avoids polluting either the
+        # before a QPaintDevice'. The ':' at the end avoids polluting either of
-        # user-defined categories (':' at end) or columns namespaces (no ':').
+        # the other namespaces (alpha, '#', or '@')
         iconmap = {}
         for key in category_icon_map:
             iconmap[key] = QIcon(I(category_icon_map[key]))
@@ -544,6 +586,9 @@ class TagsModel(QAbstractItemModel): # {{{
                     tooltip=tt, category_key=r)
         self.refresh(data=data)
 
+    def break_cycles(self):
+        self.db = self.root_item = None
+
     def mimeTypes(self):
         return ["application/calibre+from_library"]
 
@@ -681,8 +726,12 @@ class TagsModel(QAbstractItemModel): # {{{
         tb_cats = self.db.field_metadata
         for user_cat in sorted(self.db.prefs.get('user_categories', {}).keys(),
                                key=sort_key):
-            cat_name = user_cat+':' # add the ':' to avoid name collision
+            cat_name = '@' + user_cat # add the '@' to avoid name collision
-            tb_cats.add_user_category(label=cat_name, name=user_cat)
+            try:
+                tb_cats.add_user_category(label=cat_name, name=user_cat)
+            except ValueError:
+                import traceback
+                traceback.print_exc()
         if len(saved_searches().names()):
             tb_cats.add_search_category(label='search', name=_('Searches'))
 
@@ -988,7 +1037,7 @@ class TagsModel(QAbstractItemModel): # {{{
             if self.hidden_categories and self.categories[i] in self.hidden_categories:
                 continue
             row_index += 1
-            if key.endswith(':'):
+            if key.startswith('@'):
                 # User category, so skip it. The tag will be marked in its real category
                 continue
             category_item = self.root_item.children[row_index]
@@ -1007,7 +1056,7 @@ class TagsModel(QAbstractItemModel): # {{{
                         ans.append('%s%s:"=%s"'%(prefix, category, tag.name))
         return ans
 
-    def find_node(self, key, txt, start_path):
+    def find_item_node(self, key, txt, start_path):
         '''
         Search for an item (a node) in the tags browser list that matches both
         the key (exact case-insensitive match) and txt (contains case-
@@ -1061,6 +1110,22 @@ class TagsModel(QAbstractItemModel): # {{{
                 break
         return self.path_found
 
+    def find_category_node(self, key):
+        '''
+        Search for an category node (a top-level node) in the tags browser list
+        that matches the key (exact case-insensitive match). Returns the path to
+        the node. Paths are as in find_item_node.
+        '''
+        if not key:
+            return None
+
+        for i in xrange(self.rowCount(QModelIndex())):
+            idx = self.index(i, 0, QModelIndex())
+            ckey = idx.internalPointer().category_key
+            if strcmp(ckey, key) == 0:
+                return self.path_for_index(idx)
+        return None
+
     def show_item_at_path(self, path, box=False):
         '''
         Scroll the browser and open categories to show the item referenced by
@@ -1109,8 +1174,7 @@ class TagBrowserMixin(object): # {{{
 
     def __init__(self, db):
         self.library_view.model().count_changed_signal.connect(self.tags_view.recount)
-        self.tags_view.set_database(self.library_view.model().db,
+        self.tags_view.set_database(db, self.tag_match, self.sort_by)
-                self.tag_match, self.sort_by)
         self.tags_view.tags_marked.connect(self.search.set_search_string)
         self.tags_view.tag_list_edit.connect(self.do_tags_list_edit)
         self.tags_view.user_category_edit.connect(self.do_user_categories_edit)
@@ -1347,15 +1411,15 @@ class TagBrowserWidget(QWidget): # {{{
         self.search_button.setFocus(True)
         self.item_search.lineEdit().blockSignals(False)
 
-        colon = txt.find(':')
         key = None
+        colon = txt.rfind(':') if len(txt) > 2 else 0
         if colon > 0:
             key = self.parent.library_view.model().db.\
                         field_metadata.search_term_to_field_key(txt[:colon])
             txt = txt[colon+1:]
 
-        self.current_find_position = model.find_node(key, txt,
+        self.current_find_position = \
-                                                     self.current_find_position)
+            model.find_item_node(key, txt, self.current_find_position)
         if self.current_find_position:
             model.show_item_at_path(self.current_find_position, box=True)
         elif self.item_search.text():

src/calibre/gui2/ui.py

@@ -16,7 +16,7 @@ from PyQt4.Qt import Qt, SIGNAL, QTimer, \
                      QPixmap, QMenu, QIcon, pyqtSignal, \
                      QDialog, \
                      QSystemTrayIcon, QApplication, QKeySequence, \
-                     QMessageBox, QHelpEvent
+                     QMessageBox, QHelpEvent, QAction
 
 from calibre import  prints
 from calibre.constants import __appname__, isosx
@@ -198,6 +198,10 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         self.system_tray_icon.activated.connect(
             self.system_tray_icon_activated)
 
+        self.esc_action = QAction(self)
+        self.addAction(self.esc_action)
+        self.esc_action.setShortcut(QKeySequence(Qt.Key_Escape))
+        self.esc_action.triggered.connect(self.esc)
 
         ####################### Start spare job server ########################
         QTimer.singleShot(1000, self.add_spare_server)
@@ -294,6 +298,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
                         'the file: %s<p>The '
                         'log will be displayed automatically.')%self.gui_debug, show=True)
 
+    def esc(self, *args):
+        self.search.clear()
 
     def start_content_server(self):
         from calibre.library.server.main import start_threaded_server
@@ -305,7 +311,6 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         self.content_server.state_callback(True)
         self.test_server_timer = QTimer.singleShot(10000, self.test_server)
 
-
     def resizeEvent(self, ev):
         MainWindow.resizeEvent(self, ev)
         self.search.setMaximumWidth(self.width()-150)
@@ -633,8 +638,6 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         except KeyboardInterrupt:
             pass
         time.sleep(2)
-        if mb is not None:
-            mb.flush()
         self.hide_windows()
         return True
 

src/calibre/gui2/widgets.py

@@ -16,7 +16,6 @@ from PyQt4.Qt import QIcon, QFont, QLabel, QListWidget, QAction, \
                         QTimer, QRect
 
 from calibre.gui2 import NONE, error_dialog, pixmap_to_data, gprefs
-from calibre.constants import isosx
 from calibre.gui2.filename_pattern_ui import Ui_Form
 from calibre import fit_image
 from calibre.ebooks import BOOK_EXTENSIONS
@@ -327,8 +326,9 @@ class FontFamilyModel(QAbstractListModel):
             return NONE
         if role == Qt.DisplayRole:
             return QVariant(family)
-        if not isosx and role == Qt.FontRole:
+        if False and role == Qt.FontRole:
-            # Causes a Qt crash with some fonts on OS X
+            # Causes a Qt crash with some fonts
+            # so disabled.
             return QVariant(QFont(family))
         return NONE
 

src/calibre/library/caches.py

@@ -43,6 +43,11 @@ class MetadataBackup(Thread): # {{{
     def stop(self):
         self.keep_running = False
 
+    def break_cycles(self):
+        # Break cycles so that this object doesn't hold references to db
+        self.do_write = self.get_metadata_for_dump = self.clear_dirtied = \
+            self.set_dirtied = self.db = None
+
     def run(self):
         while self.keep_running:
             self.in_limbo = None
@@ -54,7 +59,10 @@ class MetadataBackup(Thread): # {{{
             except:
                 # Happens during interpreter shutdown
                 break
+            if not self.keep_running:
+                break
 
+            self.in_limbo = id_
             try:
                 path, mi = self.get_metadata_for_dump(id_)
             except:
@@ -69,10 +77,10 @@ class MetadataBackup(Thread): # {{{
                     continue
 
             # at this point the dirty indication is off
-
             if mi is None:
                 continue
-            self.in_limbo = id_
+            if not self.keep_running:
+                break
 
             # Give the GUI thread a chance to do something. Python threads don't
             # have priorities, so this thread would naturally keep the processor
@@ -86,6 +94,9 @@ class MetadataBackup(Thread): # {{{
                 traceback.print_exc()
                 continue
 
+            if not self.keep_running:
+                break
+
             time.sleep(0.1) # Give the GUI thread a chance to do something
             try:
                 self.do_write(path, raw)
@@ -99,7 +110,10 @@ class MetadataBackup(Thread): # {{{
                     prints('Failed to write backup metadata for id:', id_,
                             'again, giving up')
                     continue
-        self.in_limbo = None
+
+            self.in_limbo = None
+        self.flush()
+        self.break_cycles()
 
     def flush(self):
         'Used during shutdown to ensure that a dirtied book is not missed'
@@ -108,6 +122,7 @@ class MetadataBackup(Thread): # {{{
                 self.db.dirtied([self.in_limbo])
             except:
                 traceback.print_exc()
+            self.in_limbo = None
 
     def write(self, path, raw):
         with lopen(path, 'wb') as f:
@@ -132,7 +147,7 @@ def _match(query, value, matchkind):
             pass
     return False
 
-class CacheRow(list):
+class CacheRow(list): # {{{
 
     def __init__(self, db, composites, val):
         self.db = db
@@ -163,14 +178,16 @@ class CacheRow(list):
     def __getslice__(self, i, j):
         return self.__getitem__(slice(i, j))
 
+# }}}
 
 class ResultCache(SearchQueryParser): # {{{
 
     '''
     Stores sorted and filtered metadata in memory.
     '''
-    def __init__(self, FIELD_MAP, field_metadata):
+    def __init__(self, FIELD_MAP, field_metadata, db_prefs=None):
         self.FIELD_MAP = FIELD_MAP
+        self.db_prefs = db_prefs
         self.composites = {}
         for key in field_metadata:
             if field_metadata[key]['datatype'] == 'composite':
@@ -185,6 +202,11 @@ class ResultCache(SearchQueryParser): # {{{
         self.build_date_relop_dict()
         self.build_numeric_relop_dict()
 
+    def break_cycles(self):
+        self._data = self.field_metadata = self.FIELD_MAP = \
+            self.numeric_search_relops = self.date_search_relops = \
+            self.all_search_locations = self.db_prefs = None
+
 
     def __getitem__(self, row):
         return self._data[self._map_filtered[row]]
@@ -397,6 +419,27 @@ class ResultCache(SearchQueryParser): # {{{
                 matches.add(item[0])
         return matches
 
+    def get_user_category_matches(self, location, query, candidates):
+        res = set([])
+        if self.db_prefs is None:
+            return  res
+        user_cats = self.db_prefs.get('user_categories', [])
+        # translate the case of the location
+        for loc in user_cats:
+            if location == icu_lower(loc):
+                location = loc
+                break
+        if location not in user_cats:
+            return res
+        c = set(candidates)
+        for (item, category, ign) in user_cats[location]:
+            s = self.get_matches(category, '=' + item, candidates=c)
+            c -= s
+            res |= s
+        if query == 'false':
+            return candidates - res
+        return res
+
     def get_matches(self, location, query, allow_recursion=True, candidates=None):
         matches = set([])
         if candidates is None:
@@ -407,7 +450,7 @@ class ResultCache(SearchQueryParser): # {{{
         if query and query.strip():
             # get metadata key associated with the search term. Eliminates
             # dealing with plurals and other aliases
-            location = self.field_metadata.search_term_to_field_key(location.lower().strip())
+            location = self.field_metadata.search_term_to_field_key(icu_lower(location.strip()))
             if isinstance(location, list):
                 if allow_recursion:
                     for loc in location:
@@ -435,6 +478,10 @@ class ResultCache(SearchQueryParser): # {{{
                     return self.get_numeric_matches(location, query[1:],
                                                     candidates, val_func=vf)
 
+            # check for user categories
+            if len(location) >= 2 and location.startswith('@'):
+                return self.get_user_category_matches(location[1:], query.lower(),
+                                                      candidates)
             # everything else, or 'all' matches
             matchkind = CONTAINS_MATCH
             if (len(query) > 1):
@@ -460,6 +507,8 @@ class ResultCache(SearchQueryParser): # {{{
             for x in range(len(self.FIELD_MAP)):
                 col_datatype.append('')
             for x in self.field_metadata:
+                if x.startswith('@'):
+                    continue
                 if len(self.field_metadata[x]['search_terms']):
                     db_col[x] = self.field_metadata[x]['rec_index']
                     if self.field_metadata[x]['datatype'] not in \

src/calibre/library/catalog.py

@@ -1820,6 +1820,9 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
                 self.booksByTitle_noSeriesPrefix = nspt
 
             # Loop through the books by title
+            # Generate one divRunningTag per initial letter for the purposes of
+            # minimizing widows and orphans on readers that can handle large
+            # <divs> styled as inline-block
             title_list = self.booksByTitle
             if not self.useSeriesPrefixInTitlesSection:
                 title_list = self.booksByTitle_noSeriesPrefix
@@ -1832,7 +1835,7 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
                         divTag.insert(dtc, divRunningTag)
                         dtc += 1
                     divRunningTag = Tag(soup, 'div')
-                    divRunningTag['style'] = 'display:inline-block;width:100%'
+                    divRunningTag['class'] = "logical_group"
                     drtc = 0
                     current_letter = self.letter_or_symbol(book['title_sort'][0])
                     pIndexTag = Tag(soup, "p")
@@ -1954,6 +1957,8 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
             drtc = 0
 
             # Loop through booksByAuthor
+            # Each author/books group goes in an openingTag div (first) or
+            # a runningTag div (subsequent)
             book_count = 0
             current_author = ''
             current_letter = ''
@@ -1977,7 +1982,7 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
                     current_letter = self.letter_or_symbol(book['author_sort'][0].upper())
                     author_count = 0
                     divOpeningTag = Tag(soup, 'div')
-                    divOpeningTag['style'] = 'display:inline-block;width:100%'
+                    divOpeningTag['class'] = "logical_group"
                     dotc = 0
                     pIndexTag = Tag(soup, "p")
                     pIndexTag['class'] = "letter_index"
@@ -2001,7 +2006,7 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
 
                         # Create a divRunningTag for the rest of the authors in this letter
                         divRunningTag = Tag(soup, 'div')
-                        divRunningTag['style'] = 'display:inline-block;width:100%'
+                        divRunningTag['class'] = "logical_group"
                         drtc = 0
 
                     non_series_books = 0

src/calibre/library/database2.py

@@ -319,7 +319,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.field_metadata.remove_dynamic_categories()
         tb_cats = self.field_metadata
         for user_cat in sorted(self.prefs.get('user_categories', {}).keys(), key=sort_key):
-            cat_name = user_cat+':' # add the ':' to avoid name collision
+            cat_name = '@' + user_cat # add the '@' to avoid name collision
             tb_cats.add_user_category(label=cat_name, name=user_cat)
         if len(saved_searches().names()):
             tb_cats.add_search_category(label='search', name=_('Searches'))
@@ -332,7 +332,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 traceback.print_exc()
 
         self.book_on_device_func = None
-        self.data    = ResultCache(self.FIELD_MAP, self.field_metadata)
+        self.data    = ResultCache(self.FIELD_MAP, self.field_metadata, db_prefs=self.prefs)
         self.search  = self.data.search
         self.search_getting_ids  = self.data.search_getting_ids
         self.refresh = functools.partial(self.data.refresh, self)
@@ -362,7 +362,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.last_update_check = self.last_modified()
 
     def break_cycles(self):
-        self.data = self.field_metadata = self.prefs = self.listeners = None
+        self.data.break_cycles()
+        self.data = self.field_metadata = self.prefs = self.listeners = \
+            self.refresh_ondevice = None
 
     def initialize_database(self):
         metadata_sqlite = open(P('metadata_sqlite.sql'), 'rb').read()
@@ -1241,7 +1243,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                     if category in icon_map:
                         icon = icon_map[label]
                 else:
-                    icon = icon_map[':custom']
+                    icon = icon_map['custom:']
                     icon_map[category] = icon
 
             datatype = cat['datatype']
@@ -1337,20 +1339,19 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 if label in taglist and name in taglist[label]:
                     items.append(taglist[label][name])
                 # else: do nothing, to not include nodes w zero counts
-            if len(items):
+            cat_name = '@' + user_cat # add the '@' to avoid name collision
-                cat_name = user_cat+':' # add the ':' to avoid name collision
+            # Not a problem if we accumulate entries in the icon map
-                # Not a problem if we accumulate entries in the icon map
+            if icon_map is not None:
-                if icon_map is not None:
+                icon_map[cat_name] = icon_map['user:']
-                    icon_map[cat_name] = icon_map[':user']
+            if sort == 'popularity':
-                if sort == 'popularity':
+                categories[cat_name] = \
-                    categories[cat_name] = \
+                    sorted(items, key=lambda x: x.count, reverse=True)
-                        sorted(items, key=lambda x: x.count, reverse=True)
+            elif sort == 'name':
-                elif sort == 'name':
+                categories[cat_name] = \
-                    categories[cat_name] = \
+                    sorted(items, key=lambda x: sort_key(x.sort))
-                        sorted(items, key=lambda x: sort_key(x.sort))
+            else:
-                else:
+                categories[cat_name] = \
-                    categories[cat_name] = \
+                    sorted(items, key=lambda x:x.avg_rating, reverse=True)
-                        sorted(items, key=lambda x:x.avg_rating, reverse=True)
 
         #### Finally, the saved searches category ####
         items = []
@@ -1375,10 +1376,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
     def tags_older_than(self, tag, delta):
         tag = tag.lower().strip()
         now = nowf()
+        tindex = self.FIELD_MAP['timestamp']
+        gindex = self.FIELD_MAP['tags']
         for r in self.data._data:
             if r is not None:
-                if (now - r[self.FIELD_MAP['timestamp']]) > delta:
+                if (now - r[tindex]) > delta:
-                    tags = r[self.FIELD_MAP['tags']]
+                    tags = r[gindex]
                     if tags and tag in [x.strip() for x in
                             tags.lower().split(',')]:
                         yield r[self.FIELD_MAP['id']]

src/calibre/library/field_metadata.py

@@ -16,7 +16,7 @@ class TagsIcons(dict):
     '''
 
     category_icons = ['authors', 'series', 'formats', 'publisher', 'rating',
-                      'news',    'tags',   ':custom', ':user',     'search',]
+                      'news',    'tags',   'custom:', 'user:',     'search',]
     def __init__(self, icon_dict):
         for a in self.category_icons:
             if a not in icon_dict:
@@ -31,8 +31,8 @@ category_icon_map = {
                     'rating'    : 'rating.png',
                     'news'      : 'news.png',
                     'tags'      : 'tags.png',
-                    ':custom'   : 'column.png',
+                    'custom:'   : 'column.png',
-                    ':user'     : 'drawer.png',
+                    'user:'     : 'drawer.png',
                     'search'    : 'search.png'
             }
 
@@ -475,6 +475,8 @@ class FieldMetadata(dict):
             val = self._tb_cats[key]
             if val['is_category'] and val['kind'] in ('user', 'search'):
                 del self._tb_cats[key]
+                if key in self._search_term_map:
+                    del self._search_term_map[key]
 
     def cc_series_index_column_for(self, key):
         return self._tb_cats[key]['rec_index'] + 1
@@ -482,11 +484,12 @@ class FieldMetadata(dict):
     def add_user_category(self, label, name):
         if label in self._tb_cats:
             raise ValueError('Duplicate user field [%s]'%(label))
-        self._tb_cats[label] = {'table':None,        'column':None,
+        self._tb_cats[label] = {'table':None,          'column':None,
-                                'datatype':None,     'is_multiple':None,
+                                'datatype':None,       'is_multiple':None,
-                                'kind':'user',       'name':name,
+                                'kind':'user',         'name':name,
-                                'search_terms':[],    'is_custom':False,
+                                'search_terms':[label],'is_custom':False,
                                 'is_category':True}
+        self._add_search_terms_to_map(label, [label])
 
     def add_search_category(self, label, name):
         if label in self._tb_cats:
@@ -518,7 +521,6 @@ class FieldMetadata(dict):
     def _add_search_terms_to_map(self, key, terms):
         if terms is not None:
             for t in terms:
-                t = t.lower()
                 if t in self._search_term_map:
                     raise ValueError('Attempt to add duplicate search term "%s"'%t)
                 self._search_term_map[t] = key

src/calibre/manual/conversion.rst

@@ -587,11 +587,11 @@ TXT input supports a number of options to differentiate how paragraphs are detec
         Assumes that every paragraph starts with an indent (either a tab or 2+ spaces). Paragraphs end when
         the next line that starts with an indent is reached::
 
-            This is the
+              This is the
             first.
-            This is the second.
+              This is the second.
             
-            This is the
+              This is the
             third.
 
     :guilabel:`Paragraph Style: Unformatted`
@@ -603,7 +603,7 @@ TXT input supports a number of options to differentiate how paragraphs are detec
         formatting will be applied.
 
     :guilabel:`Formatting Style: Heuristic`
-        Analyses the document for common chapter headings, scene breaks, and italicized words and applies the
+        Analyzes the document for common chapter headings, scene breaks, and italicized words and applies the
         appropriate html markup during conversion.
 
     :guilabel:`Formatting Style: Markdown`

src/calibre/manual/faq.rst

@@ -310,7 +310,9 @@ What formats does |app| read metadata from?
 
 Where are the book files stored?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-When you first run |app|, it will ask you for a folder in which to store your books. Whenever you add a book to |app|, it will copy the book into that folder. Books in the folder are nicely arranged into sub-folders by Author and Title. Metadata about the books is stored in the file ``metadata.db`` (which is a sqlite database).
+When you first run |app|, it will ask you for a folder in which to store your books. Whenever you add a book to |app|, it will copy the book into that folder. Books in the folder are nicely arranged into sub-folders by Author and Title. Note that the contents of this folder are automatically managed by |app|, **do not** add any files/folders manually to this folder, as they may be automatically deleted. If you want to add a file associated to a particular book, use the top right area of :guilabel:`Edit metadata` dialog to do so. Then, |app| will automatically put that file into the correct folder and move it around when the title/author changes.
+
+Metadata about the books is stored in the file ``metadata.db`` at the top level of the library folder This file is is a sqlite database. When backing up your library make sure you copy the entire folder and all its sub-folders. 
 
 Why doesn't |app| let me store books in my own directory structure?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

src/calibre/manual/gui.rst

@@ -478,6 +478,8 @@ Calibre has several keyboard shortcuts to save you time and mouse movement. Thes
       - Focus the search bar
     * - :kbd:`Shift+Ctrl+F`
       - Open the advanced search dialog
+    * - :kbd:`Esc`
+      - Clear the current search
     * - :kbd:`N or F3`
       - Find the next book that matches the current search (only works if the highlight checkbox next to the search bar is checked)
     * - :kbd:`Shift+N or Shift+F3`

src/calibre/utils/localization.py

@@ -105,6 +105,7 @@ _extra_lang_codes = {
         'en_TH' : _('English (Thailand)'),
         'en_CY' : _('English (Cyprus)'),
         'en_PK' : _('English (Pakistan)'),
+        'en_HR' : _('English (Croatia)'),
         'en_IL' : _('English (Israel)'),
         'en_SG' : _('English (Singapore)'),
         'en_YE' : _('English (Yemen)'),