Sync to trunk.

Changelog.yaml

@@ -4,8 +4,89 @@
 # for important features/bug fixes.
 # Also, each release can have new and improved recipes.
 
+- version: 0.6.53
+  date: 2010-05-15
+
+  new features:
+    - title: "Clean up GUI initialization and add support for restoring corrupted databases automatically"
+
+    - title: "Make proxy detection more robust on windows and OS X. calibre now queries OS X Network Settigns if no environment variables are set. Also handle proxies with a trailing slash correctly"
+
+    - title: "Add EPUB advanced formatting demo to User Manual"
+
+    - title: "Support for the Booq Avant, Azbooka and the Samsung GT-I5700"
+
+    - title: "Backwards search in the E-book viewer"
+
+    - title: "calibredb: Add ability to create empty books in the database."
+      tickets: [5504]
+
+    - title: "Conversion pipeline: Support for the :first-letter pseudo selector"
+
+    - title: "Interpret a Keyboard interrupt (Ctrl+C) as a request to quit the main GUI"
+
+    - title: "CBC Input: Handle comics.txt encoded in UTF-16 with a BOM"
+
+  bug fixes:
+    - title: "HTML Input: Fix silly bug in case sensitivity detection"
+
+    - title: "Kobo driver: Show all sideloaded content on the device."
+      tickets: [5492]
+
+    - title: "EPUB metadata: Fix bug with relative apths in encryption detection when reading cover"
+      tickets: [5471]
+
+    - title: "E-book viewer: Fix next page scrolling when current document is just a little more than a screenfull. Also use a more robust method to insert blank space at the end of the document when the last screenfull is partially empty."
+
+    - title: "EPUB metadata: Allow deletion of series/tags/isbn from EPUB files when Saving to Disk"
+      tickets: [5518]
+
+    - title: "Fix regression that caused temporary blank line at the bottom of the books list when adding duplicates"
+      tickets: [5500]
+
+    - title: "Add icon for RTF"
+      tickets: [5503]
+
+    - title: "Amazon metadata: If ISBN is not found, don't report an error message."
+      tickets: [5501]
+
+    - title: "EPUB Input: Fix typo that caused incorrect processing of EPUB files with more than one identifier element and encrypted fonts"
+
+    - title: "Fix bug that caused send to device to send multiple copies to the device if you had previously used Prefrences"
+
+    - title: "Linux prs 500 udev rule: Use SUBSYSTEMS instead of the deprecated BUS"
+
+    - title: "PML2PMLZ plugin: Actually compress the PML file stored in the PMLZ archive"
+      tickets: [5511]
+
+    - title: "SONY drivers: Fix regression that broke collection ordering by series when sending to device. And fix another rare error condition."
+      tickets: [5487]
+
+    - title: "CHM Input: Regression that broke CHM conversion on OS X."
+      tickets: [5483]
+
+    - title: "Fix PDB created in Dropbook not convertable by Calibre"
+      tickets: [5441]
+
+
+  new recipes:
+    - title: APCOM, Leggo (it), Ansa and Punto Informatico
+      author: Gabriele Marini
+
+    - title: Scinexx.de
+      author: JSuer
+
+    - title: Various Russian news sources
+      author: Darko Miletic
+
+  improved recipes:
+    - Christian Science Monitor
+    - The Nation
+    - Physics World
+    - Discover Magazine
+
 - version: 0.6.52
-  date: 2010-07-30
+  date: 2010-05-07
 
   new features:
     - title: "Support for the Kobo Reader and the HTC Desire"

resources/recipes/Ansa.recipe

@@ -0,0 +1,72 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__author__    = 'Gabriele Marini, based on Darko Miletic'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+description   = 'Italian daily newspaper - 01-05-2010'
+'''
+http://www.ansa.it/
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Ansa(BasicNewsRecipe):
+    __author__    = 'Gabriele Marini'
+    description   = 'Italian News Agency'
+
+    cover_url      = 'http://www.ansa.it/web/images/logo_ansa_interna.gif'
+    title          = u'Ansa'
+    publisher      = 'Ansa'
+    category       = 'News, politics, culture, economy, general interest'
+
+    language       = 'it'
+    timefmt        = '[%a, %d %b, %Y]'
+
+    oldest_article = 1
+    max_articles_per_feed = 10
+    use_embedded_content  = False
+    recursion             = 10
+
+    remove_javascript = True
+    no_stylesheets        = True
+    conversion_options = {'linearize_tables':True}
+    remove_attributes = ['colspan']
+
+    keep_only_tags     = [dict(name='div', attrs={'class':['path','header-content','corpo']}),
+                         ]
+
+
+    remove_tags        = [
+                            dict(name='div', attrs={'class':'tools-bar'}),
+                            dict(name='div', attrs={'id':['rssdiv','blocco']})
+                         ]
+
+
+    feeds          = [
+                       (u'HomePage', u'http://www.ansa.it/web/ansait_web_rss_homepage.xml'),
+                       (u'Top New', u'http://www.ansa.it/web/notizie/rubriche/topnews/topnews_rss.xml'),
+                       (u'Cronaca', u'http://www.ansa.it/web/notizie/rubriche/cronaca/cronaca_rss.xml'),
+                       (u'Mondo', u'http://www.ansa.it/web/notizie/rubriche/mondo/mondo_rss.xml'),
+                       (u'Economia', u'http://www.ansa.it/web/notizie/rubriche/economia/economia_rss.xml'),
+                       (u'Politica', u'http://www.ansa.it/web/notizie/rubriche/politica/politica_rss.xml'),
+                       (u'Scienze', u'http://www.ansa.it/web/notizie/rubriche/scienza/scienza_rss.xml'),
+                       (u'Cinema', u'http://www.ansa.it/web/notizie/rubriche/cinema/cinema_rss.xml'),
+                       (u'Tecnologia e Internet', u'http://www.ansa.it/web/notizie/rubriche/tecnologia/tecnologia_rss.xml'),
+                       (u'Spettacolo', u'http://www.ansa.it/web/notizie/rubriche/spettacolo/spettacolo_rss.xml'),
+                       (u'Cultura e Tendenze', u'http://www.ansa.it/web/notizie/rubriche/cultura/cultura_rss.xml'),
+                       (u'Sport', u'http://www.ansa.it/web/notizie/rubriche/altrisport/altrisport_rss.xml'),
+                       (u'Calcio', u'http://www.ansa.it/web/notizie/rubriche/calcio/calcio_rss.xml'),
+                       (u'Lazio', u'http://www.ansa.it/web/notizie/regioni/lazio/lazio_rss.xml'),
+                       (u'Lombardia', u'http://www.ansa.it/web/notizie/regioni/lombardia/lombardia.shtml'),
+                       (u'Veneto', u'http://www.ansa.it/web/notizie/regioni/veneto/veneto.shtml'),
+                       (u'Campanioa', u'http://www.ansa.it/web/notizie/regioni/campania/campania.shtml'),
+                       (u'Sicilia', u'http://www.ansa.it/web/notizie/regioni/sicilia/sicilia.shtml'),
+                       (u'Toscana', u'http://www.ansa.it/web/notizie/regioni/toscana/toscana.shtml'),
+                       (u'Trentino', u'http://www.ansa.it/web/notizie/regioni/trentino/trentino.shtml')
+                      ]
+
+    extra_css = '''
+                   .path{font-style: italic; font-size: small}
+                   .header-content h1{font-weight: bold; font-size: xx-large}
+                   .header-content h2{font-weight: bold; font-size: x-large; font-syle: italic}
+                   .content-corpo{font-size: 14px;font-family: Times New Roman;}
+                '''

resources/recipes/aif_ru.recipe

@@ -0,0 +1,31 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.aif.ru
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AIF_ru(BasicNewsRecipe):
+    title                 = 'Arguments & Facts - Russian'
+    __author__            = 'Darko Miletic'
+    description           = 'News from Russia'
+    publisher             = 'AIF'
+    category              = 'news, politics, Russia'
+    oldest_article        = 2
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    encoding              = 'cp1251'
+    language              = 'ru'
+    publication_type      = 'magazine'
+    extra_css             = ' @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: Verdana,Arial,Helvetica,sans1,sans-serif} '
+    keep_only_tags     = [dict(name='div',attrs={'id':'inner'})]
+    remove_tags        = [
+                            dict(name=['iframe','object','link','base','input','img'])
+                           ,dict(name='div',attrs={'class':'photo'})
+                           ,dict(name='p',attrs={'class':'resizefont'})
+                         ]
+
+    feeds       = [(u'News', u'http://www.aif.ru/rss/all.php')]
+

resources/recipes/apcom.recipe

@@ -0,0 +1,48 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__author__    = 'Gabriele Marini, based on Darko Miletic'
+__copyright__ = '2009-2010, Darko Miletic <darko.miletic at gmail.com>'
+description   = 'Italian daily newspaper - 14-05-2010'
+
+'''
+http://www.apcom.NET/
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Apcom(BasicNewsRecipe):
+    __author__        = 'Marini Gabriele'
+    description   = 'Italian daily newspaper'
+
+    cover_url      = 'http://www.apcom.net/img/logoAP.gif'
+    title          = u'Apcom'
+    publisher      = 'TM News S.p.A.'
+    category       = 'News, politics, culture, economy, general interest'
+
+    language       = 'it'
+    timefmt        = '[%a, %d %b, %Y]'
+
+    oldest_article = 7
+    max_articles_per_feed = 50
+    use_embedded_content  = False
+    recursion             = 100
+
+    no_stylesheets        = True
+    conversion_options = {'linearize_tables':True}
+    remove_javascript = True
+
+    keep_only_tags     = [
+                          dict(name='div', attrs={'id':'ag_center'})
+                         ]
+
+
+
+    feeds = [
+	         (u'Globale',  u'http://www.apcom.net/rss/globale.xml '),
+             (u'Politica', u'http://www.apcom.net/rss/politica.xml'),
+             (u'Cronaca', u'http://www.apcom.net/rss/cronaca.xml'),
+             (u'Econimia', u'http://www.apcom.net/rss/economia.xml'),
+             (u'Esteri', u'http://www.apcom.net/rss/esteri.xml'),
+             (u'Cultura', u'http://www.apcom.net/rss/cultura.xml'),
+             (u'Sport', u'http://www.apcom.net/rss/sport.xml')
+             ]

resources/recipes/chr_mon.recipe

@@ -37,6 +37,7 @@ class ChristianScienceMonitor(BasicNewsRecipe):
 
     preprocess_regexps = [ (re.compile(i[0], re.IGNORECASE | re.DOTALL), i[1]) for i in
         [
+            (r'<!--.*?-->', lambda match : ''),
         (r'<body.*?<div id="story"', lambda match : '<body><div id="story"'),
         (r'<div class="pubdate">.*?</div>', lambda m: ''),
         (r'Full HTML version of this story which may include photos, graphics, and related links.*</body>',
@@ -79,7 +80,10 @@ class ChristianScienceMonitor(BasicNewsRecipe):
 
     remove_tags    = [
                         dict(name='div', attrs={'id':['story-tools','videoPlayer','storyRelatedBottom','enlarge-photo','photo-paginate']}),
-                        dict(name='div', attrs={'class':['storyToolbar cfx','podStoryRel','spacer3','divvy spacer7','comment','storyIncludeBottom']}),
+                        dict(name=['div','a'], attrs={'class':
+                            ['storyToolbar cfx','podStoryRel','spacer3',
+                                'divvy spacer7','comment','storyIncludeBottom',
+                                'hide', 'podBrdr']}),
                         dict(name='ul', attrs={'class':[ 'centerliststories']}) ,
                         dict(name='form', attrs={'id':[ 'commentform']}) ,
                     ]

resources/recipes/discover_magazine.recipe

@@ -7,13 +7,14 @@ __docformat__ = 'restructuredtext en'
 discovermagazine.com
 '''
 
+import re
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class DiscoverMagazine(BasicNewsRecipe):
 
     title = u'Discover Magazine'
-    description = u'Science, Technology and the Future' 
-    __author__ = 'Mike Diaz' 
+    description = u'Science, Technology and the Future'
+    __author__ = 'Starson17'
     language = 'en'
 
     oldest_article = 33
@@ -23,25 +24,63 @@ class DiscoverMagazine(BasicNewsRecipe):
     use_embedded_content  = False
     encoding = 'utf-8'
     extra_css = '.headline {font-size: x-large;} \n .fact {padding-top: 10pt}'
-    
-    remove_tags = [dict(name='div', attrs={'id':['searchModule', 'mainMenu', 'tool-box']}),
+
+    remove_tags = [
+                   dict(name='div', attrs={'id':['searchModule', 'mainMenu', 'tool-box']}),
+                   dict(name='div', attrs={'id':['footer','teaser','already-subscriber','teaser-suite','related-articles']}),
+                   dict(name='div', attrs={'class':['column']}),
                    dict(name='img', attrs={'src':'http://discovermagazine.com/onebyone.gif'})]
 
-    remove_tags_after = [dict(name='div', attrs={'class':'articlebody'})]
- 
+    remove_tags_after = [dict(name='div', attrs={'class':'listingBar'})]
+
+    def append_page(self, soup, appendtag, position):
+        pager = soup.find('span',attrs={'class':'next'})
+        if pager:
+           nexturl = pager.a['href']
+           soup2 = self.index_to_soup(nexturl)
+           texttag = soup2.find('div', attrs={'class':'articlebody'})
+           newpos = len(texttag.contents)
+           self.append_page(soup2,texttag,newpos)
+           texttag.extract()
+           appendtag.insert(position,texttag)
+
+    def preprocess_html(self, soup):
+        mtag = '<meta http-equiv="Content-Language" content="en-US"/>\n<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>'
+        soup.head.insert(0,mtag)
+        self.append_page(soup, soup.body, 3)
+        pager = soup.find('div',attrs={'class':'listingBar'})
+        if pager:
+           pager.extract()
+        return soup
+
+    def postprocess_html(self, soup, first_fetch):
+        for tag in soup.findAll(text=re.compile('^This article is a sample')):
+            tag.parent.extract()
+        for tag in soup.findAll(['table', 'tr', 'td']):
+            tag.name = 'div'
+        for tag in soup.findAll('div', attrs={'class':'discreet advert'}):
+            tag.extract()
+        for tag in soup.findAll('hr', attrs={'size':'1'}):
+            tag.extract()
+        for tag in soup.findAll('br'):
+            tag.extract()
+        return soup
+
     feeds = [
-             (u'Technology', u'http://discovermagazine.com/topics/technology/rss.xml'), 
-             (u'Health - Medicine', u'http://discovermagazine.com/topics/health-medicine/rss.xml'), 
-             (u'Mind Brain', u'http://discovermagazine.com/topics/mind-brain/rss.xml'), 
-             (u'Space', u'http://discovermagazine.com/topics/space/rss.xml'), 
-             (u'Human Origins', u'http://discovermagazine.com/topics/human-origins/rss.xml'), 
-             (u'Living World', u'http://discovermagazine.com/topics/living-world/rss.xml'), 
-             (u'Environment', u'http://discovermagazine.com/topics/environment/rss.xml'), 
-             (u'Physics & Math', u'http://discovermagazine.com/topics/physics-math/rss.xml'), 
-             (u'Vital Signs', u'http://discovermagazine.com/columns/vital-signs/rss.xml'), 
-             (u"20 Things you didn't know about...", u'http://discovermagazine.com/columns/20-things-you-didnt-know/rss.xml'), 
-             (u'Fuzzy Math', u'http://discovermagazine.com/columns/fuzzy-math/rss.xml'), 
-             (u'The Brain', u'http://discovermagazine.com/columns/the-brain/rss.xml'), 
-             (u'Stupid Science Word of the Month', u'http://discovermagazine.com/columns/stupid-science-word-of-the-month/rss.xml'), 
-             (u'Science Not Fiction', u'http://blogs.discovermagazine.com/sciencenotfiction/wp-rss.php')
-            ]
+             (u'Technology', u'http://discovermagazine.com/topics/technology/rss.xml'),
+             (u'Health - Medicine', u'http://discovermagazine.com/topics/health-medicine/rss.xml'),
+             (u'Mind Brain', u'http://discovermagazine.com/topics/mind-brain/rss.xml'),
+             (u'Space', u'http://discovermagazine.com/topics/space/rss.xml'),
+             (u'Human Origins', u'http://discovermagazine.com/topics/human-origins/rss.xml'),
+             (u'Living World', u'http://discovermagazine.com/topics/living-world/rss.xml'),
+             (u'Environment', u'http://discovermagazine.com/topics/environment/rss.xml'),
+             (u'Physics & Math', u'http://discovermagazine.com/topics/physics-math/rss.xml'),
+             (u"20 Things you didn't know about...", u'http://discovermagazine.com/columns/20-things-you-didnt-know/rss.xml'),
+             (u'Fuzzy Math', u'http://discovermagazine.com/columns/fuzzy-math/rss.xml'),
+             (u'The Brain', u'http://discovermagazine.com/columns/the-brain/rss.xml'),
+             (u'What is This', u'http://discovermagazine.com/columns/what-is-this/rss.xml'),
+             (u'Vital Signs', u'http://discovermagazine.com/columns/vital-signs/rss.xml'),
+             (u'Think Tech', u'http://discovermagazine.com/columns/think-tech/rss.xml'),
+             (u'Future Tech', u'http://discovermagazine.com/columns/future-tech/rss.xml'),
+             (u'Discover Interview', u'http://discovermagazine.com/columns/discover-interview/rss.xml'),
+            ]

resources/recipes/economist.recipe

@@ -69,9 +69,9 @@ class Economist(BasicNewsRecipe):
         key = None
         for tag in soup.findAll(['h1', 'h2']):
             text = ''.join(tag.findAll(text=True))
+            if tag.name in ('h1', 'h2') and 'Classified ads' in text:
+                break
             if tag.name == 'h1':
-                if 'Classified ads' in text:
-                    break
                 if 'The world this week' in text or 'The world this year' in text:
                     index_started = True
                 if not index_started:

resources/recipes/izvestia.recipe

@@ -0,0 +1,28 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+'''
+izvestia.ru
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Izvestia(BasicNewsRecipe):
+    title                 = 'Izvestia'
+    __author__            = 'Darko Miletic'
+    description           = 'News from Russia'
+    publisher             = 'Izvestia'
+    category              = 'news, politics, Russia'
+    oldest_article        = 5
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    encoding              = 'cp1251'
+    language              = 'ru'
+    publication_type      = 'newspaper'
+    masthead_url          = 'http://images.izvestia.ru/izv/sys/logo.gif'
+    extra_css             = ' @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: Verdana,Arial,Helvetica,sans1,sans-serif} '
+    keep_only_tags     = [dict(name='div', attrs={'class':'newsFull'})]
+    remove_tags        = [dict(name=['iframe','object','img','link','base'])]
+    remove_tags_before = dict(name='h1', attrs={'class':'statya'})
+
+    feeds       = [(u'Daily edition', u'http://rss.feedsportal.com/c/32171/f/424076/index.rss')]

resources/recipes/kommersant.recipe

@@ -0,0 +1,42 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.kommersant.ru
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Kommersant_ru(BasicNewsRecipe):
+    title                 = 'Kommersant'
+    __author__            = 'Darko Miletic'
+    description           = 'News from Russia'
+    publisher             = 'Kommersant'
+    category              = 'news, politics, Russia'
+    oldest_article        = 5
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    encoding              = 'cp1251'
+    language              = 'ru'
+    publication_type      = 'newspaper'
+    masthead_url          = 'http://www.kommersant.ru/CorpPics/logo_daily_1.gif'
+    extra_css             = ' @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: Arial, sans1, sans-serif} span#ctl00_ContentPlaceHolderStyle_LabelSubTitle{margin-bottom: 1em; display: block} .author{margin-bottom: 1em; display: block} .paragraph{margin-bottom: 1em; display: block} .vvodka{font-weight: bold; margin-bottom: 1em} '
+
+    conversion_options = {
+                          'comment'          : description
+                        , 'tags'             : category
+                        , 'publisher'        : publisher
+                        , 'language'         : language
+                        }
+
+    keep_only_tags = [
+                         dict(attrs={'id':'ctl00_ContentPlaceHolderStyle_PanelHeader'})
+                        ,dict(attrs={'class':['vvodka','paragraph','author']})
+                     ]
+    remove_tags        = [dict(name=['iframe','object','link','img','base'])]
+
+    feeds       = [(u'Articles', u'http://feeds.kommersant.ru/RSS_Export/RU/daily.xml')]
+
+    def print_version(self, url):
+        return url.replace('doc-rss.aspx','doc.aspx') + '&print=true'
+

resources/recipes/leggo_it.recipe

@@ -0,0 +1,49 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__author__    = 'Gabriele Marini, based on Darko Miletic'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+description   = 'Italian daily newspaper - v1.00 05-05-2010'
+
+'''
+http://www.leggo.it
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class LeggoIT(BasicNewsRecipe):
+    __author__        = 'Gabriele Marini'
+    description   = 'Italian Free daily newspaper'
+
+    cover_url      = 'http://www.leggo.it/img/logo-leggo2.gif'
+    title          = u'Leggo.it'
+    publisher      = 'Ced Caltagirone Editore S.p.A.'
+    category       = 'News, politics, culture, economy, general interest'
+
+    language       = 'it'
+    timefmt        = '[%a, %d %b, %Y]'
+
+    oldest_article = 5
+    max_articles_per_feed = 100
+    use_embedded_content  = False
+    recursion             = 100
+
+    no_stylesheets        = True
+    remove_javascript = True
+    conversion_options = {'linearize_tables':True}
+
+    keep_only_tags     = [
+                          dict(name='h1',attrs={'class':'nero22'}),
+						  dict(name='div',attrs={'id':'testodim'})
+                          ]
+    feeds          = [
+                       (u'Home Page', u'http://www.leggo.it/rss/home.xml'),
+                       (u'Italia', u'http://www.leggo.it/rss/italia.xml'),
+                       (u'Esteri', u'http://www.leggo.it/rss/esteri.xml'),
+                       (u'Economia', u'http://www.leggo.it/rss/economia.xml'),
+                       (u'Sport', u'http://www.leggo.it/rss/sport.xml'),
+                       (u'Gossip', u'http://www.leggo.it/rss/gossip.xml'),
+                       (u'Spettacoli', u'http://www.leggo.it/rss/spettacoli.xml'),
+                       (u'Salute', u'http://www.leggo.it/rss/salute.xml'),
+                       (u'Scienza', u'http://www.leggo.it/rss/scienza.xml')
+					 ]
+

resources/recipes/physics_world.recipe

@@ -10,6 +10,7 @@ class PhysicsWorld(BasicNewsRecipe):
     oldest_article = 7
     max_articles_per_feed = 100
     no_stylesheets        = True
+    cover_url = 'http://images.iop.org/cws/icons/themes/phw/header-logo.png'
     use_embedded_content  = False
     remove_javascript     = True
     needs_subscription = True
@@ -27,7 +28,7 @@ class PhysicsWorld(BasicNewsRecipe):
         br = BasicNewsRecipe.get_browser(self)
         if self.username is not None and self.password is not None:
             br.open('http://physicsworld.com/cws/sign-in')
-            br.select_form(nr=1)
+            br.select_form(nr=2)
             br['username'] = self.username
             br['password'] = self.password
             br.submit()

resources/recipes/punto_informatico.recipe

@@ -0,0 +1,38 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__author__    = 'Gabriele Marini'
+__copyright__ = 'Gabriele Marini'
+__description__ = 'Punto Informatico'
+
+'''
+http://www.punto-informatico.it/
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+
+class PuntoInformatico(BasicNewsRecipe):
+    __author__        = 'Gabriele Marini'
+    description   = 'Punto Informatico: Internet dal 1996'
+
+    cover_url      = 'http://punto-informatico.it/images/logo_8bit.png'
+    title          = u'Punto Informatico '
+    publisher      = 'italiaNews High Tech'
+    category       = 'News, Information Tecnology'
+
+    language       = 'it'
+    timefmt        = '[%a, %d %b, %Y]'
+
+    oldest_article = 15
+    max_articles_per_feed = 50
+    use_embedded_content  = False
+
+    remove_javascript  = True
+    no_stylesheets     = True
+    keep_only_tags     = [dict(name='div', attrs={'class':'box'})]
+    remove_tags        = [dict(name='div',attrs={'class':'boxadv'})]
+    def get_article_url(self, article):
+        return article.get('id', article.get('guid', None))
+
+    feeds              = [(u'Punto Informatico',u'http://feeds.punto-informatico.it/c/32288/f/438866/index.rss')]
+

resources/recipes/ria_ru.recipe

@@ -0,0 +1,43 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.rian.ru
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class RIANovosti(BasicNewsRecipe):
+    title                 = 'RIA Novosti - Russian'
+    __author__            = 'Darko Miletic'
+    description           = 'News from Russia'
+    publisher             = 'RIA'
+    category              = 'news, politics, Russia'
+    oldest_article        = 2
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    encoding              = 'utf8'
+    language              = 'ru'
+    publication_type      = 'newsportal'
+    masthead_url          = 'http://img.beta.rian.ru/images/22868/43/228684314.jpg'
+    extra_css             = ' @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: Arial,Helvetica,sans1,sans-serif} '
+    remove_tags_before = dict(name='h1')
+    remove_tags_after  = dict(name='div', attrs={'class':'text'})
+    remove_tags        = [dict(name=['iframe','object','link','img','base'])]
+
+    feeds       = [
+                     (u'Frontpage', u'http://www.rian.ru/export/rss2/lenta/index.xml')
+                    ,(u'Politics', u'http://www.rian.ru/export/rss2/politics/index.xml')
+                    ,(u'World', u'http://www.rian.ru/export/rss2/world/index.xml')
+                    ,(u'Economy', u'http://www.rian.ru/export/rss2/economy/index.xml')
+                    ,(u'Society', u'http://www.rian.ru/export/rss2/society/index.xml')
+                    ,(u'Moscow', u'http://www.rian.ru/export/rss2/moscow/index.xml')
+                    ,(u'Defense', u'http://www.rian.ru/export/rss2/defense_safety/index.xml')
+                    ,(u'Science', u'http://www.rian.ru/export/rss2/science/index.xml')
+                    ,(u'Turism', u'http://www.rian.ru/export/rss2/tourism/index.xml')
+                    ,(u'Culture', u'http://www.rian.ru/export/rss2/culture/index.xml')
+                  ]
+
+    def print_version(self, url):
+        return url.replace('.html','-print.html')
+

resources/recipes/scinexx.recipe

@@ -0,0 +1,35 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1265145870(BasicNewsRecipe):
+    title          = u'Scinexx.de'
+    language = 'de'
+    __author__ = 'JSuer'
+    cover_url    = 'http://www.g-o.de/grafiken/web_scinexx/head2.gif'
+    oldest_article = 14
+    max_articles_per_feed = 100
+    no_stylesheets = True
+    use_embedded_content  = False
+    encoding = 'ISO-8859-1'
+
+    feeds          = [(u'Scinexx.de', u'http://feeds.feedburner.com/scinexx')]
+
+    remove_tags = [{'class':['text1fett']}]
+    remove_tags = [{'href':['javascript:window.print()']}]
+
+    extra_css = '''
+                    .text2normal{font-family:Verdana,Geneva,Kalimati,sans-serif; font-size:x-small;}
+                    .text1normalblau{font-family:Verdana,Geneva,Kalimati,sans-serif; font-size:small;}
+                    .text1fett{font-color:grey; font-size:small;}
+                    .titel1{font-family:Georgia,"Times New Roman",Times,serif; font-size:large;}
+                    .titel2{font-family:Georgia,"Times New Roman",Times,serif; }
+                    .titel3{font-family:Georgia,"Times New Roman",Times,serif; font-size:larger;}
+                    h1{font-size:large;}
+                    '''
+
+
+    def print_version(self, url):
+        id_start = url.rfind('2010') - 6
+        id_end = id_start +  5
+        id = url[id_start : id_end]
+        result = 'http://www.scinexx.de/inc/artikel_drucken.php?id=' + id + '&a_flag=1'
+        return result

resources/recipes/the_escapist.recipe

@@ -15,7 +15,7 @@ class al(BasicNewsRecipe):
     description   = 'The Escapist Magazine'
 
     cover_url      = 'http://cdn.themis-media.com/themes/escapistmagazine/default/images/logo.png'
-    title          = u'the Escapist Magazine'
+    title          = u'The Escapist Magazine'
     publisher      = 'Themis media'
     category       = 'Video games news, lifestyle, gaming culture'
 

resources/recipes/the_nation.recipe

@@ -1,37 +1,40 @@
-#!/usr/bin/env  python
-
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008 - 2010, Darko Miletic <darko.miletic at gmail.com>'
 '''
 thenation.com
 '''
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Thenation(BasicNewsRecipe):
-    title                 = u'The Nation'
-    __author__            = u'Darko Miletic'
-    description           = u'Unconventional Wisdom Since 1865'
+    title                 = 'The Nation'
+    __author__            = 'Darko Miletic'
+    description           = 'Unconventional Wisdom Since 1865'
+    publisher             = 'The Nation'
+    category              = 'news, politics, USA'
     oldest_article        = 120
+    encoding              = 'utf-8'
     max_articles_per_feed = 100
     no_stylesheets        = True
-    language = 'en'
-
-    use_embedded_content  = False
-    simultaneous_downloads = 1
+    language              = 'en'
+    use_embedded_content   = False
     delay                  = 1
-    timefmt                = ' [%A, %d %B, %Y]'
-     
+    masthead_url           = 'http://www.thenation.com/sites/default/themes/thenation/images/logo-main.gif'
+    exra_css               = ' body{font-family: Arial,Helvetica,sans-serif;} .print-created{font-size: small;} .caption{display: block; font-size: x-small;} '
 
-    keep_only_tags = [ dict(name='div', attrs={'class':'main'}) ]
-    remove_tags = [
-                     dict(name='div', attrs={'class':'mod tools'})
-                    ,dict(name='div', attrs={'class':'inset'    })
-                    ,dict(name='div', attrs={'class':'share'    })
-                    ,dict(name='ol' , attrs={'id'   :'comments' })
-                    ,dict(name='p'  , attrs={'class':'info'     })
-                    ,dict(name='a'  , attrs={'class':'comments' })
-                    ,dict(name='ul' , attrs={'class':'important'})
-                    ,dict(name='object')
-                  ]
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
 
-    feeds       = [(u"Top Stories", u'http://feedproxy.google.com/TheNationEdPicks')]
+    keep_only_tags = [ dict(attrs={'class':['print-title','print-created','print-content','print-links']}) ]
+    remove_tags    = [dict(name='link')]
+
+    feeds       = [(u"Editor's Picks", u'http://www.thenation.com/rss/editors_picks')]
+
+    def print_version(self, url):
+        return url.replace('.thenation.com/','.thenation.com/print/')
+
+    def preprocess_html(self, soup):
+        return self.adeify_images(soup)

src/calibre/__init__.py

@@ -4,6 +4,7 @@ __copyright__ = '2008, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 import sys, os, re, logging, time, mimetypes, \
        __builtin__, warnings, multiprocessing
+from urllib import getproxies
 __builtin__.__dict__['dynamic_property'] = lambda(func): func(None)
 from htmlentitydefs import name2codepoint
 from math import floor
@@ -199,71 +200,54 @@ def extract(path, dir):
     extractor(path, dir)
 
 def get_proxies(debug=True):
-    proxies = {}
+    proxies = getproxies()
+    for key, proxy in list(proxies.items()):
+        if not proxy or '..' in proxy:
+            del proxies[key]
+            continue
+        if proxy.startswith(key+'://'):
+            proxy = proxy[len(key)+3:]
+        if proxy.endswith('/'):
+            proxy = proxy[:-1]
+        if len(proxy) > 4:
+            proxies[key] = proxy
+        else:
+            prints('Removing invalid', key, 'proxy:', proxy)
+            del proxies[key]
 
-    for q in ('http', 'ftp'):
-        proxy =  os.environ.get(q+'_proxy', None)
-        if not proxy: continue
-        if proxy.startswith(q+'://'):
-            proxy = proxy[7:]
-        proxies[q] = proxy
-
-    if iswindows:
-        try:
-            winreg = __import__('_winreg')
-            settings = winreg.OpenKey(winreg.HKEY_CURRENT_USER,
-                                      'Software\\Microsoft\\Windows'
-                                      '\\CurrentVersion\\Internet Settings')
-            proxy = winreg.QueryValueEx(settings, "ProxyEnable")[0]
-            if proxy:
-                server = str(winreg.QueryValueEx(settings, 'ProxyServer')[0])
-                if ';' in server:
-                    for p in server.split(';'):
-                        protocol, address = p.split('=')
-                        proxies[protocol] = address
-                else:
-                    proxies['http'] = server
-                    proxies['ftp'] =  server
-            settings.Close()
-        except Exception, e:
-            prints('Unable to detect proxy settings: %s' % str(e))
-    for x in list(proxies):
-        if len(proxies[x]) < 5:
-            prints('Removing invalid', x, 'proxy:', proxies[x])
-            del proxies[x]
     if proxies and debug:
         prints('Using proxies:', proxies)
     return proxies
 
 def get_parsed_proxy(typ='http', debug=True):
     proxies = get_proxies(debug)
-    if typ not in proxies:
-        return
-    pattern = re.compile((
-        '(?:ptype://)?' \
-        '(?:(?P<user>\w+):(?P<pass>.*)@)?' \
-        '(?P<host>[\w\-\.]+)' \
-        '(?::(?P<port>\d+))?').replace('ptype', typ)
-    )
+    proxy = proxies.get(typ, None)
+    if proxy:
+        pattern = re.compile((
+            '(?:ptype://)?' \
+            '(?:(?P<user>\w+):(?P<pass>.*)@)?' \
+            '(?P<host>[\w\-\.]+)' \
+            '(?::(?P<port>\d+))?').replace('ptype', typ)
+        )
 
-    match = pattern.match(proxies['typ'])
-    if match:
-        try:
-            ans = {
-                    'host' : match.group('host'),
-                    'port' : match.group('port'),
-                    'user' : match.group('user'),
-                    'pass' : match.group('pass')
-                }
-            if ans['port']:
-                ans['port'] = int(ans['port'])
-        except:
-            if debug:
-                traceback.print_exc()
-            return
-        if debug:
-            prints('Using http proxy', ans)
-        return ans
+        match = pattern.match(proxies[typ])
+        if match:
+            try:
+                ans = {
+                        'host' : match.group('host'),
+                        'port' : match.group('port'),
+                        'user' : match.group('user'),
+                        'pass' : match.group('pass')
+                    }
+                if ans['port']:
+                    ans['port'] = int(ans['port'])
+            except:
+                if debug:
+                    traceback.print_exc()
+            else:
+                if debug:
+                    prints('Using http proxy', str(ans))
+                return ans
 
 
 def browser(honor_time=True, max_time=2, mobile_browser=False):

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.6.52'
+__version__   = '0.6.53'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 
 import re

src/calibre/customize/__init__.py

@@ -236,6 +236,10 @@ class MetadataWriterPlugin(Plugin):
 
     type = _('Metadata writer')
 
+    def __init__(self, *args, **kwargs):
+        Plugin.__init__(self, *args, **kwargs)
+        self.apply_null = False
+
     def set_metadata(self, stream, mi, type):
         '''
         Set metadata for the file represented by stream (a file like object

src/calibre/customize/builtins.py

@@ -68,7 +68,7 @@ class PML2PMLZ(FileTypePlugin):
 
         of = self.temporary_file('_plugin_pml2pmlz.pmlz')
         pmlz = zipfile.ZipFile(of.name, 'w')
-        pmlz.write(pmlfile, os.path.basename(pmlfile))
+        pmlz.write(pmlfile, os.path.basename(pmlfile), zipfile.ZIP_DEFLATED)
 
         pml_img = os.path.splitext(pmlfile)[0] + '_img'
         i_img = os.path.join(os.path.dirname(pmlfile),'images')
@@ -329,7 +329,7 @@ class EPUBMetadataWriter(MetadataWriterPlugin):
 
     def set_metadata(self, stream, mi, type):
         from calibre.ebooks.metadata.epub import set_metadata
-        set_metadata(stream, mi)
+        set_metadata(stream, mi, apply_null=self.apply_null)
 
 class LRFMetadataWriter(MetadataWriterPlugin):
 
@@ -450,11 +450,11 @@ from calibre.devices.eslick.driver import ESLICK
 from calibre.devices.nuut2.driver import NUUT2
 from calibre.devices.iriver.driver import IRIVER_STORY
 from calibre.devices.binatone.driver import README
-from calibre.devices.hanvon.driver import N516, EB511, ALEX
+from calibre.devices.hanvon.driver import N516, EB511, ALEX, AZBOOKA
 from calibre.devices.edge.driver import EDGE
 from calibre.devices.teclast.driver import TECLAST_K3
 from calibre.devices.sne.driver import SNE
-from calibre.devices.misc import PALMPRE, KOBO
+from calibre.devices.misc import PALMPRE, KOBO, AVANT
 
 from calibre.ebooks.metadata.fetch import GoogleBooks, ISBNDB, Amazon
 from calibre.library.catalog import CSV_XML, EPUB_MOBI
@@ -538,6 +538,8 @@ plugins += [
     ALEX,
     PALMPRE,
     KOBO,
+    AZBOOKA,
+    AVANT,
 ]
 plugins += [x for x in list(locals().values()) if isinstance(x, type) and \
                                         x.__name__.endswith('MetadataReader')]

src/calibre/customize/ui.py

@@ -187,6 +187,18 @@ class QuickMetadata(object):
 
 quick_metadata = QuickMetadata()
 
+class ApplyNullMetadata(object):
+
+    def __init__(self):
+        self.apply_null = False
+
+    def __enter__(self):
+        self.apply_null = True
+
+    def __exit__(self, *args):
+        self.apply_null = False
+
+apply_null_metadata = ApplyNullMetadata()
 
 def get_file_type_metadata(stream, ftype):
     mi = MetaInformation(None, None)
@@ -214,6 +226,7 @@ def set_file_type_metadata(stream, mi, ftype):
             if not is_disabled(plugin):
                 with plugin:
                     try:
+                        plugin.apply_null = apply_null_metadata.apply_null
                         plugin.set_metadata(stream, mi, ftype.lower().strip())
                         break
                     except:

src/calibre/debug.py

@@ -34,9 +34,51 @@ Run an embedded python interpreter.
             help='Add a simple plugin (i.e. a plugin that consists of only a '
             '.py file), by specifying the path to the py file containing the '
             'plugin code.')
+    parser.add_option('--reinitialize-db', default=None,
+            help='Re-initialize the sqlite calibre database at the '
+            'specified path. Useful to recover from db corruption.')
 
     return parser
 
+def reinit_db(dbpath, callback=None):
+    if not os.path.exists(dbpath):
+        raise ValueError(dbpath + ' does not exist')
+    from calibre.library.sqlite import connect
+    from contextlib import closing
+    import shutil
+    conn = connect(dbpath, False)
+    uv = conn.get('PRAGMA user_version;', all=False)
+    conn.execute('PRAGMA writable_schema=ON')
+    conn.commit()
+    sql_lines = conn.dump()
+    conn.close()
+    dest = dbpath + '.tmp'
+    try:
+        with closing(connect(dest, False)) as nconn:
+            nconn.execute('create temporary table temp_sequence(id INTEGER PRIMARY KEY AUTOINCREMENT)')
+            nconn.commit()
+            if callable(callback):
+                callback(len(sql_lines), True)
+            for i, line in enumerate(sql_lines):
+                try:
+                    nconn.execute(line)
+                except:
+                    import traceback
+                    prints('SQL line %r failed with error:'%line)
+                    prints(traceback.format_exc())
+                    continue
+                finally:
+                    if callable(callback):
+                        callback(i, False)
+            nconn.execute('pragma user_version=%d'%int(uv))
+            nconn.commit()
+        os.remove(dbpath)
+        shutil.copyfile(dest, dbpath)
+    finally:
+        if os.path.exists(dest):
+            os.remove(dest)
+    prints('Database successfully re-initialized')
+
 def migrate(old, new):
     from calibre.utils.config import prefs
     from calibre.library.database import LibraryDatabase
@@ -122,6 +164,8 @@ def main(args=sys.argv):
         prints('CALIBRE_RESOURCES_PATH='+sys.resources_location)
         prints('CALIBRE_EXTENSIONS_PATH='+sys.extensions_location)
         prints('CALIBRE_PYTHON_PATH='+os.pathsep.join(sys.path))
+    elif opts.reinitialize_db is not None:
+        reinit_db(opts.reinitialize_db)
     else:
         from calibre import ipython
         ipython()

src/calibre/devices/android/driver.py

@@ -27,7 +27,7 @@ class ANDROID(USBMS):
             0x18d1 : { 0x4e11 : [0x0100, 0x226], 0x4e12: [0x0100, 0x226]},
 
             # Samsung
-            0x04e8 : { 0x681d : [0x0222], 0x681c : [0x0222]},
+            0x04e8 : { 0x681d : [0x0222], 0x681c : [0x0222, 0x0224]},
 
             # Acer
             0x502 : { 0x3203 : [0x0100]},
@@ -38,9 +38,9 @@ class ANDROID(USBMS):
             'be used')
     EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(EBOOK_DIR_MAIN)
 
-    VENDOR_NAME      = ['HTC', 'MOTOROLA', 'GOOGLE_', 'ANDROID', 'ACER']
+    VENDOR_NAME      = ['HTC', 'MOTOROLA', 'GOOGLE_', 'ANDROID', 'ACER', 'GT-I5700']
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
-            '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE']
+            '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD']
 
     OSX_MAIN_MEM = 'HTC Android Phone Media'
 

src/calibre/devices/hanvon/driver.py

@@ -24,7 +24,7 @@ class N516(USBMS):
 
     VENDOR_ID   = [0x0525]
     PRODUCT_ID  = [0xa4a5]
-    BCD         = [0x323, 0x326]
+    BCD         = [0x323, 0x326, 0x399]
 
     VENDOR_NAME      = 'INGENIC'
     WINDOWS_MAIN_MEM = '_FILE-STOR_GADGE'
@@ -50,6 +50,20 @@ class ALEX(N516):
     EBOOK_DIR_MAIN = 'eBooks'
     SUPPORTS_SUB_DIRS = True
 
+class AZBOOKA(ALEX):
+
+    name = 'Azbooka driver'
+    gui_name = 'Azbooka'
+    description = _('Communicate with the Azbooka')
+
+    VENDOR_NAME      = 'LINUX'
+    WINDOWS_MAIN_MEM = 'FILE-STOR_GADGET'
+
+    MAIN_MEMORY_VOLUME_LABEL  = 'Azbooka Internal Memory'
+
+    EBOOK_DIR_MAIN = ''
+
+
 class EB511(USBMS):
     name           = 'Elonex EB 511 driver'
     gui_name       = 'EB 511'

src/calibre/devices/misc.py

@@ -47,5 +47,25 @@ class KOBO(USBMS):
     VENDOR_NAME = 'KOBO_INC'
     WINDOWS_MAIN_MEM = '.KOBOEREADER'
 
-    EBOOK_DIR_MAIN = 'e-books'
+    EBOOK_DIR_MAIN = ''
+
+class AVANT(USBMS):
+    name           = 'Booq Avant Device Interface'
+    gui_name       = 'Avant'
+    description    = _('Communicate with the Booq Avant')
+    author         = 'Kovid Goyal'
+    supported_platforms = ['windows', 'osx', 'linux']
+
+    # Ordered list of supported formats
+    FORMATS     = ['epub', 'fb2', 'html', 'rtf', 'pdf', 'txt']
+
+    VENDOR_ID   = [0x0525]
+    PRODUCT_ID  = [0xa4a5]
+    BCD         = [0x0319]
+
+    VENDOR_NAME = 'E-BOOK'
+    WINDOWS_MAIN_MEM = 'READER'
+
+    EBOOK_DIR_MAIN = ''
+    SUPPORTS_SUB_DIRS = True
 

src/calibre/devices/prs505/books.py

@@ -284,7 +284,12 @@ class BookList(_BookList):
         plitems = []
         for pl in self.playlists():
             for c in pl.childNodes:
-                if hasattr(c, 'tagName')  and c.tagName.endswith('item'):
+                if hasattr(c, 'tagName') and c.tagName.endswith('item') and \
+                    hasattr(c, 'getAttribute'):
+                    try:
+                        c.getAttribute('id')
+                    except: # Unlinked node
+                        continue
                     plitems.append(c)
         return plitems
 
@@ -385,9 +390,9 @@ class BookList(_BookList):
                 continue
             db_ids = [i.getAttribute('id') for i in pl.childNodes if hasattr(i, 'getAttribute')]
             pl_book_ids = [getattr(self.book_by_id(i), 'db_id', None) for i in db_ids]
-            map = {}
+            imap = {}
             for i, j in zip(pl_book_ids, db_ids):
-                map[i] = j
+                imap[i] = j
             pl_book_ids = [i for i in pl_book_ids if i is not None]
             ordered_ids = [i for i in self.tag_order[title] if i in pl_book_ids]
 
@@ -399,7 +404,7 @@ class BookList(_BookList):
                 child.unlink()
             for id in ordered_ids:
                 item = self.document.createElement(self.prefix+'item')
-                item.setAttribute('id', str(map[id]))
+                item.setAttribute('id', str(imap[id]))
                 pl.appendChild(item)
 
 def fix_ids(main, carda, cardb):

src/calibre/devices/prs505/driver.py

@@ -121,6 +121,14 @@ class PRS505(CLI, Device):
         self.report_progress(1.0, _('Getting list of books on device...'))
         return bl
 
+    def filename_callback(self, fname, mi):
+        if getattr(mi, 'application_id', None) is not None:
+            base = fname.rpartition('.')[0]
+            suffix = '_%s'%mi.application_id
+            if not base.endswith(suffix):
+                fname = base + suffix + '.' + fname.rpartition('.')[-1]
+        return fname
+
     def upload_books(self, files, names, on_card=None, end_session=True,
                      metadata=None):
 

src/calibre/devices/usbms/device.py

@@ -825,7 +825,10 @@ class Device(DeviceConfig, DevicePlugin):
         from calibre.library.save_to_disk import get_components
         if not isinstance(template, unicode):
             template = template.decode('utf-8')
-        extra_components = get_components(template, mdata, fname)
+        app_id = str(getattr(mdata, 'application_id', ''))
+        # The SONY readers need to have the db id in the created filename
+        extra_components = get_components(template, mdata, fname,
+                length=250-len(app_id)-1)
         if not extra_components:
             extra_components.append(sanitize(self.filename_callback(fname,
                 mdata)))

src/calibre/ebooks/comic/input.py

@@ -341,8 +341,15 @@ class ComicInput(InputFormatPlugin):
             if not os.path.exists('comics.txt'):
                 raise ValueError('%s is not a valid comic collection'
                         %stream.name)
-            raw = open('comics.txt', 'rb').read().decode('utf-8')
-            raw.lstrip(unicode(codecs.BOM_UTF8, "utf8" ))
+            raw = open('comics.txt', 'rb').read()
+            if raw.startswith(codecs.BOM_UTF16_BE):
+                raw = raw.decode('utf-16-be')[1:]
+            elif raw.startswith(codecs.BOM_UTF16_LE):
+                raw = raw.decode('utf-16-le')[1:]
+            elif raw.startswith(codecs.BOM_UTF8):
+                raw = raw.decode('utf-8')[1:]
+            else:
+                raw = raw.decode('utf-8')
             for line in raw.splitlines():
                 line = line.strip()
                 if not line:

src/calibre/ebooks/epub/input.py

@@ -32,9 +32,9 @@ class EPUBInput(InputFormatPlugin):
         key = None
         for item in opf.identifier_iter():
             scheme = None
-            for key in item.attrib.keys():
-                if key.endswith('scheme'):
-                    scheme = item.get(key)
+            for xkey in item.attrib.keys():
+                if xkey.endswith('scheme'):
+                    scheme = item.get(xkey)
             if (scheme and scheme.lower() == 'uuid') or \
                     (item.text and item.text.startswith('urn:uuid:')):
                 key = str(item.text).rpartition(':')[-1]

src/calibre/ebooks/html/input.py

@@ -292,12 +292,12 @@ class HTMLInput(InputFormatPlugin):
                 encoding=opts.input_encoding)
 
     def is_case_sensitive(self, path):
-        if self._is_case_sensitive is not None:
+        if getattr(self, '_is_case_sensitive', None) is not None:
             return self._is_case_sensitive
         if not path or not os.path.exists(path):
             return islinux or isfreebsd
-        self._is_case_sensitive = os.path.exists(path.lower()) \
-                and os.path.exists(path.upper())
+        self._is_case_sensitive = not (os.path.exists(path.lower()) \
+                and os.path.exists(path.upper()))
         return self._is_case_sensitive
 
     def create_oebbook(self, htmlpath, basedir, opts, log, mi):

src/calibre/ebooks/metadata/amazon.py

@@ -19,12 +19,18 @@ AWS_NS = 'http://webservices.amazon.com/AWSECommerceService/2005-10-05'
 def AWS(tag):
     return '{%s}%s'%(AWS_NS, tag)
 
-def check_for_errors(root):
+class ISBNNotFound(ValueError):
+    pass
+
+def check_for_errors(root, isbn):
     err = root.find('.//'+AWS('Error'))
     if err is not None:
+        text = etree.tostring(err, method='text', pretty_print=True,
+                    encoding=unicode)
+        if 'AWS.InvalidParameterValue'+isbn in text:
+            raise ISBNNotFound(isbn)
         raise Exception('Failed to get metadata with error: '\
-                + etree.tostring(err, method='text', pretty_print=True,
-                    encoding=unicode))
+                + text)
 
 def get_social_metadata(title, authors, publisher, isbn):
     mi = MetaInformation(title, authors)
@@ -32,7 +38,10 @@ def get_social_metadata(title, authors, publisher, isbn):
         br = browser()
         response_xml = br.open('http://status.calibre-ebook.com/aws/metadata/'+isbn).read()
         root = etree.fromstring(response_xml)
-        check_for_errors(root)
+        try:
+            check_for_errors(root, isbn)
+        except ISBNNotFound:
+            return mi
         mi.title = root.findtext('.//'+AWS('Title'))
         authors = [x.text for x in root.findall('.//'+AWS('Author'))]
         if authors:

src/calibre/ebooks/metadata/epub.py

@@ -133,11 +133,11 @@ def get_cover(opf, opf_path, stream, reader=None):
     stream.seek(0)
     zf = ZipFile(stream)
     if raster_cover:
-        if reader is not None and \
-            reader.encryption_meta.is_encrypted(raster_cover):
-                return
         base = posixpath.dirname(opf_path)
         cpath = posixpath.normpath(posixpath.join(base, raster_cover))
+        if reader is not None and \
+            reader.encryption_meta.is_encrypted(cpath):
+                return
         try:
             member = zf.getinfo(cpath)
         except:
@@ -182,13 +182,21 @@ def get_metadata(stream, extract_cover=True):
 def get_quick_metadata(stream):
     return get_metadata(stream, False)
 
-def set_metadata(stream, mi):
+def set_metadata(stream, mi, apply_null=False):
     stream.seek(0)
     reader = OCFZipReader(stream, root=os.getcwdu())
     mi = MetaInformation(mi)
     for x in ('guide', 'toc', 'manifest', 'spine'):
         setattr(mi, x, None)
     reader.opf.smart_update(mi)
+    if apply_null:
+        if not getattr(mi, 'series', None):
+            reader.opf.series = None
+        if not getattr(mi, 'tags', []):
+            reader.opf.tags = []
+        if not getattr(mi, 'isbn', None):
+            reader.opf.isbn = None
+
     newopf = StringIO(reader.opf.render())
     safe_replace(stream, reader.container[OPF.MIMETYPE], newopf)
 

src/calibre/ebooks/metadata/opf2.py

@@ -404,6 +404,10 @@ class MetadataField(object):
 
     def __set__(self, obj, val):
         elem = obj.get_metadata_element(self.name)
+        if val is None:
+            if elem is not None:
+                elem.getparent().remove(elem)
+            return
         if elem is None:
             elem = obj.create_metadata_element(self.name, is_dc=self.is_dc)
         obj.set_text(elem, unicode(val))
@@ -722,6 +726,11 @@ class OPF(object):
 
         def fset(self, val):
             matches = self.isbn_path(self.metadata)
+            if val is None:
+                if matches:
+                    for x in matches:
+                        x.getparent().remove(x)
+                    return
             if not matches:
                 attrib = {'{%s}scheme'%self.NAMESPACES['opf']: 'ISBN'}
                 matches = [self.create_metadata_element('identifier',

src/calibre/ebooks/metadata/worker.py

@@ -235,6 +235,7 @@ def save_book(task, library_path, path, recs, notification=lambda x,y:x):
     from calibre.library.database2 import LibraryDatabase2
     db = LibraryDatabase2(library_path)
     from calibre.library.save_to_disk import config, save_to_disk
+    from calibre.customize.ui import apply_null_metadata
     opts = config().parse()
     for name in recs:
         setattr(opts, name, recs[name])
@@ -244,5 +245,6 @@ def save_book(task, library_path, path, recs, notification=lambda x,y:x):
         notification((id, title, not failed, tb))
         return True
 
-    save_to_disk(db, task, path, opts, callback)
+    with apply_null_metadata:
+        save_to_disk(db, task, path, opts, callback)
 

src/calibre/ebooks/oeb/iterator.py

@@ -92,10 +92,10 @@ class EbookIterator(object):
         ext = re.sub(r'(x{0,1})htm(l{0,1})', 'html', ext)
         self.ebook_ext = ext
 
-    def search(self, text, index):
+    def search(self, text, index, backwards=False):
         text = text.lower()
         for i, path in enumerate(self.spine):
-            if i > index:
+            if (backwards and i < index) or (not backwards and i > index):
                 if text in open(path, 'rb').read().decode(path.encoding).lower():
                     return i
 

src/calibre/ebooks/oeb/stylizer.py

@@ -176,6 +176,9 @@ class Stylizer(object):
         class_sel_pat = re.compile(r'\.[a-z]+', re.IGNORECASE)
         capital_sel_pat = re.compile(r'h|[A-Z]+')
         for _, _, cssdict, text, _ in rules:
+            fl = ':first-letter' in text
+            if fl:
+                text = text.replace(':first-letter', '')
             try:
                 selector = CSSSelector(text)
             except (AssertionError, ExpressionError, etree.XPathSyntaxError,
@@ -202,8 +205,21 @@ class Stylizer(object):
                 if found:
                     self.logger.warn('Ignoring case mismatches for CSS selector: %s in %s'
                         %(text, item.href))
-            for elem in matches:
-                self.style(elem)._update_cssdict(cssdict)
+            if fl:
+                from lxml.builder import ElementMaker
+                E = ElementMaker(namespace=XHTML_NS)
+                for elem in matches:
+                    for x in elem.iter():
+                        if x.text:
+                            span = E.span(x.text[0])
+                            span.tail = x.text[1:]
+                            x.text = None
+                            x.insert(0, span)
+                            self.style(span)._update_cssdict(cssdict)
+                            break
+            else:
+                for elem in matches:
+                    self.style(elem)._update_cssdict(cssdict)
         for elem in xpath(tree, '//h:img[@width or @height]'):
             base = elem.get('style', '').strip()
             if base:

src/calibre/gui2/__init__.py

@@ -339,6 +339,7 @@ class FileIconProvider(QFileIconProvider):
              'tan'     : 'zero',
              'epub'    : 'epub',
              'fb2'     : 'fb2',
+             'rtf'     : 'rtf',
              }
 
     def __init__(self):
@@ -410,6 +411,7 @@ class FileDialog(QObject):
                        modal = True,
                        name = '',
                        mode = QFileDialog.ExistingFiles,
+                       default_dir='~'
                        ):
         QObject.__init__(self)
         ftext = ''
@@ -428,9 +430,10 @@ class FileDialog(QObject):
         self.selected_files = None
         self.fd = None
 
-        initial_dir = dynamic.get(self.dialog_name, os.path.expanduser('~'))
+        initial_dir = dynamic.get(self.dialog_name,
+                os.path.expanduser(default_dir))
         if not isinstance(initial_dir, basestring):
-            initial_dir = os.path.expanduser('~')
+            initial_dir = os.path.expanduser(default_dir)
         self.selected_files = []
         if mode == QFileDialog.AnyFile:
             f = unicode(QFileDialog.getSaveFileName(parent, title, initial_dir, ftext, ""))
@@ -465,9 +468,10 @@ class FileDialog(QObject):
         return tuple(self.selected_files)
 
 
-def choose_dir(window, name, title):
-    fd = FileDialog(title, [], False, window, name=name,
-                    mode=QFileDialog.DirectoryOnly)
+def choose_dir(window, name, title, default_dir='~'):
+    fd = FileDialog(title=title, filters=[], add_all_files_filter=False,
+            parent=window, name=name, mode=QFileDialog.DirectoryOnly,
+            default_dir=default_dir)
     dir = fd.get_files()
     if dir:
         return dir[0]

src/calibre/gui2/add.py

@@ -199,11 +199,11 @@ class DBAdder(Thread):
                     self.add_formats(id, formats)
             else:
                 id = self.db.create_book_entry(mi, cover=cover, add_duplicates=False)
-                self.number_of_books_added += 1
                 if id is None:
                     self.duplicates.append((mi, cover, orig_formats))
                 else:
                     self.add_formats(id, formats)
+                    self.number_of_books_added += 1
         else:
             self.names.append(name)
             self.paths.append(formats[0])

src/calibre/gui2/main.py

@@ -4,15 +4,18 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import sys, os, time, socket, traceback
 from functools import partial
 
-from PyQt4.Qt import QCoreApplication, QIcon, QMessageBox
+from PyQt4.Qt import QCoreApplication, QIcon, QMessageBox, QObject, QTimer, \
+        QThread, pyqtSignal, Qt, QProgressDialog, QString
 
-from calibre import prints
-from calibre.constants import iswindows, __appname__, isosx
+from calibre import prints, plugins
+from calibre.constants import iswindows, __appname__, isosx, filesystem_encoding
 from calibre.utils.ipc import ADDRESS, RC
 from calibre.gui2 import ORG_NAME, APP_UID, initialize_file_icon_provider, \
-    Application
+    Application, choose_dir, error_dialog, question_dialog
 from calibre.gui2.main_window import option_parser as _option_parser
 from calibre.utils.config import prefs, dynamic
+from calibre.library.database2 import LibraryDatabase2
+from calibre.library.sqlite import sqlite, DatabaseException
 
 def option_parser():
     parser = _option_parser('''\
@@ -48,25 +51,186 @@ def init_qt(args):
     app.setWindowIcon(QIcon(I('library.png')))
     return app, opts, args, actions
 
+def get_library_path():
+    library_path = prefs['library_path']
+    if library_path is None: # Need to migrate to new database layout
+        base = os.path.expanduser('~')
+        if iswindows:
+            base = plugins['winutil'][0].special_folder_path(
+                    plugins['winutil'][0].CSIDL_PERSONAL)
+            if not base or not os.path.exists(base):
+                from PyQt4.Qt import QDir
+                base = unicode(QDir.homePath()).replace('/', os.sep)
+        candidate = choose_dir(None, 'choose calibre library',
+                _('Choose a location for your calibre e-book library'),
+                default_dir=base)
+        if not candidate:
+            candidate = os.path.join(base, 'Calibre Library')
+        library_path = os.path.abspath(candidate)
+    if not os.path.exists(library_path):
+        try:
+            os.makedirs(library_path)
+        except:
+            error_dialog(None, _('Failed to create library'),
+                    _('Failed to create calibre library at: %r. Aborting.')%library_path,
+                    det_msg=traceback.format_exc(), show=True)
+            library_path = None
+    return library_path
+
+class DBRepair(QThread):
+
+    repair_done = pyqtSignal(object, object)
+    progress = pyqtSignal(object, object)
+
+    def __init__(self, library_path, parent, pd):
+        QThread.__init__(self, parent)
+        self.library_path = library_path
+        self.pd = pd
+        self.progress.connect(self._callback, type=Qt.QueuedConnection)
+
+    def _callback(self, num, is_length):
+        if is_length:
+            self.pd.setRange(0, num-1)
+            num = 0
+        self.pd.setValue(num)
+
+    def callback(self, num, is_length):
+        self.progress.emit(num, is_length)
+
+    def run(self):
+        from calibre.debug import reinit_db
+        try:
+            reinit_db(os.path.join(self.library_path, 'metadata.db'),
+                    self.callback)
+            db = LibraryDatabase2(self.library_path)
+            tb = None
+        except:
+            db, tb = None, traceback.format_exc()
+        self.repair_done.emit(db, tb)
+
+class GuiRunner(QObject):
+    '''Make sure an event loop is running before starting the main work of
+    initialization'''
+
+    def __init__(self, opts, args, actions, listener, app):
+        self.opts, self.args, self.listener, self.app = opts, args, listener, app
+        self.actions = actions
+        self.main = None
+        QObject.__init__(self)
+        self.timer = QTimer.singleShot(1, self.initialize)
+
+    def start_gui(self):
+        from calibre.gui2.ui import Main
+        main = Main(self.library_path, self.db, self.listener, self.opts, self.actions)
+        add_filesystem_book = partial(main.add_filesystem_book, allow_device=False)
+        sys.excepthook = main.unhandled_exception
+        if len(self.args) > 1:
+            p = os.path.abspath(self.args[1])
+            add_filesystem_book(p)
+        self.app.file_event_hook = add_filesystem_book
+        self.main = main
+
+    def initialization_failed(self):
+        print 'Catastrophic failure initializing GUI, bailing out...'
+        QCoreApplication.exit(1)
+        raise SystemExit(1)
+
+    def initialize_db_stage2(self, db, tb):
+        repair_pd = getattr(self, 'repair_pd', None)
+        if repair_pd is not None:
+            repair_pd.cancel()
+
+        if db is None and tb is not None:
+            # DB Repair failed
+            error_dialog(None, _('Repairing failed'),
+                    _('The database repair failed. Starting with '
+                        'a new empty library.'),
+                    det_msg=tb, show=True)
+        if db is None:
+            fname = _('Calibre Library')
+            if isinstance(fname, unicode):
+                try:
+                    fname = fname.encode(filesystem_encoding)
+                except:
+                    fname = 'Calibre Library'
+            x = os.path.expanduser('~'+os.sep+fname)
+            if not os.path.exists(x):
+                try:
+                    os.makedirs(x)
+                except:
+                    x = os.path.expanduser('~')
+            candidate = choose_dir(None, 'choose calibre library',
+                _('Choose a location for your new calibre e-book library'),
+                default_dir=x)
+
+            if not candidate:
+                self.initialization_failed()
+
+            try:
+                self.library_path = candidate
+                db = LibraryDatabase2(candidate)
+            except:
+                error_dialog(None, _('Bad database location'),
+                    _('Bad database location %r. calibre will now quit.'
+                     )%self.library_path,
+                    det_msg=traceback.format_exc(), show=True)
+                self.initialization_failed()
+
+        self.db = db
+        self.start_gui()
+
+    def initialize_db(self):
+        db = None
+        try:
+            db = LibraryDatabase2(self.library_path)
+        except (sqlite.Error, DatabaseException):
+            repair = question_dialog(None, _('Corrupted database'),
+                    _('Your calibre database appears to be corrupted. Do '
+                    'you want calibre to try and repair it automatically? '
+                    'If you say No, a new empty calibre library will be created.'),
+                    det_msg=traceback.format_exc()
+                    )
+            if repair:
+                self.repair_pd = QProgressDialog(_('Repairing database. This '
+                    'can take a very long time for a large collection'), QString(),
+                    0, 0)
+                self.repair_pd.setWindowModality(Qt.WindowModal)
+                self.repair_pd.show()
+
+                self.repair = DBRepair(self.library_path, self, self.repair_pd)
+                self.repair.repair_done.connect(self.initialize_db_stage2,
+                        type=Qt.QueuedConnection)
+                self.repair.start()
+                return
+        except:
+            error_dialog(None, _('Bad database location'),
+                    _('Bad database location %r. Will start with '
+                    ' a new, empty calibre library')%self.library_path,
+                    det_msg=traceback.format_exc(), show=True)
+
+        self.initialize_db_stage2(db, None)
+
+    def initialize(self, *args):
+        self.library_path = get_library_path()
+        if self.library_path is None:
+            self.initialization_failed()
+
+        self.initialize_db()
+
+
+
 def run_gui(opts, args, actions, listener, app):
-    from calibre.gui2.ui import Main
     initialize_file_icon_provider()
     if not dynamic.get('welcome_wizard_was_run', False):
         from calibre.gui2.wizard import wizard
         wizard().exec_()
         dynamic.set('welcome_wizard_was_run', True)
-    main = Main(listener, opts, actions)
-    add_filesystem_book = partial(main.add_filesystem_book, allow_device=False)
-    sys.excepthook = main.unhandled_exception
-    if len(args) > 1:
-        args[1] = os.path.abspath(args[1])
-        add_filesystem_book(args[1])
-    app.file_event_hook = add_filesystem_book
+    runner = GuiRunner(opts, args, actions, listener, app)
     ret = app.exec_()
-    if getattr(main, 'run_wizard_b4_shutdown', False):
+    if getattr(runner.main, 'run_wizard_b4_shutdown', False):
         from calibre.gui2.wizard import wizard
         wizard().exec_()
-    if getattr(main, 'restart_after_quit', False):
+    if getattr(runner.main, 'restart_after_quit', False):
         e = sys.executable if getattr(sys, 'frozen', False) else sys.argv[0]
         print 'Restarting with:', e, sys.argv
         if hasattr(sys, 'frameworks_dir'):
@@ -78,7 +242,7 @@ def run_gui(opts, args, actions, listener, app):
     else:
         if iswindows:
             try:
-                main.system_tray_icon.hide()
+                runner.main.system_tray_icon.hide()
             except:
                 pass
     return ret

src/calibre/gui2/main_window.py

@@ -4,7 +4,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import StringIO, traceback, sys
 
 from PyQt4.Qt import QMainWindow, QString, Qt, QFont, QCoreApplication, SIGNAL,\
-                     QAction, QMenu, QMenuBar, QIcon
+                     QAction, QMenu, QMenuBar, QIcon, pyqtSignal
 from calibre.gui2.dialogs.conversion_error import ConversionErrorDialog
 from calibre.utils.config import OptionParser
 from calibre.gui2 import error_dialog
@@ -41,6 +41,8 @@ class MainWindow(QMainWindow):
     ___menu     = None
     __actions   = []
 
+    keyboard_interrupt = pyqtSignal()
+
     @classmethod
     def create_application_menubar(cls):
         mb = QMenuBar(None)
@@ -76,6 +78,9 @@ class MainWindow(QMainWindow):
         print 'Received signal:', repr(signal)
 
     def unhandled_exception(self, type, value, tb):
+        if type == KeyboardInterrupt:
+            self.keyboard_interrupt.emit()
+            return
         try:
             sio = StringIO.StringIO()
             traceback.print_exception(type, value, tb, file=sio)

src/calibre/gui2/search_box.py

@@ -137,6 +137,13 @@ class SearchBox2(QComboBox):
         if event.timerId() == self.timer:
             self.do_search()
 
+    @property
+    def smart_text(self):
+        text = unicode(self.currentText()).strip()
+        if not text or text == self.help_text:
+            return ''
+        return text
+
     def do_search(self):
         text = unicode(self.currentText()).strip()
         if not text or text == self.help_text:

src/calibre/gui2/ui.py

@@ -14,9 +14,9 @@ from xml.parsers.expat import ExpatError
 from Queue import Queue, Empty
 from threading import Thread
 from functools import partial
-from PyQt4.Qt import Qt, SIGNAL, QObject, QCoreApplication, QUrl, QTimer, \
+from PyQt4.Qt import Qt, SIGNAL, QObject, QUrl, QTimer, \
                      QModelIndex, QPixmap, QColor, QPainter, QMenu, QIcon, \
-                     QToolButton, QDialog, QDesktopServices, QFileDialog, \
+                     QToolButton, QDialog, QDesktopServices, \
                      QSystemTrayIcon, QApplication, QKeySequence, QAction, \
                      QMessageBox, QStackedLayout, QHelpEvent, QInputDialog,\
                      QThread, pyqtSignal
@@ -125,8 +125,9 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         self.default_thumbnail = (pixmap.width(), pixmap.height(),
                 pixmap_to_data(pixmap))
 
-    def __init__(self, listener, opts, actions, parent=None):
+    def __init__(self, library_path, db, listener, opts, actions, parent=None):
         self.preferences_action, self.quit_action = actions
+        self.library_path = library_path
         self.spare_servers = []
         MainWindow.__init__(self, opts, parent)
         # Initialize fontconfig in a separate thread as this can be a lengthy
@@ -395,6 +396,9 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         self.action_sync.setShortcut(Qt.Key_D)
         self.action_sync.setEnabled(True)
         self.create_device_menu()
+        self.connect(self.action_sync, SIGNAL('triggered(bool)'),
+                self._sync_action_triggered)
+
         self.action_edit.setMenu(md)
         self.action_save.setMenu(self.save_menu)
 
@@ -513,31 +517,6 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         if self.system_tray_icon.isVisible() and opts.start_in_tray:
             self.hide_windows()
         self.stack.setCurrentIndex(0)
-        try:
-            db = LibraryDatabase2(self.library_path)
-        except Exception:
-            import traceback
-            error_dialog(self, _('Bad database location'),
-                    _('Bad database location')+':'+self.library_path,
-                    det_msg=traceback.format_exc()).exec_()
-            fname = _('Calibre Library')
-            if isinstance(fname, unicode):
-                try:
-                    fname = fname.encode(filesystem_encoding)
-                except:
-                    fname = 'Calibre Library'
-            x = os.path.expanduser('~'+os.sep+fname)
-            if not os.path.exists(x):
-                os.makedirs(x)
-            dir = unicode(QFileDialog.getExistingDirectory(self,
-                            _('Choose a location for your ebook library.'),
-                            x))
-            if not dir:
-                QCoreApplication.exit(1)
-                raise SystemExit(1)
-            else:
-                self.library_path = dir
-                db = LibraryDatabase2(self.library_path)
         self.library_view.set_database(db)
         prefs['library_path'] = self.library_path
         self.library_view.sortByColumn(*dynamic.get('sort_column',
@@ -658,11 +637,17 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
             height = v.rowHeight(0)
             self.library_view.verticalHeader().setDefaultSectionSize(height)
 
+        self.keyboard_interrupt.connect(self.quit, type=Qt.QueuedConnection)
+
 
     def resizeEvent(self, ev):
         MainWindow.resizeEvent(self, ev)
         self.search.setMaximumWidth(self.width()-150)
 
+    def _sync_action_triggered(self, *args):
+        m = getattr(self, '_sync_menu', None)
+        if m is not None:
+            m.trigger_default()
 
     def create_device_menu(self):
         self._sync_menu = DeviceMenu(self)
@@ -670,8 +655,6 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         self.connect(self._sync_menu,
                 SIGNAL('sync(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'),
                 self.dispatch_sync_event)
-        self.connect(self.action_sync, SIGNAL('triggered(bool)'),
-                self._sync_menu.trigger_default)
         self._sync_menu.fetch_annotations.connect(self.fetch_annotations)
 
     def add_spare_server(self, *args):
@@ -2330,38 +2313,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         d.show()
         self._modeless_dialogs.append(d)
 
-
-    def initialize_database(self):
-        self.library_path = prefs['library_path']
-        if self.library_path is None: # Need to migrate to new database layout
-            base = os.path.expanduser('~')
-            if iswindows:
-                from calibre import plugins
-                from PyQt4.Qt import QDir
-                base = plugins['winutil'][0].special_folder_path(
-                        plugins['winutil'][0].CSIDL_PERSONAL)
-                if not base or not os.path.exists(base):
-                    base = unicode(QDir.homePath()).replace('/', os.sep)
-            dir = unicode(QFileDialog.getExistingDirectory(self,
-                        _('Choose a location for your ebook library.'), base))
-            if not dir:
-                dir = os.path.expanduser('~/Library')
-            self.library_path = os.path.abspath(dir)
-        if not os.path.exists(self.library_path):
-            try:
-                os.makedirs(self.library_path)
-            except:
-                self.library_path = os.path.expanduser('~/CalibreLibrary')
-                error_dialog(self, _('Invalid library location'),
-                     _('Could not access %s. Using %s as the library.')%
-                     (repr(self.library_path), repr(self.library_path))
-                             ).exec_()
-                if not os.path.exists(self.library_path):
-                    os.makedirs(self.library_path)
-
-
     def read_settings(self):
-        self.initialize_database()
         geometry = config['main_window_geometry']
         if geometry is not None:
             self.restoreGeometry(geometry)

src/calibre/gui2/viewer/documentview.py

@@ -384,25 +384,25 @@ class Document(QWebPage):
 
     @property
     def height(self):
-        ans = self.javascript('document.body.offsetHeight', 'int') # contentsSize gives inaccurate results
-        if ans == 0:
-            ans = self.mainFrame().contentsSize().height()
-        return ans
+        j = self.javascript('document.body.offsetHeight', 'int')
+        q = self.mainFrame().contentsSize().height()
+        if q == j:
+            return j
+        if min(j, q) <= 0:
+            return max(j, q)
+        window_height = self.window_height
+        if j == window_height:
+            return j if q < 1.2*j else q
+        return j
 
     @property
     def width(self):
         return self.mainFrame().contentsSize().width() # offsetWidth gives inaccurate results
 
     def set_bottom_padding(self, amount):
-        body = self.mainFrame().documentElement().findFirst('body')
-        if body.isNull():
-            return
-        old_padding = unicode(body.styleProperty('padding-bottom',
-            body.ComputedStyle)).strip()
-        padding = u'%dpx'%amount
-        if old_padding != padding:
-            body.setStyleProperty('padding-bottom', padding + ' !important')
-
+        s = QSize(-1, -1) if amount == 0 else QSize(self.width,
+                self.height+amount)
+        self.setPreferredContentsSize(s)
 
 class EntityDeclarationProcessor(object):
 
@@ -585,7 +585,9 @@ class DocumentView(QWebView):
         def fset(self, val): self.document.current_language = val
         return property(fget=fget, fset=fset)
 
-    def search(self, text):
+    def search(self, text, backwards=False):
+        if backwards:
+            return self.findText(text, self.document.FindBackwards)
         return self.findText(text)
 
     def path(self):
@@ -705,13 +707,21 @@ class DocumentView(QWebView):
 
     def next_page(self):
         window_height = self.document.window_height
+        document_height = self.document.height
+        ddelta = document_height - window_height
+        #print '\nWindow height:', window_height
+        #print 'Document height:', self.document.height
+
         delta_y = window_height - 25
-        if self.document.at_bottom:
+        if self.document.at_bottom or ddelta <= 0:
             if self.manager is not None:
                 self.manager.next_document()
+        elif ddelta < 25:
+            self.scroll_by(y=ddelta)
+            return
         else:
             oopos = self.document.ypos
-            #print '\nOriginal position:', oopos
+            #print 'Original position:', oopos
             self.document.set_bottom_padding(0)
             opos = self.document.ypos
             #print 'After set padding=0:', self.document.ypos
@@ -722,8 +732,14 @@ class DocumentView(QWebView):
             lower_limit = opos + delta_y # Max value of top y co-ord after scrolling
             max_y = self.document.height - window_height # The maximum possible top y co-ord
             if max_y < lower_limit:
+                padding = lower_limit - max_y
+                if padding == window_height:
+                    if self.manager is not None:
+                        self.manager.next_document()
+                    return
                 #print 'Setting padding to:', lower_limit - max_y
                 self.document.set_bottom_padding(lower_limit - max_y)
+            #print 'Document height:', self.document.height
             max_y = self.document.height - window_height
             lower_limit = min(max_y, lower_limit)
             #print 'Scroll to:', lower_limit

src/calibre/gui2/viewer/main.py

@@ -229,7 +229,11 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         self.connect(self.action_previous_page, SIGNAL('triggered(bool)'),
                      lambda x:self.view.previous_page())
         self.connect(self.action_find_next, SIGNAL('triggered(bool)'),
-                     lambda x:self.find(unicode(self.search.text()), True, repeat=True))
+                     lambda x:self.find(self.search.smart_text, True, repeat=True))
+        self.connect(self.action_find_previous, SIGNAL('triggered(bool)'),
+                     lambda x:self.find(self.search.smart_text, True,
+                         repeat=True, backwards=True))
+
         self.connect(self.action_full_screen, SIGNAL('triggered(bool)'),
                      self.toggle_fullscreen)
         self.action_full_screen.setShortcuts([Qt.Key_F11, Qt.CTRL+Qt.SHIFT+Qt.Key_F])
@@ -420,13 +424,15 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
             self.set_bookmarks(self.iterator.bookmarks)
 
 
-    def find(self, text, refinement, repeat=False):
+    def find(self, text, refinement, repeat=False, backwards=False):
         if not text:
+            self.view.search('')
             return self.search.search_done(False)
         if self.view.search(text):
             self.scrolled(self.view.scroll_fraction)
             return self.search.search_done(True)
-        index = self.iterator.search(text, self.current_index)
+        index = self.iterator.search(text, self.current_index,
+                backwards=backwards)
         if index is None:
             if self.current_index > 0:
                 index = self.iterator.search(text, 0)
@@ -444,10 +450,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
             self.scrolled(self.view.scroll_fraction)
 
     def keyPressEvent(self, event):
-        if event.key() == Qt.Key_F3:
-            text = unicode(self.search.text())
-            self.find(text, True, repeat=True)
-        elif event.key() == Qt.Key_Slash:
+        if event.key() == Qt.Key_Slash:
             self.search.setFocus(Qt.OtherFocusReason)
         else:
             return MainWindow.keyPressEvent(self, event)

src/calibre/gui2/viewer/main.ui

@@ -11,7 +11,7 @@
    </rect>
   </property>
   <property name="windowTitle">
-   <string>Ebook Viewer</string>
+   <string>E-book Viewer</string>
   </property>
   <property name="windowIcon">
    <iconset resource="../../../../resources/images.qrc">
@@ -142,6 +142,7 @@
     <bool>false</bool>
    </attribute>
    <addaction name="action_find_next"/>
+   <addaction name="action_find_previous"/>
   </widget>
   <action name="action_back">
    <property name="icon">
@@ -232,6 +233,12 @@
    <property name="text">
     <string>Find next</string>
    </property>
+   <property name="toolTip">
+    <string>Find next occurrence</string>
+   </property>
+   <property name="shortcut">
+    <string>F3</string>
+   </property>
   </action>
   <action name="action_copy">
    <property name="icon">
@@ -287,6 +294,21 @@
     <string>Print</string>
    </property>
   </action>
+  <action name="action_find_previous">
+   <property name="icon">
+    <iconset resource="../../../../resources/images.qrc">
+     <normaloff>:/images/arrow-up.svg</normaloff>:/images/arrow-up.svg</iconset>
+   </property>
+   <property name="text">
+    <string>Find previous</string>
+   </property>
+   <property name="toolTip">
+    <string>Find previous occurrence</string>
+   </property>
+   <property name="shortcut">
+    <string>Shift+F3</string>
+   </property>
+  </action>
  </widget>
  <customwidgets>
   <customwidget>

src/calibre/gui2/wizard/__init__.py

@@ -97,9 +97,18 @@ class Kobo(Device):
     manufacturer = 'Kobo'
     output_profile = 'kobo'
     output_format = 'EPUB'
-    name = 'Kobo Reader'
     id = 'kobo'
 
+class Booq(Device):
+    name = 'Booq Reader'
+    manufacturer = 'Booq'
+    output_profile = 'prs505'
+    output_format = 'EPUB'
+    id = 'booq'
+
+class Avant(Booq):
+    name = 'Booq Avant'
+
 class Sony300(Sony505):
 
     name = 'SONY Reader Pocket Edition'

src/calibre/library/__init__.py

@@ -26,3 +26,8 @@ def server_config(defaults=None):
             help=_('The maximum number of matches to return per OPDS query. '
             'This affects Stanza, WordPlayer, etc. integration.'))
     return c
+
+def db():
+    from calibre.library.database2 import LibraryDatabase2
+    from calibre.utils.config import prefs
+    return LibraryDatabase2(prefs['library_path'])

src/calibre/library/cli.py

@@ -376,12 +376,35 @@ the directory related options below.
                       help=_('Process directories recursively'))
     parser.add_option('-d', '--duplicates', action='store_true', default=False,
                       help=_('Add books to database even if they already exist. Comparison is done based on book titles.'))
+    parser.add_option('-e', '--empty', action='store_true', default=False,
+                    help=_('Add an empty book (a book with no formats)'))
+    parser.add_option('-t', '--title', default=None,
+            help=_('Set the title of the added empty book'))
+    parser.add_option('-a', '--authors', default=None,
+            help=_('Set the authors of the added empty book'))
+    parser.add_option('-i', '--isbn', default=None,
+            help=_('Set the ISBN of the added empty book'))
+
     return parser
 
+def do_add_empty(db, title, authors, isbn):
+    from calibre.ebooks.metadata import MetaInformation, string_to_authors
+    mi = MetaInformation(None)
+    if title is not None:
+        mi.title = title
+    if authors:
+        mi.authors = string_to_authors(authors)
+    if isbn:
+        mi.isbn = isbn
+    db.import_book(mi, [])
+    send_message()
 
 def command_add(args, dbpath):
     parser = add_option_parser()
     opts, args = parser.parse_args(sys.argv[:1] + args)
+    if opts.empty:
+        do_add_empty(get_db(dbpath, opts), opts.title, opts.authors, opts.isbn)
+        return 0
     if len(args) < 2:
         parser.print_help()
         print

src/calibre/library/database2.py

@@ -1399,7 +1399,7 @@ books_series_link      feeds
     def check_integrity(self, callback):
         callback(0., _('Checking SQL integrity...'))
         user_version = self.user_version
-        sql = self.conn.dump()
+        sql = '\n'.join(self.conn.dump())
         self.conn.close()
         dest = self.dbpath+'.tmp'
         if os.path.exists(dest):

src/calibre/library/db/__init__.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+

src/calibre/library/db/base.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+''' Design documentation {{{
+
+    Storage paradigm {{{
+        * Agnostic to storage paradigm (i.e. no book per folder assumptions)
+        * Two separate concepts: A store and collection
+          A store is a backend, like a sqlite database associated with a path on
+          the local filesystem, or a cloud based storage solution.
+          A collection is a user defined group of stores. Most of the logic for
+          data manipulation sorting/searching/restrictions should be in the collection
+          class. The collection class should transparently handle the
+          conversion from store name + id to row number in the collection.
+        * Not sure how feasible it is to allow many-many maps between stores
+          and collections.
+    }}}
+
+    Event system {{{
+        * Comprehensive event system that other components can subscribe to
+        * Subscribers should be able to temporarily block receiving events
+        * Should event dispatch be asynchronous?
+        * Track last modified time for metadata and each format
+    }}}
+}}}'''
+
+# Imports {{{
+# }}}
+
+
+
+

src/calibre/library/sqlite.py

@@ -116,7 +116,7 @@ class DBThread(Thread):
                     break
                 if func == 'dump':
                     try:
-                        ok, res = True, '\n'.join(self.conn.iterdump())
+                        ok, res = True, tuple(self.conn.iterdump())
                     except Exception, err:
                         ok, res = False, (err, traceback.format_exc())
                 else:

src/calibre/linux.py

@@ -313,7 +313,7 @@ class PostInstall:
             with open(os.path.join(base, '95-calibre.rules'), 'wb') as udev:
                 self.manifest.append(udev.name)
                 udev.write('''# Sony Reader PRS-500\n'''
-                        '''BUS=="usb", SYSFS{idProduct}=="029b", SYSFS{idVendor}=="054c", MODE="660", GROUP="%s"\n'''%(group,)
+                        '''SUBSYSTEMS=="usb", SYSFS{idProduct}=="029b", SYSFS{idVendor}=="054c", MODE="660", GROUP="%s"\n'''%(group,)
                         )
         except:
             if self.opts.fatal_errors:

src/calibre/manual/conversion.rst

@@ -533,3 +533,17 @@ The .cbc file will then contain::
 
 |app| will automatically convert this .cbc file into a e-book with a Table of Contents pointing to each entry in comics.txt.
 
+
+EPUB advanced formatting demo
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Various advanced formatting for EPUB files is demonstrated in this `demo file <http://calibre-ebook.com/downloads/demos/demo.epub>`_.
+The file was created from hand coded HTML using calibre and is meant to be used as a template for your own EPUB creation efforts. 
+
+The source HTML it was created from is available `here <http://calibre-ebook.com/downloads/demos/demo.zip>`_. The settings used to create the
+EPUB from the ZIP file are::
+
+    ebook-convert demo.zip .epub -vv --authors "Kovid Goyal" --language en --level1-toc '//*[@class="title"]' --disable-font-rescaling --page-breaks-before / --no-default-epub-cover
+
+Note that because this file explores the potential of EPUB, most of the advanced formatting is not going to work on readers less capable than |app|'s builtin EPUB viewer. 
+

src/calibre/manual/develop.rst

@@ -149,10 +149,10 @@ Linux development environment
 
 |app| is primarily developed on linux. You have two choices in setting up the development environment. You can install the
 |app| binary as normal and use that as a runtime environment to do your development. This approach is similar to that
-used in windows and linux. Alternatively, you can install |app| from source. Instructions for setting up a development
+used in windows and OS X. Alternatively, you can install |app| from source. Instructions for setting up a development
 environment from source are in the INSTALL file in the source tree. Here we will address using the binary a runtime.
 
-Install the |app| using the binary installer. The opena  terminal and change to the previously checked out |app| code directory, for example::
+Install the |app| using the binary installer. Then open a terminal and change to the previously checked out |app| code directory, for example::
 
     cd /home/kovid/work/calibre
 

src/calibre/manual/faq.rst

@@ -125,7 +125,7 @@ With recent reader iterations, SONY, in all its wisdom has decided to try to for
 use their software. If you install it, it auto-launches whenever you connect the reader.
 If you don't want to uninstall it altogether, there are a couple of tricks you can use. The
 simplest is to simply re-name the executable file that launches the library program. More detail
-`here <http://www.mobileread.com/forums/showthread.php?t=65809>`_.
+`in the forums <http://www.mobileread.com/forums/showthread.php?t=65809>`_.
 
 Can I use the collections feature of the SONY reader?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -262,7 +262,7 @@ I want |app| to download news from my favorite news website.
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 If you are reasonably proficient with computers, you can teach |app| to download news from any website of your choosing. To learn how to do this see :ref:`news`.
 
-Otherwise, you can register a request for a particular news site by adding a comment `here <http://bugs.calibre-ebook.com/ticket/405>`_.
+Otherwise, you can register a request for a particular news site by adding a comment `to this ticket <http://bugs.calibre-ebook.com/ticket/405>`_.
 
 Can I use web2disk to download an arbitrary website?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -335,6 +335,6 @@ You have two choices:
 
 Can I include |app| on a CD to be distributed with my product/magazine?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-|app| is licensed under the GNU General Public License v3 (an open source license). This means that you are free to redistribute |app| as long as you make the source code available. So if you want to put |app| on a CD with your product, you must also put the |app| source code on the CD. The source code is available for download `here <http://code.google.com/p/calibre-ebook/downloads/list>`_.
+|app| is licensed under the GNU General Public License v3 (an open source license). This means that you are free to redistribute |app| as long as you make the source code available. So if you want to put |app| on a CD with your product, you must also put the |app| source code on the CD. The source code is available for download `from googlecode <http://code.google.com/p/calibre-ebook/downloads/list>`_.
 
 

src/calibre/web/feeds/news.py

@@ -1266,7 +1266,7 @@ class BasicNewsRecipe(Recipe):
                 feed = Feed()
                 msg = 'Failed feed: %s'%(title if title else url)
                 feed.populate_from_preparsed_feed(msg, [])
-                feed.description = unicode(err)
+                feed.description = repr(err)
                 parsed_feeds.append(feed)
                 self.log.exception(msg)