Sync to trunk.

Changelog.yaml

@@ -19,6 +19,129 @@
 #  new recipes:
 #    - title: 
 
+- version: 0.8.7
+  date: 2011-06-24
+
+  new features:
+    - title: "Connect to iTunes: You now need to tell iTunes to keep its own copy of every ebook. Do this in iTunes by going to Preferences->Advanced and setting the 'Copy files to iTunes Media folder when adding to library' option. To learn about why this is necessary, see: http://www.mobileread.com/forums/showthread.php?t=140260"
+      type: major
+
+    - title: "Add a couple of date related functions to the calibre template langauge to get 'todays' date and create text based on the value of a date type field"
+
+    - title: "Improved reading of metadata from FB2 files, with support for reading isbns, tags, published date, etc."
+
+    - title: "Driver for the Imagine IMEB5"
+      tickets: [800642]
+
+    - title: "Show the currently used network proxies in Preferences->Miscellaneous"
+
+    - title: "Kobo Touch driver: Show Favorites as a device collection. Various other minor fixes."
+
+    - title: "Content server now sends the Content-Disposition header when sending ebook files."
+
+    - title: "Allow search and replace on comments custom columns."
+
+    - title: "Add a new action 'Quick View' to show the books in your library by the author/tags/series/etc. of the currently selected book, in a separate window. You can add it to your toolbar or right click menu by going to Preferences->Toolbars."
+
+    - title: "Get Books: Add libri.de as a book source. Fix a bug that caused some books downloads to fail. Fixes to the Legimi and  beam-ebooks.de stores"
+      tickets: [799367]
+  
+  bug fixes:
+    - title: "Fix a memory leak that could result in the leaking of several MB of memory with large libraries"
+      tickets: [800952]
+
+    - title: "Fix the read metadata from format button in the edit metadata dialog using the wrong timezone when setting published date"
+      tickets: [799777]
+
+    - title: "Generating catalog: Fix occassional file in use errors when generating catalogs on windows"
+
+    - title: "Fix clicking on News in Tag Browser not working in non English locales."
+      tickets: [799471]
+
+    - title: "HTML Input: Fix a regression in 0.8.6 that caused CSS stylesheets to be ignored"
+      tickets: [799171]
+
+    - title: "Fix a regression that caused restore database to stop working on some windows sytems"
+
+    - title: "EPUB Output: Convert <br> tags with text in them into <divs> as ADE cannot handle them."
+      tickets: [794427]
+
+  improved recipes:
+    - Le Temps
+    - Perfil
+    - Financial Times UK
+
+  new recipes:
+    - title: "Daytona Beach Journal"
+      author: BRGriff
+
+    - title: "El club del ebook and Frontline"
+      author: Darko Miletic
+
+
+- version: 0.8.6
+  date: 2011-06-17
+
+  new features:
+    - title: "Builtin support for downloading and installing/updating calibre plugins. Go to Preferences->Plugins and click 'Get new plugins'"
+      description: "When updates for installed plugins are available, calibre will automatically (unobtrusively) notify you"
+      type: major
+
+    - title: "Metadata download configuration: Allow defining a set of 'default' fields for metadata download and quichly switching to/from them"
+
+    - title: "Allow clicking on the news category in the Tag Browser to display all downloaded periodicals"
+
+    - title: "Driver for the Libre Air"
+
+    - title: "Email sending: Allow user to stop email jobs (note that stopping may not actually prevent the email from being sent, depending on when the stop happens). Also automatically abort email sending if it takes longer than 15mins."
+      tickets: [795960]
+  
+  bug fixes:
+    - title: "MOBI Output: Allow setting of background color on tables also set the border attribute on the table if the table has any border related css defined."
+      tickets: [797580]
+
+    - title: "Nook TSR: Put news sent to the device in My Files/Newspapers instaed of My Files/Books."
+      tickets: [796674]
+
+    - title: "MOBI Output: Fix a bug where linking to the very first element in an HTML file could sometimes result in the link pointing to the last element in the previous file."
+      tickets: [797214]
+
+    - title: "CSV catalog: Convert HTML comments to plain text"
+
+    - title: "HTML Input: Ignore links to text files."
+      tickets: [791568]
+
+    - title: "EPUB Output: Change orphaned <td> tags to <div> as they cause ADE to crash."
+
+    - title: "Fix 'Stop selected jobs' button trying to stop the same job multiple times"
+
+    - title: "Database: Explicitly test for case sensitivity on OS X instead of assuming a case insensitive filesystem."
+      tickets: [796258]
+
+    - title: "Get Books: More fixes to the Amazon store plugin"
+
+    - title: "FB2 Input: Do not specify font families/background colors"
+
+
+  improved recipes:
+    - Philadelphia Inquirer
+    - Macleans Magazone
+    - Metro UK
+
+  new recipes:
+    - title: "Christian Post, Down To Earth and Words Without Borders"
+      author: sexymax15
+
+    - title: "Noticias R7"
+      author: Diniz Bortolotto
+
+    - title: "UK Daily Mirror"
+      author: Dave Asbury
+
+    - title: "New Musical Express Magazine"
+      author: scissors
+
+
 - version: 0.8.5
   date: 2011-06-10
 

recipes/christian_post.recipe

@@ -0,0 +1,37 @@
+#created by sexymax15 ....sexymax15@gmail.com
+#christian post recipe
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class ChristianPost(BasicNewsRecipe):
+
+    title = 'The Christian Post'
+    __author__ = 'sexymax15'
+    description = 'Homepage'
+    language = 'en'
+    no_stylesheets = True
+    use_embedded_content	= False
+    oldest_article	= 30
+    max_articles_per_feed = 15
+
+    remove_empty_feeds = True
+    no_stylesheets = True
+    remove_javascript = True
+
+    extra_css = '''
+              h1 {color:#008852;font-family:Arial,Helvetica,sans-serif; font-size:20px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold; line-height:18px;}
+              h2 {color:#4D4D4D;font-family:Arial,Helvetica,sans-serif; font-size:16px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold; line-height:16px; } '''
+
+
+    feeds =  [
+             ('Homepage', 'http://www.christianpost.com/services/rss/feed/'),
+             ('Most Popular', 'http://www.christianpost.com/services/rss/feed/most-popular'),
+             ('Entertainment', 'http://www.christianpost.com/services/rss/feed/entertainment/'),
+             ('Politics', 'http://www.christianpost.com/services/rss/feed/politics/'),
+             ('Living', 'http://www.christianpost.com/services/rss/feed/living/'),
+             ('Business', 'http://www.christianpost.com/services/rss/feed/business/'),
+             ('Opinion', 'http://www.christianpost.com/services/rss/feed/opinion/')
+             ]
+
+    def print_version(self, url):
+        return url +'print.html'
+

recipes/daytona_beach.recipe

@@ -0,0 +1,78 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class DaytonBeachNewsJournal(BasicNewsRecipe):
+    title          ='Daytona Beach News Journal'
+    __author__ = 'BRGriff'
+    pubisher  = 'News-JournalOnline.com'
+    description           = 'Daytona Beach, Florida, Newspaper'
+    category              = 'News, Daytona Beach, Florida'
+    oldest_article = 1
+    max_articles_per_feed = 100
+    remove_javascript = True
+    use_embedded_content   = False
+    no_stylesheets = True
+    language = 'en'
+    filterDuplicates = True
+    remove_attributes = ['style']
+
+    keep_only_tags     = [dict(name='div', attrs={'class':'page-header'}),
+                                    dict(name='div', attrs={'class':'asset-body'})
+                                        ]
+    remove_tags         = [dict(name='div', attrs={'class':['byline-section', 'asset-meta']})
+                                      ]
+
+    feeds            = [
+                             #####NEWS#####
+                            (u"News", u"http://www.news-journalonline.com/rss.xml"),
+                            (u"Breaking News", u"http://www.news-journalonline.com/breakingnews/rss.xml"),
+                            (u"Local - East Volusia", u"http://www.news-journalonline.com/news/local/east-volusia/rss.xml"),
+                            (u"Local - West Volusia", u"http://www.news-journalonline.com/news/local/west-volusia/rss.xml"),
+                            (u"Local - Southeast", u"http://www.news-journalonline.com/news/local/southeast-volusia/rss.xml"),
+                            (u"Local - Flagler", u"http://www.news-journalonline.com/news/local/flagler/rss.xml"),
+                            (u"Florida", u"http://www.news-journalonline.com/news/florida/rss.xml"),
+                            (u"National/World", u"http://www.news-journalonline.com/news/nationworld/rss.xml"),
+                            (u"Politics", u"http://www.news-journalonline.com/news/politics/rss.xml"),
+                            (u"News of Record", u"http://www.news-journalonline.com/news/news-of-record/rss.xml"),
+                             ####BUSINESS####
+                            (u"Business", u"http://www.news-journalonline.com/business/rss.xml"),
+                            #(u"Jobs", u"http://www.news-journalonline.com/business/jobs/rss.xml"),
+                            #(u"Markets", u"http://www.news-journalonline.com/business/markets/rss.xml"),
+                            #(u"Real Estate", u"http://www.news-journalonline.com/business/real-estate/rss.xml"),
+                            #(u"Technology", u"http://www.news-journalonline.com/business/technology/rss.xml"),
+                              ####SPORTS####
+                            (u"Sports", u"http://www.news-journalonline.com/sports/rss.xml"),
+                            (u"Racing", u"http://www.news-journalonline.com/racing/rss.xml"),
+                            (u"Highschool", u"http://www.news-journalonline.com/sports/highschool/rss.xml"),
+                            (u"College", u"http://www.news-journalonline.com/sports/college/rss.xml"),
+                            (u"Basketball", u"http://www.news-journalonline.com/sports/basketball/rss.xml"),
+                            (u"Football", u"http://www.news-journalonline.com/sports/football/rss.xml"),
+                            (u"Golf", u"http://www.news-journalonline.com/sports/golf/rss.xml"),
+                            (u"Other Sports", u"http://www.news-journalonline.com/sports/other/rss.xml"),
+                              ####LIFESTYLE####
+                            (u"Lifestyle", u"http://www.news-journalonline.com/lifestyle/rss.xml"),
+                            #(u"Fashion", u"http://www.news-journalonline.com/lifestyle/fashion/rss.xml"),
+                            (u"Food", u"http://www.news-journalonline.com/lifestyle/food/rss.xml"),
+                            #(u"Health", u"http://www.news-journalonline.com/lifestyle/health/rss.xml"),
+                            (u"Home and Garden", u"http://www.news-journalonline.com/lifestyle/home-and-garden/rss.xml"),
+                            (u"Living", u"http://www.news-journalonline.com/lifestyle/living/rss.xml"),
+                            (u"Religion", u"http://www.news-journalonline.com/lifestyle/religion/rss.xml"),
+                            #(u"Travel", u"http://www.news-journalonline.com/lifestyle/travel/rss.xml"),
+                              ####OPINION####
+                            #(u"Opinion", u"http://www.news-journalonline.com/opinion/rss.xml"),
+                            #(u"Letters to Editor", u"http://www.news-journalonline.com/opinion/letters-to-the-editor/rss.xml"),
+                            #(u"Columns", u"http://www.news-journalonline.com/columns/rss.xml"),
+                            #(u"Podcasts", u"http://www.news-journalonline.com/podcasts/rss.xml"),
+                              ####ENTERTAINMENT####    ##Weekly Feature##
+                            (u"Entertainment", u"http://www.go386.com/rss.xml"),
+                            (u"Go Out", u"http://www.go386.com/go/rss.xml"),
+                            (u"Music", u"http://www.go386.com/music/rss.xml"),
+                            (u"Movies", u"http://www.go386.com/movies/rss.xml"),
+                            #(u"Culture", u"http://www.go386.com/culture/rss.xml"),
+
+                            ]
+
+    extra_css = '''
+                    .page-header{font-family:Arial,Helvetica,sans-serif; font-style:bold;font-size:22pt;}
+                    .asset-body{font-family:Helvetica,Arial,sans-serif; font-size:16pt;}
+
+      '''

recipes/down_to_earth.recipe

@@ -0,0 +1,18 @@
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class AdvancedUserRecipe1307834113(BasicNewsRecipe):
+
+    title          = u'Down To Earth'
+    oldest_article = 300
+    __author__ = 'sexymax15'
+    max_articles_per_feed = 30
+    no_stylesheets = True
+    remove_javascript     = True
+    remove_attributes = ['width','height']
+    use_embedded_content  = False
+    language              = 'en_IN'
+    remove_empty_feeds    = True
+    remove_tags_before = dict(name='div', id='PageContent')
+    remove_tags_after = [dict(name='div'),{'class':'box'}]
+    remove_tags =[{'class':'box'}]
+    feeds          = [(u'editor', u'http://www.downtoearth.org.in/taxonomy/term/20348/0/feed'), (u'cover story', u'http://www.downtoearth.org.in/taxonomy/term/20345/0/feed'), (u'special report', u'http://www.downtoearth.org.in/taxonomy/term/20384/0/feed'), (u'features', u'http://www.downtoearth.org.in/taxonomy/term/20350/0/feed'), (u'news', u'http://www.downtoearth.org.in/taxonomy/term/20366/0/feed'), (u'debate', u'http://www.downtoearth.org.in/taxonomy/term/20347/0/feed'), (u'natural disasters', u'http://www.downtoearth.org.in/taxonomy/term/20822/0/feed')]

recipes/elclubdelebook.recipe

@@ -0,0 +1,61 @@
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.clubdelebook.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class ElClubDelEbook(BasicNewsRecipe):
+    title                 = 'El club del ebook'
+    __author__            = 'Darko Miletic'
+    description           = 'El Club del eBook, es la primera fuente de informacion sobre ebooks de Argentina. Aca vas a encontrar noticias, tips, tutoriales, recursos y opiniones sobre el mundo de los libros electronicos.'
+    tags                  = 'ebook, libro electronico, e-book, ebooks, libros electronicos, e-books'
+    oldest_article        = 7
+    max_articles_per_feed = 100
+    language              = 'es_AR'
+    encoding              = 'utf-8'
+    no_stylesheets        = True
+    use_embedded_content  = True
+    publication_type      = 'blog'
+    masthead_url          = 'http://dl.dropbox.com/u/2845131/elclubdelebook.png'
+    extra_css             = """
+                                body{font-family: Arial,Helvetica,sans-serif}
+                                img{ margin-bottom: 0.8em;
+                                     border: 1px solid #333333;
+                                     padding: 4px; display: block
+                                   }
+                            """
+
+    conversion_options = {
+                          'comment'  : description
+                        , 'tags'     : tags
+                        , 'publisher': title
+                        , 'language' : language
+                        }
+
+    remove_tags       = [dict(attrs={'id':'crp_related'})]
+    remove_tags_after = dict(attrs={'id':'crp_related'})
+
+    feeds = [(u'Articulos', u'http://feeds.feedburner.com/ElClubDelEbook')]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll('a'):
+            limg = item.find('img')
+            if item.string is not None:
+               str = item.string
+               item.replaceWith(str)
+            else:
+               if limg:
+                  item.name = 'div'
+                  item.attrs = []
+               else:
+                   str = self.tag_to_string(item)
+                   item.replaceWith(str)
+        for item in soup.findAll('img'):
+            if not item.has_key('alt'):
+               item['alt'] = 'image'
+        return soup

recipes/financial_times_uk.recipe

@@ -1,15 +1,17 @@
 __license__   = 'GPL v3'
 __copyright__ = '2010-2011, Darko Miletic <darko.miletic at gmail.com>'
 '''
-ft.com
+www.ft.com/uk-edition
 '''
 from calibre import strftime
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class FinancialTimes(BasicNewsRecipe):
-    title                 = u'Financial Times - UK printed edition'
+    title                 = 'Financial Times - UK printed edition'
     __author__            = 'Darko Miletic'
-    description           = 'Financial world news'
+    description           = "The Financial Times (FT) is one of the world's leading business news and information organisations, recognised internationally for its authority, integrity and accuracy."
+    publisher             = 'The Financial Times Ltd.'
+    category              = 'news, finances, politics, UK, World'
     oldest_article        = 2
     language              = 'en_GB'
     max_articles_per_feed = 250
@@ -17,14 +19,24 @@ class FinancialTimes(BasicNewsRecipe):
     use_embedded_content  = False
     needs_subscription    = True
     encoding              = 'utf8'
-    simultaneous_downloads= 1
+    publication_type      = 'newspaper'
-    delay                 = 1
+    cover_url             = strftime('http://specials.ft.com/vtf_pdf/%d%m%y_FRONT1_LON.pdf')
+    masthead_url          = 'http://im.media.ft.com/m/img/masthead_main.jpg'
     LOGIN                 = 'https://registration.ft.com/registration/barrier/login'
     INDEX                 = 'http://www.ft.com/uk-edition'
     PREFIX                = 'http://www.ft.com'
 
+    conversion_options = {
+                          'comment'          : description
+                        , 'tags'             : category
+                        , 'publisher'        : publisher
+                        , 'language'         : language
+                        , 'linearize_tables' : True
+                        }
+
     def get_browser(self):
         br = BasicNewsRecipe.get_browser()
+        br.open(self.INDEX)
         if self.username is not None and self.password is not None:
             br.open(self.LOGIN)
             br.select_form(name='loginForm')
@@ -33,29 +45,34 @@ class FinancialTimes(BasicNewsRecipe):
             br.submit()
         return br
 
-    keep_only_tags    = [ dict(name='div', attrs={'id':'cont'}) ]
+    keep_only_tags = [dict(name='div', attrs={'class':['fullstory fullstoryHeader','fullstory fullstoryBody','ft-story-header','ft-story-body','index-detail']})]
-    remove_tags_after = dict(name='p', attrs={'class':'copyright'})
     remove_tags = [
                       dict(name='div', attrs={'id':'floating-con'})
                      ,dict(name=['meta','iframe','base','object','embed','link'])
+                     ,dict(attrs={'class':['storyTools','story-package','screen-copy','story-package separator','expandable-image']})
                   ]
     remove_attributes = ['width','height','lang']
 
     extra_css = """
-                body{font-family:Arial,Helvetica,sans-serif;}
+                body{font-family: Georgia,Times,"Times New Roman",serif}
-                h2{font-size:large;}
+                h2{font-size:large}
-                .ft-story-header{font-size:xx-small;}
+                .ft-story-header{font-size: x-small}
-                .ft-story-body{font-size:small;}
-                a{color:#003399;}
                 .container{font-size:x-small;}
                 h3{font-size:x-small;color:#003399;}
                 .copyright{font-size: x-small}
+                img{margin-top: 0.8em; display: block}
+                .lastUpdated{font-family: Arial,Helvetica,sans-serif; font-size: x-small}
+                .byline,.ft-story-body,.ft-story-header{font-family: Arial,Helvetica,sans-serif}
                 """
 
     def get_artlinks(self, elem):
         articles = []
         for item in elem.findAll('a',href=True):
-            url   = self.PREFIX + item['href']
+            rawlink = item['href']
+            if rawlink.startswith('http://'):
+               url = rawlink
+            else:
+               url   = self.PREFIX + rawlink
             title = self.tag_to_string(item)
             date = strftime(self.timefmt)
             articles.append({
@@ -65,7 +82,7 @@ class FinancialTimes(BasicNewsRecipe):
                              ,'description':''
                             })
         return articles
-                
+
     def parse_index(self):
         feeds = []
         soup = self.index_to_soup(self.INDEX)
@@ -80,11 +97,34 @@ class FinancialTimes(BasicNewsRecipe):
            strest.insert(0,st)
         for item in strest:
             ftitle   = self.tag_to_string(item)
-            self.report_progress(0, _('Fetching feed')+' %s...'%(ftitle))            
+            self.report_progress(0, _('Fetching feed')+' %s...'%(ftitle))
             feedarts = self.get_artlinks(item.parent.ul)
             feeds.append((ftitle,feedarts))
         return feeds
 
     def preprocess_html(self, soup):
-        return self.adeify_images(soup)
+        items = ['promo-box','promo-title',
-
+                 'promo-headline','promo-image',
+                 'promo-intro','promo-link','subhead']
+        for item in items:
+            for it in soup.findAll(item):
+                it.name = 'div'
+                it.attrs = []
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll('a'):
+            limg = item.find('img')
+            if item.string is not None:
+               str = item.string
+               item.replaceWith(str)
+            else:
+               if limg:
+                  item.name = 'div'
+                  item.attrs = []
+               else:
+                   str = self.tag_to_string(item)
+                   item.replaceWith(str)
+        for item in soup.findAll('img'):
+            if not item.has_key('alt'):
+               item['alt'] = 'image'
+        return soup

recipes/frontlineonnet.recipe

@@ -0,0 +1,81 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011, Darko Miletic <darko.miletic at gmail.com>'
+'''
+frontlineonnet.com
+'''
+
+import re
+from calibre import strftime
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Frontlineonnet(BasicNewsRecipe):
+    title                = 'Frontline'
+    __author__           = 'Darko Miletic'
+    description          = "India's national magazine"
+    publisher            = 'Frontline'
+    category             = 'news, politics, India'
+    no_stylesheets       = True
+    delay                = 1
+    INDEX                = 'http://frontlineonnet.com/'
+    use_embedded_content = False
+    encoding             = 'cp1252'
+    language             = 'en_IN'
+    publication_type     = 'magazine'
+    masthead_url         = 'http://frontlineonnet.com/images/newfline.jpg'
+    extra_css            = """
+                              body{font-family: Verdana,Arial,Helvetica,sans-serif}
+                              img{margin-top:0.5em; margin-bottom: 0.7em; display: block}
+                           """
+
+    conversion_options = {
+                          'comment'          : description
+                        , 'tags'             : category
+                        , 'publisher'        : publisher
+                        , 'language'         : language
+                        , 'linearize_tables' : True
+                        }
+
+    preprocess_regexps = [
+                           (re.compile(r'.*?<base', re.DOTALL|re.IGNORECASE),lambda match: '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"><html dir="ltr" xml:lang="en-IN"><head><title>title</title><base')
+                          ,(re.compile(r'<base .*?>', re.DOTALL|re.IGNORECASE),lambda match: '</head><body>')
+                          ,(re.compile(r'<byline>', re.DOTALL|re.IGNORECASE),lambda match: '<div class="byline">')
+                          ,(re.compile(r'</byline>', re.DOTALL|re.IGNORECASE),lambda match: '</div>')
+                          ,(re.compile(r'<center>', re.DOTALL|re.IGNORECASE),lambda match: '<div class="ctr">')
+                          ,(re.compile(r'</center>', re.DOTALL|re.IGNORECASE),lambda match: '</div>')
+                         ]
+
+    keep_only_tags= [
+                      dict(name='font', attrs={'class':'storyhead'})
+                     ,dict(attrs={'class':'byline'})
+                    ]
+    remove_attributes=['size','noshade','border']
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll('img'):
+            if not item.has_key('alt'):
+               item['alt'] = 'image'
+        return soup
+
+    def parse_index(self):
+        articles = []
+        soup = self.index_to_soup(self.INDEX)
+        for feed_link in soup.findAll('a',href=True):
+            if feed_link['href'].startswith('stories/'):
+                url   = self.INDEX + feed_link['href']
+                title = self.tag_to_string(feed_link)
+                date  = strftime(self.timefmt)
+                articles.append({
+                                  'title'      :title
+                                 ,'date'       :date
+                                 ,'url'        :url
+                                 ,'description':''
+                                })
+        return [('Frontline', articles)]
+
+    def print_version(self, url):
+        return "http://www.hinduonnet.com/thehindu/thscrip/print.pl?prd=fline&file=" + url.rpartition('/')[2]
+
+    def image_url_processor(self, baseurl, url):
+        return url.replace('../images/', self.INDEX + 'images/').strip()

recipes/le_temps.recipe

@@ -14,7 +14,7 @@ class LeTemps(BasicNewsRecipe):
      title          = u'Le Temps'
      oldest_article = 7
      max_articles_per_feed = 100
-     __author__ = 'Sujata Raman'
+     __author__ = 'Kovid Goyal'
      description = 'French news. Needs a subscription from http://www.letemps.ch'
      no_stylesheets = True
      remove_javascript = True
@@ -27,6 +27,7 @@ class LeTemps(BasicNewsRecipe):
      def get_browser(self):
          br = BasicNewsRecipe.get_browser(self)
          br.open('http://www.letemps.ch/login')
+         br.select_form(nr=1)
          br['username'] = self.username
          br['password'] = self.password
          raw = br.submit().read()

recipes/macleans.recipe

@@ -1,239 +1,28 @@
 #!/usr/bin/env  python
+from calibre.web.feeds.news import BasicNewsRecipe
 
-__license__   = 'GPL v3'
+class AdvancedUserRecipe1308306308(BasicNewsRecipe):
-
-'''
-macleans.ca
-'''
-from calibre.web.feeds.recipes import BasicNewsRecipe
-from calibre.ebooks.BeautifulSoup import Tag
-from datetime import timedelta, date
-
-class Macleans(BasicNewsRecipe):
     title          = u'Macleans Magazine'
-    __author__     = 'Nick Redding'
     language = 'en_CA'
-    description = ('Macleans Magazine')
+    __author__ = 'sexymax15'
+    oldest_article = 30
+    max_articles_per_feed = 12
 
+    use_embedded_content = False
+
+    remove_empty_feeds = True
     no_stylesheets = True
-    timefmt = ' [%b %d]'
+    remove_javascript = True
+    remove_tags = [dict(name ='img'),dict (id='header'),{'class':'postmetadata'}]
+    remove_tags_after = {'class':'postmetadata'}
 
-    # customization notes: delete sections you are not interested in
+    feeds          = [(u'Blog Central', u'http://www2.macleans.ca/category/blog-central/feed/'),
-    # set oldest_article to the maximum number of days back from today to include articles
+ (u'Canada', u'http://www2.macleans.ca/category/canada/feed/'),
-    sectionlist = [
+(u'World', u'http://www2.macleans.ca/category/world-from-the-magazine/feed/'),
-                        ['http://www2.macleans.ca/','Front Page'],
+(u'Business', u'http://www2.macleans.ca/category/business/feed/'),
-                        ['http://www2.macleans.ca/category/canada/','Canada'],
+(u'Arts & Culture', u'http://www2.macleans.ca/category/arts-culture/feed/'),
-                        ['http://www2.macleans.ca/category/world-from-the-magazine/','World'],
+(u'Opinion', u'http://www2.macleans.ca/category/opinion/feed/'),
-                        ['http://www2.macleans.ca/category/business','Business'],
+(u'Health', u'http://www2.macleans.ca/category/health-from-the-magazine/feed/'),
-                        ['http://www2.macleans.ca/category/arts-culture/','Culture'],
+ (u'Environment', u'http://www2.macleans.ca/category/environment-from-the-magazine/feed/')]
-                        ['http://www2.macleans.ca/category/opinion','Opinion'],
+    def print_version(self, url):
-                        ['http://www2.macleans.ca/category/health-from-the-magazine/','Health'],
+        return url + 'print/'
-                        ['http://www2.macleans.ca/category/environment-from-the-magazine/','Environment'],
-                        ['http://www2.macleans.ca/category/education/','On Campus'],
-                        ['http://www2.macleans.ca/category/travel-from-the-magazine/','Travel']
-                    ]
-    oldest_article = 7
-
-    # formatting for print version of articles
-    extra_css   =   '''h2{font-family:Times,serif; font-size:large;}
-                        small {font-family:Times,serif; font-size:xx-small; list-style-type: none;}
-                    '''
-
-    # tag handling for print version of articles
-    keep_only_tags = [dict(id='tw-print')]
-    remove_tags =   [dict({'class':'postmetadata'})]
-
-
-    def preprocess_html(self,soup):
-        for img_tag in soup.findAll('img'):
-            parent_tag = img_tag.parent
-            if parent_tag.name == 'a':
-                new_tag = Tag(soup,'p')
-                new_tag.insert(0,img_tag)
-                parent_tag.replaceWith(new_tag)
-            elif parent_tag.name == 'p':
-                if not self.tag_to_string(parent_tag) == '':
-                    new_div = Tag(soup,'div')
-                    new_tag = Tag(soup,'p')
-                    new_tag.insert(0,img_tag)
-                    parent_tag.replaceWith(new_div)
-                    new_div.insert(0,new_tag)
-                    new_div.insert(1,parent_tag)
-        return soup
-
-    def parse_index(self):
-
-
-
-        articles = {}
-        key = None
-        ans = []
-
-        def parse_index_page(page_url,page_title):
-
-            def decode_date(datestr):
-                dmysplit = datestr.strip().lower().split(',')
-                mdsplit = dmysplit[1].split()
-                m = ['january','february','march','april','may','june','july','august','september','october','november','december'].index(mdsplit[0])+1
-                d = int(mdsplit[1])
-                y = int(dmysplit[2].split()[0])
-                return date(y,m,d)
-
-            def article_title(tag):
-                atag = tag.find('a',href=True)
-                if not atag:
-                    return ''
-                return self.tag_to_string(atag)
-
-            def article_url(tag):
-                atag = tag.find('a',href=True)
-                if not atag:
-                    return ''
-                return atag['href']+'print/'
-
-            def article_description(tag):
-                for p_tag in tag.findAll('p'):
-                    d = self.tag_to_string(p_tag,False)
-                    if not d == '':
-                        return d
-                return ''
-
-            def compound_h4_h3_title(tag):
-                if tag.h4:
-                    if tag.h3:
-                        return self.tag_to_string(tag.h4,False)+u'\u2014'+self.tag_to_string(tag.h3,False)
-                    else:
-                        return self.tag_to_string(tag.h4,False)
-                elif tag.h3:
-                    return self.tag_to_string(tag.h3,False)
-                else:
-                    return ''
-
-            def compound_h2_h4_title(tag):
-                if tag.h2:
-                    if tag.h4:
-                        return self.tag_to_string(tag.h2,False)+u'\u2014'+self.tag_to_string(tag.h4,False)
-                    else:
-                        return self.tag_to_string(tag.h2,False)
-                elif tag.h4:
-                    return self.tag_to_string(tag.h4,False)
-                else:
-                    return ''
-
-
-            def handle_article(header_tag, outer_tag):
-                if header_tag:
-                    url = article_url(header_tag)
-                    title = article_title(header_tag)
-                    author_date_tag = outer_tag.h4
-                    if author_date_tag:
-                        author_date = self.tag_to_string(author_date_tag,False).split(' - ')
-                        author = author_date[0].strip()
-                        article_date = decode_date(author_date[1])
-                        earliest_date = date.today() - timedelta(days=self.oldest_article)
-                        if article_date < earliest_date:
-                            self.log("Skipping article dated %s" % author_date[1])
-                        else:
-                            excerpt_div = outer_tag.find('div','excerpt')
-                            if excerpt_div:
-                                description = article_description(excerpt_div)
-                            else:
-                                description = ''
-                            if not articles.has_key(page_title):
-                                articles[page_title] = []
-                            articles[page_title].append(dict(title=title,url=url,date=author_date[1],description=description,author=author,content=''))
-
-            def handle_category_article(cat, header_tag, outer_tag):
-                url = article_url(header_tag)
-                title = article_title(header_tag)
-                if not title == '':
-                    title = cat+u'\u2014'+title
-                a_tag = outer_tag.find('span','authorLink')
-                if a_tag:
-                    author = self.tag_to_string(a_tag,False)
-                    a_tag.parent.extract()
-                else:
-                    author = ''
-                description = article_description(outer_tag)
-                if not articles.has_key(page_title):
-                    articles[page_title] = []
-                articles[page_title].append(dict(title=title,url=url,date='',description=description,author=author,content=''))
-
-
-            soup = self.index_to_soup(page_url)
-
-            if page_title == 'Front Page':
-                # special processing for the front page
-                top_stories = soup.find('div',{ "id" : "macleansFeatured" })
-                if top_stories:
-                    for div_slide in top_stories.findAll('div','slide'):
-                        url = article_url(div_slide)
-                        div_title = div_slide.find('div','header')
-                        if div_title:
-                            title = self.tag_to_string(div_title,False)
-                        else:
-                            title = ''
-                        description = article_description(div_slide)
-                        if not articles.has_key(page_title):
-                             articles[page_title] = []
-                        articles[page_title].append(dict(title=title,url=url,date='',description=description,author='',content=''))
-
-                from_macleans = soup.find('div',{ "id" : "fromMacleans" })
-                if from_macleans:
-                    for li_tag in from_macleans.findAll('li','fromMacleansArticle'):
-                        title = compound_h4_h3_title(li_tag)
-                        url = article_url(li_tag)
-                        description = article_description(li_tag)
-                        if not articles.has_key(page_title):
-                            articles[page_title] = []
-                        articles[page_title].append(dict(title=title,url=url,date='',description=description,author='',content=''))
-
-                blog_central = soup.find('div',{ "id" : "bloglist" })
-                if blog_central:
-                    for li_tag in blog_central.findAll('li'):
-                        title = compound_h2_h4_title(li_tag)
-                        if li_tag.h4:
-                            url = article_url(li_tag.h4)
-                            if not articles.has_key(page_title):
-                                articles[page_title] = []
-                            articles[page_title].append(dict(title=title,url=url,date='',description='',author='',content=''))
-
-#                need_to_know = soup.find('div',{ "id" : "needToKnow" })
-#                if need_to_know:
-#                    for div_tag in need_to_know('div',attrs={'class' : re.compile("^needToKnowArticle")}):
-#                        title = compound_h4_h3_title(div_tag)
-#                        url = article_url(div_tag)
-#                        description = article_description(div_tag)
-#                        if not articles.has_key(page_title):
-#                            articles[page_title] = []
-#                        articles[page_title].append(dict(title=title,url=url,date='',description=description,author='',content=''))
-
-                for news_category in soup.findAll('div','newsCategory'):
-                    news_cat = self.tag_to_string(news_category.h4,False)
-                    handle_category_article(news_cat, news_category.find('h2'), news_category.find('div'))
-                    for news_item in news_category.findAll('li'):
-                        handle_category_article(news_cat,news_item.h3,news_item)
-
-                return
-
-            # find the div containing the highlight article
-            div_post = soup.find('div','post')
-            if div_post:
-                h1_tag = div_post.h1
-                handle_article(h1_tag,div_post)
-
-            # find the divs containing the rest of the articles
-            div_other = div_post.find('div', { "id" : "categoryOtherPosts" })
-            if div_other:
-                for div_entry in div_other.findAll('div','entry'):
-                    h2_tag = div_entry.h2
-                    handle_article(h2_tag,div_entry)
-
-
-
-        for page_name,page_title in self.sectionlist:
-            parse_index_page(page_name,page_title)
-            ans.append(page_title)
-
-        ans = [(key, articles[key]) for key in ans if articles.has_key(key)]
-        return ans

recipes/mail_and_guardian.recipe

@@ -3,7 +3,7 @@ from calibre.web.feeds.news import BasicNewsRecipe
 class AdvancedUserRecipe1295081935(BasicNewsRecipe):
      title          = u'Mail & Guardian ZA News'
      __author__     = '77ja65'
-     language = 'en'
+     language = 'en_ZA'
      oldest_article = 7
      max_articles_per_feed = 30
      no_stylesheets = True

recipes/metro_uk.recipe

@@ -1,29 +1,34 @@
+import re
 from calibre.web.feeds.news import BasicNewsRecipe
 class AdvancedUserRecipe1306097511(BasicNewsRecipe):
     title          = u'Metro UK'
-
+    description = 'News as provide by The Metro -UK'
-    no_stylesheets = True
-    oldest_article = 1
-    max_articles_per_feed = 200
 
     __author__ = 'Dave Asbury'
+    no_stylesheets = True
+    oldest_article = 1
+    max_articles_per_feed = 25
+    remove_empty_feeds = True
+    remove_javascript     = True
+
+    preprocess_regexps = [(re.compile(r'Tweet'), lambda  a : '')]
+
     language = 'en_GB'
-    simultaneous_downloads= 3
+
 
     masthead_url        = 'http://e-edition.metro.co.uk/images/metro_logo.gif'
 
+    extra_css = 'h2 {font: sans-serif medium;}'
     keep_only_tags = [
+	dict(name='h1'),dict(name='h2', attrs={'class':'h2'}),
                     dict(attrs={'class':['img-cnt figure']}),
-    dict(attrs={'class':['art-img']}),
+    	dict(attrs={'class':['art-img']}),
-                    dict(name='h1'),
+
-    dict(name='h2', attrs={'class':'h2'}),
                     dict(name='div', attrs={'class':'art-lft'})
     ]
-    remove_tags    = [dict(name='div', attrs={'class':[ 'metroCommentFormWrap',
+    remove_tags    = [dict(name='div', attrs={'class':[ 'news m12 clrd clr-b p5t shareBtm', 'commentForm', 'metroCommentInnerWrap',
-                                                                             'commentForm', 'metroCommentInnerWrap',
+                             'art-rgt','pluck-app pluck-comm','news m12 clrd clr-l p5t', 'flt-r' ]}),
-                             'art-rgt','pluck-app pluck-comm','news m12 clrd clr-l p5t', 'flt-r' ]})]
+	dict(attrs={'class':[ 'metroCommentFormWrap','commentText','commentsNav','avatar','submDateAndTime']})
-
+	          ]
     feeds          = [
         (u'News', u'http://www.metro.co.uk/rss/news/'), (u'Money', u'http://www.metro.co.uk/rss/money/'), (u'Sport', u'http://www.metro.co.uk/rss/sport/'), (u'Film', u'http://www.metro.co.uk/rss/metrolife/film/'), (u'Music', u'http://www.metro.co.uk/rss/metrolife/music/'), (u'TV', u'http://www.metro.co.uk/rss/tv/'), (u'Showbiz', u'http://www.metro.co.uk/rss/showbiz/'), (u'Weird News', u'http://www.metro.co.uk/rss/weird/'), (u'Travel', u'http://www.metro.co.uk/rss/travel/'), (u'Lifestyle', u'http://www.metro.co.uk/rss/lifestyle/'), (u'Books', u'http://www.metro.co.uk/rss/lifestyle/books/'), (u'Food', u'http://www.metro.co.uk/rss/lifestyle/restaurants/')]
-
-

recipes/noticias_r7.recipe

@@ -0,0 +1,40 @@
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class PortalR7(BasicNewsRecipe):
+    title                  = 'Noticias R7'
+    __author__             = 'Diniz Bortolotto'
+    description            = 'Noticias Portal R7'
+    oldest_article         = 2
+    max_articles_per_feed  = 20
+    encoding               = 'utf8'
+    publisher              = 'Rede Record'
+    category               = 'news, Brazil'
+    language               = 'pt_BR'
+    publication_type       = 'newsportal'
+    use_embedded_content   = False
+    no_stylesheets         = True
+    remove_javascript      = True
+    remove_attributes      = ['style']
+
+    feeds                  = [
+                              (u'Brasil', u'http://www.r7.com/data/rss/brasil.xml'),
+                              (u'Economia', u'http://www.r7.com/data/rss/economia.xml'),
+                              (u'Internacional', u'http://www.r7.com/data/rss/internacional.xml'),
+                              (u'Tecnologia e Ci\xeancia', u'http://www.r7.com/data/rss/tecnologiaCiencia.xml')
+                             ]
+    reverse_article_order  = True
+
+    keep_only_tags         = [dict(name='div', attrs={'class':'materia'})]
+    remove_tags            = [
+                              dict(id=['espalhe', 'report-erro']),
+                              dict(name='ul', attrs={'class':'controles'}),
+                              dict(name='ul', attrs={'class':'relacionados'}),
+                              dict(name='div', attrs={'class':'materia_banner'}),
+                              dict(name='div', attrs={'class':'materia_controles'})
+                             ]
+
+    preprocess_regexps     = [
+                              (re.compile(r'<div class="materia">.*<div class="materia_cabecalho">',re.DOTALL|re.IGNORECASE),
+                              lambda match: '<div class="materia"><div class="materia_cabecalho">')
+                             ]

recipes/perfil.recipe

@@ -26,6 +26,7 @@ class Perfil(BasicNewsRecipe):
                                 .foto1 h1{font-size: x-small}
                                 h1{font-family: Georgia,"Times New Roman",serif}
                                 img{margin-bottom: 0.4em}
+                                .hora{font-size: x-small; color: red}
                             """
 
     conversion_options = {
@@ -60,7 +61,26 @@ class Perfil(BasicNewsRecipe):
              ,(u'Tecnologia'     , u'http://www.perfil.com/rss/tecnologia.xml'   )
             ]
 
+    def get_article_url(self, article):
+        return article.get('guid',  None)
+            
     def preprocess_html(self, soup):
         for item in soup.findAll(style=True):
             del item['style']
+        for item in soup.findAll('a'):
+            limg = item.find('img')
+            if item.string is not None:
+               str = item.string
+               item.replaceWith(str)
+            else:
+               if limg:
+                  item.name = 'div'
+                  item.attrs = []
+               else:
+                   str = self.tag_to_string(item)
+                   item.replaceWith(str)
+        for item in soup.findAll('img'):
+            if not item.has_key('alt'):
+               item['alt'] = 'image'
         return soup
+        

recipes/philly.recipe

@@ -1,85 +1,45 @@
 #!/usr/bin/env  python
-__license__   = 'GPL v3'
+from calibre.web.feeds.news import BasicNewsRecipe
-'''
-philly.com/inquirer/
-'''
-from calibre.web.feeds.recipes import BasicNewsRecipe
 
-class Philly(BasicNewsRecipe):
+class AdvancedUserRecipe1308312288(BasicNewsRecipe):
-
+    title          = u'Philadelphia Inquirer'
-    title       = 'Philadelphia Inquirer'
+    __author__ = 'sexymax15'
-    __author__  = 'RadikalDissent and Sujata Raman'
     language = 'en'
     description = 'Daily news from the Philadelphia Inquirer'
-    no_stylesheets        = True
+    oldest_article = 15
-    use_embedded_content  = False
+    max_articles_per_feed = 20
-    oldest_article = 1
+    use_embedded_content = False
-    max_articles_per_feed = 25
+    remove_empty_feeds = True
+    no_stylesheets = True
+    remove_javascript = True
 
-    extra_css = '''
+   # remove_tags_before = {'class':'article_timestamp'}
-        h1{font-family:verdana,arial,helvetica,sans-serif; font-size: large;}
+    #remove_tags_after = {'class':'graylabel'}
-        h2{font-family:verdana,arial,helvetica,sans-serif; font-size: small;}
+    keep_only_tags= [dict(name=['h1','p'])]
-        .body-content{font-family:verdana,arial,helvetica,sans-serif; font-size: small;}
+    remove_tags = [dict(name=['hr','dl','dt','img','meta','iframe','link','script','form','input','label']),
-        .byline {font-size: small; color: #666666; font-style:italic; }
+dict(id=['toggleConfirmEmailDiv','toggleTOS','toggleUsernameMsgDiv','toggleConfirmYear','navT1_philly','secondaryNav','navPlacement','globalPrimaryNav'
-        .lastline {font-size: small; color: #666666; font-style:italic;}
+,'ugc-footer-philly','bv_footer_include','footer','header',
-        .contact {font-size: small; color: #666666;}
+'container_rag_bottom','section_rectangle','contentrightside'])
-        .contact p {font-size: small; color: #666666;}
+,{'class':['megamenu3 megamenu','container misc','container_inner misc_inner'
-        #photoCaption { font-family:verdana,arial,helvetica,sans-serif; font-size:x-small;}
+,'misccontainer_left_32','headlineonly','misccontainer_middle_32'
-        .photoCaption { font-family:verdana,arial,helvetica,sans-serif; font-size:x-small;}
+,'misccontainer_right_32','headline formBegin',
-        #photoCredit{ font-family:verdana,arial,helvetica,sans-serif; font-size:x-small; color:#666666;}
+'post_balloon','relatedlist','linkssubhead','b_sq','dotted-rule-above'
-        .photoCredit{ font-family:verdana,arial,helvetica,sans-serif; font-size:x-small; color:#666666;}
+,'container','headlines-digest','graylabel','container_inner'
-        .article_timestamp{font-size:x-small; color:#666666;}
+,'rlinks_colorbar1','rlinks_colorbar2','supercontainer','container_5col_left','container_image_left',
-        a {font-family:verdana,arial,helvetica,sans-serif; font-size: x-small;}
+'digest-headline2','digest-lead','container_5col_leftmiddle',
-                '''
+'container_5col_middlemiddle','container_5col_rightmiddle'
+,'container_5col_right','divclear','supercontainer_outer force-width',
+'supercontainer','containertitle  kicker-title',
+'pollquestion','pollchoice','photomore','pollbutton','container rssbox','containertitle video ',
+'containertitle_image ','container_tabtwo','selected'
+,'shadetabs','selected','tabcontentstyle','tabcontent','inner_container'
+,'arrow','container_ad','containertitlespacer','adUnit','tracking','sitemsg_911 clearfix']}]
 
-    keep_only_tags = [
+    extra_css             = """
-               dict(name='div', attrs={'class':'story-content'}),
+                               h1{font-family: Georgia,serif; font-size: xx-large}
-               dict(name='div', attrs={'id': 'contentinside'})
-                    ]
 
-    remove_tags = [
+                            """
-         dict(name='div', attrs={'class':['linkssubhead','post_balloon','relatedlist','pollquestion','b_sq']}),
-         dict(name='dl', attrs={'class':'relatedlist'}),
-        dict(name='div', attrs={'id':['photoNav','sidebar_adholder']}),
-        dict(name='a', attrs={'class': ['headlineonly','bl']}),
-         dict(name='img', attrs={'class':'img_noborder'})
-    ]
-   # def print_version(self, url):
-   #     return url + '?viewAll=y'
 
 
-    feeds = [
+    feeds          = [(u'News', u'http://www.philly.com/philly_news.rss')]
-        ('Front Page', 'http://www.philly.com/inquirer_front_page.rss'),
-        ('Business', 'http://www.philly.com/inq_business.rss'),
-        #('News', 'http://www.philly.com/inquirer/news/index.rss'),
-        ('Nation', 'http://www.philly.com/inq_news_world_us.rss'),
-        ('Local', 'http://www.philly.com/inquirer_local.rss'),
-        ('Health', 'http://www.philly.com/inquirer_health_science.rss'),
-        ('Education', 'http://www.philly.com/inquirer_education.rss'),
-        ('Editorial and opinion', 'http://www.philly.com/inq_news_editorial.rss'),
-        ('Sports', 'http://www.philly.com/inquirer_sports.rss')
-        ]
 
-    def get_article_url(self, article):
-        ans = article.link
-
-        try:
-            self.log('Looking for full story link in', ans)
-            soup = self.index_to_soup(ans)
-            x = soup.find(text="View All")
-
-            if x is not None:
-                ans = ans + '?viewAll=y'
-                self.log('Found full story link', ans)
-        except:
-            pass
-        return ans
-
-    def postprocess_html(self, soup,first):
-
-         for tag in soup.findAll(name='div',attrs={'class':"container_ate_qandatitle"}):
-                tag.extract()
-         for tag in soup.findAll(name='br'):
-                tag.extract()
-
-         return soup

recipes/words_without_borders.recipe

@@ -0,0 +1,25 @@
+#recipe created by sexymax15.....sexymax15@gmail.com
+#Words without Borders recipe
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1308302002(BasicNewsRecipe):
+    title          = u'Words Without Borders'
+    language = 'en'
+    __author__ = 'sexymax15'
+    oldest_article = 90
+    max_articles_per_feed = 30
+    use_embedded_content = False
+
+    remove_empty_feeds = True
+    no_stylesheets = True
+    remove_javascript = True
+    keep_only_tags = {'class':'span-14 article'}
+    remove_tags_after = [{'class':'addthis_toolbox addthis_default_style no_print'}]
+    remove_tags = [{'class':['posterous_quote_citation','button']}]
+    extra_css             = """
+                               h1{font-family: Georgia,serif; font-size: large}h2{font-family: Georgia,serif; font-size: large} """
+
+
+
+    feeds          = [(u'wwb', u'http://feeds.feedburner.com/wwborders?format=xml')]

recipes/wsj.recipe

@@ -51,7 +51,7 @@ class WallStreetJournal(BasicNewsRecipe):
             br['password'] = self.password
             res = br.submit()
             raw = res.read()
-            if 'Welcome,' not in raw:
+            if 'Welcome,' not in raw and '>Logout<' not in raw:
                 raise ValueError('Failed to log in to wsj.com, check your '
                         'username and password')
         return br

resources/content_server/read/monocle.js

@@ -1,5 +1,5 @@
 Monocle = {
-  VERSION: "1.0.0"
+  VERSION: "2.0.0"
 };
 
 
@@ -170,7 +170,8 @@ Monocle.Browser.has.iframeTouchBug = Monocle.Browser.iOSVersionBelow("4.2");
 Monocle.Browser.has.selectThruBug = Monocle.Browser.iOSVersionBelow("4.2");
 
 Monocle.Browser.has.mustScrollSheaf = Monocle.Browser.is.MobileSafari;
-Monocle.Browser.has.iframeDoubleWidthBug = Monocle.Browser.has.mustScrollSheaf;
+Monocle.Browser.has.iframeDoubleWidthBug =
+  Monocle.Browser.has.mustScrollSheaf || Monocle.Browser.on.Kindle3;
 
 Monocle.Browser.has.floatColumnBug = Monocle.Browser.is.WebKit;
 
@@ -181,6 +182,11 @@ Monocle.Browser.has.jumpFlickerBug =
   Monocle.Browser.on.MacOSX && Monocle.Browser.is.WebKit;
 
 
+Monocle.Browser.has.columnOverflowPaintBug = Monocle.Browser.is.WebKit &&
+  !Monocle.Browser.is.MobileSafari &&
+  navigator.userAgent.indexOf("AppleWebKit/534") > 0;
+
+
 if (typeof window.console == "undefined") {
   window.console = {
     messages: [],
@@ -241,6 +247,7 @@ Monocle.Factory = function (element, label, index, reader) {
 
 
   function initialize() {
+    if (!p.label) { return; }
     var node = p.reader.properties.graph;
     node[p.label] = node[p.label] || [];
     if (typeof p.index == 'undefined' && node[p.label][p.index]) {
@@ -274,7 +281,11 @@ Monocle.Factory = function (element, label, index, reader) {
 
   function make(tagName, oLabel, index_or_options, or_options) {
     var oIndex, options;
-    if (arguments.length == 2) {
+    if (arguments.length == 1) {
+      oLabel = null,
+      oIndex = 0;
+      options = {};
+    } else if (arguments.length == 2) {
       oIndex = 0;
       options = {};
     } else if (arguments.length == 4) {
@@ -376,6 +387,22 @@ Monocle.pieceLoaded('factory');
 Monocle.Events = {}
 
 
+Monocle.Events.dispatch = function (elem, evtType, data, cancelable) {
+  if (!document.createEvent) {
+    return true;
+  }
+  var evt = document.createEvent("Events");
+  evt.initEvent(evtType, false, cancelable || false);
+  evt.m = data;
+  try {
+    return elem.dispatchEvent(evt);
+  } catch(e) {
+    console.warn("Failed to dispatch event: "+evtType);
+    return false;
+  }
+}
+
+
 Monocle.Events.listen = function (elem, evtType, fn, useCapture) {
   if (elem.addEventListener) {
     return elem.addEventListener(evtType, fn, useCapture || false);
@@ -405,7 +432,7 @@ Monocle.Events.listenForContact = function (elem, fns, options) {
       pageY: ci.pageY
     };
 
-    var target = evt.target || window.srcElement;
+    var target = evt.target || evt.srcElement;
     while (target.nodeType != 1 && target.parentNode) {
       target = target.parentNode;
     }
@@ -527,13 +554,18 @@ Monocle.Events.deafenForContact = function (elem, listeners) {
 }
 
 
-Monocle.Events.listenForTap = function (elem, fn) {
+Monocle.Events.listenForTap = function (elem, fn, activeClass) {
   var startPos;
 
   if (Monocle.Browser.on.Kindle3) {
     Monocle.Events.listen(elem, 'click', function () {});
   }
 
+  var annul = function () {
+    startPos = null;
+    if (activeClass && elem.dom) { elem.dom.removeClass(activeClass); }
+  }
+
   var annulIfOutOfBounds = function (evt) {
     if (evt.type.match(/^mouse/)) {
       return;
@@ -545,7 +577,7 @@ Monocle.Events.listenForTap = function (elem, fn) {
       evt.m.registrantX < 0 || evt.m.registrantX > elem.offsetWidth ||
       evt.m.registrantY < 0 || evt.m.registrantY > elem.offsetHeight
     ) {
-      startPos = null;
+      annul();
     } else {
       evt.preventDefault();
     }
@@ -557,6 +589,7 @@ Monocle.Events.listenForTap = function (elem, fn) {
       start: function (evt) {
         startPos = [evt.m.pageX, evt.m.pageY];
         evt.preventDefault();
+        if (activeClass && elem.dom) { elem.dom.addClass(activeClass); }
       },
       move: annulIfOutOfBounds,
       end: function (evt) {
@@ -565,10 +598,9 @@ Monocle.Events.listenForTap = function (elem, fn) {
           evt.m.startOffset = startPos;
           fn(evt);
         }
+        annul();
       },
-      cancel: function (evt) {
+      cancel: annul
-        startPos = null;
-      }
     },
     {
       useCapture: false
@@ -997,6 +1029,9 @@ Monocle.Reader = function (node, bookData, options, onLoadCallback) {
     createReaderElements();
 
     p.defaultStyles = addPageStyles(k.DEFAULT_STYLE_RULES, false);
+    if (options.stylesheet) {
+      p.initialStyles = addPageStyles(options.stylesheet, false);
+    }
 
     primeFrames(options.primeURL, function () {
       applyStyles();
@@ -1077,6 +1112,7 @@ Monocle.Reader = function (node, bookData, options, onLoadCallback) {
       if (Monocle.Browser.is.WebKit) {
         frame.contentDocument.documentElement.style.overflow = "hidden";
       }
+      dispatchEvent('monocle:frameprimed', { frame: frame, pageIndex: pageCount });
       if ((pageCount += 1) == pageMax) {
         Monocle.defer(callback);
       }
@@ -1131,6 +1167,7 @@ Monocle.Reader = function (node, bookData, options, onLoadCallback) {
     var pageCount = 0;
     if (typeof callback == 'function') {
       var watcher = function (evt) {
+        dispatchEvent('monocle:firstcomponentchange', evt.m);
         if ((pageCount += 1) == p.flipper.pageCount) {
           deafen('monocle:componentchange', watcher);
           callback();
@@ -1239,7 +1276,7 @@ Monocle.Reader = function (node, bookData, options, onLoadCallback) {
         page.appendChild(runner);
         ctrlData.elements.push(runner);
       }
-    } else if (cType == "modal" || cType == "popover") {
+    } else if (cType == "modal" || cType == "popover" || cType == "hud") {
       ctrlElem = ctrl.createControlElements(overlay);
       overlay.appendChild(ctrlElem);
       ctrlData.elements.push(ctrlElem);
@@ -1312,24 +1349,33 @@ Monocle.Reader = function (node, bookData, options, onLoadCallback) {
     var controlData = dataForControl(ctrl);
     if (!controlData) {
       console.warn("No data for control: " + ctrl);
-      return;
+      return false;
     }
-    if (controlData.hidden == false) {
+
-      return;
+    if (showingControl(ctrl)) {
+      return false;
     }
+
+    var overlay = dom.find('overlay');
+    if (controlData.usesOverlay && controlData.controlType != "hud") {
+      for (var i = 0, ii = p.controls.length; i < ii; ++i) {
+        if (p.controls[i].usesOverlay && !p.controls[i].hidden) {
+          return false;
+        }
+      }
+      overlay.style.display = "block";
+    }
+
     for (var i = 0; i < controlData.elements.length; ++i) {
       controlData.elements[i].style.display = "block";
     }
-    var overlay = dom.find('overlay');
+
-    if (controlData.usesOverlay) {
-      overlay.style.display = "block";
-    }
     if (controlData.controlType == "popover") {
       overlay.listeners = Monocle.Events.listenForContact(
         overlay,
         {
           start: function (evt) {
-            obj = evt.target || window.event.srcElement;
+            var obj = evt.target || window.event.srcElement;
             do {
               if (obj == controlData.elements[0]) { return true; }
             } while (obj && (obj = obj.parentNode));
@@ -1346,22 +1392,18 @@ Monocle.Reader = function (node, bookData, options, onLoadCallback) {
       ctrl.properties.hidden = false;
     }
     dispatchEvent('controlshow', ctrl, false);
+    return true;
+  }
+
+
+  function showingControl(ctrl) {
+    var controlData = dataForControl(ctrl);
+    return controlData.hidden == false;
   }
 
 
   function dispatchEvent(evtType, data, cancelable) {
-    if (!document.createEvent) {
+    return Monocle.Events.dispatch(dom.find('box'), evtType, data, cancelable);
-      return true;
-    }
-    var evt = document.createEvent("Events");
-    evt.initEvent(evtType, false, cancelable || false);
-    evt.m = data;
-    try {
-      return dom.find('box').dispatchEvent(evt);
-    } catch(e) {
-      console.warn("Failed to dispatch event: " + evtType);
-      return false;
-    }
   }
 
 
@@ -1502,6 +1544,7 @@ Monocle.Reader = function (node, bookData, options, onLoadCallback) {
   API.addControl = addControl;
   API.hideControl = hideControl;
   API.showControl = showControl;
+  API.showingControl = showingControl;
   API.dispatchEvent = dispatchEvent;
   API.listen = listen;
   API.deafen = deafen;
@@ -1527,22 +1570,32 @@ Monocle.Reader.DEFAULT_CLASS_PREFIX = 'monelem_'
 Monocle.Reader.FLIPPER_DEFAULT_CLASS = "Slider";
 Monocle.Reader.FLIPPER_LEGACY_CLASS = "Legacy";
 Monocle.Reader.DEFAULT_STYLE_RULES = [
-  "html * {" +
+  "html#RS\\:monocle * {" +
+    "-webkit-font-smoothing: subpixel-antialiased;" +
     "text-rendering: auto !important;" +
     "word-wrap: break-word !important;" +
+    "overflow: visible !important;" +
     (Monocle.Browser.has.floatColumnBug ? "float: none !important;" : "") +
-  "}" +
+  "}",
-  "body {" +
+  "html#RS\\:monocle body {" +
     "margin: 0 !important;" +
     "padding: 0 !important;" +
     "-webkit-text-size-adjust: none;" +
-  "}" +
+  "}",
-  "table, img {" +
+  "html#RS\\:monocle body * {" +
     "max-width: 100% !important;" +
-    "max-height: 90% !important;" +
+  "}",
+  "html#RS\\:monocle img, html#RS\\:monocle video, html#RS\\:monocle object {" +
+    "max-height: 95% !important;" +
   "}"
 ]
 
+if (Monocle.Browser.has.columnOverflowPaintBug) {
+  Monocle.Reader.DEFAULT_STYLE_RULES.push(
+    "::-webkit-scrollbar { width: 0; height: 0; }"
+  )
+}
+
 
 Monocle.pieceLoaded('reader');
 /* BOOK */
@@ -1586,6 +1639,16 @@ Monocle.Book = function (dataSource) {
       locus.load = true;
       locus.componentId = p.componentIds[0];
       return locus;
+    } else if (
+      cIndex < 0 &&
+      locus.componentId &&
+      currComponent.properties.id != locus.componentId
+    ) {
+      pageDiv.m.reader.dispatchEvent(
+        "monocle:notfound",
+        { href: locus.componentId }
+      );
+      return null;
     } else if (cIndex < 0) {
       component = currComponent;
       locus.componentId = pageDiv.m.activeFrame.m.component.properties.id;
@@ -1619,6 +1682,8 @@ Monocle.Book = function (dataSource) {
       result.page += locus.direction;
     } else if (typeof(locus.anchor) == "string") {
       result.page = component.pageForChapter(locus.anchor, pageDiv);
+    } else if (typeof(locus.xpath) == "string") {
+      result.page = component.pageForXPath(locus.xpath, pageDiv);
     } else if (typeof(locus.position) == "string") {
       if (locus.position == "start") {
         result.page = 1;
@@ -1638,6 +1703,7 @@ Monocle.Book = function (dataSource) {
     if (result.page < 1) {
       if (cIndex == 0) {
         result.page = 1;
+        result.boundarystart = true;
       } else {
         result.load = true;
         result.componentId = p.componentIds[cIndex - 1];
@@ -1647,6 +1713,7 @@ Monocle.Book = function (dataSource) {
     } else if (result.page > lastPageNum['new']) {
       if (cIndex == p.lastCIndex) {
         result.page = lastPageNum['new'];
+        result.boundaryend = true;
       } else {
         result.load = true;
         result.componentId = p.componentIds[cIndex + 1];
@@ -1660,18 +1727,25 @@ Monocle.Book = function (dataSource) {
 
   function setPageAt(pageDiv, locus) {
     locus = pageNumberAt(pageDiv, locus);
-    if (!locus.load) {
+    if (locus && !locus.load) {
-      var component = p.components[p.componentIds.indexOf(locus.componentId)];
+      var evtData = { locus: locus, page: pageDiv }
-      pageDiv.m.place = pageDiv.m.place || new Monocle.Place();
+      if (locus.boundarystart) {
-      pageDiv.m.place.setPlace(component, locus.page);
+        pageDiv.m.reader.dispatchEvent('monocle:boundarystart', evtData);
+      } else if (locus.boundaryend) {
+        pageDiv.m.reader.dispatchEvent('monocle:boundaryend', evtData);
+      } else {
+        var component = p.components[p.componentIds.indexOf(locus.componentId)];
+        pageDiv.m.place = pageDiv.m.place || new Monocle.Place();
+        pageDiv.m.place.setPlace(component, locus.page);
 
-      var evtData = {
+        var evtData = {
-        page: pageDiv,
+          page: pageDiv,
-        locus: locus,
+          locus: locus,
-        pageNumber: pageDiv.m.place.pageNumber(),
+          pageNumber: pageDiv.m.place.pageNumber(),
-        componentId: locus.componentId
+          componentId: locus.componentId
+        }
+        pageDiv.m.reader.dispatchEvent("monocle:pagechange", evtData);
       }
-      pageDiv.m.reader.dispatchEvent("monocle:pagechange", evtData);
     }
     return locus;
   }
@@ -1683,6 +1757,10 @@ Monocle.Book = function (dataSource) {
       locus = pageNumberAt(pageDiv, locus);
     }
 
+    if (!locus) {
+      return;
+    }
+
     if (!locus.load) {
       callback(locus);
       return;
@@ -1690,7 +1768,9 @@ Monocle.Book = function (dataSource) {
 
     var findPageNumber = function () {
       locus = setPageAt(pageDiv, locus);
-      if (locus.load) {
+      if (!locus) {
+        return;
+      } else if (locus.load) {
         loadPageAt(pageDiv, locus, callback, progressCallback)
       } else {
         callback(locus);
@@ -1715,10 +1795,12 @@ Monocle.Book = function (dataSource) {
   }
 
 
-  function setOrLoadPageAt(pageDiv, locus, callback, progressCallback) {
+  function setOrLoadPageAt(pageDiv, locus, callback, onProgress, onFail) {
     locus = setPageAt(pageDiv, locus);
-    if (locus.load) {
+    if (!locus) {
-      loadPageAt(pageDiv, locus, callback, progressCallback);
+      if (onFail) { onFail(); }
+    } else if (locus.load) {
+      loadPageAt(pageDiv, locus, callback, onProgress);
     } else {
       callback(locus);
     }
@@ -1864,13 +1946,18 @@ Monocle.Place = function () {
   }
 
 
-  function percentageThrough() {
+  function percentAtTopOfPage() {
+    return p.percent - 1.0 / p.component.lastPageNumber();
+  }
+
+
+  function percentAtBottomOfPage() {
     return p.percent;
   }
 
 
-  function pageAtPercentageThrough(pc) {
+  function pageAtPercentageThrough(percent) {
-    return Math.max(Math.round(p.component.lastPageNumber() * pc), 1);
+    return Math.max(Math.round(p.component.lastPageNumber() * percent), 1);
   }
 
 
@@ -1911,6 +1998,8 @@ Monocle.Place = function () {
     }
     if (options.direction) {
       locus.page += options.direction;
+    } else {
+      locus.percent = percentAtBottomOfPage();
     }
     return locus;
   }
@@ -1942,7 +2031,9 @@ Monocle.Place = function () {
   API.setPlace = setPlace;
   API.setPercentageThrough = setPercentageThrough;
   API.componentId = componentId;
-  API.percentageThrough = percentageThrough;
+  API.percentAtTopOfPage = percentAtTopOfPage;
+  API.percentAtBottomOfPage = percentAtBottomOfPage;
+  API.percentageThrough = percentAtBottomOfPage;
   API.pageAtPercentageThrough = pageAtPercentageThrough;
   API.pageNumber = pageNumber;
   API.chapterInfo = chapterInfo;
@@ -2158,11 +2249,13 @@ Monocle.Component = function (book, id, index, chapters, source) {
     if (p.chapters[0] && typeof p.chapters[0].percent == "number") {
       return;
     }
+    var doc = pageDiv.m.activeFrame.contentDocument;
     for (var i = 0; i < p.chapters.length; ++i) {
       var chp = p.chapters[i];
       chp.percent = 0;
       if (chp.fragment) {
-        chp.percent = pageDiv.m.dimensions.percentageThroughOfId(chp.fragment);
+        var node = doc.getElementById(chp.fragment);
+        chp.percent = pageDiv.m.dimensions.percentageThroughOfNode(node);
       }
     }
     return p.chapters;
@@ -2187,14 +2280,37 @@ Monocle.Component = function (book, id, index, chapters, source) {
     if (!fragment) {
       return 1;
     }
-    var pc2pn = function (pc) { return Math.floor(pc * p.pageLength) + 1 }
     for (var i = 0; i < p.chapters.length; ++i) {
       if (p.chapters[i].fragment == fragment) {
-        return pc2pn(p.chapters[i].percent);
+        return percentToPageNumber(p.chapters[i].percent);
       }
     }
-    var percent = pageDiv.m.dimensions.percentageThroughOfId(fragment);
+    var doc = pageDiv.m.activeFrame.contentDocument;
-    return pc2pn(percent);
+    var node = doc.getElementById(fragment);
+    var percent = pageDiv.m.dimensions.percentageThroughOfNode(node);
+    return percentToPageNumber(percent);
+  }
+
+
+  function pageForXPath(xpath, pageDiv) {
+    var doc = pageDiv.m.activeFrame.contentDocument;
+    var percent = 0;
+    if (typeof doc.evaluate == "function") {
+      var node = doc.evaluate(
+        xpath,
+        doc,
+        null,
+        9,
+        null
+      ).singleNodeValue;
+      var percent = pageDiv.m.dimensions.percentageThroughOfNode(node);
+    }
+    return percentToPageNumber(percent);
+  }
+
+
+  function percentToPageNumber(pc) {
+    return Math.floor(pc * p.pageLength) + 1;
   }
 
 
@@ -2207,6 +2323,7 @@ Monocle.Component = function (book, id, index, chapters, source) {
   API.updateDimensions = updateDimensions;
   API.chapterForPage = chapterForPage;
   API.pageForChapter = pageForChapter;
+  API.pageForXPath = pageForXPath;
   API.lastPageNumber = lastPageNumber;
 
   return API;
@@ -2415,9 +2532,11 @@ Monocle.Dimensions.Vert = function (pageDiv) {
   }
 
 
-  function percentageThroughOfId(id) {
+  function percentageThroughOfNode(target) {
+    if (!target) {
+      return 0;
+    }
     var doc = p.page.m.activeFrame.contentDocument;
-    var target = doc.getElementById(id);
     var offset = 0;
     if (target.getBoundingClientRect) {
       offset = target.getBoundingClientRect().top;
@@ -2456,7 +2575,7 @@ Monocle.Dimensions.Vert = function (pageDiv) {
   API.hasChanged = hasChanged;
   API.measure = measure;
   API.pages = pages;
-  API.percentageThroughOfId = percentageThroughOfId;
+  API.percentageThroughOfNode = percentageThroughOfNode;
   API.locusToOffset = locusToOffset;
 
   initialize();
@@ -2713,8 +2832,7 @@ Monocle.Dimensions.Columns = function (pageDiv) {
       (!p.measurements) ||
       (p.measurements.width != newMeasurements.width) ||
       (p.measurements.height != newMeasurements.height) ||
-      (p.measurements.scrollWidth != newMeasurements.scrollWidth) ||
+      (p.measurements.scrollWidth != newMeasurements.scrollWidth)
-      (p.measurements.fontSize != newMeasurements.fontSize)
     );
   }
 
@@ -2736,10 +2854,16 @@ Monocle.Dimensions.Columns = function (pageDiv) {
       if (!lc || !lc.getBoundingClientRect) {
         console.warn('Empty document for page['+p.page.m.pageIndex+']');
         p.measurements.scrollWidth = p.measurements.width;
-      } else if (lc.getBoundingClientRect().bottom > p.measurements.height) {
-        p.measurements.scrollWidth = p.measurements.width * 2;
       } else {
-        p.measurements.scrollWidth = p.measurements.width;
+        var bcr = lc.getBoundingClientRect();
+        if (
+          bcr.right > p.measurements.width ||
+          bcr.bottom > p.measurements.height
+        ) {
+          p.measurements.scrollWidth = p.measurements.width * 2;
+        } else {
+          p.measurements.scrollWidth = p.measurements.width;
+        }
       }
     }
 
@@ -2758,12 +2882,11 @@ Monocle.Dimensions.Columns = function (pageDiv) {
   }
 
 
-  function percentageThroughOfId(id) {
+  function percentageThroughOfNode(target) {
-    var doc = p.page.m.activeFrame.contentDocument;
-    var target = doc.getElementById(id);
     if (!target) {
       return 0;
     }
+    var doc = p.page.m.activeFrame.contentDocument;
     var offset = 0;
     if (target.getBoundingClientRect) {
       offset = target.getBoundingClientRect().left;
@@ -2785,20 +2908,30 @@ Monocle.Dimensions.Columns = function (pageDiv) {
   function componentChanged(evt) {
     if (evt.m['page'] != p.page) { return; }
     var doc = evt.m['document'];
-    Monocle.Styles.applyRules(doc.body, k.BODY_STYLES);
+    if (Monocle.Browser.has.columnOverflowPaintBug) {
+      var div = doc.createElement('div');
+      Monocle.Styles.applyRules(div, k.BODY_STYLES);
+      div.style.cssText += "overflow: scroll !important;";
+      while (doc.body.childNodes.length) {
+        div.appendChild(doc.body.firstChild);
+      }
+      doc.body.appendChild(div);
+    } else {
+      Monocle.Styles.applyRules(doc.body, k.BODY_STYLES);
 
-    if (Monocle.Browser.is.WebKit) {
+      if (Monocle.Browser.is.WebKit) {
-      doc.documentElement.style.overflow = 'hidden';
+        doc.documentElement.style.overflow = 'hidden';
+      }
     }
+
     p.dirty = true;
   }
 
 
   function setColumnWidth() {
     var cw = p.page.m.sheafDiv.clientWidth;
-    var doc = p.page.m.activeFrame.contentDocument;
     if (currBodyStyleValue('column-width') != cw+"px") {
-      Monocle.Styles.affix(doc.body, 'column-width', cw+"px");
+      Monocle.Styles.affix(columnedElement(), 'column-width', cw+"px");
       p.dirty = true;
     }
   }
@@ -2809,8 +2942,7 @@ Monocle.Dimensions.Columns = function (pageDiv) {
     return {
       width: sheaf.clientWidth,
       height: sheaf.clientHeight,
-      scrollWidth: scrollerWidth(),
+      scrollWidth: scrollerWidth()
-      fontSize: currBodyStyleValue('font-size')
     }
   }
 
@@ -2819,16 +2951,24 @@ Monocle.Dimensions.Columns = function (pageDiv) {
     if (Monocle.Browser.has.mustScrollSheaf) {
       return p.page.m.sheafDiv;
     } else {
-      return p.page.m.activeFrame.contentDocument.body;
+      return columnedElement();
     }
   }
 
 
+  function columnedElement() {
+    var elem = p.page.m.activeFrame.contentDocument.body;
+    return Monocle.Browser.has.columnOverflowPaintBug ? elem.firstChild : elem;
+  }
+
+
   function scrollerWidth() {
     var bdy = p.page.m.activeFrame.contentDocument.body;
     if (Monocle.Browser.has.iframeDoubleWidthBug) {
-      if (Monocle.Browser.on.Android) {
+      if (Monocle.Browser.on.Kindle3) {
-        return bdy.scrollWidth * 1.5; // I actually have no idea why 1.5.
+        return scrollerElement().scrollWidth;
+      } else if (Monocle.Browser.on.Android) {
+        return bdy.scrollWidth;
       } else if (Monocle.Browser.iOSVersion < "4.1") {
         var hbw = bdy.scrollWidth / 2;
         var sew = scrollerElement().scrollWidth;
@@ -2838,15 +2978,18 @@ Monocle.Dimensions.Columns = function (pageDiv) {
         var hbw = bdy.scrollWidth / 2;
         return hbw;
       }
-    } else if (Monocle.Browser.is.Gecko) {
+    } else if (bdy.getBoundingClientRect) {
-      var lc = bdy.lastChild;
+      var elems = bdy.getElementsByTagName('*');
-      while (lc && lc.nodeType != 1) {
+      var bdyRect = bdy.getBoundingClientRect();
-        lc = lc.previousSibling;
+      var l = bdyRect.left, r = bdyRect.right;
-      }
+      for (var i = elems.length - 1; i >= 0; --i) {
-      if (lc && lc.getBoundingClientRect) {
+        var rect = elems[i].getBoundingClientRect();
-        return lc.getBoundingClientRect().right;
+        l = Math.min(l, rect.left);
+        r = Math.max(r, rect.right);
       }
+      return Math.abs(l) + Math.abs(r);
     }
+
     return scrollerElement().scrollWidth;
   }
 
@@ -2867,8 +3010,14 @@ Monocle.Dimensions.Columns = function (pageDiv) {
 
   function translateToLocus(locus) {
     var offset = locusToOffset(locus);
-    var bdy = p.page.m.activeFrame.contentDocument.body;
+    p.page.m.offset = 0 - offset;
-    Monocle.Styles.affix(bdy, "transform", "translateX("+offset+"px)");
+    if (k.SETX && !Monocle.Browser.has.columnOverflowPaintBug) {
+      var bdy = p.page.m.activeFrame.contentDocument.body;
+      Monocle.Styles.affix(bdy, "transform", "translateX("+offset+"px)");
+    } else {
+      var scrElem = scrollerElement();
+      scrElem.scrollLeft = 0 - offset;
+    }
     return offset;
   }
 
@@ -2876,7 +3025,7 @@ Monocle.Dimensions.Columns = function (pageDiv) {
   API.hasChanged = hasChanged;
   API.measure = measure;
   API.pages = pages;
-  API.percentageThroughOfId = percentageThroughOfId;
+  API.percentageThroughOfNode = percentageThroughOfNode;
 
   API.locusToOffset = locusToOffset;
   API.translateToLocus = translateToLocus;
@@ -2898,6 +3047,8 @@ Monocle.Dimensions.Columns.BODY_STYLES = {
   "column-fill": "auto"
 }
 
+Monocle.Dimensions.Columns.SETX = true; // Set to false for scrollLeft.
+
 if (Monocle.Browser.has.iframeDoubleWidthBug) {
   Monocle.Dimensions.Columns.BODY_STYLES["min-width"] = "200%";
 } else {
@@ -2924,6 +3075,8 @@ Monocle.Flippers.Slider = function (reader) {
 
   function addPage(pageDiv) {
     pageDiv.m.dimensions = new Monocle.Dimensions.Columns(pageDiv);
+
+    Monocle.Styles.setX(pageDiv, "0px");
   }
 
 
@@ -2963,6 +3116,7 @@ Monocle.Flippers.Slider = function (reader) {
 
 
   function interactiveMode(bState) {
+    p.reader.dispatchEvent('monocle:interactive:'+(bState ? 'on' : 'off'));
     if (!Monocle.Browser.has.selectThruBug) {
       return;
     }
@@ -2994,10 +3148,10 @@ Monocle.Flippers.Slider = function (reader) {
 
   function moveTo(locus, callback) {
     var fn = function () {
-      prepareNextPage(announceTurn);
+      prepareNextPage(function () {
-      if (typeof callback == "function") {
+        if (typeof callback == "function") { callback(); }
-        callback();
+        announceTurn();
-      }
+      });
     }
     setPage(upperPage(), locus, fn);
   }
@@ -3045,12 +3199,26 @@ Monocle.Flippers.Slider = function (reader) {
 
     if (dir == k.FORWARDS) {
       if (getPlace().onLastPageOfBook()) {
+        p.reader.dispatchEvent(
+          'monocle:boundaryend',
+          {
+            locus: getPlace().getLocus({ direction : dir }),
+            page: upperPage()
+          }
+        );
         resetTurnData();
         return;
       }
       onGoingForward(boxPointX);
     } else if (dir == k.BACKWARDS) {
       if (getPlace().onFirstPageOfBook()) {
+        p.reader.dispatchEvent(
+          'monocle:boundarystart',
+          {
+            locus: getPlace().getLocus({ direction : dir }),
+            page: upperPage()
+          }
+        );
         resetTurnData();
         return;
       }
@@ -3215,14 +3383,14 @@ Monocle.Flippers.Slider = function (reader) {
 
 
   function announceTurn() {
-    hideWaitControl(upperPage());
-    hideWaitControl(lowerPage());
     p.reader.dispatchEvent('monocle:turn');
     resetTurnData();
   }
 
 
   function resetTurnData() {
+    hideWaitControl(upperPage());
+    hideWaitControl(lowerPage());
     p.turnData = {};
   }
 
@@ -3268,7 +3436,7 @@ Monocle.Flippers.Slider = function (reader) {
           (new Date()).getTime() - stamp > duration ||
           Math.abs(currX - finalX) <= Math.abs((currX + step) - finalX)
         ) {
-          clearTimeout(elem.setXTransitionInterval)
+          clearTimeout(elem.setXTransitionInterval);
           Monocle.Styles.setX(elem, finalX);
           if (elem.setXTCB) {
             elem.setXTCB();
@@ -3366,13 +3534,17 @@ Monocle.Flippers.Slider = function (reader) {
 
   function jumpIn(pageDiv, callback) {
     var dur = Monocle.Browser.has.jumpFlickerBug ? 1 : 0;
-    setX(pageDiv, 0, { duration: dur }, callback);
+    Monocle.defer(function () {
+      setX(pageDiv, 0, { duration: dur }, callback);
+    });
   }
 
 
   function jumpOut(pageDiv, callback) {
     var dur = Monocle.Browser.has.jumpFlickerBug ? 1 : 0;
-    setX(pageDiv, 0 - pageDiv.offsetWidth, { duration: dur }, callback);
+    Monocle.defer(function () {
+      setX(pageDiv, 0 - pageDiv.offsetWidth, { duration: dur }, callback);
+    });
   }
 
 
@@ -3382,7 +3554,9 @@ Monocle.Flippers.Slider = function (reader) {
       duration: k.durations.SLIDE,
       timing: 'ease-in'
     };
-    setX(upperPage(), 0, slideOpts, callback);
+    Monocle.defer(function () {
+      setX(upperPage(), 0, slideOpts, callback);
+    });
   }
 
 
@@ -3391,7 +3565,9 @@ Monocle.Flippers.Slider = function (reader) {
       duration: k.durations.SLIDE,
       timing: 'ease-in'
     };
-    setX(upperPage(), 0 - upperPage().offsetWidth, slideOpts, callback);
+    Monocle.defer(function () {
+      setX(upperPage(), 0 - upperPage().offsetWidth, slideOpts, callback);
+    });
   }
 
 
@@ -3418,13 +3594,13 @@ Monocle.Flippers.Slider = function (reader) {
 
   function showWaitControl(page) {
     var ctrl = p.reader.dom.find('flippers_slider_wait', page.m.pageIndex);
-    ctrl.style.opacity = 0.5;
+    ctrl.style.visibility = "visible";
   }
 
 
   function hideWaitControl(page) {
     var ctrl = p.reader.dom.find('flippers_slider_wait', page.m.pageIndex);
-    ctrl.style.opacity = 0;
+    ctrl.style.visibility = "hidden";
   }
 
   API.pageCount = p.pageCount;

resources/metadata_sqlite.sql

@@ -13,8 +13,10 @@ CREATE TABLE books ( id      INTEGER PRIMARY KEY AUTOINCREMENT,
                              isbn TEXT DEFAULT "" COLLATE NOCASE,
                              lccn TEXT DEFAULT "" COLLATE NOCASE,
                              path TEXT NOT NULL DEFAULT "",
-                             flags INTEGER NOT NULL DEFAULT 1
+                             flags INTEGER NOT NULL DEFAULT 1,
-                        , uuid TEXT, has_cover BOOL DEFAULT 0, last_modified TIMESTAMP NOT NULL DEFAULT "2000-01-01 00:00:00+00:00");
+                             uuid TEXT,
+                             has_cover BOOL DEFAULT 0,
+                             last_modified TIMESTAMP NOT NULL DEFAULT "2000-01-01 00:00:00+00:00");
 CREATE TABLE books_authors_link ( id INTEGER PRIMARY KEY,
                                           book INTEGER NOT NULL,
                                           author INTEGER NOT NULL,

src/calibre/__init__.py

@@ -578,6 +578,7 @@ def url_slash_cleaner(url):
 def get_download_filename(url, cookie_file=None):
     '''
     Get a local filename for a URL using the content disposition header
+    Returns empty string if no content disposition header present
     '''
     from contextlib import closing
     from urllib2 import unquote as urllib2_unquote
@@ -591,8 +592,10 @@ def get_download_filename(url, cookie_file=None):
         cj.load(cookie_file)
         br.set_cookiejar(cj)
 
+    last_part_name = ''
     try:
         with closing(br.open(url)) as r:
+            last_part_name = r.geturl().split('/')[-1]
             disposition = r.info().get('Content-disposition', '')
             for p in disposition.split(';'):
                 if 'filename' in p:
@@ -612,7 +615,7 @@ def get_download_filename(url, cookie_file=None):
         traceback.print_exc()
 
     if not filename:
-        filename = r.geturl().split('/')[-1]
+        filename = last_part_name
 
     return filename
 

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 8, 5)
+numeric_version = (0, 8, 7)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 

src/calibre/customize/builtins.py

@@ -762,99 +762,132 @@ plugins += input_profiles + output_profiles
 class ActionAdd(InterfaceActionBase):
     name = 'Add Books'
     actual_plugin = 'calibre.gui2.actions.add:AddAction'
+    description = _('Add books to calibre or the connected device')
 
 class ActionFetchAnnotations(InterfaceActionBase):
     name = 'Fetch Annotations'
     actual_plugin = 'calibre.gui2.actions.annotate:FetchAnnotationsAction'
+    description = _('Fetch annotations from a connected Kindle (experimental)')
 
 class ActionGenerateCatalog(InterfaceActionBase):
     name = 'Generate Catalog'
     actual_plugin = 'calibre.gui2.actions.catalog:GenerateCatalogAction'
+    description = _('Generate a catalog of the books in your calibre library')
 
 class ActionConvert(InterfaceActionBase):
     name = 'Convert Books'
     actual_plugin = 'calibre.gui2.actions.convert:ConvertAction'
+    description = _('Convert books to various ebook formats')
 
 class ActionDelete(InterfaceActionBase):
     name = 'Remove Books'
     actual_plugin = 'calibre.gui2.actions.delete:DeleteAction'
+    description = _('Delete books from your calibre library or connected device')
 
 class ActionEditMetadata(InterfaceActionBase):
     name = 'Edit Metadata'
     actual_plugin = 'calibre.gui2.actions.edit_metadata:EditMetadataAction'
+    description = _('Edit the metadata of books in your calibre library')
 
 class ActionView(InterfaceActionBase):
     name = 'View'
     actual_plugin = 'calibre.gui2.actions.view:ViewAction'
+    description = _('Read books in your calibre library')
 
 class ActionFetchNews(InterfaceActionBase):
     name = 'Fetch News'
     actual_plugin = 'calibre.gui2.actions.fetch_news:FetchNewsAction'
+    description = _('Download news from the internet in ebook form')
+
+class ActionQuickview(InterfaceActionBase):
+    name = 'Show Quickview'
+    actual_plugin = 'calibre.gui2.actions.show_quickview:ShowQuickviewAction'
+    description = _('Show a list of related books quickly')
 
 class ActionSaveToDisk(InterfaceActionBase):
     name = 'Save To Disk'
     actual_plugin = 'calibre.gui2.actions.save_to_disk:SaveToDiskAction'
+    description = _('Export books from your calibre library to the hard disk')
 
 class ActionShowBookDetails(InterfaceActionBase):
     name = 'Show Book Details'
     actual_plugin = 'calibre.gui2.actions.show_book_details:ShowBookDetailsAction'
+    description = _('Show book details in a separate popup')
 
 class ActionRestart(InterfaceActionBase):
     name = 'Restart'
     actual_plugin = 'calibre.gui2.actions.restart:RestartAction'
+    description = _('Restart calibre')
 
 class ActionOpenFolder(InterfaceActionBase):
     name = 'Open Folder'
     actual_plugin = 'calibre.gui2.actions.open:OpenFolderAction'
+    description = _('Open the folder that contains the book files in your'
+            ' calibre library')
 
 class ActionSendToDevice(InterfaceActionBase):
     name = 'Send To Device'
     actual_plugin = 'calibre.gui2.actions.device:SendToDeviceAction'
+    description = _('Send books to the connected device')
 
 class ActionConnectShare(InterfaceActionBase):
     name = 'Connect Share'
     actual_plugin = 'calibre.gui2.actions.device:ConnectShareAction'
+    description = _('Send books via email or the web also connect to iTunes or'
+            ' folders on your computer as if they are devices')
 
 class ActionHelp(InterfaceActionBase):
     name = 'Help'
     actual_plugin = 'calibre.gui2.actions.help:HelpAction'
+    description = _('Browse the calibre User Manual')
 
 class ActionPreferences(InterfaceActionBase):
     name = 'Preferences'
     actual_plugin = 'calibre.gui2.actions.preferences:PreferencesAction'
+    description = _('Customize calibre')
 
 class ActionSimilarBooks(InterfaceActionBase):
     name = 'Similar Books'
     actual_plugin = 'calibre.gui2.actions.similar_books:SimilarBooksAction'
+    description = _('Easily find books similar to the currently selected one')
 
 class ActionChooseLibrary(InterfaceActionBase):
     name = 'Choose Library'
     actual_plugin = 'calibre.gui2.actions.choose_library:ChooseLibraryAction'
+    description = _('Switch between different calibre libraries and perform'
+            ' maintenance on them')
 
 class ActionAddToLibrary(InterfaceActionBase):
     name = 'Add To Library'
     actual_plugin = 'calibre.gui2.actions.add_to_library:AddToLibraryAction'
+    description = _('Copy books from the devce to your calibre library')
 
 class ActionEditCollections(InterfaceActionBase):
     name = 'Edit Collections'
     actual_plugin = 'calibre.gui2.actions.edit_collections:EditCollectionsAction'
+    description = _('Edit the collections in which books are placed on your device')
 
 class ActionCopyToLibrary(InterfaceActionBase):
     name = 'Copy To Library'
     actual_plugin = 'calibre.gui2.actions.copy_to_library:CopyToLibraryAction'
+    description = _('Copy a book from one calibre library to another')
 
 class ActionTweakEpub(InterfaceActionBase):
     name = 'Tweak ePub'
     actual_plugin = 'calibre.gui2.actions.tweak_epub:TweakEpubAction'
+    description = _('Make small tweaks to epub files in your calibre library')
 
 class ActionNextMatch(InterfaceActionBase):
     name = 'Next Match'
     actual_plugin = 'calibre.gui2.actions.next_match:NextMatchAction'
+    description = _('Find the next or previous match when searching in '
+            'your calibre library in highlight mode')
 
 class ActionStore(InterfaceActionBase):
     name = 'Store'
     author = 'John Schember'
     actual_plugin = 'calibre.gui2.actions.store:StoreAction'
+    description = _('Search for books from different book sellers')
 
     def customization_help(self, gui=False):
         return 'Customize the behavior of the store search.'
@@ -867,20 +900,20 @@ class ActionStore(InterfaceActionBase):
         from calibre.gui2.store.config.store import save_settings as save
         save(config_widget)
 
-class ActionPluginUpdates(InterfaceActionBase):
+class ActionPluginUpdater(InterfaceActionBase):
-    name = 'Plugin Updates'
+    name = 'Plugin Updater'
     author = 'Grant Drake'
-    description = 'Queries the MobileRead forums for updates to plugins to install'
+    description = _('Get new calibre plugins or update your existing ones')
-    actual_plugin = 'calibre.gui2.actions.plugin_updates:PluginUpdatesAction'
+    actual_plugin = 'calibre.gui2.actions.plugin_updates:PluginUpdaterAction'
 
 plugins += [ActionAdd, ActionFetchAnnotations, ActionGenerateCatalog,
         ActionConvert, ActionDelete, ActionEditMetadata, ActionView,
-        ActionFetchNews, ActionSaveToDisk, ActionShowBookDetails,
+        ActionFetchNews, ActionSaveToDisk, ActionQuickview,
-        ActionRestart, ActionOpenFolder, ActionConnectShare,
+        ActionShowBookDetails,ActionRestart, ActionOpenFolder, ActionConnectShare,
         ActionSendToDevice, ActionHelp, ActionPreferences, ActionSimilarBooks,
         ActionAddToLibrary, ActionEditCollections, ActionChooseLibrary,
         ActionCopyToLibrary, ActionTweakEpub, ActionNextMatch, ActionStore,
-        ActionPluginUpdates]
+        ActionPluginUpdater]
 
 # }}}
 
@@ -1307,6 +1340,16 @@ class StoreLegimiStore(StoreBase):
     headquarters = 'PL'
     formats = ['EPUB']
 
+class StoreLibreDEStore(StoreBase):
+    name = 'Libri DE'
+    author = 'Charles Haley'
+    description = u'Sicher Bücher, Hörbücher und Downloads online bestellen.'
+    actual_plugin = 'calibre.gui2.store.libri_de_plugin:LibreDEStore'
+
+    headquarters = 'DE'
+    formats = ['EPUB', 'PDF']
+    affiliate = True
+
 class StoreManyBooksStore(StoreBase):
     name = 'ManyBooks'
     description = u'Public domain and creative commons works from many sources.'
@@ -1335,6 +1378,14 @@ class StoreNextoStore(StoreBase):
     formats = ['EPUB', 'PDF']
     affiliate = True
 
+class StoreOpenBooksStore(StoreBase):
+    name = 'Open Books'
+    description = u'Comprehensive listing of DRM free ebooks from a variety of sources provided by users of calibre.'
+    actual_plugin = 'calibre.gui2.store.open_books_plugin:OpenBooksStore'
+    
+    drm_free_only = True
+    headquarters = 'US'
+
 class StoreOpenLibraryStore(StoreBase):
     name = 'Open Library'
     description = u'One web page for every book ever published. The goal is to be a true online library. Over 20 million records from a variety of large catalogs as well as single contributions, with more on the way.'
@@ -1457,9 +1508,11 @@ plugins += [
     StoreGutenbergStore,
     StoreKoboStore,
     StoreLegimiStore,
+    StoreLibreDEStore,
     StoreManyBooksStore,
     StoreMobileReadStore,
     StoreNextoStore,
+    StoreOpenBooksStore,
     StoreOpenLibraryStore,
     StoreOReillyStore,
     StorePragmaticBookshelfStore,

src/calibre/customize/conversion.py

@@ -259,6 +259,10 @@ class OutputFormatPlugin(Plugin):
     #: (option_name, recommended_value, recommendation_level)
     recommendations = set([])
 
+    @property
+    def description(self):
+        return _('Convert ebooks to the %s format')%self.file_type
+
     def __init__(self, *args):
         Plugin.__init__(self, *args)
         self.report_progress = DummyReporter()

src/calibre/db/__init__.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+'''
+Rewrite of the calibre database backend.
+
+Broad Objectives:
+
+    * Use the sqlite db only as a datastore. i.e. do not do
+      sorting/searching/concatenation or anything else in sqlite. Instead
+      mirror the sqlite tables in memory, create caches and lookup maps from
+      them and create a set_* API that updates the memory caches and the sqlite
+      correctly.
+
+    * Move from keeping a list of books in memory as a cache to a per table
+      cache. This allows much faster search and sort operations at the expense
+      of slightly slower lookup operations. That slowdown can be mitigated by
+      keeping lots of maps and updating them in the set_* API. Also
+      get_categories becomes blazingly fast.
+
+    * Separate the database layer from the cache layer more cleanly. Rather
+      than having the db layer refer to the cache layer and vice versa, the
+      cache layer will refer to the db layer only and the new API will be
+      defined on the cache layer.
+
+    * Get rid of index_is_id and other poor design decisions
+
+    * Minimize the API as much as possible and define it cleanly
+
+    * Do not change the on disk format of metadata.db at all (this is for
+      backwards compatibility)
+
+    * Get rid of the need for a separate db access thread by switching to apsw
+      to access sqlite, which is thread safe
+
+    * The new API will have methods to efficiently do bulk operations and will
+      use shared/exclusive/pending locks to serialize access to the in-mem data
+      structures. Use the same locking scheme as sqlite itself does.
+
+How this will proceed:
+
+    1. Create the new API
+    2. Create a test suite for it
+    3. Write a replacement for LibraryDatabase2 that uses the new API
+       internally
+    4. Lots of testing of calibre with the new LibraryDatabase2
+    5. Gradually migrate code to use the (much faster) new api wherever possible (the new api
+       will be exposed via db.new_api)
+
+    I plan to work on this slowly, in parallel to normal calibre development
+    work.
+
+Various things that require other things before they can be migrated:
+    1. From initialize_dynamic(): set_saved_searches,
+                    load_user_template_functions. Also add custom
+                    columns/categories/searches info into
+                    self.field_metadata. Finally, implement metadata dirtied
+                    functionality.
+
+'''

src/calibre/db/backend.py

@@ -0,0 +1,443 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+# Imports {{{
+import os, shutil, uuid, json
+from functools import partial
+
+import apsw
+
+from calibre import isbytestring, force_unicode, prints
+from calibre.constants import (iswindows, filesystem_encoding,
+        preferred_encoding)
+from calibre.ptempfile import PersistentTemporaryFile
+from calibre.library.schema_upgrades import SchemaUpgrade
+from calibre.library.field_metadata import FieldMetadata
+from calibre.ebooks.metadata import title_sort, author_to_author_sort
+from calibre.utils.icu import strcmp
+from calibre.utils.config import to_json, from_json, prefs, tweaks
+from calibre.utils.date import utcfromtimestamp
+from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable,
+        SizeTable, FormatsTable, AuthorsTable, IdentifiersTable)
+# }}}
+
+'''
+Differences in semantics from pysqlite:
+
+    1. execute/executemany/executescript operate in autocommit mode
+
+'''
+
+class DynamicFilter(object): # {{{
+
+    'No longer used, present for legacy compatibility'
+
+    def __init__(self, name):
+        self.name = name
+        self.ids = frozenset([])
+
+    def __call__(self, id_):
+        return int(id_ in self.ids)
+
+    def change(self, ids):
+        self.ids = frozenset(ids)
+# }}}
+
+class DBPrefs(dict): # {{{
+
+    'Store preferences as key:value pairs in the db'
+
+    def __init__(self, db):
+        dict.__init__(self)
+        self.db = db
+        self.defaults = {}
+        self.disable_setting = False
+        for key, val in self.db.conn.get('SELECT key,val FROM preferences'):
+            try:
+                val = self.raw_to_object(val)
+            except:
+                prints('Failed to read value for:', key, 'from db')
+                continue
+            dict.__setitem__(self, key, val)
+
+    def raw_to_object(self, raw):
+        if not isinstance(raw, unicode):
+            raw = raw.decode(preferred_encoding)
+        return json.loads(raw, object_hook=from_json)
+
+    def to_raw(self, val):
+        return json.dumps(val, indent=2, default=to_json)
+
+    def __getitem__(self, key):
+        try:
+            return dict.__getitem__(self, key)
+        except KeyError:
+            return self.defaults[key]
+
+    def __delitem__(self, key):
+        dict.__delitem__(self, key)
+        self.db.conn.execute('DELETE FROM preferences WHERE key=?', (key,))
+
+    def __setitem__(self, key, val):
+        if self.disable_setting:
+            return
+        raw = self.to_raw(val)
+        self.db.conn.execute('INSERT OR REPLACE INTO preferences (key,val) VALUES (?,?)', (key,
+            raw))
+        dict.__setitem__(self, key, val)
+
+    def set(self, key, val):
+        self.__setitem__(key, val)
+
+# }}}
+
+# Extra collators {{{
+def pynocase(one, two, encoding='utf-8'):
+    if isbytestring(one):
+        try:
+            one = one.decode(encoding, 'replace')
+        except:
+            pass
+    if isbytestring(two):
+        try:
+            two = two.decode(encoding, 'replace')
+        except:
+            pass
+    return cmp(one.lower(), two.lower())
+
+def _author_to_author_sort(x):
+    if not x: return ''
+    return author_to_author_sort(x.replace('|', ','))
+
+def icu_collator(s1, s2):
+    return strcmp(force_unicode(s1, 'utf-8'), force_unicode(s2, 'utf-8'))
+# }}}
+
+class Connection(apsw.Connection): # {{{
+
+    BUSY_TIMEOUT = 2000 # milliseconds
+
+    def __init__(self, path):
+        apsw.Connection.__init__(self, path)
+
+        self.setbusytimeout(self.BUSY_TIMEOUT)
+        self.execute('pragma cache_size=5000')
+        self.conn.execute('pragma temp_store=2')
+
+        encoding = self.execute('pragma encoding').fetchone()[0]
+        self.conn.create_collation('PYNOCASE', partial(pynocase,
+            encoding=encoding))
+
+        self.conn.create_function('title_sort', 1, title_sort)
+        self.conn.create_function('author_to_author_sort', 1,
+                _author_to_author_sort)
+
+        self.conn.create_function('uuid4', 0, lambda : str(uuid.uuid4()))
+
+        # Dummy functions for dynamically created filters
+        self.conn.create_function('books_list_filter', 1, lambda x: 1)
+        self.conn.create_collation('icucollate', icu_collator)
+
+    def create_dynamic_filter(self, name):
+        f = DynamicFilter(name)
+        self.conn.create_function(name, 1, f)
+
+    def get(self, *args, **kw):
+        ans = self.cursor().execute(*args)
+        if kw.get('all', True):
+            return ans.fetchall()
+        for row in ans:
+            return ans[0]
+
+    def execute(self, sql, bindings=None):
+        cursor = self.cursor()
+        return cursor.execute(sql, bindings)
+
+    def executemany(self, sql, sequence_of_bindings):
+        return self.cursor().executemany(sql, sequence_of_bindings)
+
+    def executescript(self, sql):
+        with self:
+            # Use an explicit savepoint so that even if this is called
+            # while a transaction is active, it is atomic
+            return self.cursor().execute(sql)
+# }}}
+
+class DB(object, SchemaUpgrade):
+
+    PATH_LIMIT = 40 if iswindows else 100
+    WINDOWS_LIBRARY_PATH_LIMIT = 75
+
+    # Initialize database {{{
+
+    def __init__(self, library_path, default_prefs=None, read_only=False):
+        try:
+            if isbytestring(library_path):
+                library_path = library_path.decode(filesystem_encoding)
+        except:
+            import traceback
+            traceback.print_exc()
+
+        self.field_metadata = FieldMetadata()
+
+        self.library_path = os.path.abspath(library_path)
+        self.dbpath = os.path.join(library_path, 'metadata.db')
+        self.dbpath = os.environ.get('CALIBRE_OVERRIDE_DATABASE_PATH',
+                self.dbpath)
+
+        if iswindows and len(self.library_path) + 4*self.PATH_LIMIT + 10 > 259:
+            raise ValueError(_(
+                'Path to library too long. Must be less than'
+                ' %d characters.')%(259-4*self.PATH_LIMIT-10))
+        exists = self._exists = os.path.exists(self.dbpath)
+        if not exists:
+            # Be more strict when creating new libraries as the old calculation
+            # allowed for max path lengths of 265 chars.
+            if (iswindows and len(self.library_path) >
+                    self.WINDOWS_LIBRARY_PATH_LIMIT):
+                raise ValueError(_(
+                    'Path to library too long. Must be less than'
+                    ' %d characters.')%self.WINDOWS_LIBRARY_PATH_LIMIT)
+
+        if read_only and os.path.exists(self.dbpath):
+            # Work on only a copy of metadata.db to ensure that
+            # metadata.db is not changed
+            pt = PersistentTemporaryFile('_metadata_ro.db')
+            pt.close()
+            shutil.copyfile(self.dbpath, pt.name)
+            self.dbpath = pt.name
+
+        self.is_case_sensitive = (not iswindows and
+            not os.path.exists(self.dbpath.replace('metadata.db',
+                'MeTAdAtA.dB')))
+
+        self._conn = None
+
+        if self.user_version == 0:
+            self.initialize_database()
+
+        with self.conn:
+            SchemaUpgrade.__init__(self)
+
+        # Guarantee that the library_id is set
+        self.library_id
+
+        self.initialize_prefs(default_prefs)
+
+        # Fix legacy triggers and columns
+        self.conn.executescript('''
+        DROP TRIGGER IF EXISTS author_insert_trg;
+        CREATE TEMP TRIGGER author_insert_trg
+            AFTER INSERT ON authors
+            BEGIN
+            UPDATE authors SET sort=author_to_author_sort(NEW.name) WHERE id=NEW.id;
+        END;
+        DROP TRIGGER IF EXISTS author_update_trg;
+        CREATE TEMP TRIGGER author_update_trg
+            BEFORE UPDATE ON authors
+            BEGIN
+            UPDATE authors SET sort=author_to_author_sort(NEW.name)
+            WHERE id=NEW.id AND name <> NEW.name;
+        END;
+        UPDATE authors SET sort=author_to_author_sort(name) WHERE sort IS NULL;
+        ''')
+
+    def initialize_prefs(self, default_prefs):
+        self.prefs = DBPrefs(self)
+
+        if default_prefs is not None and not self._exists:
+            # Only apply default prefs to a new database
+            for key in default_prefs:
+                # be sure that prefs not to be copied are listed below
+                if key not in frozenset(['news_to_be_synced']):
+                    self.prefs[key] = default_prefs[key]
+            if 'field_metadata' in default_prefs:
+                fmvals = [f for f in default_prefs['field_metadata'].values()
+                                if f['is_custom']]
+                for f in fmvals:
+                    self.create_custom_column(f['label'], f['name'],
+                            f['datatype'], f['is_multiple'] is not None,
+                            f['is_editable'], f['display'])
+
+        defs = self.prefs.defaults
+        defs['gui_restriction'] = defs['cs_restriction'] = ''
+        defs['categories_using_hierarchy'] = []
+        defs['column_color_rules'] = []
+
+        # Migrate the bool tristate tweak
+        defs['bools_are_tristate'] = \
+                tweaks.get('bool_custom_columns_are_tristate', 'yes') == 'yes'
+        if self.prefs.get('bools_are_tristate') is None:
+            self.prefs.set('bools_are_tristate', defs['bools_are_tristate'])
+
+        # Migrate column coloring rules
+        if self.prefs.get('column_color_name_1', None) is not None:
+            from calibre.library.coloring import migrate_old_rule
+            old_rules = []
+            for i in range(1, 6):
+                col = self.prefs.get('column_color_name_'+str(i), None)
+                templ = self.prefs.get('column_color_template_'+str(i), None)
+                if col and templ:
+                    try:
+                        del self.prefs['column_color_name_'+str(i)]
+                        rules = migrate_old_rule(self.field_metadata, templ)
+                        for templ in rules:
+                            old_rules.append((col, templ))
+                    except:
+                        pass
+            if old_rules:
+                self.prefs['column_color_rules'] += old_rules
+
+        # Migrate saved search and user categories to db preference scheme
+        def migrate_preference(key, default):
+            oldval = prefs[key]
+            if oldval != default:
+                self.prefs[key] = oldval
+                prefs[key] = default
+            if key not in self.prefs:
+                self.prefs[key] = default
+
+        migrate_preference('user_categories', {})
+        migrate_preference('saved_searches', {})
+
+        # migrate grouped_search_terms
+        if self.prefs.get('grouped_search_terms', None) is None:
+            try:
+                ogst = tweaks.get('grouped_search_terms', {})
+                ngst = {}
+                for t in ogst:
+                    ngst[icu_lower(t)] = ogst[t]
+                self.prefs.set('grouped_search_terms', ngst)
+            except:
+                pass
+
+        # Rename any user categories with names that differ only in case
+        user_cats = self.prefs.get('user_categories', [])
+        catmap = {}
+        for uc in user_cats:
+            ucl = icu_lower(uc)
+            if ucl not in catmap:
+                catmap[ucl] = []
+            catmap[ucl].append(uc)
+        cats_changed = False
+        for uc in catmap:
+            if len(catmap[uc]) > 1:
+                prints('found user category case overlap', catmap[uc])
+                cat = catmap[uc][0]
+                suffix = 1
+                while icu_lower((cat + unicode(suffix))) in catmap:
+                    suffix += 1
+                prints('Renaming user category %s to %s'%(cat, cat+unicode(suffix)))
+                user_cats[cat + unicode(suffix)] = user_cats[cat]
+                del user_cats[cat]
+                cats_changed = True
+        if cats_changed:
+            self.prefs.set('user_categories', user_cats)
+
+    @property
+    def conn(self):
+        if self._conn is None:
+            self._conn = apsw.Connection(self.dbpath)
+            if self._exists and self.user_version == 0:
+                self._conn.close()
+                os.remove(self.dbpath)
+                self._conn = apsw.Connection(self.dbpath)
+        return self._conn
+
+    @dynamic_property
+    def user_version(self):
+        doc = 'The user version of this database'
+
+        def fget(self):
+            return self.conn.get('pragma user_version;', all=False)
+
+        def fset(self, val):
+            self.conn.execute('pragma user_version=%d'%int(val))
+
+        return property(doc=doc, fget=fget, fset=fset)
+
+    def initialize_database(self):
+        metadata_sqlite = P('metadata_sqlite.sql', data=True,
+                allow_user_override=False).decode('utf-8')
+        self.conn.executescript(metadata_sqlite)
+        if self.user_version == 0:
+            self.user_version = 1
+    # }}}
+
+    # Database layer API {{{
+
+    @classmethod
+    def exists_at(cls, path):
+        return path and os.path.exists(os.path.join(path, 'metadata.db'))
+
+    @dynamic_property
+    def library_id(self):
+        doc = ('The UUID for this library. As long as the user only operates'
+                ' on libraries with calibre, it will be unique')
+
+        def fget(self):
+            if getattr(self, '_library_id_', None) is None:
+                ans = self.conn.get('SELECT uuid FROM library_id', all=False)
+                if ans is None:
+                    ans = str(uuid.uuid4())
+                    self.library_id = ans
+                else:
+                    self._library_id_ = ans
+            return self._library_id_
+
+        def fset(self, val):
+            self._library_id_ = unicode(val)
+            self.conn.execute('''
+                    DELETE FROM library_id;
+                    INSERT INTO library_id (uuid) VALUES (?);
+                    ''', self._library_id_)
+
+        return property(doc=doc, fget=fget, fset=fset)
+
+    def last_modified(self):
+        ''' Return last modified time as a UTC datetime object '''
+        return utcfromtimestamp(os.stat(self.dbpath).st_mtime)
+
+    def read_tables(self):
+        tables = {}
+        for col in ('title', 'sort', 'author_sort', 'series_index', 'comments',
+                'timestamp', 'published', 'uuid', 'path', 'cover',
+                'last_modified'):
+            metadata = self.field_metadata[col].copy()
+            if metadata['table'] is None:
+                metadata['table'], metadata['column'] == 'books', ('has_cover'
+                        if col == 'cover' else col)
+            tables[col] = OneToOneTable(col, metadata)
+
+        for col in ('series', 'publisher', 'rating'):
+            tables[col] = ManyToOneTable(col, self.field_metadata[col].copy())
+
+        for col in ('authors', 'tags', 'formats', 'identifiers'):
+            cls = {
+                    'authors':AuthorsTable,
+                    'formats':FormatsTable,
+                    'identifiers':IdentifiersTable,
+                  }.get(col, ManyToManyTable)
+            tables[col] = cls(col, self.field_metadata[col].copy())
+
+        tables['size'] = SizeTable('size', self.field_metadata['size'].copy())
+
+        with self.conn: # Use a single transaction, to ensure nothing modifies
+                        # the db while we are reading
+            for table in tables.itervalues():
+                try:
+                    table.read()
+                except:
+                    prints('Failed to read table:', table.name)
+                    raise
+
+        return tables
+
+   # }}}
+

src/calibre/db/errors.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+class NoSuchFormat(ValueError):
+    pass
+

src/calibre/db/tables.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from datetime import datetime
+
+from dateutil.tz import tzoffset
+
+from calibre.constants import plugins
+from calibre.utils.date import parse_date, local_tz
+from calibre.ebooks.metadata import author_to_author_sort
+
+_c_speedup = plugins['speedup'][0]
+
+def _c_convert_timestamp(val):
+    if not val:
+        return None
+    try:
+        ret = _c_speedup.parse_date(val.strip())
+    except:
+        ret = None
+    if ret is None:
+        return parse_date(val, as_utc=False)
+    year, month, day, hour, minutes, seconds, tzsecs = ret
+    return datetime(year, month, day, hour, minutes, seconds,
+                tzinfo=tzoffset(None, tzsecs)).astimezone(local_tz)
+
+class Table(object):
+
+    def __init__(self, name, metadata):
+        self.name, self.metadata = name, metadata
+
+        # self.adapt() maps values from the db to python objects
+        self.adapt = \
+            {
+                'datetime': _c_convert_timestamp,
+                'bool': bool
+            }.get(
+                metadata['datatype'], lambda x: x)
+        if name == 'authors':
+            # Legacy
+            self.adapt = lambda x: x.replace('|', ',') if x else None
+
+class OneToOneTable(Table):
+
+    def read(self, db):
+        self.book_col_map = {}
+        idcol = 'id' if self.metadata['table'] == 'books' else 'book'
+        for row in db.conn.execute('SELECT {0}, {1} FROM {2}'.format(idcol,
+            self.metadata['column'], self.metadata['table'])):
+            self.book_col_map[row[0]] = self.adapt(row[1])
+
+class SizeTable(OneToOneTable):
+
+    def read(self, db):
+        self.book_col_map = {}
+        for row in db.conn.execute(
+                'SELECT books.id, (SELECT MAX(uncompressed_size) FROM data '
+                'WHERE data.book=books.id) FROM books'):
+            self.book_col_map[row[0]] = self.adapt(row[1])
+
+class ManyToOneTable(Table):
+
+    def read(self, db):
+        self.id_map = {}
+        self.extra_map = {}
+        self.col_book_map = {}
+        self.book_col_map = {}
+        self.read_id_maps(db)
+        self.read_maps(db)
+
+    def read_id_maps(self, db):
+        for row in db.conn.execute('SELECT id, {0} FROM {1}'.format(
+            self.metadata['name'], self.metadata['table'])):
+            if row[1]:
+                self.id_map[row[0]] = self.adapt(row[1])
+
+    def read_maps(self, db):
+        for row in db.conn.execute(
+                'SELECT book, {0} FROM books_{1}_link'.format(
+                    self.metadata['link_column'], self.metadata['table'])):
+            if row[1] not in self.col_book_map:
+                self.col_book_map[row[1]] = []
+            self.col_book_map.append(row[0])
+            self.book_col_map[row[0]] = row[1]
+
+class ManyToManyTable(ManyToOneTable):
+
+    def read_maps(self, db):
+        for row in db.conn.execute(
+                'SELECT book, {0} FROM books_{1}_link'.format(
+                    self.metadata['link_column'], self.metadata['table'])):
+            if row[1] not in self.col_book_map:
+                self.col_book_map[row[1]] = []
+            self.col_book_map.append(row[0])
+            if row[0] not in self.book_col_map:
+                self.book_col_map[row[0]] = []
+            self.book_col_map[row[0]].append(row[1])
+
+class AuthorsTable(ManyToManyTable):
+
+    def read_id_maps(self, db):
+        for row in db.conn.execute(
+                'SELECT id, name, sort FROM authors'):
+            self.id_map[row[0]] = row[1]
+            self.extra_map[row[0]] = (row[2] if row[2] else
+                    author_to_author_sort(row[1]))
+
+class FormatsTable(ManyToManyTable):
+
+    def read_id_maps(self, db):
+        pass
+
+    def read_maps(self, db):
+        for row in db.conn.execute('SELECT book, format, name FROM data'):
+            if row[1] is not None:
+                if row[1] not in self.col_book_map:
+                    self.col_book_map[row[1]] = []
+                self.col_book_map.append(row[0])
+                if row[0] not in self.book_col_map:
+                    self.book_col_map[row[0]] = []
+                self.book_col_map[row[0]].append((row[1], row[2]))
+
+class IdentifiersTable(ManyToManyTable):
+
+    def read_id_maps(self, db):
+        pass
+
+    def read_maps(self, db):
+        for row in db.conn.execute('SELECT book, type, val FROM identifiers'):
+            if row[1] is not None and row[2] is not None:
+                if row[1] not in self.col_book_map:
+                    self.col_book_map[row[1]] = []
+                self.col_book_map.append(row[0])
+                if row[0] not in self.book_col_map:
+                    self.book_col_map[row[0]] = []
+                self.book_col_map[row[0]].append((row[1], row[2]))
+

src/calibre/devices/android/driver.py

@@ -45,8 +45,11 @@ class ANDROID(USBMS):
             0xfce : { 0xd12e : [0x0100]},
 
             # Google
-            0x18d1 : { 0x4e11 : [0x0100, 0x226, 0x227], 0x4e12: [0x0100, 0x226,
+            0x18d1 : {
-                0x227], 0x4e21: [0x0100, 0x226, 0x227], 0xb058: [0x0222]},
+                0x4e11 : [0x0100, 0x226, 0x227],
+                0x4e12: [0x0100, 0x226, 0x227],
+                0x4e21: [0x0100, 0x226, 0x227],
+                0xb058: [0x0222, 0x226, 0x227]},
 
             # Samsung
             0x04e8 : { 0x681d : [0x0222, 0x0223, 0x0224, 0x0400],
@@ -107,7 +110,7 @@ class ANDROID(USBMS):
     VENDOR_NAME      = ['HTC', 'MOTOROLA', 'GOOGLE_', 'ANDROID', 'ACER',
             'GT-I5700', 'SAMSUNG', 'DELL', 'LINUX', 'GOOGLE', 'ARCHOS',
             'TELECHIP', 'HUAWEI', 'T-MOBILE', 'SEMC', 'LGE', 'NVIDIA',
-            'GENERIC-', 'ZTE']
+            'GENERIC-', 'ZTE', 'MID']
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID',
@@ -116,7 +119,7 @@ class ANDROID(USBMS):
             'IDEOS_TABLET', 'MYTOUCH_4G', 'UMS_COMPOSITE', 'SCH-I800_CARD',
             '7', 'A956', 'A955', 'A43', 'ANDROID_PLATFORM', 'TEGRA_2',
             'MB860', 'MULTI-CARD', 'MID7015A', 'INCREDIBLE', 'A7EB', 'STREAK',
-            'MB525']
+            'MB525', 'ANDROID2.3']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD',

src/calibre/devices/apple/driver.py

@@ -5,7 +5,7 @@ __copyright__ = '2010, Gregory Riker'
 __docformat__ = 'restructuredtext en'
 
 
-import cStringIO, ctypes, datetime, os, re, shutil, subprocess, sys, tempfile, time
+import cStringIO, ctypes, datetime, os, re, sys, tempfile, time
 from calibre.constants import __appname__, __version__, DEBUG
 from calibre import fit_image, confirm_config_name
 from calibre.constants import isosx, iswindows
@@ -13,8 +13,7 @@ from calibre.devices.errors import OpenFeedback, UserFeedback
 from calibre.devices.usbms.deviceconfig import DeviceConfig
 from calibre.devices.interface import DevicePlugin
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
-from calibre.ebooks.metadata import authors_to_string, MetaInformation, \
+from calibre.ebooks.metadata import authors_to_string, MetaInformation, title_sort
-    title_sort
 from calibre.ebooks.metadata.book.base import Metadata
 from calibre.ebooks.metadata.epub import set_metadata
 from calibre.library.server.utils import strftime
@@ -107,6 +106,7 @@ class DriverBase(DeviceConfig, DevicePlugin):
     # Needed for config_widget to work
     FORMATS = ['epub', 'pdf']
     USER_CAN_ADD_NEW_FORMATS = False
+    KEEP_TEMP_FILES_AFTER_UPLOAD = True
 
     # Hide the standard customization widgets
     SUPPORTS_SUB_DIRS = False
@@ -135,7 +135,8 @@ class ITUNES(DriverBase):
     '''
     Calling sequences:
     Initialization:
-        can_handle() or can_handle_windows()
+        can_handle() | can_handle_windows()
+         _launch_iTunes()
         reset()
         open()
         card_prefix()
@@ -163,8 +164,12 @@ class ITUNES(DriverBase):
         settings()
         set_progress_reporter()
         upload_books()
-         _get_fpath()
+         _remove_existing_copy()
-          _update_epub_metadata()
+          _remove_from_device()
+          _remove_from_iTunes()
+         _add_new_copy()
+          _add_library_book()
+         _update_iTunes_metadata()
         add_books_to_metadata()
         use_plugboard_ext()
         set_plugboard()
@@ -181,7 +186,7 @@ class ITUNES(DriverBase):
     supported_platforms = ['osx','windows']
     author = 'GRiker'
     #: The version of this plugin as a 3-tuple (major, minor, revision)
-    version        = (1,0,0)
+    version        = (1,1,0)
 
     DISPLAY_DISABLE_DIALOG = "display_disable_apple_driver_dialog"
 
@@ -276,7 +281,6 @@ class ITUNES(DriverBase):
     description_prefix = "added by calibre"
     ejected = False
     iTunes= None
-    iTunes_media = None
     library_orphans = None
     log = Log()
     manual_sync_mode = False
@@ -412,11 +416,11 @@ class ITUNES(DriverBase):
                             this_book.datetime = parse_date(str(book.date_added())).timetuple()
                         except:
                             this_book.datetime = time.gmtime()
-                        this_book.db_id = None
                         this_book.device_collections = []
                         this_book.library_id = library_books[this_book.path] if this_book.path in library_books else None
                         this_book.size = book.size()
                         this_book.uuid = book.composer()
+                        this_book.cid = None
                         # Hack to discover if we're running in GUI environment
                         if self.report_progress is not None:
                             this_book.thumbnail = self._generate_thumbnail(this_book.path, book)
@@ -451,10 +455,10 @@ class ITUNES(DriverBase):
                                 this_book.datetime = parse_date(str(book.DateAdded)).timetuple()
                             except:
                                 this_book.datetime = time.gmtime()
-                            this_book.db_id = None
                             this_book.device_collections = []
                             this_book.library_id = library_books[this_book.path] if this_book.path in library_books else None
                             this_book.size = book.Size
+                            this_book.cid = None
                             # Hack to discover if we're running in GUI environment
                             if self.report_progress is not None:
                                 this_book.thumbnail = self._generate_thumbnail(this_book.path, book)
@@ -490,7 +494,7 @@ class ITUNES(DriverBase):
 
     def can_handle(self, device_info, debug=False):
         '''
-        Unix version of :method:`can_handle_windows`
+        OSX version of :method:`can_handle_windows`
 
         :param device_info: Is a tupe of (vid, pid, bcd, manufacturer, product,
         serial number)
@@ -1020,17 +1024,14 @@ class ITUNES(DriverBase):
 
         if DEBUG:
             self.log.info("ITUNES.upload_books()")
-            self._dump_files(files, header='upload_books()',indent=2)
-            self._dump_update_list(header='upload_books()',indent=2)
 
         if isosx:
-            for (i,file) in enumerate(files):
+            for (i,fpath) in enumerate(files):
-                format = file.rpartition('.')[2].lower()
+                format = fpath.rpartition('.')[2].lower()
                 path = self.path_template % (metadata[i].title,
                                              authors_to_string(metadata[i].authors),
                                              format)
                 self._remove_existing_copy(path, metadata[i])
-                fpath = self._get_fpath(file, metadata[i], format, update_md=True)
                 db_added, lb_added = self._add_new_copy(fpath, metadata[i])
                 thumb = self._cover_to_thumb(path, metadata[i], db_added, lb_added, format)
                 this_book = self._create_new_book(fpath, metadata[i], path, db_added, lb_added, thumb, format)
@@ -1061,13 +1062,12 @@ class ITUNES(DriverBase):
                 pythoncom.CoInitialize()
                 self.iTunes = win32com.client.Dispatch("iTunes.Application")
 
-                for (i,file) in enumerate(files):
+                for (i,fpath) in enumerate(files):
-                    format = file.rpartition('.')[2].lower()
+                    format = fpath.rpartition('.')[2].lower()
                     path = self.path_template % (metadata[i].title,
                                                  authors_to_string(metadata[i].authors),
                                                  format)
                     self._remove_existing_copy(path, metadata[i])
-                    fpath = self._get_fpath(file, metadata[i],format, update_md=True)
                     db_added, lb_added = self._add_new_copy(fpath, metadata[i])
 
                     if self.manual_sync_mode and not db_added:
@@ -1211,7 +1211,8 @@ class ITUNES(DriverBase):
         '''
         windows assumes pythoncom wrapper
         '''
-        self.log.info(" ITUNES._add_library_book()")
+        if DEBUG:
+            self.log.info(" ITUNES._add_library_book()")
         if isosx:
             added = self.iTunes.add(appscript.mactypes.File(file))
 
@@ -1274,24 +1275,59 @@ class ITUNES(DriverBase):
 
     def _add_new_copy(self, fpath, metadata):
         '''
+        fp = cached_book['lib_book'].location().path
+        fp = cached_book['lib_book'].Location
         '''
         if DEBUG:
             self.log.info(" ITUNES._add_new_copy()")
 
+        def _save_last_known_iTunes_storage(lb_added):
+            if isosx:
+                fp = lb_added.location().path
+                index = fp.rfind('/Books') + len('/Books')
+                last_known_iTunes_storage = fp[:index]
+            elif iswindows:
+                fp = lb_added.Location
+                index = fp.rfind('\Books') + len('\Books')
+                last_known_iTunes_storage = fp[:index]
+            dynamic['last_known_iTunes_storage'] = last_known_iTunes_storage
+            self.log.warning("  last_known_iTunes_storage: %s" % last_known_iTunes_storage)
+
         db_added = None
         lb_added = None
 
         if self.manual_sync_mode:
+            '''
+            This is the unsupported direct-connect mode.
+            In an attempt to avoid resetting the iTunes library Media folder, don't try to
+            add the book to iTunes if the last_known_iTunes_storage path is inaccessible.
+            This means that the path has to be set at least once, probably by using
+            'Connect to iTunes' and doing a transfer.
+            '''
+            self.log.warning("  unsupported direct connect mode")
             db_added = self._add_device_book(fpath, metadata)
-            if not getattr(fpath, 'deleted_after_upload', False):
+            last_known_iTunes_storage = dynamic.get('last_known_iTunes_storage', None)
-                lb_added = self._add_library_book(fpath, metadata)
+            if last_known_iTunes_storage is not None:
-                if lb_added:
+                if os.path.exists(last_known_iTunes_storage):
                     if DEBUG:
-                        self.log.info("  file added to Library|Books for iTunes<->iBooks tracking")
+                        self.log.warning("  iTunes storage online, adding to library")
+                    lb_added = self._add_library_book(fpath, metadata)
+                else:
+                    if DEBUG:
+                        self.log.warning("  iTunes storage not online, can't add to library")
+
+            if lb_added:
+                _save_last_known_iTunes_storage(lb_added)
+            if not lb_added and DEBUG:
+                self.log.warn("  failed to add '%s' to iTunes, iTunes Media folder inaccessible" % metadata.title)
         else:
             lb_added = self._add_library_book(fpath, metadata)
-            if DEBUG:
+            if lb_added:
-                self.log.info("  file added to Library|Books for pending sync")
+                _save_last_known_iTunes_storage(lb_added)
+            else:
+                raise UserFeedback("iTunes Media folder inaccessible",
+                                   details="Failed to add '%s' to iTunes" % metadata.title,
+                                   level=UserFeedback.WARN)
 
         return db_added, lb_added
 
@@ -1300,14 +1336,17 @@ class ITUNES(DriverBase):
         assumes pythoncom wrapper for db_added
         as of iTunes 9.2, iBooks 1.1, can't set artwork for PDF files via automation
         '''
-        self.log.info(" ITUNES._cover_to_thumb()")
+        if DEBUG:
+            self.log.info(" ITUNES._cover_to_thumb()")
 
         thumb = None
         if metadata.cover:
 
             if format == 'epub':
-                # Pre-shrink cover
+                '''
-                # self.MAX_COVER_WIDTH, self.MAX_COVER_HEIGHT
+                Pre-shrink cover
+                self.MAX_COVER_WIDTH, self.MAX_COVER_HEIGHT
+                '''
                 try:
                     img = PILImage.open(metadata.cover)
                     width = img.size[0]
@@ -1315,8 +1354,8 @@ class ITUNES(DriverBase):
                     scaled, nwidth, nheight = fit_image(width, height, self.MAX_COVER_WIDTH, self.MAX_COVER_HEIGHT)
                     if scaled:
                         if DEBUG:
-                            self.log.info("   '%s' scaled from %sx%s to %sx%s" %
+                            self.log.info("   cover scaled from %sx%s to %sx%s" %
-                                          (metadata.cover,width,height,nwidth,nheight))
+                                          (width,height,nwidth,nheight))
                         img = img.resize((nwidth, nheight), PILImage.ANTIALIAS)
                         cd = cStringIO.StringIO()
                         img.convert('RGB').save(cd, 'JPEG')
@@ -1335,9 +1374,11 @@ class ITUNES(DriverBase):
                     return thumb
 
                 if isosx:
-                    # The following commands generate an error, but the artwork does in fact
+                    '''
-                    # get sent to the device.  Seems like a bug in Apple's automation interface?
+                    The following commands generate an error, but the artwork does in fact
-                    # Could also be a problem with the integrity of the cover data?
+                    get sent to the device.  Seems like a bug in Apple's automation interface?
+                    Could also be a problem with the integrity of the cover data?
+                    '''
                     if lb_added:
                         try:
                             lb_added.artworks[1].data_.set(cover_data)
@@ -1360,9 +1401,8 @@ class ITUNES(DriverBase):
                             #ipython(user_ns=locals())
                             pass
 
-
                 elif iswindows:
-                    # Write the data to a real file for Windows iTunes
+                    ''' Write the data to a real file for Windows iTunes '''
                     tc = os.path.join(tempfile.gettempdir(), "cover.jpg")
                     with open(tc,'wb') as tmp_cover:
                         tmp_cover.write(cover_data)
@@ -1421,7 +1461,8 @@ class ITUNES(DriverBase):
 
         this_book = Book(metadata.title, authors_to_string(metadata.authors))
         this_book.datetime = time.gmtime()
-        this_book.db_id = None
+        #this_book.cid = metadata.id
+        this_book.cid = None
         this_book.device_collections = []
         this_book.format = format
         this_book.library_id = lb_added     # ??? GR
@@ -1429,7 +1470,6 @@ class ITUNES(DriverBase):
         this_book.thumbnail = thumb
         this_book.iTunes_id = lb_added      # ??? GR
         this_book.uuid = metadata.uuid
-
         if isosx:
             if lb_added:
                 this_book.size = self._get_device_book_size(fpath, lb_added.size())
@@ -1460,24 +1500,6 @@ class ITUNES(DriverBase):
 
         return this_book
 
-    def _delete_iTunesMetadata_plist(self,fpath):
-        '''
-        Delete the plist file from the file to force recache
-        '''
-        zf = ZipFile(fpath,'a')
-        fnames = zf.namelist()
-        pl_name = 'iTunesMetadata.plist'
-        try:
-            plist = [x for x in fnames if pl_name in x][0]
-        except:
-            plist = None
-        if plist:
-            if DEBUG:
-                self.log.info("  _delete_iTunesMetadata_plist():")
-                self.log.info("    deleting '%s'\n   from '%s'" % (pl_name,fpath))
-                zf.delete(pl_name)
-        zf.close()
-
     def _discover_manual_sync_mode(self, wait=0):
         '''
         Assumes pythoncom for windows
@@ -1662,18 +1684,6 @@ class ITUNES(DriverBase):
         zf.close()
         return (title, author, timestamp)
 
-    def _dump_files(self, files, header=None,indent=0):
-        if header:
-            msg = '\n%sfiles passed to %s:' % (' '*indent,header)
-            self.log.info(msg)
-            self.log.info( "%s%s" % (' '*indent,'-' * len(msg)))
-        for file in files:
-            if getattr(file, 'orig_file_path', None) is not None:
-                self.log.info(" %s%s" % (' '*indent,file.orig_file_path))
-            elif getattr(file, 'name', None) is not None:
-                self.log.info(" %s%s" % (' '*indent,file.name))
-        self.log.info()
-
     def _dump_hex(self, src, length=16):
         '''
         '''
@@ -1697,7 +1707,7 @@ class ITUNES(DriverBase):
         self.log.info()
 
     def _dump_update_list(self,header=None,indent=0):
-        if header:
+        if header and self.update_list:
             msg = '\n%sself.update_list %s' % (' '*indent,header)
             self.log.info(msg)
             self.log.info( "%s%s" % (' '*indent,'-' * len(msg)))
@@ -1716,7 +1726,6 @@ class ITUNES(DriverBase):
                  (' '*indent,
                   ub['title'],
                   ub['author']))
-        self.log.info()
 
     def _find_device_book(self, search):
         '''
@@ -2115,35 +2124,6 @@ class ITUNES(DriverBase):
                         self.log.error("  no iPad|Books playlist found")
                 return pl
 
-    def _get_fpath(self,file, metadata, format, update_md=False):
-        '''
-        If the database copy will be deleted after upload, we have to
-        use file (the PersistentTemporaryFile), which will be around until
-        calibre exits.
-        If we're using the database copy, delete the plist
-        '''
-        if DEBUG:
-            self.log.info(" ITUNES._get_fpath()")
-
-        fpath = file
-        if not getattr(fpath, 'deleted_after_upload', False):
-            if getattr(file, 'orig_file_path', None) is not None:
-                # Database copy
-                fpath = file.orig_file_path
-                self._delete_iTunesMetadata_plist(fpath)
-            elif getattr(file, 'name', None) is not None:
-                # PTF
-                fpath = file.name
-        else:
-            # Recipe - PTF
-            if DEBUG:
-                self.log.info("   file will be deleted after upload")
-
-        if format == 'epub' and update_md:
-            self._update_epub_metadata(fpath, metadata)
-
-        return fpath
-
     def _get_library_books(self):
         '''
         Populate a dict of paths from iTunes Library|Books
@@ -2347,6 +2327,7 @@ class ITUNES(DriverBase):
                 self.iTunes = appscript.app('iTunes')
                 self.initial_status = 'already running'
 
+            '''
             # Read the current storage path for iTunes media
             cmd = "defaults read com.apple.itunes NSNavLastRootDirectory"
             proc = subprocess.Popen( cmd, shell=True, cwd=os.curdir, stdout=subprocess.PIPE)
@@ -2357,12 +2338,13 @@ class ITUNES(DriverBase):
             else:
                 self.log.error("  could not confirm valid iTunes.media_dir from %s" % 'com.apple.itunes')
                 self.log.error("  media_dir: %s" % media_dir)
+            '''
+
             if DEBUG:
                 self.log.info("  %s %s" % (__appname__, __version__))
                 self.log.info("  [OSX %s - %s (%s), driver version %d.%d.%d]" %
                  (self.iTunes.name(), self.iTunes.version(), self.initial_status,
                   self.version[0],self.version[1],self.version[2]))
-                self.log.info("  iTunes_media: %s" % self.iTunes_media)
                 self.log.info("  calibre_library_path: %s" % self.calibre_library_path)
 
         if iswindows:
@@ -2402,6 +2384,7 @@ class ITUNES(DriverBase):
                              ' iTunes automation interface non-responsive, ' +
                              'recommend reinstalling iTunes')
 
+            '''
             # Read the current storage path for iTunes media from the XML file
             media_dir = ''
             string = None
@@ -2420,13 +2403,13 @@ class ITUNES(DriverBase):
                     self.log.error("  '%s' not found" % media_dir)
                 else:
                     self.log.error("  no media dir found: string: %s" % string)
+            '''
 
             if DEBUG:
                 self.log.info("  %s %s" % (__appname__, __version__))
                 self.log.info("  [Windows %s - %s (%s), driver version %d.%d.%d]" %
                  (self.iTunes.Windows[0].name, self.iTunes.Version, self.initial_status,
                   self.version[0],self.version[1],self.version[2]))
-                self.log.info("  iTunes_media: %s" % self.iTunes_media)
                 self.log.info("  calibre_library_path: %s" % self.calibre_library_path)
 
     def _purge_orphans(self,library_books, cached_books):
@@ -2476,13 +2459,14 @@ class ITUNES(DriverBase):
                    (self.cached_books[book]['title'] == metadata.title and \
                    self.cached_books[book]['author'] == authors_to_string(metadata.authors)):
                     self.update_list.append(self.cached_books[book])
-                    self._remove_from_device(self.cached_books[book])
+
                     if DEBUG:
                         self.log.info( "  deleting device book '%s'" % (metadata.title))
-                    if not getattr(file, 'deleted_after_upload', False):
+                    self._remove_from_device(self.cached_books[book])
-                        self._remove_from_iTunes(self.cached_books[book])
+
-                        if DEBUG:
+                    if DEBUG:
-                            self.log.info("  deleting library book '%s'" % metadata.title)
+                        self.log.info("  deleting library book '%s'" % metadata.title)
+                    self._remove_from_iTunes(self.cached_books[book])
                     break
             else:
                 if DEBUG:
@@ -2495,9 +2479,9 @@ class ITUNES(DriverBase):
                    (self.cached_books[book]['title'] == metadata.title and \
                     self.cached_books[book]['author'] == authors_to_string(metadata.authors)):
                     self.update_list.append(self.cached_books[book])
-                    self._remove_from_iTunes(self.cached_books[book])
                     if DEBUG:
                         self.log.info( "  deleting library book '%s'" %  metadata.title)
+                    self._remove_from_iTunes(self.cached_books[book])
                     break
             else:
                 if DEBUG:
@@ -2507,7 +2491,8 @@ class ITUNES(DriverBase):
         '''
         Windows assumes pythoncom wrapper
         '''
-        self.log.info(" ITUNES._remove_from_device()")
+        if DEBUG:
+            self.log.info(" ITUNES._remove_from_device()")
         if isosx:
             if DEBUG:
                 self.log.info("  deleting '%s' from iDevice" % cached_book['title'])
@@ -2528,96 +2513,105 @@ class ITUNES(DriverBase):
 
     def _remove_from_iTunes(self, cached_book):
         '''
-        iTunes does not delete books from storage when removing from database
+        iTunes does not delete books from storage when removing from database via automation
-        We only want to delete stored copies if the file is stored in iTunes
-        We don't want to delete files stored outside of iTunes.
-        Also confirm that storage_path does not point into calibre's storage.
         '''
         if DEBUG:
             self.log.info(" ITUNES._remove_from_iTunes():")
 
         if isosx:
+            ''' Manually remove the book from iTunes storage '''
             try:
-                storage_path = os.path.split(cached_book['lib_book'].location().path)
+                fp = cached_book['lib_book'].location().path
-                if cached_book['lib_book'].location().path.startswith(self.iTunes_media) and \
+                if DEBUG:
-                   not storage_path[0].startswith(prefs['library_path']):
+                    self.log.info("  processing %s" % fp)
-                    title_storage_path = storage_path[0]
+                if fp.startswith(prefs['library_path']):
-                    if DEBUG:
+                    self.log.info("  '%s' stored in calibre database, not removed" % cached_book['title'])
-                        self.log.info("  removing title_storage_path: %s" % title_storage_path)
-                    try:
-                        shutil.rmtree(title_storage_path)
-                    except:
-                        self.log.info("  '%s' not empty" % title_storage_path)
-
-                    # Clean up title/author directories
-                    author_storage_path = os.path.split(title_storage_path)[0]
-                    self.log.info("  author_storage_path: %s" % author_storage_path)
-                    author_files = os.listdir(author_storage_path)
-                    if '.DS_Store' in author_files:
-                        author_files.pop(author_files.index('.DS_Store'))
-                    if not author_files:
-                        shutil.rmtree(author_storage_path)
-                        if DEBUG:
-                            self.log.info("  removing empty author_storage_path")
-                    else:
-                        if DEBUG:
-                            self.log.info("  author_storage_path not empty (%d objects):" % len(author_files))
-                            self.log.info("  %s" % '\n'.join(author_files))
                 else:
-                    self.log.info("  '%s' (stored external to iTunes, no files deleted)" % cached_book['title'])
+                    if os.path.exists(fp):
+                        os.remove(fp)
+                        if DEBUG:
+                            self.log.info("   deleting from iTunes storage")
+                        author_storage_path = os.path.split(fp)[0]
+                        try:
+                            os.rmdir(author_storage_path)
+                            if DEBUG:
+                                self.log.info("   removing empty author directory")
+                        except:
+                            author_files = os.listdir(author_storage_path)
+                            if '.DS_Store' in author_files:
+                                author_files.pop(author_files.index('.DS_Store'))
+                            if not author_files:
+                                os.rmdir(author_storage_path)
+                                if DEBUG:
+                                    self.log.info("   removing empty author directory")
+                            '''
+                            else:
+                                if DEBUG:
+                                    self.log.info("   author_storage_path not empty:")
+                                    self.log.info("   %s" % '\n'.join(author_files))
+                            '''
+                    else:
+                        self.log.info("   '%s' does not exist at storage location" % cached_book['title'])
 
             except:
                 # We get here if there was an error with .location().path
                 if DEBUG:
-                    self.log.info("   '%s' not in iTunes storage" % cached_book['title'])
+                    self.log.info("   '%s' not found in iTunes storage" % cached_book['title'])
 
+            # Delete the book from the iTunes database
             try:
                 self.iTunes.delete(cached_book['lib_book'])
+                if DEBUG:
+                    self.log.info("   removing from iTunes database")
             except:
                 if DEBUG:
-                    self.log.info("   unable to remove '%s' from iTunes" % cached_book['title'])
+                    self.log.info("   unable to remove from iTunes database")
 
         elif iswindows:
             '''
             Assume we're wrapped in a pythoncom
             Windows stores the book under a common author directory, so we just delete the .epub
             '''
+            fp = None
             try:
                 book = cached_book['lib_book']
-                path = book.Location
+                fp = book.Location
             except:
                 book = self._find_library_book(cached_book)
                 if book:
-                    path = book.Location
+                    fp = book.Location
 
             if book:
-                if self.iTunes_media and path.startswith(self.iTunes_media) and \
+                if DEBUG:
-                   not path.startswith(prefs['library_path']):
+                    self.log.info("  processing %s" % fp)
-                    storage_path = os.path.split(path)
+                if fp.startswith(prefs['library_path']):
-                    if DEBUG:
+                    self.log.info("  '%s' stored in calibre database, not removed" % cached_book['title'])
-                        self.log.info("   removing '%s' at %s" %
-                            (cached_book['title'], path))
-                    try:
-                        os.remove(path)
-                    except:
-                        self.log.warning("   '%s' not in iTunes storage" % path)
-                    try:
-                        os.rmdir(storage_path[0])
-                        self.log.info("   removed folder '%s'" % storage_path[0])
-                    except:
-                        self.log.info("   folder '%s' not found or not empty" % storage_path[0])
-
-                    # Delete from iTunes database
                 else:
-                    self.log.info("   '%s' (stored external to iTunes, no files deleted)" % cached_book['title'])
+                    if os.path.exists(fp):
+                        os.remove(fp)
+                        if DEBUG:
+                            self.log.info("   deleting from iTunes storage")
+                        author_storage_path = os.path.split(fp)[0]
+                        try:
+                            os.rmdir(author_storage_path)
+                            if DEBUG:
+                                self.log.info("   removing empty author directory")
+                        except:
+                            pass
+                    else:
+                        self.log.info("   '%s' does not exist at storage location" % cached_book['title'])
             else:
                 if DEBUG:
-                    self.log.info("   '%s' not found in iTunes" % cached_book['title'])
+                    self.log.info("   '%s' not found in iTunes storage" % cached_book['title'])
+
+            # Delete the book from the iTunes database
             try:
                 book.Delete()
+                if DEBUG:
+                    self.log.info("   removing from iTunes database")
             except:
                 if DEBUG:
-                    self.log.info("   unable to remove '%s' from iTunes" % cached_book['title'])
+                    self.log.info("   unable to remove from iTunes database")
 
     def title_sorter(self, title):
         return re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', title).rstrip()
@@ -2625,7 +2619,8 @@ class ITUNES(DriverBase):
     def _update_epub_metadata(self, fpath, metadata):
         '''
         '''
-        self.log.info(" ITUNES._update_epub_metadata()")
+        if DEBUG:
+            self.log.info(" ITUNES._update_epub_metadata()")
 
         # Fetch plugboard updates
         metadata_x = self._xform_metadata_via_plugboard(metadata, 'epub')
@@ -2796,7 +2791,7 @@ class ITUNES(DriverBase):
             if metadata_x.series and self.settings().extra_customization[self.USE_SERIES_AS_CATEGORY]:
                 if DEBUG:
                     self.log.info(" ITUNES._update_iTunes_metadata()")
-                    self.log.info("   using Series name as Genre")
+                    self.log.info("   using Series name '%s' as Genre" % metadata_x.series)
 
                 # Format the index as a sort key
                 index = metadata_x.series_index
@@ -2976,8 +2971,8 @@ class ITUNES(DriverBase):
             newmi = book.deepcopy_metadata()
             newmi.template_to_attribute(book, pb)
             if pb is not None and DEBUG:
-                self.log.info(" transforming %s using %s:" % (format, pb))
+                #self.log.info(" transforming %s using %s:" % (format, pb))
-                self.log.info("       title: %s %s" % (book.title, ">>> %s" %
+                self.log.info("       title: '%s' %s" % (book.title, ">>> '%s'" %
                                            newmi.title if book.title != newmi.title else ''))
                 self.log.info("  title_sort: %s %s" % (book.title_sort, ">>> %s" %
                                            newmi.title_sort if book.title_sort != newmi.title_sort else ''))
@@ -2992,7 +2987,8 @@ class ITUNES(DriverBase):
                 self.log.info("        tags: %s %s" % (book.tags, ">>> %s" %
                                            newmi.tags if book.tags != newmi.tags else ''))
             else:
-                self.log("  matching plugboard not found")
+                if DEBUG:
+                    self.log("  matching plugboard not found")
 
         else:
             newmi = book
@@ -3081,12 +3077,12 @@ class ITUNES_ASYNC(ITUNES):
                         this_book.datetime = parse_date(str(library_books[book].date_added())).timetuple()
                     except:
                         this_book.datetime = time.gmtime()
-                    this_book.db_id = None
                     this_book.device_collections = []
                     #this_book.library_id = library_books[this_book.path] if this_book.path in library_books else None
                     this_book.library_id = library_books[book]
                     this_book.size = library_books[book].size()
                     this_book.uuid = library_books[book].composer()
+                    this_book.cid = None
                     # Hack to discover if we're running in GUI environment
                     if self.report_progress is not None:
                         this_book.thumbnail = self._generate_thumbnail(this_book.path, library_books[book])
@@ -3122,11 +3118,11 @@ class ITUNES_ASYNC(ITUNES):
                             this_book.datetime = parse_date(str(library_books[book].DateAdded)).timetuple()
                         except:
                             this_book.datetime = time.gmtime()
-                        this_book.db_id = None
                         this_book.device_collections = []
                         this_book.library_id = library_books[book]
                         this_book.size = library_books[book].Size
                         this_book.uuid = library_books[book].Composer
+                        this_book.cid = None
                         # Hack to discover if we're running in GUI environment
                         if self.report_progress is not None:
                             this_book.thumbnail = self._generate_thumbnail(this_book.path, library_books[book])

src/calibre/devices/hanvon/driver.py

@@ -61,7 +61,7 @@ class LIBREAIR(N516):
 
     BCD = [0x399]
     VENDOR_NAME      = 'ALURATEK'
-    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = '_FILE-STOR_GADGET'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'FILE-STOR_GADGET'
     EBOOK_DIR_MAIN = 'Books'
 
 class ALEX(N516):

src/calibre/devices/interface.py

@@ -327,12 +327,7 @@ class DevicePlugin(Plugin):
         free space on the device. The text of the FreeSpaceError must contain the
         word "card" if ``on_card`` is not None otherwise it must contain the word "memory".
 
-        :param files: A list of paths and/or file-like objects. If they are paths and
+        :param files: A list of paths
-                      the paths point to temporary files, they may have an additional
-                      attribute, original_file_path pointing to the originals. They may have
-                      another optional attribute, deleted_after_upload which if True means
-                      that the file pointed to by original_file_path will be deleted after
-                      being uploaded to the device.
         :param names: A list of file names that the books should have
                       once uploaded to the device. len(names) == len(files)
         :param metadata: If not None, it is a list of :class:`Metadata` objects.

src/calibre/devices/kobo/driver.py

@@ -100,7 +100,7 @@ class KOBO(USBMS):
         for idx,b in enumerate(bl):
             bl_cache[b.lpath] = idx
 
-        def update_booklist(prefix, path, title, authors, mime, date, ContentType, ImageID, readstatus, MimeType):
+        def update_booklist(prefix, path, title, authors, mime, date, ContentType, ImageID, readstatus, MimeType, expired, favouritesindex):
             changed = False
             try:
                 lpath = path.partition(self.normalize_path(prefix))[2]
@@ -111,12 +111,23 @@ class KOBO(USBMS):
 
                 playlist_map = {}
 
+                if lpath not in playlist_map:
+                    playlist_map[lpath] = []
+
                 if readstatus == 1:
-                    playlist_map[lpath]= "Im_Reading"
+                    playlist_map[lpath].append('Im_Reading')
                 elif readstatus == 2:
-                    playlist_map[lpath]= "Read"
+                    playlist_map[lpath].append('Read')
                 elif readstatus == 3:
-                    playlist_map[lpath]= "Closed"
+                    playlist_map[lpath].append('Closed')
+
+                # Related to a bug in the Kobo firmware that leaves an expired row for deleted books
+                # this shows an expired Collection so the user can decide to delete the book
+                if expired == 3:
+                    playlist_map[lpath].append('Expired')
+                # A SHORTLIST is supported on the touch but the data field is there on most earlier models
+                if favouritesindex == 1:
+                    playlist_map[lpath].append('Shortlist')
 
                 path = self.normalize_path(path)
                 # print "Normalized FileName: " + path
@@ -126,7 +137,13 @@ class KOBO(USBMS):
                     bl_cache[lpath] = None
                     if ImageID is not None:
                         imagename = self.normalize_path(self._main_prefix + '.kobo/images/' + ImageID + ' - NickelBookCover.parsed')
+                        if not os.path.exists(imagename): 
+                            # Try the Touch version if the image does not exist
+                            imagename = self.normalize_path(self._main_prefix + '.kobo/images/' + ImageID + ' - N3_LIBRARY_FULL.parsed')
+
                         #print "Image name Normalized: " + imagename
+                        if not os.path.exists(imagename):
+                            debug_print("Strange - The image name does not exist - title: ", title)
                         if imagename is not None:
                             bl[idx].thumbnail = ImageWrapper(imagename)
                     if (ContentType != '6' and MimeType != 'Shortcover'):
@@ -138,7 +155,7 @@ class KOBO(USBMS):
                              debug_print("    Strange:  The file: ", prefix, lpath, " does mot exist!")
                     if lpath in playlist_map and \
                         playlist_map[lpath] not in bl[idx].device_collections:
-                            bl[idx].device_collections.append(playlist_map[lpath])
+                            bl[idx].device_collections = playlist_map.get(lpath,[])
                 else:
                     if ContentType == '6' and MimeType == 'Shortcover':
                         book =  Book(prefix, lpath, title, authors, mime, date, ContentType, ImageID, size=1048576)
@@ -157,7 +174,7 @@ class KOBO(USBMS):
                             raise
 
                     # print 'Update booklist'
-                    book.device_collections = [playlist_map[lpath]] if lpath in playlist_map else []
+                    book.device_collections = playlist_map.get(lpath,[])# if lpath in playlist_map else []
 
                     if bl.add_book(book, replace_metadata=False):
                         changed = True
@@ -186,24 +203,30 @@ class KOBO(USBMS):
         result = cursor.fetchone()
         self.dbversion = result[0]
 
-        query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \
+        if self.dbversion >= 14:
-                'ImageID, ReadStatus from content where BookID is Null'
+            query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \
+                'ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex from content where BookID is Null'
+        else:
+            query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \
+                'ImageID, ReadStatus, ___ExpirationStatus, "-1" as FavouritesIndex from content where BookID is Null'
 
         cursor.execute (query)
 
         changed = False
         for i, row in enumerate(cursor):
         #  self.report_progress((i+1) / float(numrows), _('Getting list of books on device...'))
-
+            if row[3].startswith("file:///usr/local/Kobo/help/"):
+                # These are internal to the Kobo device and do not exist
+                continue
             path = self.path_from_contentid(row[3], row[5], row[4], oncard)
             mime = mime_type_ext(path_to_ext(path)) if path.find('kepub') == -1 else 'application/epub+zip'
             # debug_print("mime:", mime)
 
             if oncard != 'carda' and oncard != 'cardb' and not row[3].startswith("file:///mnt/sd/"):
-                changed = update_booklist(self._main_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7], row[4])
+                changed = update_booklist(self._main_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7], row[4], row[8], row[9])
                 # print "shortbook: " + path
             elif oncard == 'carda' and row[3].startswith("file:///mnt/sd/"):
-                changed = update_booklist(self._card_a_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7], row[4])
+                changed = update_booklist(self._card_a_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7], row[4], row[8], row[9])
 
             if changed:
                 need_sync = True
@@ -267,8 +290,12 @@ class KOBO(USBMS):
             cursor.execute('delete from content_keys where volumeid = ?', t)
 
         # Delete the chapters associated with the book next
-        t = (ContentID,ContentID,)
+        t = (ContentID,)
-        cursor.execute('delete from content where BookID  = ? or ContentID = ?', t)
+        # Kobo does not delete the Book row (ie the row where the BookID is Null)
+        # The next server sync should remove the row
+        cursor.execute('delete from content where BookID = ?', t)
+        cursor.execute('update content set ReadStatus=0, FirstTimeReading = \'true\', ___PercentRead=0, ___ExpirationStatus=3 ' \
+                'where BookID is Null and ContentID =?',t)
 
         connection.commit()
 
@@ -286,7 +313,7 @@ class KOBO(USBMS):
             path_prefix = '.kobo/images/'
             path = self._main_prefix + path_prefix + ImageID
 
-            file_endings = (' - iPhoneThumbnail.parsed', ' - bbMediumGridList.parsed', ' - NickelBookCover.parsed',)
+            file_endings = (' - iPhoneThumbnail.parsed', ' - bbMediumGridList.parsed', ' - NickelBookCover.parsed', ' - N3_LIBRARY_FULL.parsed', ' - N3_LIBRARY_GRID.parsed', ' - N3_LIBRARY_LIST.parsed', ' - N3_SOCIAL_CURRENTREAD.parsed',)
 
             for ending in file_endings:
                 fpath = path + ending
@@ -450,7 +477,10 @@ class KOBO(USBMS):
                 path = self._main_prefix + path + '.kobo'
                 # print "Path: " + path
             elif (ContentType == "6" or ContentType == "10") and MimeType == 'application/x-kobo-epub+zip':
-                path = self._main_prefix + '.kobo/kepub/' + path
+                if path.startswith("file:///mnt/onboard/"):
+                    path = self._main_prefix + path.replace("file:///mnt/onboard/", '')
+                else:
+                    path = self._main_prefix + '.kobo/kepub/' + path
                 # print "Internal: " + path
             else:
                 # if path.startswith("file:///mnt/onboard/"):
@@ -527,6 +557,7 @@ class KOBO(USBMS):
         if collections:
             # Process any collections that exist
             for category, books in collections.items():
+                # debug_print (category)
                 if category == 'Im_Reading':
                     # Reset Im_Reading list in the database
                     if oncard == 'carda':
@@ -545,7 +576,8 @@ class KOBO(USBMS):
 
                     for book in books:
 #                        debug_print('Title:', book.title, 'lpath:', book.path)
-                        book.device_collections = ['Im_Reading']
+                        if 'Im_Reading' not in book.device_collections:
+                            book.device_collections.append('Im_Reading') 
 
                         extension =  os.path.splitext(book.path)[1]
                         ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path)
@@ -588,7 +620,8 @@ class KOBO(USBMS):
 
                     for book in books:
 #                       debug_print('Title:', book.title, 'lpath:', book.path)
-                        book.device_collections = ['Read']
+                        if 'Read' not in book.device_collections:
+                            book.device_collections.append('Read') 
 
                         extension =  os.path.splitext(book.path)[1]
                         ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path)
@@ -624,7 +657,8 @@ class KOBO(USBMS):
 
                     for book in books:
 #                       debug_print('Title:', book.title, 'lpath:', book.path)
-                        book.device_collections = ['Closed']
+                        if 'Closed' not in book.device_collections:
+                            book.device_collections.append('Closed') 
 
                         extension =  os.path.splitext(book.path)[1]
                         ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path)
@@ -642,6 +676,44 @@ class KOBO(USBMS):
                         else:
                             connection.commit()
 #                            debug_print('Database: Commit set ReadStatus as Closed')
+                if category == 'Shortlist':
+                    # Reset FavouritesIndex list in the database
+                    if oncard == 'carda':
+                        query= 'update content set FavouritesIndex=-1 where BookID is Null and ContentID like \'file:///mnt/sd/%\''
+                    elif oncard != 'carda' and oncard != 'cardb':
+                        query= 'update content set FavouritesIndex=-1 where BookID is Null and ContentID not like \'file:///mnt/sd/%\''
+
+                    try:
+                        cursor.execute (query)
+                    except:
+                        debug_print('Database Exception:  Unable to reset Shortlist list')
+                        raise
+                    else:
+#                       debug_print('Commit: Reset Shortlist list')
+                        connection.commit()
+
+                    for book in books:
+#                        debug_print('Title:', book.title, 'lpath:', book.path)
+                        if 'Shortlist' not in book.device_collections:
+                            book.device_collections.append('Shortlist') 
+                        # debug_print ("Shortlist found for: ", book.title) 
+                        extension =  os.path.splitext(book.path)[1]
+                        ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path)
+
+                        ContentID = self.contentid_from_path(book.path, ContentType)
+#                        datelastread = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime())
+
+                        t = (ContentID,)
+
+                        try:
+                            cursor.execute('update content set FavouritesIndex=1 where BookID is Null and ContentID = ?', t)
+                        except:
+                            debug_print('Database Exception:  Unable set book as Shortlist')
+                            raise
+                        else:
+                            connection.commit()
+#                            debug_print('Database: Commit set Shortlist as Shortlist')
+
         else: # No collections
             # Since no collections exist the ReadStatus needs to be reset to 0 (Unread)
             print "Reseting ReadStatus to 0"

src/calibre/devices/nook/driver.py

@@ -88,6 +88,7 @@ class NOOK_COLOR(NOOK):
 
     WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'EBOOK_DISK'
     EBOOK_DIR_MAIN = 'My Files'
+    NEWS_IN_FOLDER = False
 
     def upload_cover(self, path, filename, metadata, filepath):
         pass

src/calibre/devices/teclast/driver.py

@@ -19,8 +19,9 @@ class TECLAST_K3(USBMS):
     PRODUCT_ID  = [0x3203]
     BCD         = [0x0000, 0x0100]
 
-    VENDOR_NAME      = 'TECLAST'
+    VENDOR_NAME      = ['TECLAST', 'IMAGIN']
-    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['DIGITAL_PLAYER', 'TL-K5']
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['DIGITAL_PLAYER', 'TL-K5',
+            'EREADER']
 
     MAIN_MEMORY_VOLUME_LABEL  = 'K3 Main Memory'
     STORAGE_CARD_VOLUME_LABEL = 'K3 Storage Card'

src/calibre/devices/usbms/device.py

@@ -101,6 +101,9 @@ class Device(DeviceConfig, DevicePlugin):
     #: The maximum length of paths created on the device
     MAX_PATH_LEN = 250
 
+    #: Put news in its own folder
+    NEWS_IN_FOLDER = True
+
     def reset(self, key='-1', log_packets=False, report_progress=None,
             detected_device=None):
         self._main_prefix = self._card_a_prefix = self._card_b_prefix = None
@@ -946,7 +949,8 @@ class Device(DeviceConfig, DevicePlugin):
             extra_components = []
             tag = special_tag
             if tag.startswith(_('News')):
-                extra_components.append('News')
+                if self.NEWS_IN_FOLDER:
+                    extra_components.append('News')
             else:
                 for c in tag.split('/'):
                     c = sanitize(c)

src/calibre/ebooks/comic/input.py

@@ -351,7 +351,9 @@ class ComicInput(InputFormatPlugin):
         comics = []
         with CurrentDir(tdir):
             if not os.path.exists('comics.txt'):
-                raise ValueError('%s is not a valid comic collection'
+                raise ValueError((
+                    '%s is not a valid comic collection'
+                    ' no comics.txt was found in the file')
                         %stream.name)
             raw = open('comics.txt', 'rb').read()
             if raw.startswith(codecs.BOM_UTF16_BE):

src/calibre/ebooks/conversion/plumber.py

@@ -59,6 +59,8 @@ class CompositeProgressReporter(object):
                 (self.global_max - self.global_min)
         self.global_reporter(global_frac, msg)
 
+ARCHIVE_FMTS = ('zip', 'rar', 'oebzip')
+
 class Plumber(object):
     '''
     The `Plumber` manages the conversion pipeline. An UI should call the methods
@@ -594,7 +596,7 @@ OptionRecommendation(name='sr3_replace',
             raise ValueError('Input file must have an extension')
         input_fmt = input_fmt[1:].lower()
         self.archive_input_tdir = None
-        if input_fmt in ('zip', 'rar', 'oebzip'):
+        if input_fmt in ARCHIVE_FMTS:
             self.log('Processing archive...')
             tdir = PersistentTemporaryDirectory('_plumber_archive')
             self.input, input_fmt = self.unarchive(self.input, tdir)

src/calibre/ebooks/epub/fix/epubcheck.py

@@ -26,6 +26,10 @@ class Epubcheck(ePubFixer):
                 'significant changes to your epub, complain to the epubcheck '
                 'project.')
 
+    @property
+    def description(self):
+        return self.long_description
+
     @property
     def fix_name(self):
         return 'epubcheck'

src/calibre/ebooks/epub/fix/unmanifested.py

@@ -22,6 +22,10 @@ class Unmanifested(ePubFixer):
         'the manifest or delete them as specified by the '
         'delete unmanifested option.')
 
+    @property
+    def description(self):
+        return self.long_description
+
     @property
     def fix_name(self):
         return 'unmanifested'

src/calibre/ebooks/html/input.py

@@ -457,7 +457,7 @@ class HTMLInput(InputFormatPlugin):
                     href=bhref)
             guessed = self.guess_type(href)[0]
             media_type = guessed or self.BINARY_MIME
-            if 'text' in media_type:
+            if media_type == 'text/plain':
                 self.log.warn('Ignoring link to text file %r'%link_)
                 return None
 

src/calibre/ebooks/metadata/fb2.py

@@ -1,96 +1,235 @@
 #!/usr/bin/env python
 from __future__ import with_statement
 __license__   = 'GPL v3'
-__copyright__ = '2008, Anatoly Shipitsin <norguhtar at gmail.com>'
+__copyright__ = '2011, Roman Mukhin <ramses_ru at hotmail.com>, '\
-
+                '2008, Anatoly Shipitsin <norguhtar at gmail.com>'
 '''Read meta information from fb2 files'''
 
 import os
+import datetime
+from functools import partial
 from base64 import b64decode
 from lxml import etree
-from calibre.ebooks.metadata import MetaInformation
+from calibre.utils.date import parse_date
+from calibre import guess_all_extensions, prints, force_unicode
+from calibre.ebooks.metadata import MetaInformation, check_isbn
 from calibre.ebooks.chardet import xml_to_unicode
-from calibre import guess_all_extensions
 
-XLINK_NS     = 'http://www.w3.org/1999/xlink'
-def XLINK(name):
-    return '{%s}%s' % (XLINK_NS, name)
 
+NAMESPACES = {
+    'fb2'   :   'http://www.gribuser.ru/xml/fictionbook/2.0',
+    'xlink' :   'http://www.w3.org/1999/xlink'  }
+
+XPath = partial(etree.XPath, namespaces=NAMESPACES)
+tostring = partial(etree.tostring, method='text', encoding=unicode)
 
 def get_metadata(stream):
-    """ Return metadata as a L{MetaInfo} object """
+    """ Return fb2 metadata as a L{MetaInformation} object """
-    XPath = lambda x : etree.XPath(x,
+
-            namespaces={'fb2':'http://www.gribuser.ru/xml/fictionbook/2.0',
+    root = _get_fbroot(stream)
-                'xlink':XLINK_NS})
+
-    tostring = lambda x : etree.tostring(x, method='text',
+    book_title = _parse_book_title(root)
-            encoding=unicode).strip()
+    authors = _parse_authors(root)
+
+    # fallback for book_title
+    if book_title:
+        book_title = unicode(book_title)
+    else:
+        book_title = force_unicode(os.path.splitext(
+            os.path.basename(getattr(stream, 'name',
+                _('Unknown'))))[0])
+    mi = MetaInformation(book_title, authors)
+
+    try:
+        _parse_cover(root, mi)
+    except:
+        pass
+    try:
+        _parse_comments(root, mi)
+    except:
+        pass
+    try:
+        _parse_tags(root, mi)
+    except:
+        pass
+    try:
+        _parse_series(root, mi)
+    except:
+        pass
+    try:
+        _parse_isbn(root, mi)
+    except:
+        pass
+    try:
+        _parse_publisher(root, mi)
+    except:
+        pass
+    try:
+        _parse_pubdate(root, mi)
+    except:
+        pass
+    try:
+        _parse_timestamp(root, mi)
+    except:
+        pass
+
+    try:
+        _parse_language(root, mi)
+    except:
+        pass
+    #_parse_uuid(root, mi)
+
+    #if DEBUG:
+    #   prints(mi)
+    return mi
+
+def _parse_authors(root):
+    authors = []
+    # pick up authors but only from 1 secrion <title-info>; otherwise it is not consistent!
+    # Those are fallbacks: <src-title-info>, <document-info>
+    for author_sec in ['title-info', 'src-title-info', 'document-info']:
+        for au in XPath('//fb2:%s/fb2:author'%author_sec)(root):
+            author = _parse_author(au)
+            if author:
+                authors.append(author)
+        if author:
+            break
+
+    # if no author so far
+    if not authors:
+        authors.append(_('Unknown'))
+
+    return authors
+
+def _parse_author(elm_author):
+    """ Returns a list of display author and sortable author"""
+
+    xp_templ = 'normalize-space(fb2:%s/text())'
+
+    author = XPath(xp_templ % 'first-name')(elm_author)
+    lname = XPath(xp_templ % 'last-name')(elm_author)
+    mname = XPath(xp_templ % 'middle-name')(elm_author)
+
+    if mname:
+        author = (author + ' ' + mname).strip()
+    if lname:
+        author = (author + ' ' + lname).strip()
+
+    # fallback to nickname
+    if not author:
+        nname = XPath(xp_templ % 'nickname')(elm_author)
+        if nname:
+            author = nname
+
+    return author
+
+
+def _parse_book_title(root):
+    # <title-info> has a priority.   (actually <title-info>  is mandatory)
+    # other are backup solution (sequence is important. other then in fb2-doc)
+    xp_ti = '//fb2:title-info/fb2:book-title/text()'
+    xp_pi = '//fb2:publish-info/fb2:book-title/text()'
+    xp_si = '//fb2:src-title-info/fb2:book-title/text()'
+    book_title = XPath('normalize-space(%s|%s|%s)' % (xp_ti, xp_pi, xp_si))(root)
+
+    return book_title
+
+def _parse_cover(root, mi):
+    # pickup from <title-info>, if not exists it fallbacks to <src-title-info>
+    imgid = XPath('substring-after(string(//fb2:coverpage/fb2:image/@xlink:href), "#")')(root)
+    if imgid:
+        try:
+            _parse_cover_data(root, imgid, mi)
+        except:
+            pass
+
+def _parse_cover_data(root, imgid, mi):
+    elm_binary = XPath('//fb2:binary[@id="%s"]'%imgid)(root)
+    if elm_binary:
+        mimetype = elm_binary[0].get('content-type', 'image/jpeg')
+        mime_extensions = guess_all_extensions(mimetype)
+        if mime_extensions:
+            pic_data = elm_binary[0].text
+            if pic_data:
+                mi.cover_data = (mime_extensions[0][1:], b64decode(pic_data))
+        else:
+            prints("WARNING: Unsupported coverpage mime-type '%s' (id=#%s)" % (mimetype, imgid) )
+
+def _parse_tags(root, mi):
+    # pick up genre but only from 1 secrion <title-info>; otherwise it is not consistent!
+    # Those are fallbacks: <src-title-info>
+    for genre_sec in ['title-info', 'src-title-info']:
+        # -- i18n Translations-- ?
+        tags = XPath('//fb2:%s/fb2:genre/text()' % genre_sec)(root)
+        if tags:
+            mi.tags = list(map(unicode, tags))
+            break
+
+def _parse_series(root, mi):
+    #calibri supports only 1 series: use the 1-st one
+    # pick up sequence but only from 1 secrion in prefered order
+    # except <src-title-info>
+    xp_ti = '//fb2:title-info/fb2:sequence[1]'
+    xp_pi = '//fb2:publish-info/fb2:sequence[1]'
+
+    elms_sequence = XPath('%s|%s' % (xp_ti, xp_pi))(root)
+    if elms_sequence:
+        mi.series = elms_sequence[0].get('name', None)
+        if mi.series:
+            mi.series_index = elms_sequence[0].get('number', None)
+
+def _parse_isbn(root, mi):
+    # some people try to put several isbn in this field, but it is not allowed.  try to stick to the 1-st one in this case
+    isbn = XPath('normalize-space(//fb2:publish-info/fb2:isbn/text())')(root)
+    # some people try to put several isbn in this field, but it is not allowed.  try to stick to the 1-st one in this case
+    if ',' in isbn:
+        isbn = isbn[:isbn.index(',')]
+    if check_isbn(isbn):
+        mi.isbn = isbn
+
+def _parse_comments(root, mi):
+    # pick up annotation but only from 1 secrion <title-info>;  fallback: <src-title-info>
+    for annotation_sec in ['title-info', 'src-title-info']:
+        elms_annotation = XPath('//fb2:%s/fb2:annotation' % annotation_sec)(root)
+        if elms_annotation:
+            mi.comments = tostring(elms_annotation[0])
+            # TODO: tags i18n, xslt?
+            break
+
+def _parse_publisher(root, mi):
+    publisher = XPath('string(//fb2:publish-info/fb2:publisher/text())')(root)
+    if publisher:
+        mi.publisher = publisher
+
+def _parse_pubdate(root, mi):
+    year = XPath('number(//fb2:publish-info/fb2:year/text())')(root)
+    if float.is_integer(year):
+        # only year is available, so use 1-st of Jan
+        mi.pubdate = datetime.date(int(year), 1, 1)
+
+def _parse_timestamp(root, mi):
+    #<date value="1996-12-03">03.12.1996</date>
+    xp ='//fb2:document-info/fb2:date/@value|'\
+        '//fb2:document-info/fb2:date/text()'
+    docdate = XPath('string(%s)' % xp)(root)
+    if docdate:
+        mi.timestamp = parse_date(docdate)
+
+def _parse_language(root, mi):
+    language = XPath('string(//fb2:title-info/fb2:lang/text())')(root)
+    if language:
+        mi.language = language
+        mi.languages = [ language ]
+
+def _parse_uuid(root, mi):
+    uuid = XPath('normalize-space(//document-info/fb2:id/text())')(root)
+    if uuid:
+        mi.uuid = uuid
+
+def _get_fbroot(stream):
     parser = etree.XMLParser(recover=True, no_network=True)
     raw = stream.read()
-    raw = xml_to_unicode(raw, strip_encoding_pats=True,
+    raw = xml_to_unicode(raw, strip_encoding_pats=True)[0]
-            assume_utf8=True)[0]
     root = etree.fromstring(raw, parser=parser)
-    authors, author_sort = [], None
+    return root
-    for au in XPath('//fb2:author')(root):
-        fname = lname = author = None
-        fe = XPath('descendant::fb2:first-name')(au)
-        if fe:
-            fname = tostring(fe[0])
-            author = fname
-        le = XPath('descendant::fb2:last-name')(au)
-        if le:
-            lname = tostring(le[0])
-            if author:
-                author += ' '+lname
-            else:
-                author = lname
-        if author:
-            authors.append(author)
-        if len(authors) == 1 and author is not None:
-            if lname:
-                author_sort = lname
-            if fname:
-                if author_sort: author_sort += ', '+fname
-                else: author_sort = fname
-    title = os.path.splitext(os.path.basename(getattr(stream, 'name',
-        _('Unknown'))))[0]
-    for x in XPath('//fb2:book-title')(root):
-        title = tostring(x)
-        break
-    comments = ''
-    for x in XPath('//fb2:annotation')(root):
-        comments += tostring(x)
-    if not comments:
-        comments = None
-    tags = list(map(tostring, XPath('//fb2:genre')(root)))
 
-    cp = XPath('//fb2:coverpage')(root)
-    cdata = None
-    if cp:
-        cimage = XPath('descendant::fb2:image[@xlink:href]')(cp[0])
-        if cimage:
-            id = cimage[0].get(XLINK('href')).replace('#', '')
-            binary = XPath('//fb2:binary[@id="%s"]'%id)(root)
-            if binary:
-                mt = binary[0].get('content-type', 'image/jpeg')
-                exts = guess_all_extensions(mt)
-                if not exts:
-                    exts = ['.jpg']
-                cdata = (exts[0][1:], b64decode(tostring(binary[0])))
-
-    series = None
-    series_index = 1.0
-    for x in XPath('//fb2:sequence')(root):
-        series = x.get('name', None)
-        if series is not None:
-            series_index = x.get('number', 1.0)
-            break
-    mi = MetaInformation(title, authors)
-    mi.comments = comments
-    mi.author_sort = author_sort
-    if tags:
-        mi.tags = tags
-    mi.series = series
-    mi.series_index = series_index
-    if cdata:
-        mi.cover_data = cdata
-    return mi

src/calibre/ebooks/metadata/sources/base.py

@@ -21,6 +21,7 @@ from calibre.ebooks.metadata import check_isbn
 msprefs = JSONConfig('metadata_sources/global.json')
 msprefs.defaults['txt_comments'] = False
 msprefs.defaults['ignore_fields'] = []
+msprefs.defaults['user_default_ignore_fields'] = []
 msprefs.defaults['max_tags'] = 20
 msprefs.defaults['wait_after_first_identify_result'] = 30 # seconds
 msprefs.defaults['wait_after_first_cover_result'] = 60 # seconds

src/calibre/ebooks/metadata/sources/identify.py

@@ -85,7 +85,11 @@ class ISBNMerge(object):
                 isbns, min_year = xisbn.get_isbn_pool(isbn)
                 if not isbns:
                     isbns = frozenset([isbn])
-                self.pools[isbns] = pool = (min_year, [])
+                if isbns in self.pools:
+                    # xISBN had a brain fart
+                    pool = self.pools[isbns]
+                else:
+                    self.pools[isbns] = pool = (min_year, [])
 
             if not self.pool_has_result_from_same_source(pool, result):
                 pool[1].append(result)

src/calibre/ebooks/metadata/worker.py

@@ -15,7 +15,7 @@ from calibre.utils.ipc.server import Server
 from calibre.ptempfile import PersistentTemporaryDirectory, TemporaryDirectory
 from calibre import prints, isbytestring
 from calibre.constants import filesystem_encoding
-
+from calibre.db.errors import NoSuchFormat
 
 def debug(*args):
     prints(*args)
@@ -201,27 +201,35 @@ class SaveWorker(Thread):
         self.spare_server = spare_server
         self.start()
 
-    def collect_data(self, ids):
+    def collect_data(self, ids, tdir):
         from calibre.ebooks.metadata.opf2 import metadata_to_opf
         data = {}
         for i in set(ids):
-            mi = self.db.get_metadata(i, index_is_id=True, get_cover=True)
+            mi = self.db.get_metadata(i, index_is_id=True, get_cover=True,
+                    cover_as_data=True)
             opf = metadata_to_opf(mi)
             if isbytestring(opf):
                 opf = opf.decode('utf-8')
             cpath = None
-            if mi.cover:
+            if mi.cover_data and mi.cover_data[1]:
-                cpath = mi.cover
+                cpath = os.path.join(tdir, 'cover_%s.jpg'%i)
+                with lopen(cpath, 'wb') as f:
+                    f.write(mi.cover_data[1])
                 if isbytestring(cpath):
                     cpath = cpath.decode(filesystem_encoding)
             formats = {}
             if mi.formats:
                 for fmt in mi.formats:
-                    fpath = self.db.format_abspath(i, fmt, index_is_id=True)
+                    fpath = os.path.join(tdir, 'fmt_%s.%s'%(i, fmt.lower()))
-                    if fpath is not None:
+                    with lopen(fpath, 'wb') as f:
-                        if isbytestring(fpath):
+                        try:
-                            fpath = fpath.decode(filesystem_encoding)
+                            self.db.copy_format_to(i, fmt, f, index_is_id=True)
-                        formats[fmt.lower()] = fpath
+                        except NoSuchFormat:
+                            continue
+                        else:
+                            if isbytestring(fpath):
+                                fpath = fpath.decode(filesystem_encoding)
+                            formats[fmt.lower()] = fpath
             data[i] = [opf, cpath, formats, mi.last_modified.isoformat()]
         return data
 
@@ -244,7 +252,7 @@ class SaveWorker(Thread):
 
         for i, task in enumerate(tasks):
             tids = [x[-1] for x in task]
-            data = self.collect_data(tids)
+            data = self.collect_data(tids, tdir)
             dpath = os.path.join(tdir, '%d.json'%i)
             with open(dpath, 'wb') as f:
                 f.write(json.dumps(data, ensure_ascii=False).encode('utf-8'))

src/calibre/ebooks/metadata/xisbn.py

@@ -45,6 +45,11 @@ class xISBN(object):
                 ans.append(rec)
         return ans
 
+    def isbns_in_data(self, data):
+        for rec in data:
+            for i in rec.get('isbn', []):
+                yield i
+
     def get_data(self, isbn):
         isbn = self.purify(isbn)
         with self.lock:
@@ -57,9 +62,8 @@ class xISBN(object):
                     data = []
                 id_ = len(self._data)
                 self._data.append(data)
-                for rec in data:
+                for i in self.isbns_in_data(data):
-                    for i in rec.get('isbn', []):
+                    self._map[i] = id_
-                        self._map[i] = id_
                 self._map[isbn] = id_
             return self._data[self._map[isbn]]
 

src/calibre/ebooks/mobi/mobiml.py

@@ -443,11 +443,15 @@ class MobiMLizer(object):
             tag = 'span' if tag == 'td' else 'div'
 
         if tag == 'table':
+            col = style.backgroundColor
+            if col:
+                elem.set('bgcolor', col)
             css = style.cssdict()
             if 'border' in css or 'border-width' in css:
                 elem.set('border', '1')
         if tag in TABLE_TAGS:
-            for attr in ('rowspan', 'colspan', 'width', 'border', 'scope'):
+            for attr in ('rowspan', 'colspan', 'width', 'border', 'scope',
+                    'bgcolor'):
                 if attr in elem.attrib:
                     istate.attrib[attr] = elem.attrib[attr]
         if tag == 'q':

src/calibre/ebooks/mobi/writer.py

@@ -241,6 +241,7 @@ class Serializer(object):
         if self.write_page_breaks_after_item:
             buffer.write('<mbp:pagebreak/>')
         buffer.write('</div>')
+        self.anchor_offset = None
 
     def serialize_elem(self, elem, item, nsrmap=NSRMAP):
         buffer = self.buffer

src/calibre/ebooks/oeb/base.py

@@ -1055,6 +1055,12 @@ class Manifest(object):
                         and len(a) == 0 and not a.text:
                     remove_elem(a)
 
+            # Convert <br>s with content into paragraphs as ADE can't handle
+            # them
+            for br in xpath(data, '//h:br'):
+                if len(br) > 0 or br.text:
+                    br.tag = XHTML('div')
+
             return data
 
         def _parse_txt(self, data):
@@ -1156,7 +1162,7 @@ class Manifest(object):
                     data = self._parse_xml(data)
                 elif self.media_type.lower() in OEB_STYLES:
                     data = self._parse_css(data)
-                elif 'text' in self.media_type.lower():
+                elif self.media_type.lower() == 'text/plain':
                     self.oeb.log.warn('%s contains data in TXT format'%self.href,
                             'converting to HTML')
                     data = self._parse_txt(data)

src/calibre/ebooks/oeb/stylizer.py

@@ -11,7 +11,6 @@ __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
 import os, itertools, re, logging, copy, unicodedata
 from weakref import WeakKeyDictionary
 from xml.dom import SyntaxErr as CSSSyntaxError
-import cssutils
 from cssutils.css import (CSSStyleRule, CSSPageRule, CSSStyleDeclaration,
     CSSFontFaceRule, cssproperties)
 try:
@@ -20,7 +19,8 @@ try:
 except ImportError:
     # cssutils >= 0.9.8
     from cssutils.css import PropertyValue as CSSValueList
-from cssutils import profile as cssprofiles
+from cssutils import (profile as cssprofiles, parseString, parseStyle, log as
+        cssutils_log, CSSParser, profiles)
 from lxml import etree
 from lxml.cssselect import css_to_xpath, ExpressionError, SelectorSyntaxError
 from calibre import force_unicode
@@ -28,7 +28,7 @@ from calibre.ebooks import unit_convert
 from calibre.ebooks.oeb.base import XHTML, XHTML_NS, CSS_MIME, OEB_STYLES
 from calibre.ebooks.oeb.base import XPNSMAP, xpath, urlnormalize
 
-cssutils.log.setLevel(logging.WARN)
+cssutils_log.setLevel(logging.WARN)
 
 _html_css_stylesheet = None
 
@@ -36,7 +36,7 @@ def html_css_stylesheet():
     global _html_css_stylesheet
     if _html_css_stylesheet is None:
         html_css = open(P('templates/html.css'), 'rb').read()
-        _html_css_stylesheet = cssutils.parseString(html_css)
+        _html_css_stylesheet = parseString(html_css)
         _html_css_stylesheet.namespaces['h'] = XHTML_NS
     return _html_css_stylesheet
 
@@ -157,11 +157,11 @@ class Stylizer(object):
 
         # Add cssutils parsing profiles from output_profile
         for profile in self.opts.output_profile.extra_css_modules:
-            cssutils.profile.addProfile(profile['name'],
+            cssprofiles.addProfile(profile['name'],
                                         profile['props'],
                                         profile['macros'])
 
-        parser = cssutils.CSSParser(fetcher=self._fetch_css_file,
+        parser = CSSParser(fetcher=self._fetch_css_file,
                 log=logging.getLogger('calibre.css'))
         self.font_face_rules = []
         for elem in head:
@@ -473,6 +473,7 @@ class Style(object):
         self._width = None
         self._height = None
         self._lineHeight = None
+        self._bgcolor = None
         stylizer._styles[element] = self
 
     def set(self, prop, val):
@@ -533,6 +534,48 @@ class Style(object):
     def pt_to_px(self, value):
         return (self._profile.dpi / 72.0) * value
 
+    @property
+    def backgroundColor(self):
+        '''
+        Return the background color by parsing both the background-color and
+        background shortcut properties. Note that inheritance/default values
+        are not used. None is returned if no background color is set.
+        '''
+
+        def validate_color(col):
+            return cssprofiles.validateWithProfile('color',
+                        col,
+                        profiles=[profiles.Profiles.CSS_LEVEL_2])[1]
+
+        if self._bgcolor is None:
+            col = None
+            val = self._style.get('background-color', None)
+            if val and validate_color(val):
+                col = val
+            else:
+                val = self._style.get('background', None)
+                if val is not None:
+                    try:
+                        style = parseStyle('background: '+val)
+                        val = style.getProperty('background').cssValue
+                        try:
+                            val = list(val)
+                        except:
+                            # val is CSSPrimitiveValue
+                            val = [val]
+                        for c in val:
+                            c = c.cssText
+                            if validate_color(c):
+                                col = c
+                                break
+                    except:
+                        pass
+            if col is None:
+                self._bgcolor = False
+            else:
+                self._bgcolor = col
+        return self._bgcolor if self._bgcolor else None
+
     @property
     def fontSize(self):
         def normalize_fontsize(value, base):

src/calibre/gui2/__init__.py

@@ -248,10 +248,11 @@ def error_dialog(parent, title, msg, det_msg='', show=False,
         return d.exec_()
     return d
 
-def question_dialog(parent, title, msg, det_msg='', show_copy_button=False):
+def question_dialog(parent, title, msg, det_msg='', show_copy_button=False,
+        default_yes=True):
     from calibre.gui2.dialogs.message_box import MessageBox
     d = MessageBox(MessageBox.QUESTION, title, msg, det_msg, parent=parent,
-                    show_copy_button=show_copy_button)
+                    show_copy_button=show_copy_button, default_yes=default_yes)
     return d.exec_() == d.Accepted
 
 def info_dialog(parent, title, msg, det_msg='', show=False,

src/calibre/gui2/actions/choose_library.py

@@ -5,7 +5,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, shutil
+import os
 from functools import partial
 
 from PyQt4.Qt import QMenu, Qt, QInputDialog, QToolButton
@@ -14,7 +14,7 @@ from calibre import isbytestring
 from calibre.constants import filesystem_encoding, iswindows
 from calibre.utils.config import prefs
 from calibre.gui2 import (gprefs, warning_dialog, Dispatcher, error_dialog,
-    question_dialog, info_dialog)
+    question_dialog, info_dialog, open_local_file)
 from calibre.library.database2 import LibraryDatabase2
 from calibre.gui2.actions import InterfaceAction
 
@@ -107,7 +107,7 @@ class ChooseLibraryAction(InterfaceAction):
             self.quick_menu_action = self.choose_menu.addMenu(self.quick_menu)
             self.rename_menu = QMenu(_('Rename library'))
             self.rename_menu_action = self.choose_menu.addMenu(self.rename_menu)
-            self.delete_menu = QMenu(_('Delete library'))
+            self.delete_menu = QMenu(_('Remove library'))
             self.delete_menu_action = self.choose_menu.addMenu(self.delete_menu)
 
         ac = self.create_action(spec=(_('Pick a random book'), 'catalog.png',
@@ -252,21 +252,15 @@ class ChooseLibraryAction(InterfaceAction):
 
     def delete_requested(self, name, location):
         loc = location.replace('/', os.sep)
-        if not question_dialog(self.gui, _('Are you sure?'), '<p>'+
-                _('<b style="color: red">All files</b> (not just ebooks) '
-                    'from <br><br><b>%s</b><br><br> will be '
-                '<b>permanently deleted</b>. Are you sure?') % loc,
-                show_copy_button=False):
-            return
-        exists = self.gui.library_view.model().db.exists_at(loc)
-        if exists:
-            try:
-                shutil.rmtree(loc, ignore_errors=True)
-            except:
-                pass
         self.stats.remove(location)
         self.build_menus()
         self.gui.iactions['Copy To Library'].build_menus()
+        info_dialog(self.gui, _('Library removed'),
+                _('The library %s has been removed from calibre. '
+                    'The files remain on your computer, if you want '
+                    'to delete them, you will have to do so manually.') % loc,
+                show=True)
+        open_local_file(loc)
 
     def backup_status(self, location):
         dirty_text = 'no'
@@ -287,6 +281,18 @@ class ChooseLibraryAction(InterfaceAction):
               'rate of approximately 1 book every three seconds.'), show=True)
 
     def restore_database(self):
+        m = self.gui.library_view.model()
+        db = m.db
+        if (iswindows and len(db.library_path) >
+                LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT):
+            return error_dialog(self.gui, _('Too long'),
+                    _('Path to library too long. Must be less than'
+                    ' %d characters. Move your library to a location with'
+                    ' a shorter path using Windows Explorer, then point'
+                    ' calibre to the new location and try again.')%
+                    LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT,
+                    show=True)
+
         from calibre.gui2.dialogs.restore_library import restore_database
         m = self.gui.library_view.model()
         m.stop_metadata_backup()

src/calibre/gui2/actions/copy_to_library.py

@@ -53,13 +53,18 @@ class Worker(Thread): # {{{
         from calibre.library.database2 import LibraryDatabase2
         newdb = LibraryDatabase2(self.loc)
         for i, x in enumerate(self.ids):
-            mi = self.db.get_metadata(x, index_is_id=True, get_cover=True)
+            mi = self.db.get_metadata(x, index_is_id=True, get_cover=True,
+                    cover_as_data=True)
             self.progress(i, mi.title)
             fmts = self.db.formats(x, index_is_id=True)
             if not fmts: fmts = []
             else: fmts = fmts.split(',')
-            paths = [self.db.format_abspath(x, fmt, index_is_id=True) for fmt in
+            paths = []
-                    fmts]
+            for fmt in fmts:
+                p = self.db.format(x, fmt, index_is_id=True,
+                    as_path=True)
+                if p:
+                    paths.append(p)
             added = False
             if prefs['add_formats_to_existing']:
                 identical_book_list = newdb.find_identical_books(mi)
@@ -75,6 +80,11 @@ class Worker(Thread): # {{{
                 if co is not None:
                     newdb.set_conversion_options(x, 'PIPE', co)
             self.processed.add(x)
+            for path in paths:
+                try:
+                    os.remove(path)
+                except:
+                    pass
 # }}}
 
 class CopyToLibraryAction(InterfaceAction):

src/calibre/gui2/actions/edit_metadata.py

@@ -17,6 +17,7 @@ from calibre.gui2.dialogs.tag_list_editor import TagListEditor
 from calibre.gui2.actions import InterfaceAction
 from calibre.ebooks.metadata import authors_to_string
 from calibre.utils.icu import sort_key
+from calibre.db.errors import NoSuchFormat
 
 class EditMetadataAction(InterfaceAction):
 
@@ -265,7 +266,7 @@ class EditMetadataAction(InterfaceAction):
                                 +'</p>', 'merge_too_many_books', self.gui):
                 return
 
-        dest_id, src_books, src_ids = self.books_to_merge(rows)
+        dest_id, src_ids = self.books_to_merge(rows)
         title = self.gui.library_view.model().db.title(dest_id, index_is_id=True)
         if safe_merge:
             if not confirm('<p>'+_(
@@ -277,7 +278,7 @@ class EditMetadataAction(InterfaceAction):
                 'Please confirm you want to proceed.')%title
             +'</p>', 'merge_books_safe', self.gui):
                 return
-            self.add_formats(dest_id, src_books)
+            self.add_formats(dest_id, self.formats_for_books(rows))
             self.merge_metadata(dest_id, src_ids)
         elif merge_only_formats:
             if not confirm('<p>'+_(
@@ -293,7 +294,7 @@ class EditMetadataAction(InterfaceAction):
                 'Are you <b>sure</b> you want to proceed?')%title
             +'</p>', 'merge_only_formats', self.gui):
                 return
-            self.add_formats(dest_id, src_books)
+            self.add_formats(dest_id, self.formats_for_books(rows))
             self.delete_books_after_merge(src_ids)
         else:
             if not confirm('<p>'+_(
@@ -308,7 +309,7 @@ class EditMetadataAction(InterfaceAction):
                 'Are you <b>sure</b> you want to proceed?')%title
             +'</p>', 'merge_books', self.gui):
                 return
-            self.add_formats(dest_id, src_books)
+            self.add_formats(dest_id, self.formats_for_books(rows))
             self.merge_metadata(dest_id, src_ids)
             self.delete_books_after_merge(src_ids)
             # leave the selection highlight on first selected book
@@ -329,8 +330,22 @@ class EditMetadataAction(InterfaceAction):
                     self.gui.library_view.model().db.add_format(dest_id, fmt, f, index_is_id=True,
                             notify=False, replace=replace)
 
+    def formats_for_books(self, rows):
+        m = self.gui.library_view.model()
+        ans = []
+        for id_ in map(m.id, rows):
+            dbfmts = m.db.formats(id_, index_is_id=True)
+            if dbfmts:
+                for fmt in dbfmts.split(','):
+                    try:
+                        path = m.db.format(id_, fmt, index_is_id=True,
+                                as_path=True)
+                        ans.append(path)
+                    except NoSuchFormat:
+                        continue
+        return ans
+
     def books_to_merge(self, rows):
-        src_books = []
         src_ids = []
         m = self.gui.library_view.model()
         for i, row in enumerate(rows):
@@ -339,22 +354,19 @@ class EditMetadataAction(InterfaceAction):
                 dest_id = id_
             else:
                 src_ids.append(id_)
-                dbfmts = m.db.formats(id_, index_is_id=True)
+        return [dest_id, src_ids]
-                if dbfmts:
-                    for fmt in dbfmts.split(','):
-                        src_books.append(m.db.format_abspath(id_, fmt,
-                            index_is_id=True))
-        return [dest_id, src_books, src_ids]
 
     def delete_books_after_merge(self, ids_to_delete):
         self.gui.library_view.model().delete_books_by_id(ids_to_delete)
 
     def merge_metadata(self, dest_id, src_ids):
         db = self.gui.library_view.model().db
-        dest_mi = db.get_metadata(dest_id, index_is_id=True, get_cover=True)
+        dest_mi = db.get_metadata(dest_id, index_is_id=True)
         orig_dest_comments = dest_mi.comments
+        dest_cover = db.cover(dest_id, index_is_id=True)
+        had_orig_cover = bool(dest_cover)
         for src_id in src_ids:
-            src_mi = db.get_metadata(src_id, index_is_id=True, get_cover=True)
+            src_mi = db.get_metadata(src_id, index_is_id=True)
             if src_mi.comments and orig_dest_comments != src_mi.comments:
                 if not dest_mi.comments:
                     dest_mi.comments = src_mi.comments
@@ -372,8 +384,10 @@ class EditMetadataAction(InterfaceAction):
                     dest_mi.tags = src_mi.tags
                 else:
                     dest_mi.tags.extend(src_mi.tags)
-            if src_mi.cover and not dest_mi.cover:
+            if not dest_cover:
-                dest_mi.cover = src_mi.cover
+                src_cover = db.cover(src_id, index_is_id=True)
+                if src_cover:
+                    dest_cover = src_cover
             if not dest_mi.publisher:
                 dest_mi.publisher = src_mi.publisher
             if not dest_mi.rating:
@@ -382,6 +396,8 @@ class EditMetadataAction(InterfaceAction):
                 dest_mi.series = src_mi.series
                 dest_mi.series_index = src_mi.series_index
         db.set_metadata(dest_id, dest_mi, ignore_errors=False)
+        if not had_orig_cover and dest_cover:
+            db.set_cover(dest_id, dest_cover)
 
         for key in db.field_metadata: #loop thru all defined fields
           if db.field_metadata[key]['is_custom']:

src/calibre/gui2/actions/plugin_updates.py

@@ -12,10 +12,10 @@ from calibre.gui2.actions import InterfaceAction
 from calibre.gui2.dialogs.plugin_updater import (PluginUpdaterDialog,
                                                  FILTER_ALL, FILTER_UPDATE_AVAILABLE)
 
-class PluginUpdatesAction(InterfaceAction):
+class PluginUpdaterAction(InterfaceAction):
 
-    name = 'Plugin Updates'
+    name = 'Plugin Updater'
-    action_spec = (_('Plugin Updates'), None, None, None)
+    action_spec = (_('Plugin Updater'), None, None, None)
     action_type = 'current'
 
     def genesis(self):

src/calibre/gui2/actions/show_quickview.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+from calibre.gui2.actions import InterfaceAction
+from calibre.gui2.dialogs.quickview import Quickview
+from calibre.gui2 import error_dialog
+
+class ShowQuickviewAction(InterfaceAction):
+
+    name = 'Show quickview'
+    action_spec = (_('Show quickview'), 'search.png', None, _('Q'))
+    dont_add_to = frozenset(['menubar-device', 'toolbar-device', 'context-menu-device'])
+    action_type = 'current'
+
+    current_instance = None
+
+    def genesis(self):
+        self.qaction.triggered.connect(self.show_quickview)
+
+    def show_quickview(self, *args):
+        if self.current_instance:
+            if not self.current_instance.is_closed:
+                return
+            self.current_instance = None
+        if self.gui.current_view() is not self.gui.library_view:
+            error_dialog(self.gui, _('No quickview available'),
+                _('Quickview is not available for books '
+                  'on the device.')).exec_()
+            return
+        index = self.gui.library_view.currentIndex()
+        if index.isValid():
+            self.current_instance = \
+                Quickview(self.gui, self.gui.library_view, index)
+            self.current_instance.show()
+

src/calibre/gui2/actions/tweak_epub.py

@@ -5,6 +5,8 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
+import os
+
 from calibre.gui2 import error_dialog
 from calibre.gui2.actions import InterfaceAction
 from calibre.gui2.dialogs.tweak_epub import TweakEpub
@@ -30,8 +32,8 @@ class TweakEpubAction(InterfaceAction):
         # Confirm 'EPUB' in formats
         book_id = self.gui.library_view.model().id(row)
         try:
-            path_to_epub = self.gui.library_view.model().db.format_abspath(
+            path_to_epub = self.gui.library_view.model().db.format(
-                    book_id, 'EPUB', index_is_id=True)
+                    book_id, 'EPUB', index_is_id=True, as_path=True)
         except:
             path_to_epub = None
 
@@ -45,6 +47,7 @@ class TweakEpubAction(InterfaceAction):
         if dlg.exec_() == dlg.Accepted:
             self.update_db(book_id, dlg._output)
         dlg.cleanup()
+        os.remove(path_to_epub)
 
     def update_db(self, book_id, rebuilt):
         '''

src/calibre/gui2/add.py

@@ -445,12 +445,14 @@ class Saver(QObject): # {{{
         self.pd.setModal(True)
         self.pd.show()
         self.pd.set_min(0)
+        self.pd.set_msg(_('Collecting data, please wait...'))
         self._parent = parent
         self.callback = callback
         self.callback_called = False
         self.rq = Queue()
         self.ids = [x for x in map(db.id, [r.row() for r in rows]) if x is not None]
-        self.pd.set_max(len(self.ids))
+        self.pd_max = len(self.ids)
+        self.pd.set_max(0)
         self.pd.value = 0
         self.failures = set([])
 
@@ -509,6 +511,8 @@ class Saver(QObject): # {{{
             id, title, ok, tb = self.rq.get_nowait()
         except Empty:
             return
+        if self.pd.max != self.pd_max:
+            self.pd.max = self.pd_max
         self.pd.value += 1
         self.ids.remove(id)
         if not isinstance(title, unicode):

src/calibre/gui2/convert/regex_builder.py

@@ -4,7 +4,7 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
-import re
+import re, os
 
 from PyQt4.QtCore import SIGNAL, Qt, pyqtSignal
 from PyQt4.QtGui import QDialog, QWidget, QDialogButtonBox, \
@@ -134,7 +134,17 @@ class RegexBuilder(QDialog, Ui_RegexBuilder):
                          _('Cannot build regex using the GUI builder without a book.'),
                          show=True)
             return False
-        self.open_book(db.format_abspath(book_id, format, index_is_id=True))
+        fpath = db.format(book_id, format, index_is_id=True,
+            as_path=True)
+        try:
+            self.open_book(fpath)
+        finally:
+            try:
+                os.remove(fpath)
+            except:
+                # Fails on windows if the input plugin for this format keeps the file open
+                # Happens for LIT files
+                pass
         return True
 
     def open_book(self, pathtoebook):
@@ -143,7 +153,8 @@ class RegexBuilder(QDialog, Ui_RegexBuilder):
         text = [u'']
         preprocessor = HTMLPreProcessor(None, False)
         for path in self.iterator.spine:
-            html = open(path, 'rb').read().decode('utf-8', 'replace')
+            with open(path, 'rb') as f:
+                html = f.read().decode('utf-8', 'replace')
             html = preprocessor(html, get_preprocess_html=True)
             text.append(html)
         self.preview.setPlainText('\n---\n'.join(text))

src/calibre/gui2/convert/single.py

@@ -11,8 +11,8 @@ import sys, cPickle, shutil, importlib
 from PyQt4.Qt import QString, SIGNAL, QAbstractListModel, Qt, QVariant, QFont
 
 from calibre.gui2 import ResizableDialog, NONE
-from calibre.ebooks.conversion.config import GuiRecommendations, save_specifics, \
+from calibre.ebooks.conversion.config import (GuiRecommendations, save_specifics,
-        load_specifics
+        load_specifics)
 from calibre.gui2.convert.single_ui import Ui_Dialog
 from calibre.gui2.convert.metadata import MetadataWidget
 from calibre.gui2.convert.look_and_feel import LookAndFeelWidget
@@ -24,7 +24,8 @@ from calibre.gui2.convert.toc import TOCWidget
 from calibre.gui2.convert.debug import DebugWidget
 
 
-from calibre.ebooks.conversion.plumber import Plumber, supported_input_formats
+from calibre.ebooks.conversion.plumber import (Plumber,
+        supported_input_formats, ARCHIVE_FMTS)
 from calibre.ebooks.conversion.config import delete_specifics
 from calibre.customize.ui import available_output_formats
 from calibre.customize.conversion import OptionRecommendation
@@ -106,7 +107,6 @@ class Config(ResizableDialog, Ui_Dialog):
     Configuration dialog for single book conversion. If accepted, has the
     following important attributes
 
-    input_path - Path to input file
     output_format - Output format (without a leading .)
     input_format  - Input format (without a leading .)
     opf_path - Path to OPF file with user specified metadata
@@ -156,13 +156,13 @@ class Config(ResizableDialog, Ui_Dialog):
         oidx = self.groups.currentIndex().row()
         input_format = self.input_format
         output_format = self.output_format
-        input_path = self.db.format_abspath(self.book_id, input_format,
-                index_is_id=True)
-        self.input_path = input_path
         output_path = 'dummy.'+output_format
         log = Log()
         log.outputs = []
-        self.plumber = Plumber(input_path, output_path, log)
+        input_file = 'dummy.'+input_format
+        if input_format in ARCHIVE_FMTS:
+            input_file = 'dummy.html'
+        self.plumber = Plumber(input_file, output_path, log)
 
         def widget_factory(cls):
             return cls(self.stack, self.plumber.get_option_by_name,

src/calibre/gui2/custom_column_widgets.py

@@ -25,7 +25,7 @@ class Base(object):
     def __init__(self, db, col_id, parent=None):
         self.db, self.col_id = db, col_id
         self.col_metadata = db.custom_column_num_map[col_id]
-        self.initial_val = None
+        self.initial_val = self.widgets = None
         self.setup_ui(parent)
 
     def initialize(self, book_id):
@@ -54,6 +54,9 @@ class Base(object):
     def normalize_ui_val(self, val):
         return val
 
+    def break_cycles(self):
+        self.db = self.widgets = self.initial_val = None
+
 class Bool(Base):
 
     def setup_ui(self, parent):

src/calibre/gui2/device.py

@@ -49,16 +49,26 @@ class DeviceJob(BaseJob): # {{{
         self._aborted = False
 
     def start_work(self):
+        if DEBUG:
+            prints('Job:', self.id, self.description, 'started',
+                safe_encode=True)
         self.start_time = time.time()
         self.job_manager.changed_queue.put(self)
 
     def job_done(self):
         self.duration = time.time() - self.start_time
         self.percent = 1
+        if DEBUG:
+            prints('DeviceJob:', self.id, self.description,
+                    'done, calling callback', safe_encode=True)
+
         try:
             self.callback_on_done(self)
         except:
             pass
+        if DEBUG:
+            prints('DeviceJob:', self.id, self.description,
+                    'callback returned', safe_encode=True)
         self.job_manager.changed_queue.put(self)
 
     def report_progress(self, percent, msg=''):
@@ -119,6 +129,7 @@ class DeviceManager(Thread): # {{{
         self.sleep_time     = sleep_time
         self.connected_slot = connected_slot
         self.jobs           = Queue.Queue(0)
+        self.job_steps      = Queue.Queue(0)
         self.keep_going     = True
         self.job_manager    = job_manager
         self.reported_errors = set([])
@@ -235,6 +246,12 @@ class DeviceManager(Thread): # {{{
                 self.connected_device.unmount_device()
 
     def next(self):
+        if not self.job_steps.empty():
+            try:
+                return self.job_steps.get_nowait()
+            except Queue.Empty:
+                pass
+
         if not self.jobs.empty():
             try:
                 return self.jobs.get_nowait()
@@ -271,13 +288,20 @@ class DeviceManager(Thread): # {{{
                     break
             time.sleep(self.sleep_time)
 
-    def create_job(self, func, done, description, args=[], kwargs={}):
+    def create_job_step(self, func, done, description, to_job, args=[], kwargs={}):
         job = DeviceJob(func, done, self.job_manager,
                         args=args, kwargs=kwargs, description=description)
         self.job_manager.add_job(job)
-        self.jobs.put(job)
+        if (done is None or isinstance(done, FunctionDispatcher)) and \
+                    (to_job is not None and to_job == self.current_job):
+            self.job_steps.put(job)
+        else:
+            self.jobs.put(job)
         return job
 
+    def create_job(self, func, done, description, args=[], kwargs={}):
+        return self.create_job_step(func, done, description, None, args, kwargs)
+
     def has_card(self):
         try:
             return bool(self.device.card_prefix())
@@ -295,10 +319,10 @@ class DeviceManager(Thread): # {{{
         self._device_information = {'info': info, 'prefixes': cp, 'freespace': fs}
         return info, cp, fs
 
-    def get_device_information(self, done):
+    def get_device_information(self, done, add_as_step_to_job=None):
         '''Get device information and free space on device'''
-        return self.create_job(self._get_device_information, done,
+        return self.create_job_step(self._get_device_information, done,
-                    description=_('Get device information'))
+                    description=_('Get device information'), to_job=add_as_step_to_job)
 
     def get_current_device_information(self):
         return self._device_information
@@ -310,36 +334,38 @@ class DeviceManager(Thread): # {{{
         cardblist = self.device.books(oncard='cardb')
         return (mainlist, cardalist, cardblist)
 
-    def books(self, done):
+    def books(self, done, add_as_step_to_job=None):
         '''Return callable that returns the list of books on device as two booklists'''
-        return self.create_job(self._books, done, description=_('Get list of books on device'))
+        return self.create_job_step(self._books, done,
+                description=_('Get list of books on device'), to_job=add_as_step_to_job)
 
     def _annotations(self, path_map):
         return self.device.get_annotations(path_map)
 
-    def annotations(self, done, path_map):
+    def annotations(self, done, path_map, add_as_step_to_job=None):
         '''Return mapping of ids to annotations. Each annotation is of the
         form (type, location_info, content). path_map is a mapping of
         ids to paths on the device.'''
-        return self.create_job(self._annotations, done, args=[path_map],
+        return self.create_job_step(self._annotations, done, args=[path_map],
-                description=_('Get annotations from device'))
+                description=_('Get annotations from device'), to_job=add_as_step_to_job)
 
     def _sync_booklists(self, booklists):
         '''Sync metadata to device'''
         self.device.sync_booklists(booklists, end_session=False)
         return self.device.card_prefix(end_session=False), self.device.free_space()
 
-    def sync_booklists(self, done, booklists, plugboards):
+    def sync_booklists(self, done, booklists, plugboards, add_as_step_to_job=None):
         if hasattr(self.connected_device, 'set_plugboards') and \
                 callable(self.connected_device.set_plugboards):
             self.connected_device.set_plugboards(plugboards, find_plugboard)
-        return self.create_job(self._sync_booklists, done, args=[booklists],
+        return self.create_job_step(self._sync_booklists, done, args=[booklists],
-                        description=_('Send metadata to device'))
+                        description=_('Send metadata to device'), to_job=add_as_step_to_job)
 
-    def upload_collections(self, done, booklist, on_card):
+    def upload_collections(self, done, booklist, on_card, add_as_step_to_job=None):
-        return self.create_job(booklist.rebuild_collections, done,
+        return self.create_job_step(booklist.rebuild_collections, done,
                                args=[booklist, on_card],
-                        description=_('Send collections to device'))
+                        description=_('Send collections to device'),
+                        to_job=add_as_step_to_job)
 
     def _upload_books(self, files, names, on_card=None, metadata=None, plugboards=None):
         '''Upload books to device: '''
@@ -370,15 +396,25 @@ class DeviceManager(Thread): # {{{
                             if DEBUG:
                                 prints(traceback.format_exc(), file=sys.__stdout__)
 
-        return self.device.upload_books(files, names, on_card,
+        try:
-                                        metadata=metadata, end_session=False)
+            return self.device.upload_books(files, names, on_card,
+                    metadata=metadata, end_session=False)
+        finally:
+            if metadata:
+                for mi in metadata:
+                    try:
+                        if mi.cover:
+                            os.remove(mi.cover)
+                    except:
+                        pass
 
     def upload_books(self, done, files, names, on_card=None, titles=None,
-                     metadata=None, plugboards=None):
+                     metadata=None, plugboards=None, add_as_step_to_job=None):
         desc = _('Upload %d books to device')%len(names)
         if titles:
             desc += u':' + u', '.join(titles)
-        return self.create_job(self._upload_books, done, args=[files, names],
+        return self.create_job_step(self._upload_books, done, to_job=add_as_step_to_job,
+                               args=[files, names],
                 kwargs={'on_card':on_card,'metadata':metadata,'plugboards':plugboards}, description=desc)
 
     def add_books_to_metadata(self, locations, metadata, booklists):
@@ -388,9 +424,10 @@ class DeviceManager(Thread): # {{{
         '''Remove books from device'''
         self.device.delete_books(paths, end_session=True)
 
-    def delete_books(self, done, paths):
+    def delete_books(self, done, paths, add_as_step_to_job=None):
-        return self.create_job(self._delete_books, done, args=[paths],
+        return self.create_job_step(self._delete_books, done, args=[paths],
-                        description=_('Delete books from device'))
+                        description=_('Delete books from device'),
+                        to_job=add_as_step_to_job)
 
     def remove_books_from_metadata(self, paths, booklists):
         self.device.remove_books_from_metadata(paths, booklists)
@@ -405,9 +442,10 @@ class DeviceManager(Thread): # {{{
                 self.device.get_file(path, f)
                 f.close()
 
-    def save_books(self, done, paths, target):
+    def save_books(self, done, paths, target, add_as_step_to_job=None):
-        return self.create_job(self._save_books, done, args=[paths, target],
+        return self.create_job_step(self._save_books, done, args=[paths, target],
-                        description=_('Download books from device'))
+                        description=_('Download books from device'),
+                        to_job=add_as_step_to_job)
 
     def _view_book(self, path, target):
         f = open(target, 'wb')
@@ -415,9 +453,9 @@ class DeviceManager(Thread): # {{{
         f.close()
         return target
 
-    def view_book(self, done, path, target):
+    def view_book(self, done, path, target, add_as_step_to_job=None):
-        return self.create_job(self._view_book, done, args=[path, target],
+        return self.create_job_step(self._view_book, done, args=[path, target],
-                        description=_('View book on device'))
+                        description=_('View book on device'), to_job=add_as_step_to_job)
 
     def set_current_library_uuid(self, uuid):
         self.current_library_uuid = uuid
@@ -778,7 +816,8 @@ class DeviceMixin(object): # {{{
                 self.device_manager.device.icon)
         self.bars_manager.update_bars()
         self.status_bar.device_connected(info[0])
-        self.device_manager.books(FunctionDispatcher(self.metadata_downloaded))
+        self.device_manager.books(FunctionDispatcher(self.metadata_downloaded),
+                                  add_as_step_to_job=job)
 
     def metadata_downloaded(self, job):
         '''
@@ -788,7 +827,7 @@ class DeviceMixin(object): # {{{
             self.device_job_exception(job)
             return
         # set_books_in_library might schedule a sync_booklists job
-        self.set_books_in_library(job.result, reset=True)
+        self.set_books_in_library(job.result, reset=True, add_as_step_to_job=job)
         mainlist, cardalist, cardblist = job.result
         self.memory_view.set_database(mainlist)
         self.memory_view.set_editable(self.device_manager.device.CAN_SET_METADATA,
@@ -843,8 +882,8 @@ class DeviceMixin(object): # {{{
         # set_books_in_library even though books were not added because
         # the deleted book might have been an exact match. Upload the booklists
         # if set_books_in_library did not.
-        if not self.set_books_in_library(self.booklists(), reset=True):
+        if not self.set_books_in_library(self.booklists(), reset=True, add_as_step_to_job=job):
-            self.upload_booklists()
+            self.upload_booklists(job)
         self.book_on_device(None, reset=True)
         # We need to reset the ondevice flags in the library. Use a big hammer,
         # so we don't need to worry about whether some succeeded or not.
@@ -1042,8 +1081,6 @@ class DeviceMixin(object): # {{{
                             'the device?'), autos):
                         self.iactions['Convert Books'].auto_convert_news(auto, format)
             files = [f for f in files if f is not None]
-            for f in files:
-                f.deleted_after_upload = del_on_upload
             if not files:
                 self.news_to_be_synced = set([])
                 return
@@ -1193,13 +1230,14 @@ class DeviceMixin(object): # {{{
         self.device_manager.sync_booklists(Dispatcher(lambda x: x),
                                            self.booklists(), plugboards)
 
-    def upload_booklists(self):
+    def upload_booklists(self, add_as_step_to_job=None):
         '''
         Upload metadata to device.
         '''
         plugboards = self.library_view.model().db.prefs.get('plugboards', {})
         self.device_manager.sync_booklists(FunctionDispatcher(self.metadata_synced),
-                                           self.booklists(), plugboards)
+                                           self.booklists(), plugboards,
+                                           add_as_step_to_job=add_as_step_to_job)
 
     def metadata_synced(self, job):
         '''
@@ -1274,8 +1312,8 @@ class DeviceMixin(object): # {{{
         # because the UUID changed. Force both the device and the library view
         # to refresh the flags. Set_books_in_library could upload the booklists.
         # If it does not, then do it here.
-        if not self.set_books_in_library(self.booklists(), reset=True):
+        if not self.set_books_in_library(self.booklists(), reset=True, add_as_step_to_job=job):
-            self.upload_booklists()
+            self.upload_booklists(job)
         with self.library_view.preserve_selected_books:
             self.book_on_device(None, reset=True)
             self.refresh_ondevice()
@@ -1284,8 +1322,17 @@ class DeviceMixin(object): # {{{
             self.card_b_view if on_card == 'cardb' else self.memory_view
         view.model().resort(reset=False)
         view.model().research()
-        for f in files:
+        if files:
-            getattr(f, 'close', lambda : True)()
+            for f in files:
+                # Remove temporary files
+                try:
+                    rem = not getattr(
+                            self.device_manager.device,
+                            'KEEP_TEMP_FILES_AFTER_UPLOAD', False)
+                    if rem and 'caltmpfmt.' in f:
+                        os.remove(f)
+                except:
+                    pass
 
     def book_on_device(self, id, reset=False):
         '''
@@ -1335,7 +1382,7 @@ class DeviceMixin(object): # {{{
                 loc[4] |= self.book_db_uuid_path_map[id]
         return loc
 
-    def set_books_in_library(self, booklists, reset=False):
+    def set_books_in_library(self, booklists, reset=False, add_as_step_to_job=None):
         '''
         Set the ondevice indications in the device database.
         This method should be called before book_on_device is called, because
@@ -1487,7 +1534,7 @@ class DeviceMixin(object): # {{{
                 plugboards = self.library_view.model().db.prefs.get('plugboards', {})
                 self.device_manager.sync_booklists(
                                     FunctionDispatcher(self.metadata_synced), booklists,
-                                    plugboards)
+                                    plugboards, add_as_step_to_job)
         return update_metadata
     # }}}
 

src/calibre/gui2/dialogs/choose_library.ui

@@ -41,7 +41,7 @@
    <item row="4" column="0" colspan="4">
     <widget class="QRadioButton" name="existing_library">
      <property name="text">
-      <string>Use &amp;existing library at the new location</string>
+      <string>Use the previously &amp;existing library at the new location</string>
      </property>
      <property name="checked">
       <bool>true</bool>

src/calibre/gui2/dialogs/message_box.py

@@ -23,7 +23,7 @@ class MessageBox(QDialog, Ui_Dialog): # {{{
                  det_msg='',
                  q_icon=None,
                  show_copy_button=True,
-                 parent=None):
+                 parent=None, default_yes=True):
         QDialog.__init__(self, parent)
         if q_icon is None:
             icon = {
@@ -65,7 +65,9 @@ class MessageBox(QDialog, Ui_Dialog): # {{{
         self.is_question = type_ == self.QUESTION
         if self.is_question:
             self.bb.setStandardButtons(self.bb.Yes|self.bb.No)
-            self.bb.button(self.bb.Yes).setDefault(True)
+            self.bb.button(self.bb.Yes if default_yes else self.bb.No
+                    ).setDefault(True)
+            self.default_yes = default_yes
         else:
             self.bb.button(self.bb.Ok).setDefault(True)
 
@@ -101,7 +103,8 @@ class MessageBox(QDialog, Ui_Dialog): # {{{
         ret = QDialog.showEvent(self, ev)
         if self.is_question:
             try:
-                self.bb.button(self.bb.Yes).setFocus(Qt.OtherFocusReason)
+                self.bb.button(self.bb.Yes if self.default_yes else self.bb.No
+                        ).setFocus(Qt.OtherFocusReason)
             except:
                 pass# Buttons were changed
         else:

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -24,7 +24,7 @@ from calibre.utils.config import prefs, tweaks
 from calibre.utils.magick.draw import identify_data
 from calibre.utils.date import qt_to_dt
 
-def get_cover_data(path): # {{{
+def get_cover_data(stream, ext): # {{{
     from calibre.ebooks.metadata.meta import get_metadata
     old = prefs['read_file_metadata']
     if not old:
@@ -32,8 +32,8 @@ def get_cover_data(path): # {{{
     cdata = area = None
 
     try:
-        mi = get_metadata(open(path, 'rb'),
+        with stream:
-                os.path.splitext(path)[1][1:].lower())
+            mi = get_metadata(stream, ext)
         if mi.cover and os.access(mi.cover, os.R_OK):
             cdata = open(mi.cover).read()
         elif mi.cover_data[1] is not None:
@@ -186,9 +186,10 @@ class MyBlockingBusy(QDialog): # {{{
                 if fmts:
                     covers = []
                     for fmt in fmts.split(','):
-                        fmt = self.db.format_abspath(id, fmt, index_is_id=True)
+                        fmtf = self.db.format(id, fmt, index_is_id=True,
-                        if not fmt: continue
+                                as_file=True)
-                        cdata, area = get_cover_data(fmt)
+                        if fmtf is None: continue
+                        cdata, area = get_cover_data(fmtf, fmt)
                         if cdata:
                             covers.append((cdata, area))
                     covers.sort(key=lambda x: x[1])
@@ -361,7 +362,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         fm = self.db.field_metadata
         for f in fm:
             if (f in ['author_sort'] or
-                    (fm[f]['datatype'] in ['text', 'series', 'enumeration']
+                    (fm[f]['datatype'] in ['text', 'series', 'enumeration', 'comments']
                      and fm[f].get('search_terms', None)
                      and f not in ['formats', 'ondevice']) or
                     (fm[f]['datatype'] in ['int', 'float', 'bool'] and

src/calibre/gui2/dialogs/progress.py

@@ -53,6 +53,13 @@ class ProgressDialog(QDialog, Ui_Dialog):
     def set_max(self, max):
         self.bar.setMaximum(max)
 
+    @dynamic_property
+    def max(self):
+        def fget(self): return self.bar.maximum()
+        def fset(self, val): self.bar.setMaximum(val)
+        return property(fget=fget, fset=fset)
+
+
     def _canceled(self, *args):
         self.canceled = True
         self.button_box.setDisabled(True)

src/calibre/gui2/dialogs/quickview.py

@@ -0,0 +1,254 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
+__docformat__ = 'restructuredtext en'
+
+
+from PyQt4.Qt import (Qt, QDialog, QAbstractItemView, QTableWidgetItem,
+                      QListWidgetItem, QByteArray, QCoreApplication,
+                      QApplication)
+
+from calibre.customize.ui import find_plugin
+from calibre.gui2 import gprefs
+from calibre.gui2.dialogs.quickview_ui import Ui_Quickview
+from calibre.utils.icu import sort_key
+
+class TableItem(QTableWidgetItem):
+    '''
+    A QTableWidgetItem that sorts on a separate string and uses ICU rules
+    '''
+
+    def __init__(self, val, sort):
+        self.sort = sort
+        QTableWidgetItem.__init__(self, val)
+        self.setFlags(Qt.ItemIsEnabled|Qt.ItemIsSelectable)
+
+    def __ge__(self, other):
+        return sort_key(self.sort) >= sort_key(other.sort)
+
+    def __lt__(self, other):
+        return sort_key(self.sort) < sort_key(other.sort)
+
+class Quickview(QDialog, Ui_Quickview):
+
+    def __init__(self, gui, view, row):
+        QDialog.__init__(self, gui, flags=Qt.Window)
+        Ui_Quickview.__init__(self)
+        self.setupUi(self)
+        self.isClosed = False
+
+        self.books_table_column_widths = None
+        try:
+            self.books_table_column_widths = \
+                        gprefs.get('quickview_dialog_books_table_widths', None)
+            geom = gprefs.get('quickview_dialog_geometry', bytearray(''))
+            self.restoreGeometry(QByteArray(geom))
+        except:
+            pass
+
+        # Remove the help button from the window title bar
+        icon = self.windowIcon()
+        self.setWindowFlags(self.windowFlags()&(~Qt.WindowContextHelpButtonHint))
+        self.setWindowIcon(icon)
+
+        self.db = view.model().db
+        self.view = view
+        self.gui = gui
+        self.is_closed = False
+        self.current_book_id = None
+        self.current_key = None
+        self.last_search = None
+        self.current_column = None
+        self.current_item = None
+
+        self.items.setSelectionMode(QAbstractItemView.SingleSelection)
+        self.items.currentTextChanged.connect(self.item_selected)
+
+        # Set up the books table columns
+        self.books_table.setSelectionBehavior(QAbstractItemView.SelectRows)
+        self.books_table.setSelectionMode(QAbstractItemView.SingleSelection)
+        self.books_table.setColumnCount(3)
+        t = QTableWidgetItem(_('Title'))
+        self.books_table.setHorizontalHeaderItem(0, t)
+        t = QTableWidgetItem(_('Authors'))
+        self.books_table.setHorizontalHeaderItem(1, t)
+        t = QTableWidgetItem(_('Series'))
+        self.books_table.setHorizontalHeaderItem(2, t)
+        self.books_table_header_height = self.books_table.height()
+        self.books_table.cellDoubleClicked.connect(self.book_doubleclicked)
+        self.books_table.sortByColumn(0, Qt.AscendingOrder)
+
+        # get the standard table row height. Do this here because calling
+        # resizeRowsToContents can word wrap long cell contents, creating
+        # double-high rows
+        self.books_table.setRowCount(1)
+        self.books_table.setItem(0, 0, TableItem('A', ''))
+        self.books_table.resizeRowsToContents()
+        self.books_table_row_height = self.books_table.rowHeight(0)
+        self.books_table.setRowCount(0)
+
+        # Add the data
+        self.refresh(row)
+
+        self.view.clicked.connect(self.slave)
+        QCoreApplication.instance().aboutToQuit.connect(self.save_state)
+        self.search_button.clicked.connect(self.do_search)
+        view.model().new_bookdisplay_data.connect(self.book_was_changed)
+
+    # search button
+    def do_search(self):
+        if self.last_search is not None:
+            self.gui.search.set_search_string(self.last_search)
+
+    # Called when book information is changed in the library view. Make that
+    # book current. This means that prev and next in edit metadata will move
+    # the current book.
+    def book_was_changed(self, mi):
+        if self.is_closed or self.current_column is None:
+            return
+        self.refresh(self.view.model().index(self.db.row(mi.id), self.current_column))
+
+    # clicks on the items listWidget
+    def item_selected(self, txt):
+        self.fill_in_books_box(unicode(txt))
+
+    # Given a cell in the library view, display the information
+    def refresh(self, idx):
+        bv_row = idx.row()
+        self.current_column = idx.column()
+        key = self.view.model().column_map[self.current_column]
+        book_id = self.view.model().id(bv_row)
+
+        # Only show items for categories
+        if not self.db.field_metadata[key]['is_category']:
+            if self.current_key is None:
+                return
+            key = self.current_key
+        self.items_label.setText('{0} ({1})'.format(
+                                    self.db.field_metadata[key]['name'], key))
+
+        self.items.blockSignals(True)
+        self.items.clear()
+        self.books_table.setRowCount(0)
+
+        mi = self.db.get_metadata(book_id, index_is_id=True, get_user_categories=False)
+        vals = mi.get(key, None)
+
+        if vals:
+            if not isinstance(vals, list):
+                vals = [vals]
+            vals.sort(key=sort_key)
+
+            for v in vals:
+                a = QListWidgetItem(v)
+                self.items.addItem(a)
+            self.items.setCurrentRow(0)
+
+            self.current_book_id = book_id
+            self.current_key = key
+
+            self.fill_in_books_box(vals[0])
+        self.items.blockSignals(False)
+
+    def fill_in_books_box(self, selected_item):
+        self.current_item = selected_item
+        # Do a bit of fix-up on the items so that the search works.
+        if selected_item.startswith('.'):
+            sv = '.' + selected_item
+        else:
+            sv = selected_item
+        sv = sv.replace('"', r'\"')
+        self.last_search = self.current_key+':"=' + sv + '"'
+        books = self.db.search_getting_ids(self.last_search,
+                                           self.db.data.search_restriction)
+
+        self.books_table.setRowCount(len(books))
+        self.books_label.setText(_('Books with selected item: {0}').format(len(books)))
+
+        select_item = None
+        self.books_table.setSortingEnabled(False)
+        tt = ('<p>' +
+            _('Double-click on a book to change the selection in the library view. '
+               'Shift- or control-double-click to edit the metadata of a book')
+              + '</p>')
+        for row, b in enumerate(books):
+            mi = self.db.get_metadata(b, index_is_id=True, get_user_categories=False)
+            a = TableItem(mi.title, mi.title_sort)
+            a.setData(Qt.UserRole, b)
+            a.setToolTip(tt)
+            self.books_table.setItem(row, 0, a)
+            if b == self.current_book_id:
+                select_item = a
+            a = TableItem(' & '.join(mi.authors), mi.author_sort)
+            a.setToolTip(tt)
+            self.books_table.setItem(row, 1, a)
+            series = mi.format_field('series')[1]
+            if series is None:
+                series = ''
+            a = TableItem(series, series)
+            a.setToolTip(tt)
+            self.books_table.setItem(row, 2, a)
+            self.books_table.setRowHeight(row, self.books_table_row_height)
+
+        self.books_table.setSortingEnabled(True)
+        if select_item is not None:
+            self.books_table.setCurrentItem(select_item)
+            self.books_table.scrollToItem(select_item, QAbstractItemView.PositionAtCenter)
+
+    # Deal with sizing the table columns. Done here because the numbers are not
+    # correct until the first paint.
+    def resizeEvent(self, *args):
+        QDialog.resizeEvent(self, *args)
+        if self.books_table_column_widths is not None:
+            for c,w in enumerate(self.books_table_column_widths):
+                self.books_table.setColumnWidth(c, w)
+        else:
+            # the vertical scroll bar might not be rendered, so might not yet
+            # have a width. Assume 25. Not a problem because user-changed column
+            # widths will be remembered
+            w = self.books_table.width() - 25 - self.books_table.verticalHeader().width()
+            w /= self.books_table.columnCount()
+            for c in range(0, self.books_table.columnCount()):
+                self.books_table.setColumnWidth(c, w)
+        self.save_state()
+
+    def book_doubleclicked(self, row, column):
+        book_id = self.books_table.item(row, 0).data(Qt.UserRole).toInt()[0]
+        self.view.select_rows([book_id])
+        modifiers = int(QApplication.keyboardModifiers())
+        if modifiers in (Qt.CTRL, Qt.SHIFT):
+            em = find_plugin('Edit Metadata')
+            if em is not None:
+                em.actual_plugin_.edit_metadata(None)
+
+    # called when a book is clicked on the library view
+    def slave(self, current):
+        if self.is_closed:
+            return
+        self.refresh(current)
+        self.view.activateWindow()
+
+    def save_state(self):
+        if self.is_closed:
+            return
+        self.books_table_column_widths = []
+        for c in range(0, self.books_table.columnCount()):
+            self.books_table_column_widths.append(self.books_table.columnWidth(c))
+        gprefs['quickview_dialog_books_table_widths'] = self.books_table_column_widths
+        gprefs['quickview_dialog_geometry'] = bytearray(self.saveGeometry())
+
+    def close(self):
+        self.save_state()
+        # clean up to prevent memory leaks
+        self.db = self.view = self.gui = None
+        self.is_closed = True
+
+    # called by the window system
+    def closeEvent(self, *args):
+        self.close()
+        QDialog.closeEvent(self, *args)
+
+    # called by the close button
+    def reject(self):
+        self.close()
+        QDialog.reject(self)

src/calibre/gui2/dialogs/quickview.ui

@@ -0,0 +1,131 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Quickview</class>
+ <widget class="QDialog" name="Quickview">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>768</width>
+    <height>342</height>
+   </rect>
+  </property>
+  <property name="sizePolicy">
+   <sizepolicy hsizetype="Expanding" vsizetype="MinimumExpanding">
+    <horstretch>0</horstretch>
+    <verstretch>0</verstretch>
+   </sizepolicy>
+  </property>
+  <property name="windowTitle">
+   <string>Quickview</string>
+  </property>
+  <layout class="QGridLayout">
+   <item row="0" column="0">
+    <widget class="QLabel" name="items_label">
+     <property name="text">
+      <string>Items</string>
+     </property>
+    </widget>
+   </item>
+   <item row="1" column="0">
+    <widget class="QListWidget" name="items">
+     <property name="sizePolicy">
+      <sizepolicy hsizetype="Expanding" vsizetype="Expanding" >
+       <horstretch>1</horstretch>
+       <verstretch>0</verstretch>
+      </sizepolicy>
+     </property>
+    </widget>
+   </item>
+   <item row="0" column="1">
+    <widget class="QLabel" name="books_label">
+    </widget>
+   </item>
+   <item row="1" column="1">
+    <widget class="QTableWidget" name="books_table">
+     <property name="sizePolicy">
+      <sizepolicy hsizetype="Expanding" vsizetype="Expanding" >
+       <horstretch>4</horstretch>
+       <verstretch>0</verstretch>
+      </sizepolicy>
+     </property>
+     <property name="columnCount">
+      <number>0</number>
+     </property>
+     <property name="rowCount">
+      <number>0</number>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="1">
+    <spacer>
+     <property name="orientation">
+      <enum>Qt::Vertical</enum>
+     </property>
+     <property name="sizeHint" stdset="0">
+      <size>
+       <width>0</width>
+       <height>0</height>
+      </size>
+     </property>
+    </spacer>
+   </item>
+   <item row="3" column="0" colspan="2">
+    <layout class="QHBoxLayout">
+     <item>
+      <widget class="QPushButton" name="search_button">
+       <property name="text">
+        <string>Search</string>
+       </property>
+       <property name="toolTip">
+        <string>Search in the library view for the selected item</string>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <spacer>
+       <property name="orientation">
+        <enum>Qt::Horizontal</enum>
+       </property>
+       <property name="sizeHint" stdset="0">
+        <size>
+         <width>0</width>
+         <height>0</height>
+        </size>
+       </property>
+      </spacer>
+     </item>
+     <item>
+      <widget class="QDialogButtonBox" name="buttonBox">
+       <property name="standardButtons">
+        <set>QDialogButtonBox::Close</set>
+       </property>
+       <property name="centerButtons">
+        <bool>false</bool>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>rejected()</signal>
+   <receiver>Quickview</receiver>
+   <slot>reject()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>297</x>
+     <y>217</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>286</x>
+     <y>234</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
+</ui>

src/calibre/gui2/email.py

@@ -174,7 +174,8 @@ class EmailMixin(object): # {{{
         else:
             _auto_ids = []
 
-        full_metadata = self.library_view.model().metadata_for(ids)
+        full_metadata = self.library_view.model().metadata_for(ids,
+                get_cover=False)
 
         bad, remove_ids, jobnames = [], [], []
         texts, subjects, attachments, attachment_names = [], [], [], []

src/calibre/gui2/init.py

@@ -16,7 +16,7 @@ from calibre.constants import isosx, __appname__, preferred_encoding, \
 from calibre.gui2 import config, is_widescreen, gprefs
 from calibre.gui2.library.views import BooksView, DeviceBooksView
 from calibre.gui2.widgets import Splitter
-from calibre.gui2.tag_view import TagBrowserWidget
+from calibre.gui2.tag_browser.ui import TagBrowserWidget
 from calibre.gui2.book_details import BookDetails
 from calibre.gui2.notify import get_notifier
 

src/calibre/gui2/jobs.py

@@ -432,6 +432,10 @@ class JobsDialog(QDialog, Ui_JobsDialog):
             self.jobs_view.horizontalHeader().restoreState(QByteArray(state))
         except:
             pass
+        idx = self.jobs_view.model().index(0, 0)
+        if idx.isValid():
+            sm = self.jobs_view.selectionModel()
+            sm.select(idx, sm.ClearAndSelect|sm.Rows)
 
     def save_state(self):
         try:

src/calibre/gui2/library/models.py

@@ -5,8 +5,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import shutil, functools, re, os, traceback
+import functools, re, os, traceback
-from contextlib import closing
 from collections import defaultdict
 
 from PyQt4.Qt import (QAbstractTableModel, Qt, pyqtSignal, QIcon, QImage,
@@ -36,14 +35,6 @@ TIME_FMT = '%d %b %Y'
 ALIGNMENT_MAP = {'left': Qt.AlignLeft, 'right': Qt.AlignRight, 'center':
         Qt.AlignHCenter}
 
-class FormatPath(unicode):
-
-    def __new__(cls, path, orig_file_path):
-        ans = unicode.__new__(cls, path)
-        ans.orig_file_path = orig_file_path
-        ans.deleted_after_upload = False
-        return ans
-
 _default_image = None
 
 def default_image():
@@ -391,10 +382,14 @@ class BooksModel(QAbstractTableModel): # {{{
         data = self.current_changed(index, None, False)
         return data
 
-    def metadata_for(self, ids):
+    def metadata_for(self, ids, get_cover=True):
+        '''
+        WARNING: if get_cover=True temp files are created for mi.cover.
+        Remember to delete them once you are done with them.
+        '''
         ans = []
         for id in ids:
-            mi = self.db.get_metadata(id, index_is_id=True, get_cover=True)
+            mi = self.db.get_metadata(id, index_is_id=True, get_cover=get_cover)
             ans.append(mi)
         return ans
 
@@ -449,18 +444,14 @@ class BooksModel(QAbstractTableModel): # {{{
                     format = f
                     break
             if format is not None:
-                pt = PersistentTemporaryFile(suffix='.'+format)
+                pt = PersistentTemporaryFile(suffix='caltmpfmt.'+format)
-                with closing(self.db.format(id, format, index_is_id=True,
+                self.db.copy_format_to(id, format, pt, index_is_id=True)
-                    as_file=True)) as src:
-                    shutil.copyfileobj(src, pt)
-                    pt.flush()
-                    if getattr(src, 'name', None):
-                        pt.orig_file_path = os.path.abspath(src.name)
                 pt.seek(0)
                 if set_metadata:
                     try:
-                        _set_metadata(pt, self.db.get_metadata(id, get_cover=True, index_is_id=True),
+                        _set_metadata(pt, self.db.get_metadata(
-                                  format)
+                            id, get_cover=True, index_is_id=True,
+                            cover_as_data=True), format)
                     except:
                         traceback.print_exc()
                 pt.close()
@@ -468,9 +459,7 @@ class BooksModel(QAbstractTableModel): # {{{
                     if isbytestring(x):
                         x = x.decode(filesystem_encoding)
                     return x
-                name, op = map(to_uni, map(os.path.abspath, (pt.name,
+                ans.append(to_uni(os.path.abspath(pt.name)))
-                    pt.orig_file_path)))
-                ans.append(FormatPath(name, op))
             else:
                 need_auto.append(id)
                 if not exclude_auto:
@@ -499,13 +488,11 @@ class BooksModel(QAbstractTableModel): # {{{
                     break
             if format is not None:
                 pt = PersistentTemporaryFile(suffix='.'+format)
-                with closing(self.db.format(row, format, as_file=True)) as src:
+                self.db.copy_format_to(id, format, pt, index_is_id=True)
-                    shutil.copyfileobj(src, pt)
-                    pt.flush()
                 pt.seek(0)
                 if set_metadata:
-                    _set_metadata(pt, self.db.get_metadata(row, get_cover=True),
+                    _set_metadata(pt, self.db.get_metadata(row, get_cover=True,
-                                  format)
+                        cover_as_data=True), format)
                 pt.close() if paths else pt.seek(0)
                 ans.append(pt)
             else:

src/calibre/gui2/library/views.py

@@ -584,14 +584,17 @@ class BooksView(QTableView): # {{{
         m = self.model()
         db = m.db
         rows = self.selectionModel().selectedRows()
-        selected = map(m.id, rows)
+        selected = list(map(m.id, rows))
         ids = ' '.join(map(str, selected))
         md = QMimeData()
         md.setData('application/calibre+from_library', ids)
         fmt = prefs['output_format']
 
         def url_for_id(i):
-            ans = db.format_abspath(i, fmt, index_is_id=True)
+            try:
+                ans = db.format_path(i, fmt, index_is_id=True)
+            except:
+                ans = None
             if ans is None:
                 fmts = db.formats(i, index_is_id=True)
                 if fmts:
@@ -599,9 +602,10 @@ class BooksView(QTableView): # {{{
                 else:
                     fmts = []
                 for f in fmts:
-                    ans = db.format_abspath(i, f, index_is_id=True)
+                    try:
-                    if ans is not None:
+                        ans = db.format_path(i, f, index_is_id=True)
-                        break
+                    except:
+                        ans = None
             if ans is None:
                 ans = db.abspath(i, index_is_id=True)
             return QUrl.fromLocalFile(ans)

src/calibre/gui2/metadata/basic_widgets.py

@@ -21,9 +21,10 @@ from calibre.utils.config import tweaks, prefs
 from calibre.ebooks.metadata import (title_sort, authors_to_string,
         string_to_authors, check_isbn, authors_to_sort_string)
 from calibre.ebooks.metadata.meta import get_metadata
-from calibre.gui2 import (file_icon_provider, UNDEFINED_QDATE, UNDEFINED_DATE,
+from calibre.gui2 import (file_icon_provider, UNDEFINED_QDATE,
         choose_files, error_dialog, choose_images)
-from calibre.utils.date import local_tz, qt_to_dt
+from calibre.utils.date import (local_tz, qt_to_dt, as_local_time,
+        UNDEFINED_DATE)
 from calibre import strftime
 from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.customize.ui import run_plugins_on_import
@@ -125,6 +126,9 @@ class TitleEdit(EnLineEdit):
 
         return property(fget=fget, fset=fset)
 
+    def break_cycles(self):
+        self.dialog = None
+
 class TitleSortEdit(TitleEdit):
 
     TITLE_ATTR = 'title_sort'
@@ -150,6 +154,7 @@ class TitleSortEdit(TitleEdit):
         self.title_edit.textChanged.connect(self.update_state)
         self.textChanged.connect(self.update_state)
 
+        self.autogen_button = autogen_button
         autogen_button.clicked.connect(self.auto_generate)
         self.update_state()
 
@@ -169,6 +174,20 @@ class TitleSortEdit(TitleEdit):
     def auto_generate(self, *args):
         self.current_val = title_sort(self.title_edit.current_val)
 
+    def break_cycles(self):
+        try:
+            self.title_edit.textChanged.disconnect()
+        except:
+            pass
+        try:
+            self.textChanged.disconnect()
+        except:
+            pass
+        try:
+            self.autogen_button.clicked.disconnect()
+        except:
+            pass
+
 # }}}
 
 # Authors {{{
@@ -185,6 +204,7 @@ class AuthorsEdit(MultiCompleteComboBox):
         self.setWhatsThis(self.TOOLTIP)
         self.setEditable(True)
         self.setSizeAdjustPolicy(self.AdjustToMinimumContentsLengthWithIcon)
+        self.manage_authors_signal = manage_authors
         manage_authors.triggered.connect(self.manage_authors)
 
     def manage_authors(self):
@@ -269,6 +289,13 @@ class AuthorsEdit(MultiCompleteComboBox):
 
         return property(fget=fget, fset=fset)
 
+    def break_cycles(self):
+        self.db = self.dialog = None
+        try:
+            self.manage_authors_signal.triggered.disconnect()
+        except:
+            pass
+
 class AuthorSortEdit(EnLineEdit):
 
     TOOLTIP = _('Specify how the author(s) of this book should be sorted. '
@@ -297,6 +324,10 @@ class AuthorSortEdit(EnLineEdit):
         self.authors_edit.editTextChanged.connect(self.update_state_and_val)
         self.textChanged.connect(self.update_state)
 
+        self.autogen_button = autogen_button
+        self.copy_a_to_as_action = copy_a_to_as_action
+        self.copy_as_to_a_action = copy_as_to_a_action
+
         autogen_button.clicked.connect(self.auto_generate)
         copy_a_to_as_action.triggered.connect(self.auto_generate)
         copy_as_to_a_action.triggered.connect(self.copy_to_authors)
@@ -368,6 +399,30 @@ class AuthorSortEdit(EnLineEdit):
         db.set_author_sort(id_, aus, notify=False, commit=False)
         return True
 
+    def break_cycles(self):
+        self.db = None
+        try:
+            self.authors_edit.editTextChanged.disconnect()
+        except:
+            pass
+        try:
+            self.textChanged.disconnect()
+        except:
+            pass
+        try:
+            self.autogen_button.clicked.disconnect()
+        except:
+            pass
+        try:
+            self.copy_a_to_as_action.triggered.disconnect()
+        except:
+            pass
+        try:
+            self.copy_as_to_a_action.triggered.disconnect()
+        except:
+            pass
+        self.authors_edit = None
+
 # }}}
 
 # Series {{{
@@ -427,6 +482,10 @@ class SeriesEdit(MultiCompleteComboBox):
                                             commit=True, allow_case_change=True)
         return True
 
+    def break_cycles(self):
+        self.dialog = None
+
+
 class SeriesIndexEdit(QDoubleSpinBox):
 
     TOOLTIP = ''
@@ -488,6 +547,20 @@ class SeriesIndexEdit(QDoubleSpinBox):
                 import traceback
                 traceback.print_exc()
 
+    def break_cycles(self):
+        try:
+            self.series_edit.currentIndexChanged.disconnect()
+        except:
+            pass
+        try:
+            self.series_edit.editTextChanged.disconnect()
+        except:
+            pass
+        try:
+            self.series_edit.lineEdit().editingFinished.disconnect()
+        except:
+            pass
+        self.db = self.series_edit = self.dialog = None
 
 # }}}
 
@@ -688,7 +761,8 @@ class FormatsManager(QWidget): # {{{
             else:
                 stream = open(fmt.path, 'r+b')
             try:
-                mi = get_metadata(stream, ext)
+                with stream:
+                    mi = get_metadata(stream, ext)
                 return mi, ext
             except:
                 error_dialog(self, _('Could not read metadata'),
@@ -698,6 +772,8 @@ class FormatsManager(QWidget): # {{{
             if old != prefs['read_file_metadata']:
                 prefs['read_file_metadata'] = old
 
+    def break_cycles(self):
+        self.dialog = None
 # }}}
 
 class Cover(ImageView): # {{{
@@ -859,6 +935,13 @@ class Cover(ImageView): # {{{
                 db.remove_cover(id_, notify=False, commit=False)
         return True
 
+    def break_cycles(self):
+        try:
+            self.cover_changed.disconnect()
+        except:
+            pass
+        self.dialog = self._cdata = self.current_val = self.original_val = None
+
 # }}}
 
 class CommentsEdit(Editor): # {{{
@@ -1210,6 +1293,7 @@ class DateEdit(QDateEdit): # {{{
         def fset(self, val):
             if val is None:
                 val = UNDEFINED_DATE
+            val = as_local_time(val)
             self.setDate(QDate(val.year, val.month, val.day))
         return property(fget=fget, fset=fset)
 

src/calibre/gui2/metadata/single.py

@@ -388,6 +388,10 @@ class MetadataSingleDialogBase(ResizableDialog):
 
     def apply_changes(self):
         self.changed.add(self.book_id)
+        if self.db is None:
+            # break_cycles has already been called, don't know why this should
+            # happen but a user reported it
+            return True
         for widget in self.basic_metadata_widgets:
             try:
                 if not widget.commit(self.db, self.book_id):
@@ -477,6 +481,13 @@ class MetadataSingleDialogBase(ResizableDialog):
             x = getattr(self, b, None)
             if x is not None:
                 disconnect(x.clicked)
+        for widget in self.basic_metadata_widgets:
+            bc = getattr(widget, 'break_cycles', None)
+            if bc is not None and callable(bc):
+                bc()
+        for widget in getattr(self, 'custom_metadata_widgets', []):
+            widget.break_cycles()
+
     # }}}
 
 class Splitter(QSplitter):

src/calibre/gui2/metadata/single_download.py

@@ -236,6 +236,11 @@ class ResultsView(QTableView): # {{{
         self.resizeRowsToContents()
         self.resizeColumnsToContents()
         self.setFocus(Qt.OtherFocusReason)
+        idx = self.model().index(0, 0)
+        if idx.isValid() and self.model().rowCount() > 0:
+            self.show_details(idx)
+            sm = self.selectionModel()
+            sm.select(idx, sm.ClearAndSelect|sm.Rows)
 
     def currentChanged(self, current, previous):
         ret = QTableView.currentChanged(self, current, previous)
@@ -480,12 +485,6 @@ class IdentifyWidget(QWidget): # {{{
             return
 
         self.results_view.show_results(self.worker.results)
-
-        self.comments_view.show_data('''
-            <div style="margin-bottom:2ex">Found <b>%d</b> results</div>
-            <div>To see <b>details</b>, click on any result</div>''' %
-                len(self.worker.results))
-
         self.results_found.emit()
 
 

src/calibre/gui2/preferences/coloring.py

@@ -72,19 +72,27 @@ class ConditionEditor(QWidget): # {{{
         self.l = l = QGridLayout(self)
         self.setLayout(l)
 
-        self.l1 = l1 = QLabel(_('If the '))
+        texts = _('If the ___ column ___ values')
+        try:
+            one, two, three = texts.split('___')
+        except:
+            one, two, three = 'If the ', ' column ', ' value '
+
+        self.l1 = l1 = QLabel(one)
         l.addWidget(l1, 0, 0)
 
         self.column_box = QComboBox(self)
         l.addWidget(self.column_box, 0, 1)
 
-        self.l2 = l2 = QLabel(_(' column '))
+
+
+        self.l2 = l2 = QLabel(two)
         l.addWidget(l2, 0, 2)
 
         self.action_box = QComboBox(self)
         l.addWidget(self.action_box, 0, 3)
 
-        self.l3 = l3 = QLabel(_(' value '))
+        self.l3 = l3 = QLabel(three)
         l.addWidget(l3, 0, 4)
 
         self.value_box = QLineEdit(self)

src/calibre/gui2/preferences/metadata_sources.py

@@ -224,6 +224,20 @@ class FieldsModel(QAbstractListModel): # {{{
             Qt.Unchecked])
         msprefs['ignore_fields'] = list(ignored_fields.union(changed))
 
+    def user_default_state(self, field):
+        return (Qt.Unchecked if field in msprefs.get('user_default_ignore_fields',[])
+                    else Qt.Checked)
+
+    def select_user_defaults(self):
+        self.overrides = dict([(f, self.user_default_state(f)) for f in self.fields])
+        self.reset()
+
+    def commit_user_defaults(self):
+        default_ignored_fields = set([x for x in msprefs['user_default_ignore_fields'] if x not in
+            self.overrides])
+        changed = set([k for k, v in self.overrides.iteritems() if v ==
+            Qt.Unchecked])
+        msprefs['user_default_ignore_fields'] = list(default_ignored_fields.union(changed))
 
 # }}}
 
@@ -286,6 +300,9 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.select_all_button.clicked.connect(self.changed_signal)
         self.clear_all_button.clicked.connect(self.fields_model.clear_all)
         self.clear_all_button.clicked.connect(self.changed_signal)
+        self.select_default_button.clicked.connect(self.fields_model.select_user_defaults)
+        self.select_default_button.clicked.connect(self.changed_signal)
+        self.set_as_default_button.clicked.connect(self.fields_model.commit_user_defaults)
 
     def configure_plugin(self):
         for index in self.sources_view.selectionModel().selectedRows():

src/calibre/gui2/preferences/metadata_sources.ui

@@ -102,6 +102,26 @@
             </property>
            </widget>
           </item>
+          <item row="2" column="0">
+           <widget class="QPushButton" name="select_default_button">
+            <property name="text">
+             <string>&amp;Select default</string>
+            </property>
+             <property name="toolTip">
+             <string>Restore your own subset of checked fields that you define using the 'Set as default' button</string>
+            </property>
+           </widget>
+          </item>
+          <item row="2" column="1">
+           <widget class="QPushButton" name="set_as_default_button">
+            <property name="text">
+             <string>&amp;Set as default</string>
+            </property>
+             <property name="toolTip">
+             <string>Store the currently checked fields as a default you can restore using the 'Select default' button</string>
+            </property>
+           </widget>
+          </item>
          </layout>
         </widget>
        </item>

src/calibre/gui2/preferences/misc.py

@@ -10,6 +10,7 @@ from calibre.gui2.preferences import ConfigWidgetBase, test_widget, Setting
 from calibre.gui2.preferences.misc_ui import Ui_Form
 from calibre.gui2 import error_dialog, config, open_local_file, info_dialog
 from calibre.constants import isosx
+from calibre import get_proxies
 
 class WorkersSetting(Setting):
 
@@ -33,6 +34,13 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.user_defined_device_button.clicked.connect(self.user_defined_device)
         self.button_osx_symlinks.clicked.connect(self.create_symlinks)
         self.button_osx_symlinks.setVisible(isosx)
+        proxies = get_proxies(debug=False)
+        txt = _('No proxies used')
+        if proxies:
+            lines = ['<br><code>%s: %s</code>'%(t, p) for t, p in
+                    proxies.iteritems()]
+            txt = _('<b>Using proxies:</b>') + ''.join(lines)
+        self.proxies.setText(txt)
 
     def debug_device_detection(self, *args):
         from calibre.gui2.preferences.device_debug import DebugDevice

src/calibre/gui2/preferences/misc.ui

@@ -118,7 +118,7 @@
      </property>
     </widget>
    </item>
-   <item row="20" column="0">
+   <item row="21" column="0">
     <spacer name="verticalSpacer_9">
      <property name="orientation">
       <enum>Qt::Vertical</enum>
@@ -131,6 +131,13 @@
      </property>
     </spacer>
    </item>
+   <item row="10" column="0" colspan="2">
+    <widget class="QLabel" name="proxies">
+     <property name="text">
+      <string/>
+     </property>
+    </widget>
+   </item>
   </layout>
  </widget>
  <resources/>

src/calibre/gui2/preferences/plugins.ui

@@ -122,13 +122,13 @@
    <item>
     <layout class="QHBoxLayout" name="horizontalLayout_7">
      <item>
-      <widget class="QPushButton" name="button_plugin_add">
+      <widget class="QPushButton" name="button_plugin_new">
        <property name="text">
-        <string>&amp;Add a new plugin</string>
+        <string>Get &amp;new plugins</string>
        </property>
        <property name="icon">
         <iconset resource="../../../../resources/images.qrc">
-         <normaloff>:/images/plugins.png</normaloff>:/images/plugins.png</iconset>
+         <normaloff>:/images/plugins/plugin_new.png</normaloff>:/images/plugins/plugin_new.png</iconset>
        </property>
       </widget>
      </item>
@@ -144,13 +144,13 @@
       </widget>
      </item>
      <item>
-      <widget class="QPushButton" name="button_plugin_new">
+      <widget class="QPushButton" name="button_plugin_add">
        <property name="text">
-        <string>Get &amp;new plugins</string>
+        <string>&amp;Load plugin from file</string>
        </property>
        <property name="icon">
         <iconset resource="../../../../resources/images.qrc">
-         <normaloff>:/images/plugins/plugin_new.png</normaloff>:/images/plugins/plugin_new.png</iconset>
+         <normaloff>:/images/document_open.png</normaloff>:/images/document_open.png</iconset>
        </property>
       </widget>
      </item>

src/calibre/gui2/store/beam_ebooks_de_plugin.py

@@ -51,19 +51,19 @@ class BeamEBooksDEStore(BasicStoreConfig, StorePlugin):
                 if counter <= 0:
                     break
 
-                id = ''.join(data.xpath('./tr/td/div[@class="stil2"]/a/@href')).strip()
+                id = ''.join(data.xpath('./tr/td[1]/a/@href')).strip()
                 if not id:
                     continue
                 id = id[7:]
                 cover_url = ''.join(data.xpath('./tr/td[1]/a/img/@src'))
                 if cover_url:
                     cover_url = 'http://www.beam-ebooks.de' + cover_url
-                title = ''.join(data.xpath('./tr/td/div[@class="stil2"]/a/b/text()'))
+                temp = ''.join(data.xpath('./tr/td[1]/a/img/@alt'))
-                author = ' '.join(data.xpath('./tr/td/div[@class="stil2"]/'
+                colon = temp.find(':')
-                                             'child::b/text()'
+                if not temp.startswith('eBook') or colon < 0:
-                                             '|'
+                    continue
-                                             './tr/td/div[@class="stil2"]/'
+                author = temp[5:colon]
-                                             'child::strong/text()'))
+                title = temp[colon+1:]
                 price = ''.join(data.xpath('./tr/td[3]/text()'))
                 pdf = data.xpath(
                         'boolean(./tr/td[3]/a/img[contains(@alt, "PDF")]/@alt)')

src/calibre/gui2/store/config/chooser/models.py

@@ -133,7 +133,7 @@ class Matches(QAbstractItemModel):
                 return QVariant('<p>%s</p>' % result.description)
             elif col == 2:
                 if result.drm_free_only:
-                    return QVariant('<p>' + _('This store only distributes ebooks with DRM.') + '</p>')
+                    return QVariant('<p>' + _('This store only distributes ebooks without DRM.') + '</p>')
                 else:
                     return QVariant('<p>' + _('This store distributes ebooks with DRM. It may have some titles without DRM, but you will need to check on a per title basis.') + '</p>')
             elif col == 3:

src/calibre/gui2/store/declined.txt

@@ -1,9 +1,6 @@
 This is a list of stores that objected, declined
 or asked not to be included in the store integration.
 
-* Borders (http://www.borders.com/)
+* Borders (http://www.borders.com/).
-* Indigo (http://www.chapters.indigo.ca/)
+* Indigo (http://www.chapters.indigo.ca/).
 * Libraria Rizzoli (http://libreriarizzoli.corriere.it/).
-  No reply with two attempts over 2 weeks
-* WH Smith (http://www.whsmith.co.uk/)
-  Refused to permit signing up for the affiliate program

src/calibre/gui2/store/legimi_plugin.py

@@ -58,6 +58,8 @@ class LegimiStore(BasicStoreConfig, StorePlugin):
                 cover_url = ''.join(data.xpath('.//div[@class="item_cover_container"]/a/img/@src'))
                 title = ''.join(data.xpath('.//div[@class="item_entries"]/h2/a/text()'))
                 author = ''.join(data.xpath('.//div[@class="item_entries"]/span[1]/a/text()'))
+                author = re.sub(',','',author)
+                author = re.sub(';',',',author)
                 price = ''.join(data.xpath('.//div[@class="item_entries"]/span[3]/text()'))
                 price = re.sub(r'[^0-9,]*','',price) + ' zł'
 

src/calibre/gui2/store/libri_de_plugin.py

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import urllib2
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class LibreDEStore(BasicStoreConfig, StorePlugin):
+
+    def open(self, parent=None, detail_item=None, external=False):
+        url = 'http://ad.zanox.com/ppc/?18817073C15644254T'
+        url_details = ('http://ad.zanox.com/ppc/?18845780C1371495675T&ULP=[['
+                       'http://www.libri.de/shop/action/productDetails?artiId={0}]]')
+
+        if external or self.config.get('open_external', False):
+            if detail_item:
+                url = url_details.format(detail_item)
+            open_url(QUrl(url))
+        else:
+            detail_url = None
+            if detail_item:
+                detail_url = url_details.format(detail_item)
+            d = WebStoreDialog(self.gui, url, parent, detail_url)
+            d.setWindowTitle(self.name)
+            d.set_tags(self.config.get('tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        url = ('http://www.libri.de/shop/action/quickSearch?facetNodeId=6'
+               '&mainsearchSubmit=Los!&searchString=' + urllib2.quote(query))
+        br = browser()
+
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//div[contains(@class, "item")]'):
+                if counter <= 0:
+                    break
+
+                details = data.xpath('./div[@class="beschreibungContainer"]')
+                if not details:
+                    continue
+                details = details[0]
+                id = ''.join(details.xpath('./div[@class="text"]/a/@name')).strip()
+                if not id:
+                    continue
+                cover_url = ''.join(details.xpath('./div[@class="bild"]/a/img/@src'))
+                title = ''.join(details.xpath('./div[@class="text"]/span[@class="titel"]/a/text()')).strip()
+                author = ''.join(details.xpath('./div[@class="text"]/span[@class="author"]/text()')).strip()
+                pdf = details.xpath(
+                        'boolean(.//span[@class="format" and contains(text(), "pdf")]/text())')
+                epub = details.xpath(
+                        'boolean(.//span[@class="format" and contains(text(), "epub")]/text())')
+                mobi = details.xpath(
+                        'boolean(.//span[@class="format" and contains(text(), "mobipocket")]/text())')
+                price = (''.join(data.xpath('.//span[@class="preis"]/text()'))).replace('*', '')
+                counter -= 1
+
+                s = SearchResult()
+                s.cover_url = cover_url
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price
+                s.drm = SearchResult.DRM_UNKNOWN
+                s.detail_item = id
+                formats = []
+                if epub:
+                    formats.append('ePub')
+                if pdf:
+                    formats.append('PDF')
+                if mobi:
+                    formats.append('MOBI')
+                s.formats = ', '.join(formats)
+
+                yield s

src/calibre/gui2/store/open_books_plugin.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import urllib
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class OpenBooksStore(BasicStoreConfig, StorePlugin):
+
+    def open(self, parent=None, detail_item=None, external=False):
+        url = 'http://drmfree.calibre-ebook.com/'
+
+        if external or self.config.get('open_external', False):
+            open_url(QUrl(url_slash_cleaner(detail_item if detail_item else url)))
+        else:
+            d = WebStoreDialog(self.gui, self.url, parent, detail_item)
+            d.setWindowTitle(self.name)
+            d.set_tags(self.config.get('tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        url = 'http://drmfree.calibre-ebook.com/search/?q=' + urllib.quote_plus(query)
+
+        br = browser()
+
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//ul[@id="object_list"]//li'):
+                if counter <= 0:
+                    break
+    
+                id = ''.join(data.xpath('.//div[@class="links"]/a[1]/@href'))
+                id = id.strip()
+                if not id:
+                    continue
+
+                cover_url = ''.join(data.xpath('.//div[@class="cover"]/img/@src'))
+
+                price = ''.join(data.xpath('.//div[@class="price"]/text()'))
+                a, b, price = price.partition('Price:')
+                price = price.strip()
+                if not price:
+                    continue
+
+                title = ''.join(data.xpath('.//div/strong/text()'))
+                author = ''.join(data.xpath('.//div[@class="author"]//text()'))
+                author = author.partition('by')[-1]
+
+                counter -= 1
+
+                s = SearchResult()
+                s.cover_url = cover_url
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price.strip()
+                s.detail_item = id.strip()
+                s.drm = SearchResult.DRM_UNLOCKED
+
+                yield s

src/calibre/gui2/store/woblink_plugin.py

@@ -57,7 +57,7 @@ class WoblinkStore(BasicStoreConfig, StorePlugin):
 
                 cover_url = ''.join(data.xpath('.//td[@class="w10 va-t"]/a[1]/img/@src'))
                 title = ''.join(data.xpath('.//h3[@class="title"]/a[1]/text()'))
-                author = ''.join(data.xpath('.//p[@class="author"]/a[1]/text()'))
+                author = ', '.join(data.xpath('.//p[@class="author"]/a/text()'))
                 price = ''.join(data.xpath('.//div[@class="prices"]/p[1]/span/text()'))
                 price = re.sub('PLN', ' zł', price)
                 price = re.sub('\.', ',', price)

src/calibre/gui2/store/zixo_plugin.py

@@ -53,7 +53,7 @@ class ZixoStore(BasicStoreConfig, StorePlugin):
 
                 cover_url = ''.join(data.xpath('.//a[@class="productThumb"]/img/@src'))
                 title = ''.join(data.xpath('.//a[@class="title"]/text()'))
-                author = ''.join(data.xpath('.//div[@class="productDescription"]/span[1]/a/text()'))
+                author = ','.join(data.xpath('.//div[@class="productDescription"]/span[1]/a/text()'))
                 price = ''.join(data.xpath('.//div[@class="priceList"]/span/text()'))
                 price = re.sub('\.', ',', price)
 

src/calibre/gui2/tag_browser/__init__.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+

src/calibre/gui2/tag_browser/ui.py

@@ -0,0 +1,458 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from functools import partial
+
+from PyQt4.Qt import (Qt, QIcon, QWidget, QHBoxLayout, QVBoxLayout, QShortcut,
+        QKeySequence, QToolButton, QString, QLabel, QFrame, QTimer, QComboBox,
+        QMenu, QPushButton)
+
+from calibre.gui2 import error_dialog, question_dialog
+from calibre.gui2.widgets import HistoryLineEdit
+from calibre.library.field_metadata import category_icon_map
+from calibre.utils.icu import sort_key
+from calibre.gui2.tag_browser.view import TagsView
+from calibre.ebooks.metadata import title_sort
+from calibre.gui2.dialogs.tag_categories import TagCategories
+from calibre.gui2.dialogs.tag_list_editor import TagListEditor
+from calibre.gui2.dialogs.edit_authors_dialog import EditAuthorsDialog
+
+class TagBrowserMixin(object): # {{{
+
+    def __init__(self, db):
+        self.library_view.model().count_changed_signal.connect(self.tags_view.recount)
+        self.tags_view.set_database(db, self.tag_match, self.sort_by)
+        self.tags_view.tags_marked.connect(self.search.set_search_string)
+        self.tags_view.tag_list_edit.connect(self.do_tags_list_edit)
+        self.tags_view.edit_user_category.connect(self.do_edit_user_categories)
+        self.tags_view.delete_user_category.connect(self.do_delete_user_category)
+        self.tags_view.del_item_from_user_cat.connect(self.do_del_item_from_user_cat)
+        self.tags_view.add_subcategory.connect(self.do_add_subcategory)
+        self.tags_view.add_item_to_user_cat.connect(self.do_add_item_to_user_cat)
+        self.tags_view.saved_search_edit.connect(self.do_saved_search_edit)
+        self.tags_view.rebuild_saved_searches.connect(self.do_rebuild_saved_searches)
+        self.tags_view.author_sort_edit.connect(self.do_author_sort_edit)
+        self.tags_view.tag_item_renamed.connect(self.do_tag_item_renamed)
+        self.tags_view.search_item_renamed.connect(self.saved_searches_changed)
+        self.tags_view.drag_drop_finished.connect(self.drag_drop_finished)
+        self.tags_view.restriction_error.connect(self.do_restriction_error,
+                                                 type=Qt.QueuedConnection)
+
+        for text, func, args, cat_name in (
+             (_('Manage Authors'),
+                        self.do_author_sort_edit, (self, None), 'authors'),
+             (_('Manage Series'),
+                        self.do_tags_list_edit, (None, 'series'), 'series'),
+             (_('Manage Publishers'),
+                        self.do_tags_list_edit, (None, 'publisher'), 'publisher'),
+             (_('Manage Tags'),
+                        self.do_tags_list_edit, (None, 'tags'), 'tags'),
+             (_('Manage User Categories'),
+                        self.do_edit_user_categories, (None,), 'user:'),
+             (_('Manage Saved Searches'),
+                        self.do_saved_search_edit, (None,), 'search')
+            ):
+            self.manage_items_button.menu().addAction(
+                                        QIcon(I(category_icon_map[cat_name])),
+                                        text, partial(func, *args))
+
+    def do_restriction_error(self):
+        error_dialog(self.tags_view, _('Invalid search restriction'),
+                         _('The current search restriction is invalid'), show=True)
+
+    def do_add_subcategory(self, on_category_key, new_category_name=None):
+        '''
+        Add a subcategory to the category 'on_category'. If new_category_name is
+        None, then a default name is shown and the user is offered the
+        opportunity to edit the name.
+        '''
+        db = self.library_view.model().db
+        user_cats = db.prefs.get('user_categories', {})
+
+        # Ensure that the temporary name we will use is not already there
+        i = 0
+        if new_category_name is not None:
+            new_name = new_category_name.replace('.', '')
+        else:
+            new_name = _('New Category').replace('.', '')
+        n = new_name
+        while True:
+            new_cat = on_category_key[1:] + '.' + n
+            if new_cat not in user_cats:
+                break
+            i += 1
+            n = new_name + unicode(i)
+        # Add the new category
+        user_cats[new_cat] = []
+        db.prefs.set('user_categories', user_cats)
+        self.tags_view.set_new_model()
+        m = self.tags_view.model()
+        idx = m.index_for_path(m.find_category_node('@' + new_cat))
+        m.show_item_at_index(idx)
+        # Open the editor on the new item to rename it
+        if new_category_name is None:
+            self.tags_view.edit(idx)
+
+    def do_edit_user_categories(self, on_category=None):
+        '''
+        Open the user categories editor.
+        '''
+        db = self.library_view.model().db
+        d = TagCategories(self, db, on_category)
+        if d.exec_() == d.Accepted:
+            db.prefs.set('user_categories', d.categories)
+            db.field_metadata.remove_user_categories()
+            for k in d.categories:
+                db.field_metadata.add_user_category('@' + k, k)
+            db.data.change_search_locations(db.field_metadata.get_search_terms())
+            self.tags_view.set_new_model()
+
+    def do_delete_user_category(self, category_name):
+        '''
+        Delete the user category named category_name. Any leading '@' is removed
+        '''
+        if category_name.startswith('@'):
+            category_name = category_name[1:]
+        db = self.library_view.model().db
+        user_cats = db.prefs.get('user_categories', {})
+        cat_keys = sorted(user_cats.keys(), key=sort_key)
+        has_children = False
+        found = False
+        for k in cat_keys:
+            if k == category_name:
+                found = True
+                has_children = len(user_cats[k])
+            elif k.startswith(category_name + '.'):
+                has_children = True
+        if not found:
+            return error_dialog(self.tags_view, _('Delete user category'),
+                         _('%s is not a user category')%category_name, show=True)
+        if has_children:
+            if not question_dialog(self.tags_view, _('Delete user category'),
+                                   _('%s contains items. Do you really '
+                                     'want to delete it?')%category_name):
+                return
+        for k in cat_keys:
+            if k == category_name:
+                del user_cats[k]
+            elif k.startswith(category_name + '.'):
+                del user_cats[k]
+        db.prefs.set('user_categories', user_cats)
+        self.tags_view.set_new_model()
+
+    def do_del_item_from_user_cat(self, user_cat, item_name, item_category):
+        '''
+        Delete the item (item_name, item_category) from the user category with
+        key user_cat. Any leading '@' characters are removed
+        '''
+        if user_cat.startswith('@'):
+            user_cat = user_cat[1:]
+        db = self.library_view.model().db
+        user_cats = db.prefs.get('user_categories', {})
+        if user_cat not in user_cats:
+            error_dialog(self.tags_view, _('Remove category'),
+                         _('User category %s does not exist')%user_cat,
+                         show=True)
+            return
+        self.tags_view.model().delete_item_from_user_category(user_cat,
+                                                      item_name, item_category)
+        self.tags_view.recount()
+
+    def do_add_item_to_user_cat(self, dest_category, src_name, src_category):
+        '''
+        Add the item src_name in src_category to the user category
+        dest_category. Any leading '@' is removed
+        '''
+        db = self.library_view.model().db
+        user_cats = db.prefs.get('user_categories', {})
+
+        if dest_category and dest_category.startswith('@'):
+            dest_category = dest_category[1:]
+
+        if dest_category not in user_cats:
+            return error_dialog(self.tags_view, _('Add to user category'),
+                    _('A user category %s does not exist')%dest_category, show=True)
+
+        # Now add the item to the destination user category
+        add_it = True
+        if src_category == 'news':
+            src_category = 'tags'
+        for tup in user_cats[dest_category]:
+            if src_name == tup[0] and src_category == tup[1]:
+                add_it = False
+        if add_it:
+            user_cats[dest_category].append([src_name, src_category, 0])
+        db.prefs.set('user_categories', user_cats)
+        self.tags_view.recount()
+
+    def do_tags_list_edit(self, tag, category):
+        '''
+        Open the 'manage_X' dialog where X == category. If tag is not None, the
+        dialog will position the editor on that item.
+        '''
+        db=self.library_view.model().db
+        if category == 'tags':
+            result = db.get_tags_with_ids()
+            key = sort_key
+        elif category == 'series':
+            result = db.get_series_with_ids()
+            key = lambda x:sort_key(title_sort(x))
+        elif category == 'publisher':
+            result = db.get_publishers_with_ids()
+            key = sort_key
+        else: # should be a custom field
+            cc_label = None
+            if category in db.field_metadata:
+                cc_label = db.field_metadata[category]['label']
+                result = db.get_custom_items_with_ids(label=cc_label)
+            else:
+                result = []
+            key = sort_key
+
+        d = TagListEditor(self, tag_to_match=tag, data=result, key=key)
+        d.exec_()
+        if d.result() == d.Accepted:
+            to_rename = d.to_rename # dict of new text to old id
+            to_delete = d.to_delete # list of ids
+            orig_name = d.original_names # dict of id: name
+
+            rename_func = None
+            if category == 'tags':
+                rename_func = db.rename_tag
+                delete_func = db.delete_tag_using_id
+            elif category == 'series':
+                rename_func = db.rename_series
+                delete_func = db.delete_series_using_id
+            elif category == 'publisher':
+                rename_func = db.rename_publisher
+                delete_func = db.delete_publisher_using_id
+            else:
+                rename_func = partial(db.rename_custom_item, label=cc_label)
+                delete_func = partial(db.delete_custom_item_using_id, label=cc_label)
+            m = self.tags_view.model()
+            if rename_func:
+                for item in to_delete:
+                    delete_func(item)
+                    m.delete_item_from_all_user_categories(orig_name[item], category)
+                for old_id in to_rename:
+                    rename_func(old_id, new_name=unicode(to_rename[old_id]))
+                    m.rename_item_in_all_user_categories(orig_name[old_id],
+                                            category, unicode(to_rename[old_id]))
+
+            # Clean up the library view
+            self.do_tag_item_renamed()
+            self.tags_view.recount()
+
+    def do_tag_item_renamed(self):
+        # Clean up library view and search
+        # get information to redo the selection
+        rows = [r.row() for r in \
+                self.library_view.selectionModel().selectedRows()]
+        m = self.library_view.model()
+        ids = [m.id(r) for r in rows]
+
+        m.refresh(reset=False)
+        m.research()
+        self.library_view.select_rows(ids)
+        # refreshing the tags view happens at the emit()/call() site
+
+    def do_author_sort_edit(self, parent, id, select_sort=True):
+        '''
+        Open the manage authors dialog
+        '''
+        db = self.library_view.model().db
+        editor = EditAuthorsDialog(parent, db, id, select_sort)
+        d = editor.exec_()
+        if d:
+            for (id, old_author, new_author, new_sort) in editor.result:
+                if old_author != new_author:
+                    # The id might change if the new author already exists
+                    id = db.rename_author(id, new_author)
+                db.set_sort_field_for_author(id, unicode(new_sort),
+                                             commit=False, notify=False)
+            db.commit()
+            self.library_view.model().refresh()
+            self.tags_view.recount()
+
+    def drag_drop_finished(self, ids):
+        self.library_view.model().refresh_ids(ids)
+
+# }}}
+
+class TagBrowserWidget(QWidget): # {{{
+
+    def __init__(self, parent):
+        QWidget.__init__(self, parent)
+        self.parent = parent
+        self._layout = QVBoxLayout()
+        self.setLayout(self._layout)
+        self._layout.setContentsMargins(0,0,0,0)
+
+        # Set up the find box & button
+        search_layout = QHBoxLayout()
+        self._layout.addLayout(search_layout)
+        self.item_search = HistoryLineEdit(parent)
+        try:
+            self.item_search.lineEdit().setPlaceholderText(
+                                                _('Find item in tag browser'))
+        except:
+            pass             # Using Qt < 4.7
+        self.item_search.setToolTip(_(
+        'Search for items. This is a "contains" search; items containing the\n'
+        'text anywhere in the name will be found. You can limit the search\n'
+        'to particular categories using syntax similar to search. For example,\n'
+        'tags:foo will find foo in any tag, but not in authors etc. Entering\n'
+        '*foo will filter all categories at once, showing only those items\n'
+        'containing the text "foo"'))
+        search_layout.addWidget(self.item_search)
+        # Not sure if the shortcut should be translatable ...
+        sc = QShortcut(QKeySequence(_('ALT+f')), parent)
+        sc.activated.connect(self.set_focus_to_find_box)
+
+        self.search_button = QToolButton()
+        self.search_button.setText(_('F&ind'))
+        self.search_button.setToolTip(_('Find the first/next matching item'))
+        search_layout.addWidget(self.search_button)
+
+        self.expand_button = QToolButton()
+        self.expand_button.setText('-')
+        self.expand_button.setToolTip(_('Collapse all categories'))
+        search_layout.addWidget(self.expand_button)
+        search_layout.setStretch(0, 10)
+        search_layout.setStretch(1, 1)
+        search_layout.setStretch(2, 1)
+
+        self.current_find_position = None
+        self.search_button.clicked.connect(self.find)
+        self.item_search.initialize('tag_browser_search')
+        self.item_search.lineEdit().returnPressed.connect(self.do_find)
+        self.item_search.lineEdit().textEdited.connect(self.find_text_changed)
+        self.item_search.activated[QString].connect(self.do_find)
+        self.item_search.completer().setCaseSensitivity(Qt.CaseSensitive)
+
+        parent.tags_view = TagsView(parent)
+        self.tags_view = parent.tags_view
+        self.expand_button.clicked.connect(self.tags_view.collapseAll)
+        self._layout.addWidget(parent.tags_view)
+
+        # Now the floating 'not found' box
+        l = QLabel(self.tags_view)
+        self.not_found_label = l
+        l.setFrameStyle(QFrame.StyledPanel)
+        l.setAutoFillBackground(True)
+        l.setText('<p><b>'+_('No More Matches.</b><p> Click Find again to go to first match'))
+        l.setAlignment(Qt.AlignVCenter)
+        l.setWordWrap(True)
+        l.resize(l.sizeHint())
+        l.move(10,20)
+        l.setVisible(False)
+        self.not_found_label_timer = QTimer()
+        self.not_found_label_timer.setSingleShot(True)
+        self.not_found_label_timer.timeout.connect(self.not_found_label_timer_event,
+                                                   type=Qt.QueuedConnection)
+
+        parent.sort_by = QComboBox(parent)
+        # Must be in the same order as db2.CATEGORY_SORTS
+        for x in (_('Sort by name'), _('Sort by popularity'),
+                  _('Sort by average rating')):
+            parent.sort_by.addItem(x)
+        parent.sort_by.setToolTip(
+                _('Set the sort order for entries in the Tag Browser'))
+        parent.sort_by.setStatusTip(parent.sort_by.toolTip())
+        parent.sort_by.setCurrentIndex(0)
+        self._layout.addWidget(parent.sort_by)
+
+        # Must be in the same order as db2.MATCH_TYPE
+        parent.tag_match = QComboBox(parent)
+        for x in (_('Match any'), _('Match all')):
+            parent.tag_match.addItem(x)
+        parent.tag_match.setCurrentIndex(0)
+        self._layout.addWidget(parent.tag_match)
+        parent.tag_match.setToolTip(
+                _('When selecting multiple entries in the Tag Browser '
+                    'match any or all of them'))
+        parent.tag_match.setStatusTip(parent.tag_match.toolTip())
+
+
+        l = parent.manage_items_button = QPushButton(self)
+        l.setStyleSheet('QPushButton {text-align: left; }')
+        l.setText(_('Manage authors, tags, etc'))
+        l.setToolTip(_('All of these category_managers are available by right-clicking '
+                       'on items in the tag browser above'))
+        l.m = QMenu()
+        l.setMenu(l.m)
+        self._layout.addWidget(l)
+
+        # self.leak_test_timer = QTimer(self)
+        # self.leak_test_timer.timeout.connect(self.test_for_leak)
+        # self.leak_test_timer.start(5000)
+
+    def set_pane_is_visible(self, to_what):
+        self.tags_view.set_pane_is_visible(to_what)
+
+    def find_text_changed(self, str):
+        self.current_find_position = None
+
+    def set_focus_to_find_box(self):
+        self.item_search.setFocus()
+        self.item_search.lineEdit().selectAll()
+
+    def do_find(self, str=None):
+        self.current_find_position = None
+        self.find()
+
+    def find(self):
+        model = self.tags_view.model()
+        model.clear_boxed()
+        txt = unicode(self.item_search.currentText()).strip()
+
+        if txt.startswith('*'):
+            self.tags_view.set_new_model(filter_categories_by=txt[1:])
+            self.current_find_position = None
+            return
+        if model.get_filter_categories_by():
+            self.tags_view.set_new_model(filter_categories_by=None)
+            self.current_find_position = None
+            model = self.tags_view.model()
+
+        if not txt:
+            return
+
+        self.item_search.lineEdit().blockSignals(True)
+        self.search_button.setFocus(True)
+        self.item_search.lineEdit().blockSignals(False)
+
+        key = None
+        colon = txt.rfind(':') if len(txt) > 2 else 0
+        if colon > 0:
+            key = self.parent.library_view.model().db.\
+                        field_metadata.search_term_to_field_key(txt[:colon])
+            txt = txt[colon+1:]
+
+        self.current_find_position = \
+            model.find_item_node(key, txt, self.current_find_position)
+        if self.current_find_position:
+            model.show_item_at_path(self.current_find_position, box=True)
+        elif self.item_search.text():
+            self.not_found_label.setVisible(True)
+            if self.tags_view.verticalScrollBar().isVisible():
+                sbw = self.tags_view.verticalScrollBar().width()
+            else:
+                sbw = 0
+            width = self.width() - 8 - sbw
+            height = self.not_found_label.heightForWidth(width) + 20
+            self.not_found_label.resize(width, height)
+            self.not_found_label.move(4, 10)
+            self.not_found_label_timer.start(2000)
+
+    def not_found_label_timer_event(self):
+        self.not_found_label.setVisible(False)
+
+# }}}
+

src/calibre/gui2/tag_browser/view.py

@@ -0,0 +1,578 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import cPickle, traceback
+from functools import partial
+
+from PyQt4.Qt import (QItemDelegate, Qt, QTreeView, pyqtSignal, QSize, QIcon,
+        QApplication, QMenu, QPoint)
+
+from calibre.gui2.tag_browser.model import (TagTreeItem, TAG_SEARCH_STATES,
+        TagsModel)
+from calibre.gui2 import config, gprefs
+from calibre.utils.search_query_parser import saved_searches
+from calibre.utils.icu import sort_key
+
+class TagDelegate(QItemDelegate): # {{{
+
+    def paint(self, painter, option, index):
+        item = index.data(Qt.UserRole).toPyObject()
+        if item.type != TagTreeItem.TAG:
+            QItemDelegate.paint(self, painter, option, index)
+            return
+        r = option.rect
+        model = self.parent().model()
+        icon = model.data(index, Qt.DecorationRole).toPyObject()
+        painter.save()
+        if item.tag.state != 0 or not config['show_avg_rating'] or \
+                item.tag.avg_rating is None:
+            icon.paint(painter, r, Qt.AlignLeft)
+        else:
+            painter.setOpacity(0.3)
+            icon.paint(painter, r, Qt.AlignLeft)
+            painter.setOpacity(1)
+            rating = item.tag.avg_rating
+            painter.setClipRect(r.left(), r.bottom()-int(r.height()*(rating/5.0)),
+                    r.width(), r.height())
+            icon.paint(painter, r, Qt.AlignLeft)
+            painter.setClipRect(r)
+
+        # Paint the text
+        if item.boxed:
+            painter.drawRoundedRect(r.adjusted(1,1,-1,-1), 5, 5)
+        r.setLeft(r.left()+r.height()+3)
+        painter.drawText(r, Qt.AlignLeft|Qt.AlignVCenter,
+                        model.data(index, Qt.DisplayRole).toString())
+        painter.restore()
+
+    # }}}
+
+class TagsView(QTreeView): # {{{
+
+    refresh_required        = pyqtSignal()
+    tags_marked             = pyqtSignal(object)
+    edit_user_category      = pyqtSignal(object)
+    delete_user_category    = pyqtSignal(object)
+    del_item_from_user_cat  = pyqtSignal(object, object, object)
+    add_item_to_user_cat    = pyqtSignal(object, object, object)
+    add_subcategory         = pyqtSignal(object)
+    tag_list_edit           = pyqtSignal(object, object)
+    saved_search_edit       = pyqtSignal(object)
+    rebuild_saved_searches  = pyqtSignal()
+    author_sort_edit        = pyqtSignal(object, object)
+    tag_item_renamed        = pyqtSignal()
+    search_item_renamed     = pyqtSignal()
+    drag_drop_finished      = pyqtSignal(object)
+    restriction_error       = pyqtSignal()
+
+    def __init__(self, parent=None):
+        QTreeView.__init__(self, parent=None)
+        self.tag_match = None
+        self.disable_recounting = False
+        self.setUniformRowHeights(True)
+        self.setCursor(Qt.PointingHandCursor)
+        self.setIconSize(QSize(30, 30))
+        self.setTabKeyNavigation(True)
+        self.setAlternatingRowColors(True)
+        self.setAnimated(True)
+        self.setHeaderHidden(True)
+        self.setItemDelegate(TagDelegate(self))
+        self.made_connections = False
+        self.setAcceptDrops(True)
+        self.setDragEnabled(True)
+        self.setDragDropMode(self.DragDrop)
+        self.setDropIndicatorShown(True)
+        self.setAutoExpandDelay(500)
+        self.pane_is_visible = False
+        if gprefs['tags_browser_collapse_at'] == 0:
+            self.collapse_model = 'disable'
+        else:
+            self.collapse_model = gprefs['tags_browser_partition_method']
+        self.search_icon = QIcon(I('search.png'))
+        self.user_category_icon = QIcon(I('tb_folder.png'))
+        self.delete_icon = QIcon(I('list_remove.png'))
+        self.rename_icon = QIcon(I('edit-undo.png'))
+
+    def set_pane_is_visible(self, to_what):
+        pv = self.pane_is_visible
+        self.pane_is_visible = to_what
+        if to_what and not pv:
+            self.recount()
+
+    def reread_collapse_parameters(self):
+        if gprefs['tags_browser_collapse_at'] == 0:
+            self.collapse_model = 'disable'
+        else:
+            self.collapse_model = gprefs['tags_browser_partition_method']
+        self.set_new_model(self._model.get_filter_categories_by())
+
+    def set_database(self, db, tag_match, sort_by):
+        hidden_cats = db.prefs.get('tag_browser_hidden_categories', None)
+        self.hidden_categories = []
+        # migrate from config to db prefs
+        if hidden_cats is None:
+            hidden_cats = config['tag_browser_hidden_categories']
+        # strip out any non-existence field keys
+        for cat in hidden_cats:
+            if cat in db.field_metadata:
+                self.hidden_categories.append(cat)
+        db.prefs.set('tag_browser_hidden_categories', list(self.hidden_categories))
+        self.hidden_categories = set(self.hidden_categories)
+
+        old = getattr(self, '_model', None)
+        if old is not None:
+            old.break_cycles()
+        self._model = TagsModel(db, parent=self,
+                                hidden_categories=self.hidden_categories,
+                                search_restriction=None,
+                                drag_drop_finished=self.drag_drop_finished,
+                                collapse_model=self.collapse_model,
+                                state_map={})
+        self.pane_is_visible = True # because TagsModel.init did a recount
+        self.sort_by = sort_by
+        self.tag_match = tag_match
+        self.db = db
+        self.search_restriction = None
+        self.setModel(self._model)
+        self.setContextMenuPolicy(Qt.CustomContextMenu)
+        pop = config['sort_tags_by']
+        self.sort_by.setCurrentIndex(self.db.CATEGORY_SORTS.index(pop))
+        try:
+            match_pop = self.db.MATCH_TYPE.index(config['match_tags_type'])
+        except ValueError:
+            match_pop = 0
+        self.tag_match.setCurrentIndex(match_pop)
+        if not self.made_connections:
+            self.clicked.connect(self.toggle)
+            self.customContextMenuRequested.connect(self.show_context_menu)
+            self.refresh_required.connect(self.recount, type=Qt.QueuedConnection)
+            self.sort_by.currentIndexChanged.connect(self.sort_changed)
+            self.tag_match.currentIndexChanged.connect(self.match_changed)
+            self.made_connections = True
+        self.refresh_signal_processed = True
+        db.add_listener(self.database_changed)
+        self.expanded.connect(self.item_expanded)
+
+    def database_changed(self, event, ids):
+        if self.refresh_signal_processed:
+            self.refresh_signal_processed = False
+            self.refresh_required.emit()
+
+    @property
+    def match_all(self):
+        return self.tag_match and self.tag_match.currentIndex() > 0
+
+    def sort_changed(self, pop):
+        config.set('sort_tags_by', self.db.CATEGORY_SORTS[pop])
+        self.recount()
+
+    def match_changed(self, pop):
+        try:
+            config.set('match_tags_type', self.db.MATCH_TYPE[pop])
+        except:
+            pass
+
+    def set_search_restriction(self, s):
+        if s:
+            self.search_restriction = s
+        else:
+            self.search_restriction = None
+        self.set_new_model()
+
+    def mouseReleaseEvent(self, event):
+        # Swallow everything except leftButton so context menus work correctly
+        if event.button() == Qt.LeftButton:
+            QTreeView.mouseReleaseEvent(self, event)
+
+    def mouseDoubleClickEvent(self, event):
+        # swallow these to avoid toggling and editing at the same time
+        pass
+
+    @property
+    def search_string(self):
+        tokens = self._model.tokens()
+        joiner = ' and ' if self.match_all else ' or '
+        return joiner.join(tokens)
+
+    def toggle(self, index):
+        self._toggle(index, None)
+
+    def _toggle(self, index, set_to):
+        '''
+        set_to: if None, advance the state. Otherwise must be one of the values
+        in TAG_SEARCH_STATES
+        '''
+        modifiers = int(QApplication.keyboardModifiers())
+        exclusive = modifiers not in (Qt.CTRL, Qt.SHIFT)
+        if self._model.toggle(index, exclusive, set_to=set_to):
+            self.tags_marked.emit(self.search_string)
+
+    def conditional_clear(self, search_string):
+        if search_string != self.search_string:
+            self.clear()
+
+    def context_menu_handler(self, action=None, category=None,
+                             key=None, index=None, search_state=None):
+        if not action:
+            return
+        try:
+            if action == 'edit_item':
+                self.edit(index)
+                return
+            if action == 'open_editor':
+                self.tag_list_edit.emit(category, key)
+                return
+            if action == 'manage_categories':
+                self.edit_user_category.emit(category)
+                return
+            if action == 'search':
+                self._toggle(index, set_to=search_state)
+                return
+            if action == 'add_to_category':
+                tag = index.tag
+                if len(index.children) > 0:
+                    for c in index.children:
+                        self.add_item_to_user_cat.emit(category, c.tag.original_name,
+                                               c.tag.category)
+                self.add_item_to_user_cat.emit(category, tag.original_name,
+                                               tag.category)
+                return
+            if action == 'add_subcategory':
+                self.add_subcategory.emit(key)
+                return
+            if action == 'search_category':
+                self._toggle(index, set_to=search_state)
+                return
+            if action == 'delete_user_category':
+                self.delete_user_category.emit(key)
+                return
+            if action == 'delete_search':
+                saved_searches().delete(key)
+                self.rebuild_saved_searches.emit()
+                return
+            if action == 'delete_item_from_user_category':
+                tag = index.tag
+                if len(index.children) > 0:
+                    for c in index.children:
+                        self.del_item_from_user_cat.emit(key, c.tag.original_name,
+                                               c.tag.category)
+                self.del_item_from_user_cat.emit(key, tag.original_name, tag.category)
+                return
+            if action == 'manage_searches':
+                self.saved_search_edit.emit(category)
+                return
+            if action == 'edit_author_sort':
+                self.author_sort_edit.emit(self, index)
+                return
+
+            if action == 'hide':
+                self.hidden_categories.add(category)
+            elif action == 'show':
+                self.hidden_categories.discard(category)
+            elif action == 'categorization':
+                changed = self.collapse_model != category
+                self.collapse_model = category
+                if changed:
+                    self.set_new_model(self._model.get_filter_categories_by())
+                    gprefs['tags_browser_partition_method'] = category
+            elif action == 'defaults':
+                self.hidden_categories.clear()
+            self.db.prefs.set('tag_browser_hidden_categories', list(self.hidden_categories))
+            self.set_new_model()
+        except:
+            return
+
+    def show_context_menu(self, point):
+        def display_name( tag):
+            if tag.category == 'search':
+                n = tag.name
+                if len(n) > 45:
+                    n = n[:45] + '...'
+                return "'" + n + "'"
+            return tag.name
+
+        index = self.indexAt(point)
+        self.context_menu = QMenu(self)
+
+        if index.isValid():
+            item = index.data(Qt.UserRole).toPyObject()
+            tag = None
+
+            if item.type == TagTreeItem.TAG:
+                tag_item = item
+                tag = item.tag
+                while item.type != TagTreeItem.CATEGORY:
+                    item = item.parent
+
+            if item.type == TagTreeItem.CATEGORY:
+                if not item.category_key.startswith('@'):
+                    while item.parent != self._model.root_item:
+                        item = item.parent
+                category = unicode(item.name.toString())
+                key = item.category_key
+                # Verify that we are working with a field that we know something about
+                if key not in self.db.field_metadata:
+                    return True
+
+                # Did the user click on a leaf node?
+                if tag:
+                    # If the user right-clicked on an editable item, then offer
+                    # the possibility of renaming that item.
+                    if tag.is_editable:
+                        # Add the 'rename' items
+                        self.context_menu.addAction(self.rename_icon,
+                                                    _('Rename %s')%display_name(tag),
+                            partial(self.context_menu_handler, action='edit_item',
+                                    index=index))
+                        if key == 'authors':
+                            self.context_menu.addAction(_('Edit sort for %s')%display_name(tag),
+                                    partial(self.context_menu_handler,
+                                            action='edit_author_sort', index=tag.id))
+
+                        # is_editable is also overloaded to mean 'can be added
+                        # to a user category'
+                        m = self.context_menu.addMenu(self.user_category_icon,
+                                        _('Add %s to user category')%display_name(tag))
+                        nt = self.model().category_node_tree
+                        def add_node_tree(tree_dict, m, path):
+                            p = path[:]
+                            for k in sorted(tree_dict.keys(), key=sort_key):
+                                p.append(k)
+                                n = k[1:] if k.startswith('@') else k
+                                m.addAction(self.user_category_icon, n,
+                                    partial(self.context_menu_handler,
+                                            'add_to_category',
+                                            category='.'.join(p), index=tag_item))
+                                if len(tree_dict[k]):
+                                    tm = m.addMenu(self.user_category_icon,
+                                                   _('Children of %s')%n)
+                                    add_node_tree(tree_dict[k], tm, p)
+                                p.pop()
+                        add_node_tree(nt, m, [])
+                    elif key == 'search':
+                        self.context_menu.addAction(self.rename_icon,
+                                                    _('Rename %s')%display_name(tag),
+                            partial(self.context_menu_handler, action='edit_item',
+                                    index=index))
+                        self.context_menu.addAction(self.delete_icon,
+                                _('Delete search %s')%display_name(tag),
+                                partial(self.context_menu_handler,
+                                        action='delete_search', key=tag.name))
+                    if key.startswith('@') and not item.is_gst:
+                        self.context_menu.addAction(self.user_category_icon,
+                                _('Remove %s from category %s')%
+                                            (display_name(tag), item.py_name),
+                                partial(self.context_menu_handler,
+                                        action='delete_item_from_user_category',
+                                        key = key, index = tag_item))
+                    # Add the search for value items. All leaf nodes are searchable
+                    self.context_menu.addAction(self.search_icon,
+                            _('Search for %s')%display_name(tag),
+                            partial(self.context_menu_handler, action='search',
+                                    search_state=TAG_SEARCH_STATES['mark_plus'],
+                                    index=index))
+                    self.context_menu.addAction(self.search_icon,
+                            _('Search for everything but %s')%display_name(tag),
+                            partial(self.context_menu_handler, action='search',
+                                    search_state=TAG_SEARCH_STATES['mark_minus'],
+                                    index=index))
+                    self.context_menu.addSeparator()
+                elif key.startswith('@') and not item.is_gst:
+                    if item.can_be_edited:
+                        self.context_menu.addAction(self.rename_icon,
+                            _('Rename %s')%item.py_name,
+                            partial(self.context_menu_handler, action='edit_item',
+                                    index=index))
+                    self.context_menu.addAction(self.user_category_icon,
+                            _('Add sub-category to %s')%item.py_name,
+                            partial(self.context_menu_handler,
+                                    action='add_subcategory', key=key))
+                    self.context_menu.addAction(self.delete_icon,
+                            _('Delete user category %s')%item.py_name,
+                            partial(self.context_menu_handler,
+                                    action='delete_user_category', key=key))
+                    self.context_menu.addSeparator()
+                # Hide/Show/Restore categories
+                self.context_menu.addAction(_('Hide category %s') % category,
+                    partial(self.context_menu_handler, action='hide',
+                            category=key))
+                if self.hidden_categories:
+                    m = self.context_menu.addMenu(_('Show category'))
+                    for col in sorted(self.hidden_categories,
+                            key=lambda x: sort_key(self.db.field_metadata[x]['name'])):
+                        m.addAction(self.db.field_metadata[col]['name'],
+                            partial(self.context_menu_handler, action='show', category=col))
+
+                # search by category. Some categories are not searchable, such
+                # as search and news
+                if item.tag.is_searchable:
+                    self.context_menu.addAction(self.search_icon,
+                            _('Search for books in category %s')%category,
+                            partial(self.context_menu_handler,
+                                    action='search_category',
+                                    index=self._model.createIndex(item.row(), 0, item),
+                                    search_state=TAG_SEARCH_STATES['mark_plus']))
+                    self.context_menu.addAction(self.search_icon,
+                            _('Search for books not in category %s')%category,
+                            partial(self.context_menu_handler,
+                                    action='search_category',
+                                    index=self._model.createIndex(item.row(), 0, item),
+                                    search_state=TAG_SEARCH_STATES['mark_minus']))
+                # Offer specific editors for tags/series/publishers/saved searches
+                self.context_menu.addSeparator()
+                if key in ['tags', 'publisher', 'series'] or \
+                            self.db.field_metadata[key]['is_custom']:
+                    self.context_menu.addAction(_('Manage %s')%category,
+                            partial(self.context_menu_handler, action='open_editor',
+                                    category=tag.original_name if tag else None,
+                                    key=key))
+                elif key == 'authors':
+                    self.context_menu.addAction(_('Manage %s')%category,
+                            partial(self.context_menu_handler, action='edit_author_sort'))
+                elif key == 'search':
+                    self.context_menu.addAction(_('Manage Saved Searches'),
+                        partial(self.context_menu_handler, action='manage_searches',
+                                category=tag.name if tag else None))
+
+                # Always show the user categories editor
+                self.context_menu.addSeparator()
+                if key.startswith('@') and \
+                        key[1:] in self.db.prefs.get('user_categories', {}).keys():
+                    self.context_menu.addAction(_('Manage User Categories'),
+                            partial(self.context_menu_handler, action='manage_categories',
+                                    category=key[1:]))
+                else:
+                    self.context_menu.addAction(_('Manage User Categories'),
+                            partial(self.context_menu_handler, action='manage_categories',
+                                    category=None))
+
+        if self.hidden_categories:
+            if not self.context_menu.isEmpty():
+                self.context_menu.addSeparator()
+            self.context_menu.addAction(_('Show all categories'),
+                        partial(self.context_menu_handler, action='defaults'))
+
+        m = self.context_menu.addMenu(_('Change sub-categorization scheme'))
+        da = m.addAction('Disable',
+            partial(self.context_menu_handler, action='categorization', category='disable'))
+        fla = m.addAction('By first letter',
+            partial(self.context_menu_handler, action='categorization', category='first letter'))
+        pa = m.addAction('Partition',
+            partial(self.context_menu_handler, action='categorization', category='partition'))
+        if self.collapse_model == 'disable':
+            da.setCheckable(True)
+            da.setChecked(True)
+        elif self.collapse_model == 'first letter':
+            fla.setCheckable(True)
+            fla.setChecked(True)
+        else:
+            pa.setCheckable(True)
+            pa.setChecked(True)
+
+        if not self.context_menu.isEmpty():
+            self.context_menu.popup(self.mapToGlobal(point))
+        return True
+
+    def dragMoveEvent(self, event):
+        QTreeView.dragMoveEvent(self, event)
+        self.setDropIndicatorShown(False)
+        index = self.indexAt(event.pos())
+        if not index.isValid():
+            return
+        src_is_tb = event.mimeData().hasFormat('application/calibre+from_tag_browser')
+        item = index.data(Qt.UserRole).toPyObject()
+        flags = self._model.flags(index)
+        if item.type == TagTreeItem.TAG and flags & Qt.ItemIsDropEnabled:
+            self.setDropIndicatorShown(not src_is_tb)
+            return
+        if item.type == TagTreeItem.CATEGORY and not item.is_gst:
+            fm_dest = self.db.metadata_for_field(item.category_key)
+            if fm_dest['kind'] == 'user':
+                if src_is_tb:
+                    if event.dropAction() == Qt.MoveAction:
+                        data = str(event.mimeData().data('application/calibre+from_tag_browser'))
+                        src = cPickle.loads(data)
+                        for s in src:
+                            if s[0] == TagTreeItem.TAG and \
+                                    (not s[1].startswith('@') or s[2]):
+                                return
+                    self.setDropIndicatorShown(True)
+                    return
+                md = event.mimeData()
+                if hasattr(md, 'column_name'):
+                    fm_src = self.db.metadata_for_field(md.column_name)
+                    if md.column_name in ['authors', 'publisher', 'series'] or \
+                            (fm_src['is_custom'] and (
+                             (fm_src['datatype'] in ['series', 'text', 'enumeration'] and
+                              not fm_src['is_multiple']) or
+                             (fm_src['datatype'] == 'composite' and
+                              fm_src['display'].get('make_category', False)))):
+                        self.setDropIndicatorShown(True)
+
+    def clear(self):
+        if self.model():
+            self.model().clear_state()
+
+    def is_visible(self, idx):
+        item = idx.data(Qt.UserRole).toPyObject()
+        if getattr(item, 'type', None) == TagTreeItem.TAG:
+            idx = idx.parent()
+        return self.isExpanded(idx)
+
+    def recount(self, *args):
+        '''
+        Rebuild the category tree, expand any categories that were expanded,
+        reset the search states, and reselect the current node.
+        '''
+        if self.disable_recounting or not self.pane_is_visible:
+            return
+        self.refresh_signal_processed = True
+        ci = self.currentIndex()
+        if not ci.isValid():
+            ci = self.indexAt(QPoint(10, 10))
+        path = self.model().path_for_index(ci) if self.is_visible(ci) else None
+        expanded_categories, state_map = self.model().get_state()
+        self.set_new_model(state_map=state_map)
+        for category in expanded_categories:
+            self.expand(self.model().index_for_category(category))
+        self._model.show_item_at_path(path)
+
+    def item_expanded(self, idx):
+        '''
+        Called by the expanded signal
+        '''
+        self.setCurrentIndex(idx)
+
+    def set_new_model(self, filter_categories_by=None, state_map={}):
+        '''
+        There are cases where we need to rebuild the category tree without
+        attempting to reposition the current node.
+        '''
+        try:
+            old = getattr(self, '_model', None)
+            if old is not None:
+                old.break_cycles()
+            self._model = TagsModel(self.db, parent=self,
+                                    hidden_categories=self.hidden_categories,
+                                    search_restriction=self.search_restriction,
+                                    drag_drop_finished=self.drag_drop_finished,
+                                    filter_categories_by=filter_categories_by,
+                                    collapse_model=self.collapse_model,
+                                    state_map=state_map)
+            self.setModel(self._model)
+        except:
+            # The DB must be gone. Set the model to None and hope that someone
+            # will call set_database later. I don't know if this in fact works.
+            # But perhaps a Bad Thing Happened, so print the exception
+            traceback.print_exc()
+            self._model = None
+            self.setModel(None)
+    # }}}
+
+

src/calibre/gui2/tools.py

@@ -51,12 +51,15 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{
                 #    continue
 
                 mi = db.get_metadata(book_id, True)
-                in_file = db.format_abspath(book_id, d.input_format, True)
+                in_file = PersistentTemporaryFile('.'+d.input_format)
+                with in_file:
+                    db.copy_format_to(book_id, d.input_format, in_file,
+                            index_is_id=True)
 
                 out_file = PersistentTemporaryFile('.' + d.output_format)
                 out_file.write(d.output_format)
                 out_file.close()
-                temp_files = []
+                temp_files = [in_file]
 
                 try:
                     dtitle = unicode(mi.title)
@@ -74,7 +77,7 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{
                     recs.append(('cover', d.cover_file.name,
                         OptionRecommendation.HIGH))
                     temp_files.append(d.cover_file)
-                args = [in_file, out_file.name, recs]
+                args = [in_file.name, out_file.name, recs]
                 temp_files.append(out_file)
                 jobs.append(('gui_convert_override', args, desc, d.output_format.upper(), book_id, temp_files))
 
@@ -142,12 +145,15 @@ class QueueBulk(QProgressDialog):
         try:
             input_format = get_input_format_for_book(self.db, book_id, None)[0]
             mi, opf_file = create_opf_file(self.db, book_id)
-            in_file = self.db.format_abspath(book_id, input_format, True)
+            in_file = PersistentTemporaryFile('.'+input_format)
+            with in_file:
+                self.db.copy_format_to(book_id, input_format, in_file,
+                        index_is_id=True)
 
             out_file = PersistentTemporaryFile('.' + self.output_format)
             out_file.write(self.output_format)
             out_file.close()
-            temp_files = []
+            temp_files = [in_file]
 
             combined_recs = GuiRecommendations()
             default_recs = bulk_defaults_for_input_format(input_format)
@@ -183,7 +189,7 @@ class QueueBulk(QProgressDialog):
             self.setLabelText(_('Queueing ')+dtitle)
             desc = _('Convert book %d of %d (%s)') % (self.i, len(self.book_ids), dtitle)
 
-            args = [in_file, out_file.name, lrecs]
+            args = [in_file.name, out_file.name, lrecs]
             temp_files.append(out_file)
             self.jobs.append(('gui_convert_override', args, desc, self.output_format.upper(), book_id, temp_files))
 

src/calibre/gui2/ui.py

@@ -39,7 +39,7 @@ from calibre.gui2.jobs import JobManager, JobsDialog, JobsButton
 from calibre.gui2.init import LibraryViewMixin, LayoutMixin
 from calibre.gui2.search_box import SearchBoxMixin, SavedSearchBoxMixin
 from calibre.gui2.search_restriction_mixin import SearchRestrictionMixin
-from calibre.gui2.tag_view import TagBrowserMixin
+from calibre.gui2.tag_browser.ui import TagBrowserMixin
 
 
 class Listener(Thread): # {{{

src/calibre/gui2/update.py

@@ -72,9 +72,7 @@ class UpdateNotification(QDialog):
         self.label = QLabel(('<p>'+
             _('%s has been updated to version <b>%s</b>. '
             'See the <a href="http://calibre-ebook.com/whats-new'
-            '">new features</a>.') + '<p>'+_('Update <b>only</b> if one of the '
+            '">new features</a>.'))%(
-            'new features or bug fixes is important to you. '
-            'If the current version works well for you, do not update.'))%(
                 __appname__, calibre_version))
         self.label.setOpenExternalLinks(True)
         self.label.setWordWrap(True)
@@ -179,7 +177,7 @@ class UpdateMixin(object):
 
     def plugin_update_found(self, number_of_updates):
         # Change the plugin icon to indicate there are updates available
-        plugin = self.iactions.get('Plugin Updates', None)
+        plugin = self.iactions.get('Plugin Updater', None)
         if not plugin:
             return
         if number_of_updates:

src/calibre/library/__init__.py

@@ -61,22 +61,4 @@ def generate_test_db(library_path, # {{{
     print 'Time per record:', t/float(num_of_records)
 # }}}
 
-def cover_load_timing(path=None):
-    from PyQt4.Qt import QApplication, QImage
-    import os, time
-    app = QApplication([])
-    app
-    d = db(path)
-    paths = [d.cover(i, index_is_id=True, as_path=True) for i in
-            d.data.iterallids()]
-    paths = [p for p in paths if (p and os.path.exists(p) and os.path.isfile(p))]
-
-    start = time.time()
-
-    for p in paths:
-        with open(p, 'rb') as f:
-            img = QImage()
-            img.loadFromData(f.read())
-
-    print 'Average load time:', (time.time() - start)/len(paths), 'seconds'
 

src/calibre/library/caches.py

@@ -145,7 +145,7 @@ def _match(query, value, matchkind):
                             return True
                 elif query == t:
                     return True
-            elif ((matchkind == REGEXP_MATCH and re.search(query, t, re.I)) or ### search unanchored
+            elif ((matchkind == REGEXP_MATCH and re.search(query, t, re.I|re.UNICODE)) or ### search unanchored
                   (matchkind == CONTAINS_MATCH and query in t)):
                     return True
         except re.error: