sync with Kovid's branch

manual/conversion.rst

@@ -672,6 +672,7 @@ Some limitations of PDF input are:
     * Links and Tables of Contents are not supported
     * PDFs that use embedded non-unicode fonts to represent non-English characters will result in garbled output for those characters
     * Some PDFs are made up of photographs of the page with OCRed text behind them. In such cases |app| uses the OCRed text, which can be very different from what you see when you view the PDF file
+    * PDFs that are used to display complex text, like right to left languages and math typesetting will not convert correctly
 
 To re-iterate **PDF is a really, really bad** format to use as input. If you absolutely must use PDF, then be prepared for an
 output ranging anywhere from decent to unusable, depending on the input PDF.

recipes/biweekly.recipe

@@ -6,46 +6,50 @@ __copyright__ = u'Łukasz Grąbczewski 2011'
 __version__ = '2.0'
 
 import re, os
+from calibre import walk
+from calibre.utils.zipfile import ZipFile
 from calibre.ptempfile import PersistentTemporaryFile
-from calibre.ebooks.conversion.cli import main
+from calibre.web.feeds.news import BasicNewsRecipe
 
 class biweekly(BasicNewsRecipe):
-	__author__ = u'Łukasz Grąbczewski'
-	title = 'Biweekly'
-	language = 'en_EN'
-	publisher = 'National Audiovisual Institute'
-	publication_type = 'magazine'
-	description = u'link with culture [English edition of Polish magazine]: literature, theatre, film, art, music, views, talks'
+    __author__ = u'Łukasz Grąbczewski'
+    title = 'Biweekly'
+    language = 'en'
+    publisher = 'National Audiovisual Institute'
+    publication_type = 'magazine'
+    description = u'link with culture [English edition of Polish magazine]: literature, theatre, film, art, music, views, talks'
 
-	conversion_options = {
-		'authors' : 'Biweekly.pl'
-		,'publisher' : publisher
-		,'language' : language
-		,'comments' : description
-		,'no_default_epub_cover' : True
-		,'preserve_cover_aspect_ratio': True
-	}
+    conversion_options = {
+        'authors' : 'Biweekly.pl'
+        ,'publisher' : publisher
+        ,'language' : language
+        ,'comments' : description
+        ,'no_default_epub_cover' : True
+        ,'preserve_cover_aspect_ratio': True
+    }
 
-	def build_index(self):
-		browser = self.get_browser()
-		rc = browser.open('http://www.biweekly.pl/')
+    def build_index(self):
+        browser = self.get_browser()
+        browser.open('http://www.biweekly.pl/')
 
-		# find the link
-		epublink = browser.find_link(text_regex=re.compile('ePUB VERSION'))
+        # find the link
+        epublink = browser.find_link(text_regex=re.compile('ePUB VERSION'))
 
-		# download ebook
-		self.report_progress(0,_('Downloading ePUB'))
-		response = browser.follow_link(epublink)
-		book_file = PersistentTemporaryFile(suffix='.epub')
-		book_file.write(response.read())
-		book_file.close()
+        # download ebook
+        self.report_progress(0,_('Downloading ePUB'))
+        response = browser.follow_link(epublink)
+        book_file = PersistentTemporaryFile(suffix='.epub')
+        book_file.write(response.read())
+        book_file.close()
 
-		# convert
-		self.report_progress(0.2,_('Converting to OEB'))
-		oebdir = self.output_dir + '/INPUT/'
-		main(['ebook-convert', book_file.name, oebdir])
+        # convert
+        self.report_progress(0.2,_('Converting to OEB'))
+        oeb = self.output_dir + '/INPUT/'
+        if not os.path.exists(oeb):
+            os.makedirs(oeb)
+        with ZipFile(book_file.name) as f:
+            f.extractall(path=oeb)
 
-		# feed calibre
-		index = os.path.join(oebdir, 'content.opf')
-		
-		return index
+        for f in walk(oeb):
+            if f.endswith('.opf'):
+                return f

recipes/dwutygodnik.recipe

@@ -6,46 +6,51 @@ __copyright__ = u'Łukasz Grąbczewski 2011'
 __version__ = '2.0'
 
 import re, os
+from calibre import walk
+from calibre.utils.zipfile import ZipFile
 from calibre.ptempfile import PersistentTemporaryFile
-from calibre.ebooks.conversion.cli import main
+from calibre.web.feeds.news import BasicNewsRecipe
 
 class dwutygodnik(BasicNewsRecipe):
-	__author__ = u'Łukasz Grąbczewski'
-	title = 'Dwutygodnik'
-	language = 'pl_PL'
-	publisher = 'Narodowy Instytut Audiowizualny'
-	publication_type = 'magazine'
-	description = u'Strona Kultury: literatura, teatr, film, sztuka, muzyka, felietony, rozmowy'
-	
-	conversion_options = {
-		'authors' : 'Dwutygodnik.com'
-		,'publisher' : publisher
-		,'language' : language
-		,'comments' : description
-		,'no_default_epub_cover' : True
-		,'preserve_cover_aspect_ratio': True
-	}
+    __author__ = u'Łukasz Grąbczewski'
+    title = 'Dwutygodnik'
+    language = 'pl'
+    publisher = 'Narodowy Instytut Audiowizualny'
+    publication_type = 'magazine'
+    description = u'Strona Kultury: literatura, teatr, film, sztuka, muzyka, felietony, rozmowy'
 
-	def build_index(self):
-		browser = self.get_browser()
-		rc = browser.open('http://www.dwutygodnik.com/')
+    conversion_options = {
+        'authors' : 'Dwutygodnik.com'
+        ,'publisher' : publisher
+        ,'language' : language
+        ,'comments' : description
+        ,'no_default_epub_cover' : True
+        ,'preserve_cover_aspect_ratio': True
+    }
 
-		# find the link
-		epublink = browser.find_link(text_regex=re.compile('Wersja ePub'))
+    def build_index(self):
+        browser = self.get_browser()
+        browser.open('http://www.dwutygodnik.com/')
 
-		# download ebook
-		self.report_progress(0,_('Downloading ePUB'))
-		response = browser.follow_link(epublink)
-		book_file = PersistentTemporaryFile(suffix='.epub')
-		book_file.write(response.read())
-		book_file.close()
+        # find the link
+        epublink = browser.find_link(text_regex=re.compile('Wersja ePub'))
 
-		# convert
-		self.report_progress(0.2,_('Converting to OEB'))
-		oebdir = self.output_dir + '/INPUT/'
-		main(['ebook-convert', book_file.name, oebdir])
+        # download ebook
+        self.report_progress(0,_('Downloading ePUB'))
+        response = browser.follow_link(epublink)
+        book_file = PersistentTemporaryFile(suffix='.epub')
+        book_file.write(response.read())
+        book_file.close()
+
+        # convert
+        self.report_progress(0.2,_('Converting to OEB'))
+        oeb = self.output_dir + '/INPUT/'
+        if not os.path.exists(oeb):
+            os.makedirs(oeb)
+        with ZipFile(book_file.name) as f:
+            f.extractall(path=oeb)
+
+        for f in walk(oeb):
+            if f.endswith('.opf'):
+                return f
 
-		# feed calibre
-		index = os.path.join(oebdir, 'content.opf')
-		
-		return index

recipes/hatalska.recipe

@@ -8,7 +8,6 @@ hatalska.com
 '''
 
 from calibre.web.feeds.news import BasicNewsRecipe
-import re
 
 class hatalska(BasicNewsRecipe):
     title          = u'Hatalska'

recipes/hindu.recipe

@@ -41,13 +41,16 @@ class TheHindu(BasicNewsRecipe):
             if current_section and x.get('class', '') == 'tpaper':
                 a = x.find('a', href=True)
                 if a is not None:
+                    title = self.tag_to_string(a)
+                    self.log('\tFound article:', title)
                     current_articles.append({'url':a['href']+'?css=print',
-                        'title':self.tag_to_string(a), 'date': '',
+                        'title':title, 'date': '',
                         'description':''})
             if x.name == 'h3':
                 if current_section and current_articles:
                     feeds.append((current_section, current_articles))
                 current_section = self.tag_to_string(x)
+                self.log('Found section:', current_section)
                 current_articles = []
         return feeds
 

recipes/jazzpress.recipe

@@ -5,46 +5,51 @@ __license__ = 'GPL v3'
 __copyright__ = u'Łukasz Grąbczewski 2011-2013'
 __version__ = '2.0'
 
-import re, zipfile, os
+import re, os
+from calibre import walk
+from calibre.utils.zipfile import ZipFile
 from calibre.ptempfile import PersistentTemporaryFile
-from calibre.ebooks.conversion.cli import main
+from calibre.web.feeds.news import BasicNewsRecipe
 
 class jazzpress(BasicNewsRecipe):
-	__author__ = u'Łukasz Grąbczewski'
-	title = 'JazzPRESS'
-	language = 'pl'
-	publisher = 'Fundacja Popularyzacji Muzyki Jazzowej EuroJAZZ'
-	publication_type = 'magazine'
-	description = u'Internetowa gazeta poświęcona muzyce improwizowanej'
-	
-	conversion_options = {
-		'authors' : 'Fundacja Popularyzacji Muzyki Jazzowej EuroJAZZ'
-		,'publisher' : publisher
-		,'language' : language
-		,'preserve_cover_aspect_ratio': True
-		,'remove_first_image': True
-	}
-	
-	def build_index(self):
-		browser = self.get_browser()
-		rc = browser.open('http://radiojazz.fm/')
+    __author__ = u'Łukasz Grąbczewski'
+    title = 'JazzPRESS'
+    language = 'pl'
+    publisher = 'Fundacja Popularyzacji Muzyki Jazzowej EuroJAZZ'
+    publication_type = 'magazine'
+    description = u'Internetowa gazeta poświęcona muzyce improwizowanej'
 
-		# find the link
-		epublink = browser.find_link(url_regex=re.compile('e_jazzpress\d\d\d\d\_epub'))
+    conversion_options = {
+        'authors' : 'Fundacja Popularyzacji Muzyki Jazzowej EuroJAZZ'
+        ,'publisher' : publisher
+        ,'language' : language
+        ,'preserve_cover_aspect_ratio': True
+        ,'remove_first_image': True
+    }
 
-		# download ebook
-		self.report_progress(0,_('Downloading ePUB'))
-		response = browser.follow_link(epublink)
-		book_file = PersistentTemporaryFile(suffix='.epub')
-		book_file.write(response.read())
-		book_file.close()
+    def build_index(self):
+        browser = self.get_browser()
+        browser.open('http://radiojazz.fm/')
 
-		# convert
-		self.report_progress(0.2,_('Converting to OEB'))
-		oebdir = self.output_dir + '/INPUT/'
-		main(['ebook-convert', book_file.name, oebdir])
+        # find the link
+        epublink = browser.find_link(url_regex=re.compile('e_jazzpress\d\d\d\d\_epub'))
+
+        # download ebook
+        self.report_progress(0,_('Downloading ePUB'))
+        response = browser.follow_link(epublink)
+        book_file = PersistentTemporaryFile(suffix='.epub')
+        book_file.write(response.read())
+        book_file.close()
+
+        # convert
+        self.report_progress(0.2,_('Converting to OEB'))
+        oeb = self.output_dir + '/INPUT/'
+        if not os.path.exists(oeb):
+            os.makedirs(oeb)
+        with ZipFile(book_file.name) as f:
+            f.extractall(path=oeb)
+
+        for f in walk(oeb):
+            if f.endswith('.opf'):
+                return f    # convert
 
-		# feed calibre
-		index = os.path.join(oebdir, 'content.opf')
-		
-		return index

recipes/kyungyhang.recipe

@@ -16,14 +16,14 @@ class Kyungyhang(BasicNewsRecipe):
     max_articles_per_feed = 20
     no_stylesheets         = True
     remove_javascript     = True
-	
+
     keep_only_tags    = [
                         dict(name='div', attrs ={'class':['article_title_wrap']}),
-                        dict(name='div', attrs ={'class':['article_txt']})
+                        dict(name='span', attrs ={'class':['article_txt']})
                         ]
 
     remove_tags_after = dict(id={'sub_bottom'})
-	
+
     remove_tags = [
        dict(name='iframe'),
        dict(id={'TdHot'}),
@@ -31,7 +31,7 @@ class Kyungyhang(BasicNewsRecipe):
        dict(name='dl', attrs={'class':['CL']}),
        dict(name='ul', attrs={'class':['tab']}),
        ]
-	
+
     feeds = [
 	('All News','http://www.khan.co.kr/rss/rssdata/total_news.xml'),
-           ]
+           ]

recipes/ledevoir.recipe

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __author__    = 'Lorenzo Vigentini and Olivier Daigle'
 __copyright__ = '2012, Lorenzo Vigentini <l.vigentini at gmail.com>, Olivier Daigle <odaigle _at nuvucameras __dot__ com>'
 __version__     = 'v1.01'
-__date__        = '12, February 2012'
+__date__        = '22, December 2012'
 __description__   = 'Canadian Paper '
 
 '''
@@ -32,41 +32,50 @@ class ledevoir(BasicNewsRecipe):
     recursion             = 10
     needs_subscription    = 'optional'
 
+    filterDuplicates = False
     url_list = []
 
     remove_javascript     = True
     no_stylesheets        = True
-    auto_cleanup = True
 
     preprocess_regexps = [(re.compile(r'(title|alt)=".*?>.*?"', re.DOTALL), lambda m: '')]
 
-    #keep_only_tags  = [
+    keep_only_tags  = [
                         #dict(name='div', attrs={'id':'article_detail'}),
-                        #dict(name='div', attrs={'id':'colonne_principale'})
-                    #]
+                        #dict(name='div', attrs={'id':'colonne_principale'}),
+                        dict(name='article', attrs={'id':'article', 'class':'clearfix'}),
+                        dict(name='article', attrs={'id':'article', 'class':'clearfix portrait'})
+                    ]
 
-    #remove_tags     = [
-                        #dict(name='div', attrs={'id':'dialog'}),
-                        #dict(name='div', attrs={'class':['interesse_actions','reactions','taille_du_texte right clearfix','partage_sociaux clearfix']}),
-                        #dict(name='aside', attrs={'class':['article_actions clearfix','reactions','partage_sociaux_wrapper']}),
-                        #dict(name='ul', attrs={'class':'mots_cles'}),
-                        #dict(name='ul', attrs={'id':'commentaires'}),
-                        #dict(name='a', attrs={'class':'haut'}),
-                        #dict(name='h5', attrs={'class':'interesse_actions'})
-                    #]
+    remove_tags     = [
+                        dict(name='div', attrs={'id':'prive'}),
+                        dict(name='div', attrs={'class':'acheter_article'}),
+                        dict(name='div', attrs={'id':'col_complement'}),
+                        dict(name='div', attrs={'id':'raccourcis','class':'clearfix'}),
+                        dict(name='div', attrs={'id':'dialog'}),
+                        dict(name='div', attrs={'id':'liste_photos_article','class':'clearfix'}),
+                        dict(name='script', attrs={'type':'text/javascript'}),
+                        dict(name='div', attrs={'class':['interesse_actions','reactions','taille_du_texte right clearfix','partage_sociaux clearfix']}),
+                        dict(name='aside', attrs={'class':['article_actions clearfix','partage_sociaux_wrapper']}),
+                        dict(name='aside', attrs={'class':'reactions', 'id':'reactions'}),
+                        dict(name='ul', attrs={'class':'mots_cles'}),
+                        dict(name='ul', attrs={'id':'commentaires'}),
+                        dict(name='a', attrs={'class':'haut'}),
+                        dict(name='h5', attrs={'class':'interesse_actions'})
+                    ]
 
     feeds          = [
                        (u'A la une', 'http://www.ledevoir.com/rss/manchettes.xml'),
-                       (u'Édition complete', 'http://feeds2.feedburner.com/fluxdudevoir'),
-                       (u'Opinions', 'http://www.ledevoir.com/rss/opinions.xml'),
-                       (u'Chroniques', 'http://www.ledevoir.com/rss/chroniques.xml'),
-                       (u'Politique', 'http://www.ledevoir.com/rss/section/politique.xml?id=51'),
-                       (u'International', 'http://www.ledevoir.com/rss/section/international.xml?id=76'),
-                       (u'Culture', 'http://www.ledevoir.com/rss/section/culture.xml?id=48'),
-                       (u'Environnement', 'http://www.ledevoir.com/rss/section/environnement.xml?id=78'),
-                       (u'Societe', 'http://www.ledevoir.com/rss/section/societe.xml?id=52'),
-                       (u'Economie', 'http://www.ledevoir.com/rss/section/economie.xml?id=49'),
-                       (u'Sports', 'http://www.ledevoir.com/rss/section/sports.xml?id=85'),
+#                       (u'Édition complete', 'http://feeds2.feedburner.com/fluxdudevoir'),
+#                       (u'Opinions', 'http://www.ledevoir.com/rss/opinions.xml'),
+#                       (u'Chroniques', 'http://www.ledevoir.com/rss/chroniques.xml'),
+#                       (u'Politique', 'http://www.ledevoir.com/rss/section/politique.xml?id=51'),
+#                       (u'International', 'http://www.ledevoir.com/rss/section/international.xml?id=76'),
+#                       (u'Culture', 'http://www.ledevoir.com/rss/section/culture.xml?id=48'),
+#                       (u'Environnement', 'http://www.ledevoir.com/rss/section/environnement.xml?id=78'),
+#                       (u'Societe', 'http://www.ledevoir.com/rss/section/societe.xml?id=52'),
+#                       (u'Economie', 'http://www.ledevoir.com/rss/section/economie.xml?id=49'),
+#                       (u'Sports', 'http://www.ledevoir.com/rss/section/sports.xml?id=85'),
                        (u'Art de vivre', 'http://www.ledevoir.com/rss/section/art-de-vivre.xml?id=50')
                      ]
 
@@ -88,7 +97,7 @@ class ledevoir(BasicNewsRecipe):
                 .texte {font-size:1.15em;line-height:1.4em;margin-bottom:17px;}
                 '''
     def get_browser(self):
-        br = BasicNewsRecipe.get_browser(self)
+        br = BasicNewsRecipe.get_browser()
         if self.username is not None and self.password is not None:
             br.open('http://www.ledevoir.com')
             br.select_form(nr=0)
@@ -97,4 +106,10 @@ class ledevoir(BasicNewsRecipe):
             br.submit()
         return br
 
+    def print_version(self, url):
+        if self.filterDuplicates:
+            if url in self.url_list:
+                return
+        self.url_list.append(url)
+        return url
 

recipes/lifehacker_pl.recipe

@@ -8,7 +8,6 @@ www.lifehacking.pl
 '''
 
 from calibre.web.feeds.news import BasicNewsRecipe
-import re
 
 class lifehacking(BasicNewsRecipe):
     title = u'Lifehacker Polska'

recipes/nytimesbook.recipe

@@ -1,21 +1,56 @@
+
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class NewYorkTimesBookReview(BasicNewsRecipe):
     title          = u'New York Times Book Review'
     language       = 'en'
-    __author__     = 'Krittika Goyal'
-    oldest_article = 8 #days
-    max_articles_per_feed = 1000
-    #recursions = 2
-    #encoding = 'latin1'
-    use_embedded_content = False
+    description    = 'The New York Times Sunday Book Review. Best downloaded on Fridays to avoid the ads that the New York Times shows of the first few days of the week.'
+    __author__     = 'Kovid Goyal'
+
 
     no_stylesheets = True
-    auto_cleanup = True
+    no_javascript = True
+    keep_only_tags = [dict(id='article'), dict(id=lambda x:x and x.startswith('entry-'))]
+    remove_tags = [
+        dict(attrs={'class':['articleBottomExtra', 'shareToolsBox', 'singleAd']}),
+        dict(attrs={'class':lambda x: x and ('shareTools' in x or 'enlargeThis' in x)}),
+    ]
+
+    def parse_index(self):
+        soup = self.index_to_soup('http://www.nytimes.com/pages/books/review/index.html')
+
+        # Find TOC
+        toc = soup.find('div', id='main').find(
+            'div', attrs={'class':'abColumn'})
+        feeds = []
+        articles = []
+        section_title = 'Features'
+        for x in toc.findAll(['div', 'h3', 'h6'], attrs={'class':['story', 'sectionHeader', 'ledeStory']}):
+            if x['class'] == 'sectionHeader':
+                if articles:
+                    feeds.append((section_title, articles))
+                section_title = self.tag_to_string(x)
+                articles = []
+                self.log('Found section:', section_title)
+                continue
+            if x['class'] in {'story', 'ledeStory'}:
+                tt = 'h3' if x['class'] == 'story' else 'h1'
+                a = x.find(tt).find('a', href=True)
+                title = self.tag_to_string(a)
+                url = a['href'] + '&pagewanted=all'
+                self.log('\tFound article:', title, url)
+                desc = ''
+                byline = x.find('h6', attrs={'class':'byline'})
+                if byline is not None:
+                    desc = self.tag_to_string(byline)
+                summary = x.find('p', attrs={'class':'summary'})
+                if summary is not None:
+                    desc += self.tag_to_string(summary)
+                if desc:
+                    self.log('\t\t', desc)
+                articles.append({'title':title, 'url':url, 'date':'',
+                    'description':desc})
+
+        return feeds
 
 
-    feeds          = [
-                     ('New York Times Sunday Book Review',
-                      'http://feeds.nytimes.com/nyt/rss/SundayBookReview'),
-                     ]
-

recipes/telepolis_pl.recipe

@@ -3,7 +3,6 @@
 __license__ = 'GPL v3'
 
 from calibre.web.feeds.news import BasicNewsRecipe
-import re
 
 
 class telepolis(BasicNewsRecipe):

recipes/tvn24.recipe

@@ -30,11 +30,6 @@ class tvn24(BasicNewsRecipe):
     feeds          = [(u'Najnowsze', u'http://www.tvn24.pl/najnowsze.xml'), ]
                 #(u'Polska', u'www.tvn24.pl/polska.xml'), (u'\u015awiat', u'http://www.tvn24.pl/swiat.xml'), (u'Sport', u'http://www.tvn24.pl/sport.xml'), (u'Biznes', u'http://www.tvn24.pl/biznes.xml'), (u'Meteo', u'http://www.tvn24.pl/meteo.xml'), (u'Micha\u0142ki', u'http://www.tvn24.pl/michalki.xml'), (u'Kultura', u'http://www.tvn24.pl/kultura.xml')]
 
-    def preprocess_html(self, soup):
-        for item in soup.findAll(style=True):
-            del item['style']
-        return soup
-
     def preprocess_html(self, soup):
         for alink in soup.findAll('a'):
             if alink.string is not None:

src/calibre/customize/builtins.py

@@ -634,7 +634,7 @@ from calibre.devices.apple.driver import ITUNES
 from calibre.devices.hanlin.driver import HANLINV3, HANLINV5, BOOX, SPECTRA
 from calibre.devices.blackberry.driver import BLACKBERRY, PLAYBOOK
 from calibre.devices.cybook.driver import CYBOOK, ORIZON
-from calibre.devices.eb600.driver import (EB600, COOL_ER, SHINEBOOK,
+from calibre.devices.eb600.driver import (EB600, COOL_ER, SHINEBOOK, TOLINO,
                 POCKETBOOK360, GER2, ITALICA, ECLICTO, DBOOK, INVESBOOK,
                 BOOQ, ELONEX, POCKETBOOK301, MENTOR, POCKETBOOK602,
                 POCKETBOOK701, POCKETBOOK360P, PI2, POCKETBOOK622)
@@ -704,7 +704,7 @@ plugins += [
     INVESBOOK,
     BOOX,
     BOOQ,
-    EB600,
+    EB600, TOLINO,
     README,
     N516, KIBANO,
     THEBOOK, LIBREAIR,

src/calibre/db/cache.py

@@ -217,6 +217,8 @@ class Cache(object):
                     field.series_field = self.fields[name[:-len('_index')]]
                 elif name == 'series_index':
                     field.series_field = self.fields['series']
+                elif name == 'authors':
+                    field.author_sort_field = self.fields['author_sort']
 
     @read_api
     def field_for(self, name, book_id, default_value=None):

src/calibre/db/fields.py

@@ -402,6 +402,13 @@ class AuthorsField(ManyToManyField):
     def category_sort_value(self, item_id, book_ids, lang_map):
         return self.table.asort_map[item_id]
 
+    def db_author_sort_for_book(self, book_id):
+        return self.author_sort_field.for_book(book_id)
+
+    def author_sort_for_book(self, book_id):
+        return ' & '.join(self.table.asort_map[k] for k in
+                          self.table.book_col_map[book_id])
+
 class FormatsField(ManyToManyField):
 
     def for_book(self, book_id, default_value=None):

src/calibre/db/tables.py

@@ -168,7 +168,7 @@ class AuthorsTable(ManyToManyTable):
         self.asort_map  = {}
         for row in db.conn.execute(
                 'SELECT id, name, sort, link FROM authors'):
-            self.id_map[row[0]] = row[1]
+            self.id_map[row[0]] = self.unserialize(row[1])
             self.asort_map[row[0]] = (row[2] if row[2] else
                     author_to_author_sort(row[1]))
             self.alink_map[row[0]] = row[3]

src/calibre/db/tests/writing.py

@@ -203,14 +203,63 @@ class WritingTest(BaseTest):
 
     # }}}
 
-
     def test_many_many_basic(self): # {{{
-        'Test the different code paths for writing to a many-one field'
-        # Fields: identifiers, authors, tags, languages, #authors, #tags
+        'Test the different code paths for writing to a many-many field'
+        cl = self.cloned_library
+        cache = self.init_cache(cl)
+        ae, af, sf = self.assertEqual, self.assertFalse, cache.set_field
+
+        # Tags
+        ae(sf('#tags', {1:cache.field_for('tags', 1), 2:cache.field_for('tags', 2)}),
+            {1, 2})
+        for name in ('tags', '#tags'):
+            f = cache.fields[name]
+            af(sf(name, {1:('tag one', 'News')}, allow_case_change=False))
+            ae(sf(name, {1:'tag one, News'}), {1, 2})
+            ae(sf(name, {3:('tag two', 'sep,sep2')}), {2, 3})
+            ae(len(f.table.id_map), 4)
+            ae(sf(name, {1:None}), set([1]))
+            cache2 = self.init_cache(cl)
+            for c in (cache, cache2):
+                ae(c.field_for(name, 3), ('tag two', 'sep;sep2'))
+                ae(len(c.fields[name].table.id_map), 3)
+                ae(len(c.fields[name].table.id_map), 3)
+                ae(c.field_for(name, 1), ())
+                ae(c.field_for(name, 2), ('tag one', 'tag two'))
+            del cache2
+
+        # Authors
+        ae(sf('#authors', {k:cache.field_for('authors', k) for k in (1,2,3)}),
+           {1,2,3})
+
+        for name in ('authors', '#authors'):
+            f = cache.fields[name]
+            ae(len(f.table.id_map), 3)
+            af(cache.set_field(name, {3:None if name == 'authors' else 'Unknown'}))
+            ae(cache.set_field(name, {3:'Kovid Goyal & Divok Layog'}), set([3]))
+            ae(cache.set_field(name, {1:'', 2:'An, Author'}), {1,2})
+            cache2 = self.init_cache(cl)
+            for c in (cache, cache2):
+                ae(len(c.fields[name].table.id_map), 4 if name =='authors' else 3)
+                ae(c.field_for(name, 3), ('Kovid Goyal', 'Divok Layog'))
+                ae(c.field_for(name, 2), ('An, Author',))
+                ae(c.field_for(name, 1), ('Unknown',) if name=='authors' else ())
+                ae(c.field_for('author_sort', 1), 'Unknown')
+                ae(c.field_for('author_sort', 2), 'An, Author')
+                ae(c.field_for('author_sort', 3), 'Goyal, Kovid & Layog, Divok')
+            del cache2
+        ae(cache.set_field('authors', {1:'KoviD GoyaL'}), {1, 3})
+        ae(cache.field_for('author_sort', 1), 'GoyaL, KoviD')
+        ae(cache.field_for('author_sort', 3), 'GoyaL, KoviD & Layog, Divok')
+
+        # TODO: identifiers, languages
+
     # }}}
 
 def tests():
-    return unittest.TestLoader().loadTestsFromTestCase(WritingTest)
+    tl = unittest.TestLoader()
+    # return tl.loadTestsFromName('writing.WritingTest.test_many_many_basic')
+    return tl.loadTestsFromTestCase(WritingTest)
 
 def run():
     unittest.TextTestRunner(verbosity=2).run(tests())

src/calibre/db/write.py

@@ -12,8 +12,11 @@ from functools import partial
 from datetime import datetime
 
 from calibre.constants import preferred_encoding, ispy3
+from calibre.ebooks.metadata import author_to_author_sort
 from calibre.utils.date import (parse_only_date, parse_date, UNDEFINED_DATE,
                                 isoformat)
+from calibre.utils.icu import strcmp
+
 if ispy3:
     unicode = str
 
@@ -185,28 +188,42 @@ def safe_lower(x):
         return x
 
 def get_db_id(val, db, m, table, kmap, rid_map, allow_case_change,
-              case_changes, val_map, sql_val_map=lambda x:x):
+              case_changes, val_map, is_authors=False):
     ''' Get the db id for the value val. If val does not exist in the db it is
     inserted into the db. '''
     kval = kmap(val)
     item_id = rid_map.get(kval, None)
     if item_id is None:
-        db.conn.execute('INSERT INTO %s(%s) VALUES (?)'%(
-            m['table'], m['column']), (sql_val_map(val),))
+        if is_authors:
+            aus = author_to_author_sort(val)
+            db.conn.execute('INSERT INTO authors(name,sort) VALUES (?,?)',
+                            (val.replace(',', '|'), aus))
+        else:
+            db.conn.execute('INSERT INTO %s(%s) VALUES (?)'%(
+                m['table'], m['column']), (val,))
         item_id = rid_map[kval] = db.conn.last_insert_rowid()
         table.id_map[item_id] = val
         table.col_book_map[item_id] = set()
+        if is_authors:
+            table.asort_map[item_id] = aus
+            table.alink_map[item_id] = ''
     elif allow_case_change and val != table.id_map[item_id]:
         case_changes[item_id] = val
     val_map[val] = item_id
 
-def change_case(case_changes, dirtied, db, table, m, sql_val_map=lambda x:x):
+def change_case(case_changes, dirtied, db, table, m, is_authors=False):
+    if is_authors:
+        vals = ((val.replace(',', '|'), item_id) for item_id, val in
+                case_changes.iteritems())
+    else:
+        vals = ((val, item_id) for item_id, val in case_changes.iteritems())
     db.conn.executemany(
-        'UPDATE %s SET %s=? WHERE id=?'%(m['table'], m['column']),
-        ((sql_val_map(val), item_id) for item_id, val in case_changes.iteritems()))
+        'UPDATE %s SET %s=? WHERE id=?'%(m['table'], m['column']), vals)
     for item_id, val in case_changes.iteritems():
         table.id_map[item_id] = val
         dirtied.update(table.col_book_map[item_id])
+        if is_authors:
+            table.asort_map[item_id] = author_to_author_sort(val)
 
 def many_one(book_id_val_map, db, field, allow_case_change, *args):
     dirtied = set()
@@ -288,17 +305,24 @@ def many_many(book_id_val_map, db, field, allow_case_change, *args):
     # Map values to db ids, including any new values
     kmap = safe_lower if dt == 'text' else lambda x:x
     rid_map = {kmap(item):item_id for item_id, item in table.id_map.iteritems()}
-    sql_val_map = (lambda x:x.replace(',', '|')) if is_authors else lambda x:x
     val_map = {}
     case_changes = {}
     for vals in book_id_val_map.itervalues():
         for val in vals:
             get_db_id(val, db, m, table, kmap, rid_map, allow_case_change,
-                      case_changes, val_map, sql_val_map=sql_val_map)
+                      case_changes, val_map, is_authors=is_authors)
 
     if case_changes:
-        change_case(case_changes, dirtied, db, table, m,
-                    sql_val_map=sql_val_map)
+        change_case(case_changes, dirtied, db, table, m, is_authors=is_authors)
+        if is_authors:
+            for item_id, val in case_changes.iteritems():
+                for book_id in table.col_book_map[item_id]:
+                    current_sort = field.db_author_sort_for_book(book_id)
+                    new_sort = field.author_sort_for_book(book_id)
+                    if strcmp(current_sort, new_sort) == 0:
+                        # The sort strings differ only by case, update the db
+                        # sort
+                        field.author_sort_field.writer.set_books({book_id:new_sort}, db)
 
     book_id_item_id_map = {k:tuple(val_map[v] for v in vals)
                            for k, vals in book_id_val_map.iteritems()}
@@ -338,6 +362,10 @@ def many_many(book_id_val_map, db, field, allow_case_change, *args):
                             ((k,) for k in updated))
         db.conn.executemany('INSERT INTO {0}(book,{1}) VALUES(?, ?)'.format(
             table.link_table, m['link_column']), vals)
+        if is_authors:
+            aus_map = {book_id:field.author_sort_for_book(book_id) for book_id
+                       in updated}
+            field.author_sort_field.writer.set_books(aus_map, db)
 
     # Remove no longer used items
     remove = {item_id for item_id in table.id_map if not
@@ -348,6 +376,9 @@ def many_many(book_id_val_map, db, field, allow_case_change, *args):
         for item_id in remove:
             del table.id_map[item_id]
             table.col_book_map.pop(item_id, None)
+            if is_authors:
+                table.asort_map.pop(item_id, None)
+                table.alink_map.pop(item_id, None)
 
     return dirtied
 

src/calibre/devices/apple/driver.py

@@ -7,9 +7,10 @@ __docformat__ = 'restructuredtext en'
 
 import cStringIO, ctypes, datetime, os, platform, re, shutil, sys, tempfile, time
 
-from calibre.constants import __appname__, __version__, DEBUG, cache_dir
 from calibre import fit_image, confirm_config_name, strftime as _strftime
-from calibre.constants import isosx, iswindows
+from calibre.constants import (
+    __appname__, __version__, DEBUG as CALIBRE_DEBUG, isosx, iswindows,
+    cache_dir as _cache_dir)
 from calibre.devices.errors import OpenFeedback, UserFeedback
 from calibre.devices.usbms.deviceconfig import DeviceConfig
 from calibre.devices.interface import DevicePlugin
@@ -20,6 +21,7 @@ from calibre.utils.config import config_dir, dynamic, prefs
 from calibre.utils.date import now, parse_date
 from calibre.utils.zipfile import ZipFile
 
+DEBUG = CALIBRE_DEBUG
 
 def strftime(fmt='%Y/%m/%d %H:%M:%S', dt=None):
 
@@ -309,7 +311,7 @@ class ITUNES(DriverBase):
 
     @property
     def cache_dir(self):
-        return os.path.join(cache_dir(), 'itunes')
+        return os.path.join(_cache_dir(), 'itunes')
 
     @property
     def archive_path(self):
@@ -858,7 +860,6 @@ class ITUNES(DriverBase):
         Note that most of the initialization is necessarily performed in can_handle(), as
         we need to talk to iTunes to discover if there's a connected iPod
         '''
-
         if self.iTunes is None:
             raise OpenFeedback(self.ITUNES_SANDBOX_LOCKOUT_MESSAGE)
 
@@ -887,8 +888,9 @@ class ITUNES(DriverBase):
                     logger().info(" %s" % self.UNSUPPORTED_DIRECT_CONNECT_MODE_MESSAGE)
 
         # Log supported DEVICE_IDs and BCDs
-        logger().info(" BCD: %s" % ['0x%x' % x for x in sorted(self.BCD)])
-        logger().info(" PRODUCT_ID: %s" % ['0x%x' % x for x in sorted(self.PRODUCT_ID)])
+        if DEBUG:
+            logger().info(" BCD: %s" % ['0x%x' % x for x in sorted(self.BCD)])
+            logger().info(" PRODUCT_ID: %s" % ['0x%x' % x for x in sorted(self.PRODUCT_ID)])
 
         # Confirm/create thumbs archive
         if not os.path.exists(self.cache_dir):
@@ -1035,7 +1037,7 @@ class ITUNES(DriverBase):
         self.plugboard_func = pb_func
 
     def shutdown(self):
-        if DEBUG:
+        if False and DEBUG:
             logger().info("%s.shutdown()\n" % self.__class__.__name__)
 
     def sync_booklists(self, booklists, end_session=True):
@@ -1673,7 +1675,8 @@ class ITUNES(DriverBase):
                 except:
                     self.manual_sync_mode = False
 
-        logger().info("   iTunes.manual_sync_mode: %s" % self.manual_sync_mode)
+        if DEBUG:
+            logger().info("   iTunes.manual_sync_mode: %s" % self.manual_sync_mode)
 
     def _dump_booklist(self, booklist, header=None, indent=0):
         '''
@@ -2151,27 +2154,28 @@ class ITUNES(DriverBase):
             if 'iPod' in self.sources:
                 connected_device = self.sources['iPod']
                 device = self.iTunes.sources[connected_device]
-                dev_books = None
-                for pl in device.playlists():
-                    if pl.special_kind() == appscript.k.Books:
-                        if DEBUG:
-                            logger().info("  Book playlist: '%s'" % (pl.name()))
-                        dev_books = pl.file_tracks()
-                        break
-                else:
-                    logger().error("  book_playlist not found")
-
-                for book in dev_books:
-                    if book.kind() in self.Audiobooks:
-                        if DEBUG:
-                            logger().info("   ignoring '%s' of type '%s'" % (book.name(), book.kind()))
+                if device.playlists() is not None:
+                    dev_books = None
+                    for pl in device.playlists():
+                        if pl.special_kind() == appscript.k.Books:
+                            if DEBUG:
+                                logger().info("  Book playlist: '%s'" % (pl.name()))
+                            dev_books = pl.file_tracks()
+                            break
                     else:
-                        if DEBUG:
-                            logger().info("  %-40.40s %-30.30s %-40.40s [%s]" %
-                                          (book.name(), book.artist(), book.composer(), book.kind()))
-                        device_books.append(book)
-                if DEBUG:
-                    logger().info()
+                        logger().error("  book_playlist not found")
+
+                    for book in dev_books:
+                        if book.kind() in self.Audiobooks:
+                            if DEBUG:
+                                logger().info("   ignoring '%s' of type '%s'" % (book.name(), book.kind()))
+                        else:
+                            if DEBUG:
+                                logger().info("  %-40.40s %-30.30s %-40.40s [%s]" %
+                                              (book.name(), book.artist(), book.composer(), book.kind()))
+                            device_books.append(book)
+                    if DEBUG:
+                        logger().info()
 
         elif iswindows:
             import pythoncom
@@ -2181,29 +2185,29 @@ class ITUNES(DriverBase):
                     pythoncom.CoInitialize()
                     connected_device = self.sources['iPod']
                     device = self.iTunes.sources.ItemByName(connected_device)
-
-                    dev_books = None
-                    for pl in device.Playlists:
-                        if pl.Kind == self.PlaylistKind.index('User') and \
-                           pl.SpecialKind == self.PlaylistSpecialKind.index('Books'):
-                            if DEBUG:
-                                logger().info("  Books playlist: '%s'" % (pl.Name))
-                            dev_books = pl.Tracks
-                            break
-                    else:
-                        if DEBUG:
-                            logger().info("  no Books playlist found")
-
-                    for book in dev_books:
-                        if book.KindAsString in self.Audiobooks:
-                            if DEBUG:
-                                logger().info("   ignoring '%s' of type '%s'" % (book.Name, book.KindAsString))
+                    if device.Playlists is not None:
+                        dev_books = None
+                        for pl in device.Playlists:
+                            if pl.Kind == self.PlaylistKind.index('User') and \
+                               pl.SpecialKind == self.PlaylistSpecialKind.index('Books'):
+                                if DEBUG:
+                                    logger().info("  Books playlist: '%s'" % (pl.Name))
+                                dev_books = pl.Tracks
+                                break
                         else:
                             if DEBUG:
-                                logger().info("   %-40.40s %-30.30s %-40.40s [%s]" % (book.Name, book.Artist, book.Composer, book.KindAsString))
-                            device_books.append(book)
-                    if DEBUG:
-                        logger().info()
+                                logger().info("  no Books playlist found")
+
+                        for book in dev_books:
+                            if book.KindAsString in self.Audiobooks:
+                                if DEBUG:
+                                    logger().info("   ignoring '%s' of type '%s'" % (book.Name, book.KindAsString))
+                            else:
+                                if DEBUG:
+                                    logger().info("   %-40.40s %-30.30s %-40.40s [%s]" % (book.Name, book.Artist, book.Composer, book.KindAsString))
+                                device_books.append(book)
+                        if DEBUG:
+                            logger().info()
 
                 finally:
                     pythoncom.CoUninitialize()

src/calibre/devices/eb600/driver.py

@@ -49,6 +49,13 @@ class EB600(USBMS):
     EBOOK_DIR_CARD_A = ''
     SUPPORTS_SUB_DIRS = True
 
+class TOLINO(EB600):
+
+    name = 'Tolino Shine Device Interface'
+    gui_name = 'Tolino Shine'
+    description    = _('Communicate with the Tolino Shine reader.')
+    FORMATS = ['epub', 'pdf', 'txt']
+    BCD         = [0x226]
 
 class COOL_ER(EB600):
 

src/calibre/devices/kobo/books.py

@@ -62,6 +62,7 @@ class Book(Book_):
         self.kobo_collections   = []
         self.kobo_series        = None
         self.kobo_series_number = None
+        self.can_put_on_shelves = True
 
         if thumbnail_name is not None:
             self.thumbnail = ImageWrapper(thumbnail_name)
@@ -141,7 +142,7 @@ class KTCollectionsBookList(CollectionsBookList):
                     if show_debug:
                         debug_print("KTCollectionsBookList:get_collections - adding book.device_collections", book.device_collections)
                 # If the book is not in the current library, we don't want to use the metadtaa for the collections
-                elif book.application_id is None:
+                elif book.application_id is None or not book.can_put_on_shelves:
 #                    debug_print("KTCollectionsBookList:get_collections - Book not in current library")
                     continue
                 else:

src/calibre/devices/kobo/driver.py

@@ -35,7 +35,7 @@ class KOBO(USBMS):
     gui_name = 'Kobo Reader'
     description = _('Communicate with the Kobo Reader')
     author = 'Timothy Legge and David Forrester'
-    version = (2, 0, 6)
+    version = (2, 0, 7)
 
     dbversion = 0
     fwversion = 0
@@ -1207,6 +1207,7 @@ class KOBOTOUCH(KOBO):
     supported_dbversion     = 75
     min_supported_dbversion = 53
     min_dbversion_series    = 65
+    min_dbversion_archive   = 71
 
     booklist_class = KTCollectionsBookList
     book_class = Book
@@ -1384,7 +1385,7 @@ class KOBOTOUCH(KOBO):
         for idx,b in enumerate(bl):
             bl_cache[b.lpath] = idx
 
-        def update_booklist(prefix, path, title, authors, mime, date, ContentID, ContentType, ImageID, readstatus, MimeType, expired, favouritesindex, accessibility, isdownloaded, series, seriesnumber, bookshelves):
+        def update_booklist(prefix, path, title, authors, mime, date, ContentID, ContentType, ImageID, readstatus, MimeType, expired, favouritesindex, accessibility, isdownloaded, series, seriesnumber, userid, bookshelves):
             show_debug = self.is_debugging_title(title)
 #            show_debug = authors == 'L. Frank Baum'
             if show_debug:
@@ -1404,6 +1405,7 @@ class KOBOTOUCH(KOBO):
                 if lpath not in playlist_map:
                     playlist_map[lpath] = []
 
+                allow_shelves = True
                 if readstatus == 1:
                     playlist_map[lpath].append('Im_Reading')
                 elif readstatus == 2:
@@ -1415,6 +1417,7 @@ class KOBOTOUCH(KOBO):
                 # this shows an expired Collection so the user can decide to delete the book
                 if expired == 3:
                     playlist_map[lpath].append('Expired')
+                    allow_shelves = False
                 # A SHORTLIST is supported on the touch but the data field is there on most earlier models
                 if favouritesindex == 1:
                     playlist_map[lpath].append('Shortlist')
@@ -1426,21 +1429,31 @@ class KOBOTOUCH(KOBO):
                 if isdownloaded == 'false':
                     if self.dbversion < 56 and accessibility <= 1 or self.dbversion >= 56 and accessibility == -1:
                         playlist_map[lpath].append('Deleted')
+                        allow_shelves = False
                         if show_debug:
                             debug_print("KoboTouch:update_booklist - have a deleted book")
-                # Label Previews
+                    elif self.supports_kobo_archive() and (accessibility == 1 or accessibility == 2):
+                        playlist_map[lpath].append('Archived')
+                        allow_shelves = True
+
+                # Label Previews and Recommendations
                 if accessibility == 6:
-                    if isdownloaded == 'false':
+                    if userid == '':
                         playlist_map[lpath].append('Recommendation')
+                        allow_shelves = False
                     else:
                         playlist_map[lpath].append('Preview')
-                elif accessibility == 4:
+                        allow_shelves = False
+                elif accessibility == 4:        # Pre 2.x.x firmware
                     playlist_map[lpath].append('Recommendation')
+                    allow_shelves = False
 
                 kobo_collections = playlist_map[lpath][:]
 
-                if len(bookshelves) > 0:
-                    playlist_map[lpath].extend(bookshelves)
+                if allow_shelves:
+#                    debug_print('KoboTouch:update_booklist - allowing shelves - title=%s' % title)
+                    if len(bookshelves) > 0:
+                        playlist_map[lpath].extend(bookshelves)
 
                 if show_debug:
                     debug_print('KoboTouch:update_booklist - playlist_map=', playlist_map)
@@ -1481,11 +1494,12 @@ class KOBOTOUCH(KOBO):
                     bl[idx].contentID           = ContentID
                     bl[idx].kobo_series         = series
                     bl[idx].kobo_series_number  = seriesnumber
+                    bl[idx].can_put_on_shelves  = allow_shelves
 
                     if lpath in playlist_map:
                         bl[idx].device_collections  = playlist_map.get(lpath,[])
-                        bl[idx].current_shelves = bookshelves
-                        bl[idx].kobo_collections = kobo_collections
+                        bl[idx].current_shelves     = bookshelves
+                        bl[idx].kobo_collections    = kobo_collections
 
                     if show_debug:
                         debug_print('KoboTouch:update_booklist - updated bl[idx].device_collections=', bl[idx].device_collections)
@@ -1524,12 +1538,13 @@ class KOBOTOUCH(KOBO):
                         debug_print("    kobo_collections:", kobo_collections)
 
                     # print 'Update booklist'
-                    book.device_collections  = playlist_map.get(lpath,[])# if lpath in playlist_map else []
-                    book.current_shelves     = bookshelves
-                    book.kobo_collections    = kobo_collections
-                    book.contentID           = ContentID
-                    book.kobo_series         = series
-                    book.kobo_series_number  = seriesnumber
+                    book.device_collections = playlist_map.get(lpath,[])# if lpath in playlist_map else []
+                    book.current_shelves    = bookshelves
+                    book.kobo_collections   = kobo_collections
+                    book.contentID          = ContentID
+                    book.kobo_series        = series
+                    book.kobo_series_number = seriesnumber
+                    book.can_put_on_shelves = allow_shelves
 #                    debug_print('KoboTouch:update_booklist - title=', title, 'book.device_collections', book.device_collections)
 
                     if bl.add_book(book, replace_metadata=False):
@@ -1585,20 +1600,22 @@ class KOBOTOUCH(KOBO):
 
             opts = self.settings()
             if self.supports_series():
-                query= ('select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \
-                    'ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex, Accessibility, ' \
-                    'IsDownloaded, Series, SeriesNumber ' \
-                    ' from content ' \
-                    ' where BookID is Null %(previews)s %(recomendations)s and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s') % \
+                query= ("select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, " \
+                    "ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex, Accessibility, " \
+                    "IsDownloaded, Series, SeriesNumber, ___UserID " \
+                    " from content " \
+                    " where BookID is Null " \
+                    " and ((Accessibility = -1 and IsDownloaded in ('true', 1)) or (Accessibility in (1,2)) %(previews)s %(recomendations)s )" \
+                    " and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s") % \
                         dict(\
-                             expiry=' and ContentType = 6)' if opts.extra_customization[self.OPT_SHOW_EXPIRED_BOOK_RECORDS] else ')', \
-                             previews=' and Accessibility <> 6' if opts.extra_customization[self.OPT_SHOW_PREVIEWS] == False else '', \
-                             recomendations=' and IsDownloaded in (\'true\', 1)' if opts.extra_customization[self.OPT_SHOW_RECOMMENDATIONS] == False else ''\
+                             expiry=" and ContentType = 6)" if opts.extra_customization[self.OPT_SHOW_EXPIRED_BOOK_RECORDS] else ")", \
+                             previews=" or (Accessibility in (6) and ___UserID <> '')" if opts.extra_customization[self.OPT_SHOW_PREVIEWS] else "", \
+                             recomendations=" or (Accessibility in (-1, 4, 6) and ___UserId = '')" if opts.extra_customization[self.OPT_SHOW_RECOMMENDATIONS] else "" \
                              )
             elif self.dbversion >= 33:
                 query= ('select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \
                     'ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex, Accessibility, ' \
-                    'IsDownloaded, null as Series, null as SeriesNumber' \
+                    'IsDownloaded, null as Series, null as SeriesNumber, ___UserID' \
                     ' from content ' \
                     ' where BookID is Null %(previews)s %(recomendations)s and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s') % \
                         dict(\
@@ -1609,14 +1626,14 @@ class KOBOTOUCH(KOBO):
             elif self.dbversion >= 16 and self.dbversion < 33:
                 query= ('select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \
                     'ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex, Accessibility, ' \
-                    '"1" as IsDownloaded, null as Series, null as SeriesNumber' \
+                    '"1" as IsDownloaded, null as Series, null as SeriesNumber, ___UserID' \
                     ' from content where ' \
                     'BookID is Null and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s') % dict(expiry=' and ContentType = 6)' \
                     if opts.extra_customization[self.OPT_SHOW_EXPIRED_BOOK_RECORDS] else ')')
             else:
                 query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \
                     'ImageID, ReadStatus, "-1" as ___ExpirationStatus, "-1" as FavouritesIndex, "-1" as Accessibility, ' \
-                    '"1" as IsDownloaded, null as Series, null as SeriesNumber' \
+                    '"1" as IsDownloaded, null as Series, null as SeriesNumber, ___UserID' \
                     ' from content where BookID is Null'
 
             debug_print("KoboTouch:books - query=", query)
@@ -1657,10 +1674,10 @@ class KOBOTOUCH(KOBO):
                 bookshelves = get_bookshelvesforbook(connection, row[3])
 
                 if oncard != 'carda' and oncard != 'cardb' and not row[3].startswith("file:///mnt/sd/"):
-                    changed = update_booklist(self._main_prefix,   path, row[0], row[1], mime, row[2], row[3], row[5], row[6], row[7], row[4], row[8], row[9], row[10], row[11], row[12], row[13], bookshelves)
+                    changed = update_booklist(self._main_prefix,   path, row[0], row[1], mime, row[2], row[3], row[5], row[6], row[7], row[4], row[8], row[9], row[10], row[11], row[12], row[13], row[14], bookshelves)
                     # print "shortbook: " + path
                 elif oncard == 'carda' and row[3].startswith("file:///mnt/sd/"):
-                    changed = update_booklist(self._card_a_prefix, path, row[0], row[1], mime, row[2], row[3], row[5], row[6], row[7], row[4], row[8], row[9], row[10], row[11], row[12], row[13], bookshelves)
+                    changed = update_booklist(self._card_a_prefix, path, row[0], row[1], mime, row[2], row[3], row[5], row[6], row[7], row[4], row[8], row[9], row[10], row[11], row[12], row[13], row[14], bookshelves)
 
                 if changed:
                     need_sync = True
@@ -1870,10 +1887,11 @@ class KOBOTOUCH(KOBO):
 
         # Only process categories in this list
         supportedcategories = {
-            "Im_Reading":1,
-            "Read":2,
-            "Closed":3,
-            "Shortlist":4,
+            "Im_Reading":   1,
+            "Read":         2,
+            "Closed":       3,
+            "Shortlist":    4,
+            "Archived":     5,
             # "Preview":99, # Unsupported as we don't want to change it
             }
 
@@ -2496,12 +2514,16 @@ class KOBOTOUCH(KOBO):
         opts = self.settings()
         return opts.extra_customization[self.OPT_KEEP_COVER_ASPECT_RATIO]
 
+
     def supports_bookshelves(self):
         return self.dbversion >= self.min_supported_dbversion
 
     def supports_series(self):
         return self.dbversion >= self.min_dbversion_series
 
+    def supports_kobo_archive(self):
+        return self.dbversion >= self.min_dbversion_archive
+
 
     @classmethod
     def is_debugging_title(cls, title):

src/calibre/ebooks/chm/reader.py

@@ -53,6 +53,7 @@ class CHMReader(CHMFile):
         self._playorder = 0
         self._metadata = False
         self._extracted = False
+        self.re_encoded_files = set()
 
         # location of '.hhc' file, which is the CHM TOC.
         if self.topics is None:
@@ -147,8 +148,8 @@ class CHMReader(CHMFile):
                 f.write(data)
 
         self._extracted = True
-        files = [x for x in os.listdir(output_dir) if
-                os.path.isfile(os.path.join(output_dir, x))]
+        files = [y for y in os.listdir(output_dir) if
+                os.path.isfile(os.path.join(output_dir, y))]
         if self.hhc_path not in files:
             for f in files:
                 if f.lower() == self.hhc_path.lower():
@@ -249,7 +250,9 @@ class CHMReader(CHMFile):
             pass
         # do not prettify, it would reformat the <pre> tags!
         try:
-            return str(soup)
+            ans = str(soup)
+            self.re_encoded_files.add(os.path.abspath(htmlpath))
+            return ans
         except RuntimeError:
             return data
 

src/calibre/ebooks/conversion/plugins/chm_input.py

@@ -25,7 +25,6 @@ class CHMInput(InputFormatPlugin):
         self._chm_reader = rdr
         return rdr.hhc_path
 
-
     def convert(self, stream, options, file_ext, log, accelerators):
         from calibre.ebooks.chm.metadata import get_metadata_from_reader
         from calibre.customize.ui import plugin_for_input_format
@@ -63,7 +62,10 @@ class CHMInput(InputFormatPlugin):
 
             options.debug_pipeline = None
             options.input_encoding = 'utf-8'
-            htmlpath, toc = self._create_html_root(mainpath, log, encoding)
+            uenc = encoding
+            if os.path.abspath(mainpath) in self._chm_reader.re_encoded_files:
+                uenc = 'utf-8'
+            htmlpath, toc = self._create_html_root(mainpath, log, uenc)
             oeb = self._create_oebbook_html(htmlpath, tdir, options, log, metadata)
             options.debug_pipeline = odi
             if toc.count() > 1:

src/calibre/ebooks/metadata/opf2.py

@@ -941,9 +941,19 @@ class OPF(object): # {{{
                 return self.get_text(match) or None
 
         def fset(self, val):
+            removed_ids = set()
             for x in tuple(self.application_id_path(self.metadata)):
+                removed_ids.add(x.get('id', None))
                 x.getparent().remove(x)
+
+            uuid_id = None
+            for attr in self.root.attrib:
+                if attr.endswith('unique-identifier'):
+                    uuid_id = self.root.attrib[attr]
+                    break
             attrib = {'{%s}scheme'%self.NAMESPACES['opf']: 'calibre'}
+            if uuid_id and uuid_id in removed_ids:
+                attrib['id'] = uuid_id
             self.set_text(self.create_metadata_element(
                 'identifier', attrib=attrib), unicode(val))
 

src/calibre/ebooks/metadata/sources/amazon.py

@@ -452,7 +452,7 @@ class Worker(Thread): # Get details {{{
 
 
     def parse_cover(self, root):
-        imgs = root.xpath('//img[(@id="prodImage" or @id="original-main-image") and @src]')
+        imgs = root.xpath('//img[(@id="prodImage" or @id="original-main-image" or @id="main-image") and @src]')
         if imgs:
             src = imgs[0].get('src')
             if '/no-image-avail' not in src:
@@ -895,6 +895,13 @@ if __name__ == '__main__': # tests {{{
             isbn_test, title_test, authors_test, comments_test, series_test)
     com_tests = [ # {{{
 
+            ( # + in title and uses id="main-image" for cover
+             {'title':'C++ Concurrency in Action'},
+             [title_test('C++ Concurrency in Action: Practical Multithreading',
+                         exact=True),
+              ]
+             ),
+
             ( # Series
                 {'identifiers':{'amazon':'0756407117'}},
                 [title_test(

src/calibre/ebooks/metadata/sources/base.py

@@ -373,7 +373,7 @@ class Source(Plugin):
                 # Remove single quotes not followed by 's'
                 (r"'(?!s)", ''),
                 # Replace other special chars with a space
-                (r'''[:,;+!@$%^&*(){}.`~"\s\[\]/]''', ' '),
+                (r'''[:,;!@$%^&*(){}.`~"\s\[\]/]''', ' '),
             ]]
 
             for pat, repl in title_patterns:

src/calibre/ebooks/metadata/toc.py

@@ -157,12 +157,13 @@ class TOC(list):
                     toc = m[0]
                     self.read_ncx_toc(toc)
 
-    def read_ncx_toc(self, toc):
+    def read_ncx_toc(self, toc, root=None):
         self.base_path = os.path.dirname(toc)
-        raw  = xml_to_unicode(open(toc, 'rb').read(), assume_utf8=True,
-                strip_encoding_pats=True)[0]
-        root = etree.fromstring(raw, parser=etree.XMLParser(recover=True,
-            no_network=True))
+        if root is None:
+            raw  = xml_to_unicode(open(toc, 'rb').read(), assume_utf8=True,
+                    strip_encoding_pats=True)[0]
+            root = etree.fromstring(raw, parser=etree.XMLParser(recover=True,
+                no_network=True))
         xpn = {'re': 'http://exslt.org/regular-expressions'}
         XPath = functools.partial(etree.XPath, namespaces=xpn)
 

src/calibre/ebooks/oeb/polish/choose.coffee

@@ -0,0 +1,41 @@
+#!/usr/bin/env coffee
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+###
+ Copyright 2013, Kovid Goyal <kovid at kovidgoyal.net>
+ Released under the GPLv3 License
+###
+
+
+if window?.calibre_utils
+    log = window.calibre_utils.log
+
+class AnchorLocator
+
+    ###
+    # Allow the user to click on any block level element to choose it as the
+    # location for an anchor.
+    ###
+    constructor: () ->
+        if not this instanceof arguments.callee
+            throw new Error('AnchorLocator constructor called as function')
+
+    find_blocks: () =>
+        for elem in document.body.getElementsByTagName('*')
+            style = window.getComputedStyle(elem)
+            if style.display in ['block', 'flex-box', 'box']
+                elem.className += " calibre_toc_hover"
+                elem.onclick = this.onclick
+
+    onclick: (event) ->
+        # We dont want this event to trigger onclick on this element's parent
+        # block, if any.
+        event.stopPropagation()
+        frac = window.pageYOffset/document.body.scrollHeight
+        window.py_bridge.onclick(this, frac)
+        return false
+
+calibre_anchor_locator = new AnchorLocator()
+calibre_anchor_locator.find_blocks()
+
+

src/calibre/ebooks/oeb/polish/container.py

@@ -8,6 +8,7 @@ __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import os, logging, sys, hashlib, uuid, re
+from collections import defaultdict
 from io import BytesIO
 from urllib import unquote as urlunquote, quote as urlquote
 from urlparse import urlparse
@@ -71,6 +72,7 @@ class Container(object):
         self.mime_map = {}
         self.name_path_map = {}
         self.dirtied = set()
+        self.encoding_map = {}
 
         # Map of relative paths with '/' separators from root of unzipped ePub
         # to absolute paths on filesystem with os-specific separators
@@ -93,7 +95,9 @@ class Container(object):
         # Update mime map with data from the OPF
         for item in self.opf_xpath('//opf:manifest/opf:item[@href and @media-type]'):
             href = item.get('href')
-            self.mime_map[self.href_to_name(href, self.opf_name)] = item.get('media-type')
+            name = self.href_to_name(href, self.opf_name)
+            if name in self.mime_map:
+                self.mime_map[name] = item.get('media-type')
 
     def abspath_to_name(self, fullpath):
         return self.relpath(os.path.abspath(fullpath)).replace(os.sep, '/')
@@ -159,27 +163,29 @@ class Container(object):
             data = data[3:]
         if bom_enc is not None:
             try:
+                self.used_encoding = bom_enc
                 return fix_data(data.decode(bom_enc))
             except UnicodeDecodeError:
                 pass
         try:
+            self.used_encoding = 'utf-8'
             return fix_data(data.decode('utf-8'))
         except UnicodeDecodeError:
             pass
-        data, _ = xml_to_unicode(data)
+        data, self.used_encoding = xml_to_unicode(data)
         return fix_data(data)
 
     def parse_xml(self, data):
-        data = xml_to_unicode(data, strip_encoding_pats=True, assume_utf8=True,
-                             resolve_entities=True)[0].strip()
+        data, self.used_encoding = xml_to_unicode(
+            data, strip_encoding_pats=True, assume_utf8=True, resolve_entities=True)
         return etree.fromstring(data, parser=RECOVER_PARSER)
 
     def parse_xhtml(self, data, fname):
         try:
-            return parse_html(data, log=self.log,
-                    decoder=self.decode,
-                    preprocessor=self.html_preprocessor,
-                    filename=fname, non_html_file_tags={'ncx'})
+            return parse_html(
+                data, log=self.log, decoder=self.decode,
+                preprocessor=self.html_preprocessor, filename=fname,
+                non_html_file_tags={'ncx'})
         except NotHTML:
             return self.parse_xml(data)
 
@@ -209,9 +215,11 @@ class Container(object):
     def parsed(self, name):
         ans = self.parsed_cache.get(name, None)
         if ans is None:
+            self.used_encoding = None
             mime = self.mime_map.get(name, guess_type(name))
             ans = self.parse(self.name_path_map[name], mime)
             self.parsed_cache[name] = ans
+            self.encoding_map[name] = self.used_encoding
         return ans
 
     @property
@@ -230,6 +238,14 @@ class Container(object):
         return {item.get('id'):self.href_to_name(item.get('href'), self.opf_name)
             for item in self.opf_xpath('//opf:manifest/opf:item[@href and @id]')}
 
+    @property
+    def manifest_type_map(self):
+        ans = defaultdict(list)
+        for item in self.opf_xpath('//opf:manifest/opf:item[@href and @media-type]'):
+            ans[item.get('media-type').lower()].append(self.href_to_name(
+                item.get('href'), self.opf_name))
+        return {mt:tuple(v) for mt, v in ans.iteritems()}
+
     @property
     def guide_type_map(self):
         return {item.get('type', ''):self.href_to_name(item.get('href'), self.opf_name)
@@ -380,9 +396,12 @@ class Container(object):
         remove = set()
         for child in mdata:
             child.tail = '\n    '
-            if (child.get('name', '').startswith('calibre:') and
-                child.get('content', '').strip() in {'{}', ''}):
-                remove.add(child)
+            try:
+                if (child.get('name', '').startswith('calibre:') and
+                    child.get('content', '').strip() in {'{}', ''}):
+                    remove.add(child)
+            except AttributeError:
+                continue # Happens for XML comments
         for child in remove: mdata.remove(child)
         if len(mdata) > 0:
             mdata[-1].tail = '\n  '

src/calibre/ebooks/oeb/polish/cover.py

@@ -192,7 +192,7 @@ def remove_cover_image_in_page(container, page, cover_images):
         href = img.get('src')
         name = container.href_to_name(href, page)
         if name in cover_images:
-            img.getparent.remove(img)
+            img.getparent().remove(img)
         break
 
 def set_epub_cover(container, cover_path, report):

src/calibre/ebooks/oeb/polish/toc.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from urlparse import urlparse
+
+from lxml import etree
+
+from calibre.ebooks.oeb.base import XPath
+from calibre.ebooks.oeb.polish.container import guess_type
+
+ns = etree.FunctionNamespace('calibre_xpath_extensions')
+ns.prefix = 'calibre'
+ns['lower-case'] = lambda c, x: x.lower() if hasattr(x, 'lower') else x
+
+class TOC(object):
+
+    def __init__(self, title=None, dest=None, frag=None):
+        self.title, self.dest, self.frag = title, dest, frag
+        self.dest_exists = self.dest_error = None
+        if self.title: self.title = self.title.strip()
+        self.parent = None
+        self.children = []
+
+    def add(self, title, dest, frag=None):
+        c = TOC(title, dest, frag)
+        self.children.append(c)
+        c.parent = self
+        return c
+
+    def __iter__(self):
+        for c in self.children:
+            yield c
+
+    def iterdescendants(self):
+        for child in self:
+            yield child
+            for gc in child.iterdescendants():
+                yield gc
+
+def child_xpath(tag, name):
+    return tag.xpath('./*[calibre:lower-case(local-name()) = "%s"]'%name)
+
+def add_from_navpoint(container, navpoint, parent, ncx_name):
+    dest = frag = text = None
+    nl = child_xpath(navpoint, 'navlabel')
+    if nl:
+        nl = nl[0]
+        text = ''
+        for txt in child_xpath(nl, 'text'):
+            text += etree.tostring(txt, method='text',
+                    encoding=unicode, with_tail=False)
+    content = child_xpath(navpoint, 'content')
+    if content:
+        content = content[0]
+        href = content.get('src', None)
+        if href:
+            dest = container.href_to_name(href, base=ncx_name)
+            frag = urlparse(href).fragment or None
+    return parent.add(text or None, dest or None, frag or None)
+
+def process_ncx_node(container, node, toc_parent, ncx_name):
+    for navpoint in node.xpath('./*[calibre:lower-case(local-name()) = "navpoint"]'):
+        child = add_from_navpoint(container, navpoint, toc_parent, ncx_name)
+        if child is not None:
+            process_ncx_node(container, navpoint, child, ncx_name)
+
+def parse_ncx(container, ncx_name):
+    root = container.parsed(ncx_name)
+    toc_root = TOC()
+    navmaps = root.xpath('//*[calibre:lower-case(local-name()) = "navmap"]')
+    if navmaps:
+        process_ncx_node(container, navmaps[0], toc_root, ncx_name)
+    return toc_root
+
+def verify_toc_destinations(container, toc):
+    anchor_map = {}
+    anchor_xpath = XPath('//*/@id|//h:a/@name')
+    for item in toc.iterdescendants():
+        name = item.dest
+        if not name:
+            item.dest_exists = False
+            item.dest_error = _('No file named %s exists')%name
+            continue
+        try:
+            root = container.parsed(name)
+        except KeyError:
+            item.dest_exists = False
+            item.dest_error = _('No file named %s exists')%name
+            continue
+        if not hasattr(root, 'xpath'):
+            item.dest_exists = False
+            item.dest_error = _('No HTML file named %s exists')%name
+            continue
+        if not item.frag:
+            item.dest_exists = True
+            continue
+        if name not in anchor_map:
+            anchor_map[name] = frozenset(anchor_xpath(root))
+        item.dest_exists = item.frag in anchor_map[name]
+        if not item.dest_exists:
+            item.dest_error = _('The anchor %s does not exist in file %s')%(
+                item.frag, name)
+
+def get_toc(container, verify_destinations=True):
+    toc = container.opf_xpath('//opf:spine/@toc')
+    if toc:
+        toc = container.manifest_id_map.get(toc[0], None)
+    if not toc:
+        ncx = guess_type('a.ncx')
+        toc = container.manifest_type_map.get(ncx, [None])[0]
+    if not toc:
+        return None
+    ans = parse_ncx(container, toc)
+    if verify_destinations:
+        verify_toc_destinations(container, ans)
+    return ans
+
+

src/calibre/ebooks/oeb/stylizer.py

@@ -242,16 +242,31 @@ class Stylizer(object):
                     if t:
                         text += u'\n\n' + force_unicode(t, u'utf-8')
                 if text:
-                    text = XHTML_CSS_NAMESPACE + text
-                    text = oeb.css_preprocessor(text)
+                    text = oeb.css_preprocessor(text, add_namespace=True)
+                    # We handle @import rules separately
+                    parser.setFetcher(lambda x: ('utf-8', b''))
                     stylesheet = parser.parseString(text, href=cssname,
                             validate=False)
+                    parser.setFetcher(self._fetch_css_file)
                     stylesheet.namespaces['h'] = XHTML_NS
-                    stylesheets.append(stylesheet)
+                    for rule in stylesheet.cssRules:
+                        if rule.type == rule.IMPORT_RULE:
+                            ihref = item.abshref(rule.href)
+                            if rule.media.mediaText == 'amzn-mobi': continue
+                            hrefs = self.oeb.manifest.hrefs
+                            if ihref not in hrefs:
+                                self.logger.warn('Ignoring missing stylesheet in @import rule:', rule.href)
+                                continue
+                            sitem = hrefs[ihref]
+                            if sitem.media_type not in OEB_STYLES:
+                                self.logger.warn('CSS @import of non-CSS file %r' % rule.href)
+                                continue
+                            stylesheets.append(sitem.data)
                     # Make links to resources absolute, since these rules will
                     # be folded into a stylesheet at the root
                     replaceUrls(stylesheet, item.abshref,
                             ignoreImportRules=True)
+                    stylesheets.append(stylesheet)
             elif elem.tag == XHTML('link') and elem.get('href') \
                  and elem.get('rel', 'stylesheet').lower() == 'stylesheet' \
                  and elem.get('type', CSS_MIME).lower() in OEB_STYLES:
@@ -555,8 +570,8 @@ class Style(object):
             return
         css = attrib['style'].split(';')
         css = filter(None, (x.strip() for x in css))
-        css = [x.strip() for x in css]
-        css = [x for x in css if self.MS_PAT.match(x) is None]
+        css = [y.strip() for y in css]
+        css = [y for y in css if self.MS_PAT.match(y) is None]
         css = '; '.join(css)
         try:
             style = parseStyle(css, validate=False)

src/calibre/gui2/actions/__init__.py

@@ -101,6 +101,11 @@ class InterfaceAction(QObject):
     #: on calibre as a whole
     action_type = 'global'
 
+    #: If True, then this InterfaceAction will have the opportunity to interact
+    #: with drag and drop events. See the methods, :meth:`accept_enter_event`,
+    #: :meth`:accept_drag_move_event`, :meth:`drop_event` for details.
+    accepts_drops = False
+
     def __init__(self, parent, site_customization):
         QObject.__init__(self, parent)
         self.setObjectName(self.name)
@@ -108,6 +113,27 @@ class InterfaceAction(QObject):
         self.site_customization = site_customization
         self.interface_action_base_plugin = None
 
+    def accept_enter_event(self, event, mime_data):
+        ''' This method should return True iff this interface action is capable
+        of handling the drag event. Do not call accept/ignore on the event,
+        that will be taken care of by the calibre UI.'''
+        return False
+
+    def accept_drag_move_event(self, event, mime_data):
+        ''' This method should return True iff this interface action is capable
+        of handling the drag event. Do not call accept/ignore on the event,
+        that will be taken care of by the calibre UI.'''
+        return False
+
+    def drop_event(self, event, mime_data):
+        ''' This method should perform some useful action and return True
+        iff this interface action is capable of handling the drop event. Do not
+        call accept/ignore on the event, that will be taken care of by the
+        calibre UI. You should not perform blocking/long operations in this
+        function. Instead emit a signal or use QTimer.singleShot and return
+        quickly. See the builtin actions for examples.'''
+        return False
+
     def do_genesis(self):
         self.Dispatcher = partial(Dispatcher, parent=self)
         self.create_action()

src/calibre/gui2/actions/catalog.py

@@ -18,7 +18,8 @@ from calibre import sanitize_file_name_unicode
 class GenerateCatalogAction(InterfaceAction):
 
     name = 'Generate Catalog'
-    action_spec = (_('Create catalog'), 'catalog.png', 'Catalog builder', ())
+    action_spec = (_('Create catalog'), 'catalog.png',
+                   _('Create a catalog of the books in your calibre library in different formats'), ())
     dont_add_to = frozenset(['context-menu-device'])
 
     def genesis(self):

src/calibre/gui2/actions/convert.py

@@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
 import os
 from functools import partial
 
-from PyQt4.Qt import QModelIndex
+from PyQt4.Qt import QModelIndex, QTimer
 
 from calibre.gui2 import error_dialog, Dispatcher
 from calibre.gui2.tools import convert_single_ebook, convert_bulk_ebook
@@ -19,11 +19,36 @@ from calibre.customize.ui import plugin_for_input_format
 class ConvertAction(InterfaceAction):
 
     name = 'Convert Books'
-    action_spec = (_('Convert books'), 'convert.png', None, _('C'))
+    action_spec = (_('Convert books'), 'convert.png', _('Convert books between different ebook formats'), _('C'))
     dont_add_to = frozenset(['context-menu-device'])
     action_type = 'current'
     action_add_menu = True
 
+    accepts_drops = True
+
+    def accept_enter_event(self, event, mime_data):
+        if mime_data.hasFormat("application/calibre+from_library"):
+            return True
+        return False
+
+    def accept_drag_move_event(self, event, mime_data):
+        if mime_data.hasFormat("application/calibre+from_library"):
+            return True
+        return False
+
+    def drop_event(self, event, mime_data):
+        mime = 'application/calibre+from_library'
+        if mime_data.hasFormat(mime):
+            self.dropped_ids = tuple(map(int, str(mime_data.data(mime)).split()))
+            QTimer.singleShot(1, self.do_drop)
+            return True
+        return False
+
+    def do_drop(self):
+        book_ids = self.dropped_ids
+        del self.dropped_ids
+        self.do_convert(book_ids)
+
     def genesis(self):
         m = self.convert_menu = self.qaction.menu()
         cm = partial(self.create_menu_action, self.convert_menu)
@@ -112,6 +137,9 @@ class ConvertAction(InterfaceAction):
     def convert_ebook(self, checked, bulk=None):
         book_ids = self.get_books_for_conversion()
         if book_ids is None: return
+        self.do_convert(book_ids, bulk=bulk)
+
+    def do_convert(self, book_ids, bulk=None):
         previous = self.gui.library_view.currentIndex()
         rows = [x.row() for x in \
                 self.gui.library_view.selectionModel().selectedRows()]

src/calibre/gui2/actions/delete.py

@@ -83,11 +83,37 @@ class MultiDeleter(QObject): # {{{
 class DeleteAction(InterfaceAction):
 
     name = 'Remove Books'
-    action_spec = (_('Remove books'), 'trash.png', None, 'Del')
+    action_spec = (_('Remove books'), 'trash.png', _('Delete books'), 'Del')
     action_type = 'current'
     action_add_menu = True
     action_menu_clone_qaction = _('Remove selected books')
 
+    accepts_drops = True
+
+    def accept_enter_event(self, event, mime_data):
+        if mime_data.hasFormat("application/calibre+from_library"):
+            return True
+        return False
+
+    def accept_drag_move_event(self, event, mime_data):
+        if mime_data.hasFormat("application/calibre+from_library"):
+            return True
+        return False
+
+    def drop_event(self, event, mime_data):
+        mime = 'application/calibre+from_library'
+        if mime_data.hasFormat(mime):
+            self.dropped_ids = tuple(map(int, str(mime_data.data(mime)).split()))
+            QTimer.singleShot(1, self.do_drop)
+            return True
+        return False
+
+    def do_drop(self):
+        book_ids = self.dropped_ids
+        del self.dropped_ids
+        if book_ids:
+            self.do_library_delete(book_ids)
+
     def genesis(self):
         self.qaction.triggered.connect(self.delete_books)
         self.delete_menu = self.qaction.menu()
@@ -296,6 +322,44 @@ class DeleteAction(InterfaceAction):
             current_row = rmap.get(next_id, None)
         self.library_ids_deleted(ids_deleted, current_row=current_row)
 
+    def do_library_delete(self, to_delete_ids):
+        view = self.gui.current_view()
+        # Ask the user if they want to delete the book from the library or device if it is in both.
+        if self.gui.device_manager.is_device_connected:
+            on_device = False
+            on_device_ids = self._get_selected_ids()
+            for id in on_device_ids:
+                res = self.gui.book_on_device(id)
+                if res[0] or res[1] or res[2]:
+                    on_device = True
+                if on_device:
+                    break
+            if on_device:
+                loc = confirm_location('<p>' + _('Some of the selected books are on the attached device. '
+                                            '<b>Where</b> do you want the selected files deleted from?'),
+                            self.gui)
+                if not loc:
+                    return
+                elif loc == 'dev':
+                    self.remove_matching_books_from_device()
+                    return
+                elif loc == 'both':
+                    self.remove_matching_books_from_device()
+        # The following will run if the selected books are not on a connected device.
+        # The user has selected to delete from the library or the device and library.
+        if not confirm('<p>'+_('The selected books will be '
+                                '<b>permanently deleted</b> and the files '
+                                'removed from your calibre library. Are you sure?')
+                            +'</p>', 'library_delete_books', self.gui):
+            return
+        next_id = view.next_id
+        if len(to_delete_ids) < 5:
+            view.model().delete_books_by_id(to_delete_ids)
+            self.library_ids_deleted2(to_delete_ids, next_id=next_id)
+        else:
+            self.__md = MultiDeleter(self.gui, to_delete_ids,
+                    partial(self.library_ids_deleted2, next_id=next_id))
+
     def delete_books(self, *args):
         '''
         Delete selected books from device or library.
@@ -307,41 +371,7 @@ class DeleteAction(InterfaceAction):
         # Library view is visible.
         if self.gui.stack.currentIndex() == 0:
             to_delete_ids = [view.model().id(r) for r in rows]
-            # Ask the user if they want to delete the book from the library or device if it is in both.
-            if self.gui.device_manager.is_device_connected:
-                on_device = False
-                on_device_ids = self._get_selected_ids()
-                for id in on_device_ids:
-                    res = self.gui.book_on_device(id)
-                    if res[0] or res[1] or res[2]:
-                        on_device = True
-                    if on_device:
-                        break
-                if on_device:
-                    loc = confirm_location('<p>' + _('Some of the selected books are on the attached device. '
-                                               '<b>Where</b> do you want the selected files deleted from?'),
-                                self.gui)
-                    if not loc:
-                        return
-                    elif loc == 'dev':
-                        self.remove_matching_books_from_device()
-                        return
-                    elif loc == 'both':
-                        self.remove_matching_books_from_device()
-            # The following will run if the selected books are not on a connected device.
-            # The user has selected to delete from the library or the device and library.
-            if not confirm('<p>'+_('The selected books will be '
-                                   '<b>permanently deleted</b> and the files '
-                                   'removed from your calibre library. Are you sure?')
-                                +'</p>', 'library_delete_books', self.gui):
-                return
-            next_id = view.next_id
-            if len(rows) < 5:
-                view.model().delete_books_by_id(to_delete_ids)
-                self.library_ids_deleted2(to_delete_ids, next_id=next_id)
-            else:
-                self.__md = MultiDeleter(self.gui, to_delete_ids,
-                        partial(self.library_ids_deleted2, next_id=next_id))
+            self.do_library_delete(to_delete_ids)
         # Device view is visible.
         else:
             if self.gui.stack.currentIndex() == 1:

src/calibre/gui2/actions/device.py

@@ -177,7 +177,8 @@ class SendToDeviceAction(InterfaceAction):
 class ConnectShareAction(InterfaceAction):
 
     name = 'Connect Share'
-    action_spec = (_('Connect/share'), 'connect_share.png', None, None)
+    action_spec = (_('Connect/share'), 'connect_share.png',
+                   _('Share books using a web server or email. Connect to special devices, etc.'), None)
     popup_type = QToolButton.InstantPopup
 
     def genesis(self):

src/calibre/gui2/actions/edit_metadata.py

@@ -23,10 +23,38 @@ from calibre.db.errors import NoSuchFormat
 class EditMetadataAction(InterfaceAction):
 
     name = 'Edit Metadata'
-    action_spec = (_('Edit metadata'), 'edit_input.png', None, _('E'))
+    action_spec = (_('Edit metadata'), 'edit_input.png', _('Change the title/author/cover etc. of books'), _('E'))
     action_type = 'current'
     action_add_menu = True
 
+    accepts_drops = True
+
+    def accept_enter_event(self, event, mime_data):
+        if mime_data.hasFormat("application/calibre+from_library"):
+            return True
+        return False
+
+    def accept_drag_move_event(self, event, mime_data):
+        if mime_data.hasFormat("application/calibre+from_library"):
+            return True
+        return False
+
+    def drop_event(self, event, mime_data):
+        mime = 'application/calibre+from_library'
+        if mime_data.hasFormat(mime):
+            self.dropped_ids = tuple(map(int, str(mime_data.data(mime)).split()))
+            QTimer.singleShot(1, self.do_drop)
+            return True
+        return False
+
+    def do_drop(self):
+        book_ids = self.dropped_ids
+        del self.dropped_ids
+        if book_ids:
+            db = self.gui.library_view.model().db
+            rows = [db.row(i) for i in book_ids]
+            self.edit_metadata_for(rows, book_ids)
+
     def genesis(self):
         md = self.qaction.menu()
         cm = partial(self.create_menu_action, md)
@@ -186,18 +214,23 @@ class EditMetadataAction(InterfaceAction):
         Edit metadata of selected books in library.
         '''
         rows = self.gui.library_view.selectionModel().selectedRows()
-        previous = self.gui.library_view.currentIndex()
         if not rows or len(rows) == 0:
             d = error_dialog(self.gui, _('Cannot edit metadata'),
                              _('No books selected'))
             d.exec_()
             return
-
-        if bulk or (bulk is None and len(rows) > 1):
-            return self.edit_bulk_metadata(checked)
-
         row_list = [r.row() for r in rows]
+        m = self.gui.library_view.model()
+        ids = [m.id(r) for r in rows]
+        self.edit_metadata_for(row_list, ids, bulk=bulk)
+
+    def edit_metadata_for(self, rows, book_ids, bulk=None):
+        previous = self.gui.library_view.currentIndex()
+        if bulk or (bulk is None and len(rows) > 1):
+            return self.do_edit_bulk_metadata(rows, book_ids)
+
         current_row = 0
+        row_list = rows
 
         if len(row_list) == 1:
             cr = row_list[0]
@@ -242,7 +275,6 @@ class EditMetadataAction(InterfaceAction):
             db = self.gui.library_view.model().db
             view.view_format(db.row(id_), fmt)
 
-
     def edit_bulk_metadata(self, checked):
         '''
         Edit metadata of selected books in library in bulk.
@@ -256,6 +288,9 @@ class EditMetadataAction(InterfaceAction):
                     _('No books selected'))
             d.exec_()
             return
+        self.do_edit_bulk_metadata(rows, ids)
+
+    def do_edit_bulk_metadata(self, rows, book_ids):
         # Prevent the TagView from updating due to signals from the database
         self.gui.tags_view.blockSignals(True)
         changed = False
@@ -278,7 +313,7 @@ class EditMetadataAction(InterfaceAction):
             self.gui.tags_view.recount()
             if self.gui.cover_flow:
                 self.gui.cover_flow.dataChanged()
-            self.gui.library_view.select_rows(ids)
+            self.gui.library_view.select_rows(book_ids)
 
     # Merge books {{{
     def merge_books(self, safe_merge=False, merge_only_formats=False):

src/calibre/gui2/actions/fetch_news.py

@@ -16,7 +16,7 @@ from calibre.gui2.actions import InterfaceAction
 class FetchNewsAction(InterfaceAction):
 
     name = 'Fetch News'
-    action_spec = (_('Fetch news'), 'news.png', None, _('F'))
+    action_spec = (_('Fetch news'), 'news.png', _('Download news in ebook form from various websites all over the world'), _('F'))
 
     def location_selected(self, loc):
         enabled = loc == 'library'

src/calibre/gui2/actions/open.py

@@ -11,8 +11,8 @@ from calibre.gui2.actions import InterfaceAction
 class OpenFolderAction(InterfaceAction):
 
     name = 'Open Folder'
-    action_spec = (_('Open containing folder'), 'document_open.png', None,
-            _('O'))
+    action_spec = (_('Open containing folder'), 'document_open.png',
+                   _('Open the folder containing the current book\'s files'), _('O'))
     dont_add_to = frozenset(['context-menu-device'])
     action_type = 'current'
 

src/calibre/gui2/actions/plugin_updates.py

@@ -15,7 +15,7 @@ from calibre.gui2.dialogs.plugin_updater import (PluginUpdaterDialog,
 class PluginUpdaterAction(InterfaceAction):
 
     name = 'Plugin Updater'
-    action_spec = (_('Plugin Updater'), None, None, ())
+    action_spec = (_('Plugin Updater'), None, _('Update any plugins you have installed in calibre'), ())
     action_type = 'current'
 
     def genesis(self):

src/calibre/gui2/actions/polish.py

@@ -10,6 +10,7 @@ __docformat__ = 'restructuredtext en'
 import os, weakref, shutil, textwrap
 from collections import OrderedDict
 from functools import partial
+from future_builtins import map
 
 from PyQt4.Qt import (QDialog, QGridLayout, QIcon, QCheckBox, QLabel, QFrame,
                       QApplication, QDialogButtonBox, Qt, QSize, QSpacerItem,
@@ -364,9 +365,35 @@ class Report(QDialog): # {{{
 class PolishAction(InterfaceAction):
 
     name = 'Polish Books'
-    action_spec = (_('Polish books'), 'polish.png', None, _('P'))
+    action_spec = (_('Polish books'), 'polish.png',
+                   _('Apply the shine of perfection to your books'), _('P'))
     dont_add_to = frozenset(['context-menu-device'])
     action_type = 'current'
+    accepts_drops = True
+
+    def accept_enter_event(self, event, mime_data):
+        if mime_data.hasFormat("application/calibre+from_library"):
+            return True
+        return False
+
+    def accept_drag_move_event(self, event, mime_data):
+        if mime_data.hasFormat("application/calibre+from_library"):
+            return True
+        return False
+
+    def drop_event(self, event, mime_data):
+        mime = 'application/calibre+from_library'
+        if mime_data.hasFormat(mime):
+            self.dropped_ids = tuple(map(int, str(mime_data.data(mime)).split()))
+            QTimer.singleShot(1, self.do_drop)
+            return True
+        return False
+
+    def do_drop(self):
+        book_id_map = self.get_supported_books(self.dropped_ids)
+        del self.dropped_ids
+        if book_id_map:
+            self.do_polish(book_id_map)
 
     def genesis(self):
         self.qaction.triggered.connect(self.polish_books)
@@ -377,7 +404,6 @@ class PolishAction(InterfaceAction):
         self.qaction.setEnabled(enabled)
 
     def get_books_for_polishing(self):
-        from calibre.ebooks.oeb.polish.main import SUPPORTED
         rows = [r.row() for r in
                 self.gui.library_view.selectionModel().selectedRows()]
         if not rows or len(rows) == 0:
@@ -387,11 +413,16 @@ class PolishAction(InterfaceAction):
             return None
         db = self.gui.library_view.model().db
         ans = (db.id(r) for r in rows)
+        return self.get_supported_books(ans)
+
+    def get_supported_books(self, book_ids):
+        from calibre.ebooks.oeb.polish.main import SUPPORTED
+        db = self.gui.library_view.model().db
         supported = set(SUPPORTED)
         for x in SUPPORTED:
             supported.add('ORIGINAL_'+x)
         ans = [(x, set( (db.formats(x, index_is_id=True) or '').split(',') )
-               .intersection(supported)) for x in ans]
+               .intersection(supported)) for x in book_ids]
         ans = [x for x in ans if x[1]]
         if not ans:
             error_dialog(self.gui, _('Cannot polish'),
@@ -409,6 +440,9 @@ class PolishAction(InterfaceAction):
         book_id_map = self.get_books_for_polishing()
         if not book_id_map:
             return
+        self.do_polish(book_id_map)
+
+    def do_polish(self, book_id_map):
         d = Polish(self.gui.library_view.model().db, book_id_map, parent=self.gui)
         if d.exec_() == d.Accepted and d.jobs:
             show_reports = bool(d.show_reports.isChecked())

src/calibre/gui2/actions/preferences.py

@@ -17,7 +17,7 @@ from calibre.constants import DEBUG, isosx
 class PreferencesAction(InterfaceAction):
 
     name = 'Preferences'
-    action_spec = (_('Preferences'), 'config.png', None, _('Ctrl+P'))
+    action_spec = (_('Preferences'), 'config.png', _('Configure calibre'), _('Ctrl+P'))
     action_add_menu = True
     action_menu_clone_qaction = _('Change calibre behavior')
 

src/calibre/gui2/actions/restart.py

@@ -11,7 +11,7 @@ from calibre.gui2.actions import InterfaceAction
 class RestartAction(InterfaceAction):
 
     name = 'Restart'
-    action_spec = (_('Restart'), None, None, _('Ctrl+R'))
+    action_spec = (_('Restart'), None, _('Restart calibre'), _('Ctrl+R'))
 
     def genesis(self):
         self.qaction.triggered.connect(self.restart)

src/calibre/gui2/actions/save_to_disk.py

@@ -17,7 +17,8 @@ from calibre.gui2.actions import InterfaceAction
 class SaveToDiskAction(InterfaceAction):
 
     name = "Save To Disk"
-    action_spec = (_('Save to disk'), 'save.png', None, _('S'))
+    action_spec = (_('Save to disk'), 'save.png',
+                   _('Export ebook files from the calibre library'), _('S'))
     action_type = 'current'
     action_add_menu = True
     action_menu_clone_qaction = True

src/calibre/gui2/actions/show_book_details.py

@@ -13,8 +13,8 @@ from calibre.gui2 import error_dialog
 class ShowBookDetailsAction(InterfaceAction):
 
     name = 'Show Book Details'
-    action_spec = (_('Show book details'), 'dialog_information.png', None,
-            _('I'))
+    action_spec = (_('Show book details'), 'dialog_information.png',
+                   _('Show the detailed metadata for the current book in a separate window'), _('I'))
     dont_add_to = frozenset(['context-menu-device'])
     action_type = 'current'
 

src/calibre/gui2/actions/similar_books.py

@@ -14,7 +14,7 @@ from calibre.gui2.actions import InterfaceAction
 class SimilarBooksAction(InterfaceAction):
 
     name = 'Similar Books'
-    action_spec = (_('Similar books...'), None, None, None)
+    action_spec = (_('Similar books...'), None, _('Show books similar to the current book'), None)
     popup_type = QToolButton.InstantPopup
     action_type = 'current'
     action_add_menu = True

src/calibre/gui2/actions/store.py

@@ -17,7 +17,7 @@ from calibre.gui2.dialogs.confirm_delete import confirm
 class StoreAction(InterfaceAction):
 
     name = 'Store'
-    action_spec = (_('Get books'), 'store.png', None, _('G'))
+    action_spec = (_('Get books'), 'store.png', _('Search dozens of online ebook retailers for the cheapest books'), _('G'))
     action_add_menu = True
     action_menu_clone_qaction = _('Search for ebooks')
 

src/calibre/gui2/actions/tweak_epub.py

@@ -64,7 +64,7 @@ class TweakBook(QDialog):
         self.fmt_choice_box = QGroupBox(_('Choose the format to tweak:'), self)
         self._fl = fl = QHBoxLayout()
         self.fmt_choice_box.setLayout(self._fl)
-        self.fmt_choice_buttons = [QRadioButton(x, self) for x in fmts]
+        self.fmt_choice_buttons = [QRadioButton(y, self) for y in fmts]
         for x in self.fmt_choice_buttons:
             fl.addWidget(x, stretch=10 if x is self.fmt_choice_buttons[-1] else
                     0)
@@ -291,6 +291,32 @@ class TweakEpubAction(InterfaceAction):
     dont_add_to = frozenset(['context-menu-device'])
     action_type = 'current'
 
+    accepts_drops = True
+
+    def accept_enter_event(self, event, mime_data):
+        if mime_data.hasFormat("application/calibre+from_library"):
+            return True
+        return False
+
+    def accept_drag_move_event(self, event, mime_data):
+        if mime_data.hasFormat("application/calibre+from_library"):
+            return True
+        return False
+
+    def drop_event(self, event, mime_data):
+        mime = 'application/calibre+from_library'
+        if mime_data.hasFormat(mime):
+            self.dropped_ids = tuple(map(int, str(mime_data.data(mime)).split()))
+            QTimer.singleShot(1, self.do_drop)
+            return True
+        return False
+
+    def do_drop(self):
+        book_ids = self.dropped_ids
+        del self.dropped_ids
+        if book_ids:
+            self.do_tweak(book_ids[0])
+
     def genesis(self):
         self.qaction.triggered.connect(self.tweak_book)
 
@@ -301,6 +327,9 @@ class TweakEpubAction(InterfaceAction):
                     _('No book selected'), show=True)
 
         book_id = self.gui.library_view.model().id(row)
+        self.do_tweak(book_id)
+
+    def do_tweak(self, book_id):
         db = self.gui.library_view.model().db
         fmts = db.formats(book_id, index_is_id=True) or ''
         fmts = [x.lower().strip() for x in fmts.split(',')]

src/calibre/gui2/actions/view.py

@@ -34,7 +34,7 @@ class HistoryAction(QAction):
 class ViewAction(InterfaceAction):
 
     name = 'View'
-    action_spec = (_('View'), 'view.png', None, _('V'))
+    action_spec = (_('View'), 'view.png', _('Read books'), _('V'))
     action_type = 'current'
     action_add_menu = True
     action_menu_clone_qaction = True

src/calibre/gui2/bars.py

@@ -8,8 +8,8 @@ __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 
-from PyQt4.Qt import (QObject, QToolBar, Qt, QSize, QToolButton, QVBoxLayout,
-        QLabel, QWidget, QAction, QMenuBar, QMenu)
+from PyQt4.Qt import (Qt, QAction, QLabel, QMenu, QMenuBar, QObject,
+    QToolBar, QToolButton, QSize, QVBoxLayout, QWidget)
 
 from calibre.constants import isosx
 from calibre.gui2 import gprefs
@@ -116,20 +116,38 @@ class ToolBar(QToolBar): # {{{
             ch.setPopupMode(menu_mode)
         return ch
 
-    #support drag&drop from/to library from/to reader/card
+    # support drag&drop from/to library, from/to reader/card, enabled plugins
+    def check_iactions_for_drag(self, event, md, func):
+        if self.added_actions:
+            pos = event.pos()
+            for iac in self.gui.iactions.itervalues():
+                if iac.accepts_drops:
+                    aa = iac.qaction
+                    w = self.widgetForAction(aa)
+                    m = aa.menu()
+                    if (( (w is not None and w.geometry().contains(pos)) or
+                          (m is not None and m.isVisible() and m.geometry().contains(pos)) ) and
+                         getattr(iac, func)(event, md)):
+                        return True
+        return False
+
     def dragEnterEvent(self, event):
         md = event.mimeData()
         if md.hasFormat("application/calibre+from_library") or \
            md.hasFormat("application/calibre+from_device"):
             event.setDropAction(Qt.CopyAction)
             event.accept()
+            return
+
+        if self.check_iactions_for_drag(event, md, 'accept_enter_event'):
+            event.accept()
         else:
             event.ignore()
 
     def dragMoveEvent(self, event):
         allowed = False
         md = event.mimeData()
-        #Drop is only allowed in the location manager widget's different from the selected one
+        # Drop is only allowed in the location manager widget's different from the selected one
         for ac in self.location_manager.available_actions:
             w = self.widgetForAction(ac)
             if w is not None:
@@ -141,12 +159,15 @@ class ToolBar(QToolBar): # {{{
                     break
         if allowed:
             event.acceptProposedAction()
+            return
+
+        if self.check_iactions_for_drag(event, md, 'accept_drag_move_event'):
+            event.acceptProposedAction()
         else:
             event.ignore()
 
     def dropEvent(self, event):
         data = event.mimeData()
-
         mime = 'application/calibre+from_library'
         if data.hasFormat(mime):
             ids = list(map(int, str(data.data(mime)).split()))
@@ -160,6 +181,7 @@ class ToolBar(QToolBar): # {{{
                     tgt = None
                 self.gui.sync_to_device(tgt, False, send_ids=ids)
                 event.accept()
+                return
 
         mime = 'application/calibre+from_device'
         if data.hasFormat(mime):
@@ -168,6 +190,13 @@ class ToolBar(QToolBar): # {{{
                 self.gui.iactions['Add Books'].add_books_from_device(
                         self.gui.current_view(), paths=paths)
                 event.accept()
+                return
+
+        # Give added_actions an opportunity to process the drag&drop event
+        if self.check_iactions_for_drag(event, data, 'drop_event'):
+            event.accept()
+        else:
+            event.ignore()
 
 # }}}
 

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -712,7 +712,8 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         dest_mode = self.replace_mode.currentIndex()
 
         if self.destination_field_fm['is_csp']:
-            if not unicode(self.s_r_dst_ident.text()):
+            dest_ident = unicode(self.s_r_dst_ident.text())
+            if not dest_ident or (src == 'identifiers' and dest_ident == '*'):
                 raise Exception(_('You must specify a destination identifier type'))
 
         if self.destination_field_fm['is_multiple']:
@@ -816,13 +817,18 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
                 # convert the colon-separated pair strings back into a dict,
                 # which is what set_identifiers wants
                 dst_id_type = unicode(self.s_r_dst_ident.text())
-                if dst_id_type:
+                if dst_id_type and dst_id_type != '*':
                     v = ''.join(val)
                     ids = mi.get(dest)
                     ids[dst_id_type] = v
                     val = ids
                 else:
-                    val = dict([(t.split(':')) for t in val])
+                    try:
+                        val = dict([(t.split(':')) for t in val])
+                    except:
+                        raise Exception(_('Invalid identifier string. It must be a '
+                                          'comma-separated list of pairs of '
+                                          'strings separated by a colon'))
         else:
             val = self.s_r_replace_mode_separator().join(val)
             if dest == 'title' and len(val) == 0:

src/calibre/gui2/dialogs/metadata_bulk.ui

@@ -504,7 +504,7 @@ from the value in the box</string>
              <item>
               <widget class="QCheckBox" name="restore_original">
                <property name="toolTip">
-                <string>When doing a same format to same format conversion, 
+                <string>When doing a same format to same format conversion,
 for e.g., EPUB to EPUB, calibre saves the original EPUB
  as ORIGINAL_EPUB. This option tells calibre to restore
  the EPUB from ORIGINAL_EPUB. Useful if you did a bulk
@@ -1006,7 +1006,10 @@ not multiple and the destination field is multiple</string>
               </sizepolicy>
              </property>
              <property name="toolTip">
-              <string>Choose which identifier type to operate upon</string>
+              <string>&lt;p&gt;Choose which identifier type to operate upon. When the
+              source field is something other than 'identifiers' you can enter
+              a * if you want to replace the entire set of identifiers with
+              the result of the search/replace.&lt;/p&gt;</string>
              </property>
             </widget>
            </item>

src/calibre/gui2/preferences/toolbar.py

@@ -28,9 +28,10 @@ class BaseModel(QAbstractListModel):
 
     def name_to_action(self, name, gui):
         if name == 'Donate':
-            return FakeAction('Donate', _('Donate'), 'donate.png',
-                    dont_add_to=frozenset(['context-menu',
-                        'context-menu-device']))
+            return FakeAction(
+                'Donate', _('Donate'), 'donate.png', tooltip=
+                _('Donate to support the development of calibre'),
+                dont_add_to=frozenset(['context-menu', 'context-menu-device']))
         if name == 'Location Manager':
             return FakeAction('Location Manager', _('Location Manager'), 'reader.png',
                     _('Switch between library and device views'),
@@ -247,6 +248,18 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.remove_action_button.clicked.connect(self.remove_action)
         self.action_up_button.clicked.connect(partial(self.move, -1))
         self.action_down_button.clicked.connect(partial(self.move, 1))
+        self.all_actions.setMouseTracking(True)
+        self.current_actions.setMouseTracking(True)
+        self.all_actions.entered.connect(self.all_entered)
+        self.current_actions.entered.connect(self.current_entered)
+
+    def all_entered(self, index):
+        tt = self.all_actions.model().data(index, Qt.ToolTipRole).toString()
+        self.help_text.setText(tt)
+
+    def current_entered(self, index):
+       tt = self.current_actions.model().data(index, Qt.ToolTipRole).toString()
+       self.help_text.setText(tt)
 
     def what_changed(self, idx):
         key = unicode(self.what.itemData(idx).toString())
@@ -264,7 +277,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         names = self.all_actions.model().names(x)
         if names:
             not_added = self.current_actions.model().add(names)
-            ns = set([x.name for x in not_added])
+            ns = set([y.name for y in not_added])
             added = set(names) - ns
             self.all_actions.model().remove(x, added)
             if not_added:
@@ -283,7 +296,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         names = self.current_actions.model().names(x)
         if names:
             not_removed = self.current_actions.model().remove(x)
-            ns = set([x.name for x in not_removed])
+            ns = set([y.name for y in not_removed])
             removed = set(names) - ns
             self.all_actions.model().add(removed)
             if not_removed:

src/calibre/gui2/preferences/toolbar.ui

@@ -234,6 +234,13 @@
      </layout>
     </widget>
    </item>
+   <item>
+    <widget class="QLabel" name="help_text">
+     <property name="text">
+      <string/>
+     </property>
+    </widget>
+   </item>
    <item>
     <widget class="QWidget" name="spacer_widget" native="true">
      <layout class="QVBoxLayout" name="verticalLayout_5">

src/calibre/gui2/toc/__init__.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+

src/calibre/gui2/toc/location.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from base64 import b64encode
+
+from PyQt4.Qt import (QWidget, QGridLayout, QListWidget, QSize, Qt, QUrl,
+                      pyqtSlot, pyqtSignal, QVBoxLayout, QFrame, QLabel,
+                      QLineEdit)
+from PyQt4.QtWebKit import QWebView, QWebPage, QWebElement
+
+from calibre.ebooks.oeb.display.webview import load_html
+from calibre.utils.logging import default_log
+
+class Page(QWebPage): # {{{
+
+    elem_clicked = pyqtSignal(object, object, object, object)
+
+    def __init__(self):
+        self.log = default_log
+        QWebPage.__init__(self)
+        self.js = None
+        self.evaljs = self.mainFrame().evaluateJavaScript
+        self.bridge_value = None
+        nam = self.networkAccessManager()
+        nam.setNetworkAccessible(nam.NotAccessible)
+        self.setLinkDelegationPolicy(self.DelegateAllLinks)
+
+    def javaScriptConsoleMessage(self, msg, lineno, msgid):
+        self.log(u'JS:', unicode(msg))
+
+    def javaScriptAlert(self, frame, msg):
+        self.log(unicode(msg))
+
+    def shouldInterruptJavaScript(self):
+        return True
+
+    @pyqtSlot(QWebElement, float)
+    def onclick(self, elem, frac):
+        elem_id = unicode(elem.attribute('id')) or None
+        tag = unicode(elem.tagName()).lower()
+        parent = elem
+        loc = []
+        while unicode(parent.tagName()).lower() != 'body':
+            num = 0
+            sibling = parent.previousSibling()
+            while not sibling.isNull():
+                num += 1
+                sibling = sibling.previousSibling()
+            loc.insert(0, num)
+            parent = parent.parent()
+        self.elem_clicked.emit(tag, frac, elem_id, tuple(loc))
+
+    def load_js(self):
+        if self.js is None:
+            from calibre.utils.resources import compiled_coffeescript
+            self.js = compiled_coffeescript('ebooks.oeb.display.utils')
+            self.js += compiled_coffeescript('ebooks.oeb.polish.choose')
+        self.mainFrame().addToJavaScriptWindowObject("py_bridge", self)
+        self.evaljs(self.js)
+# }}}
+
+class WebView(QWebView): # {{{
+
+    elem_clicked = pyqtSignal(object, object, object, object)
+
+    def __init__(self, parent):
+        QWebView.__init__(self, parent)
+        self._page = Page()
+        self._page.elem_clicked.connect(self.elem_clicked)
+        self.setPage(self._page)
+        raw = '''
+        body { background-color: white  }
+        .calibre_toc_hover:hover { cursor: pointer !important; border-top: solid 5px green !important }
+        '''
+        raw = '::selection {background:#ffff00; color:#000;}\n'+raw
+        data = 'data:text/css;charset=utf-8;base64,'
+        data += b64encode(raw.encode('utf-8'))
+        self.settings().setUserStyleSheetUrl(QUrl(data))
+
+    def load_js(self):
+        self.page().load_js()
+
+    def sizeHint(self):
+        return QSize(1500, 300)
+# }}}
+
+class ItemEdit(QWidget):
+
+    def __init__(self, parent):
+        QWidget.__init__(self, parent)
+        self.l = l = QGridLayout()
+        self.setLayout(l)
+
+        self.la = la = QLabel('<b>'+_(
+            'Select a destination for the Table of Contents entry'))
+        l.addWidget(la, 0, 0, 1, 3)
+
+        self.dest_list = dl = QListWidget(self)
+        dl.setMinimumWidth(250)
+        dl.currentItemChanged.connect(self.current_changed)
+        l.addWidget(dl, 1, 0)
+
+        self.view = WebView(self)
+        self.view.elem_clicked.connect(self.elem_clicked)
+        l.addWidget(self.view, 1, 1)
+
+        self.f = f = QFrame()
+        f.setFrameShape(f.StyledPanel)
+        f.setMinimumWidth(250)
+        l.addWidget(f, 1, 2)
+        l = f.l = QVBoxLayout()
+        f.setLayout(l)
+
+        f.la = la = QLabel('<p>'+_(
+            'Here you can choose a destination for the Table of Contents\' entry'
+            ' to point to. First choose a file from the book in the left-most panel. The'
+            ' file will open in the central panel.<p>'
+
+            'Then choose a location inside the file. To do so, simply click on'
+            ' the place in the central panel that you want to use as the'
+            ' destination. As you move the mouse around the central panel, a'
+            ' thick green line appears, indicating the precise location'
+            ' that will be selected when you click.'))
+        la.setStyleSheet('QLabel { margin-bottom: 20px }')
+        la.setWordWrap(True)
+        l.addWidget(la)
+
+        f.la2 = la = QLabel(_('&Name of the ToC entry:'))
+        l.addWidget(la)
+        self.name = QLineEdit(self)
+        la.setBuddy(self.name)
+        l.addWidget(self.name)
+
+        self.base_msg = _('Currently selected destination:')
+        self.dest_label = la = QLabel(self.base_msg)
+        la.setTextFormat(Qt.PlainText)
+        la.setWordWrap(True)
+        la.setStyleSheet('QLabel { margin-top: 20px }')
+        l.addWidget(la)
+
+        l.addStretch()
+
+    def load(self, container):
+        self.container = container
+        spine_names = [container.abspath_to_name(p) for p in
+                       container.spine_items]
+        spine_names = [n for n in spine_names if container.has_name(n)]
+        self.dest_list.addItems(spine_names)
+
+    def current_changed(self, item):
+        name = self.current_name = unicode(item.data(Qt.DisplayRole).toString())
+        path = self.container.name_to_abspath(name)
+        # Ensure encoding map is populated
+        self.container.parsed(name)
+        encoding = self.container.encoding_map.get(name, None) or 'utf-8'
+
+        load_html(path, self.view, codec=encoding,
+                  mime_type=self.container.mime_map[name])
+        self.view.load_js()
+        self.dest_label.setText(self.base_msg + '\n' + _('File:') + ' ' +
+                                name + '\n' + _('Top of the file'))
+
+    def __call__(self, item, where):
+        self.current_item, self.current_where = item, where
+        self.current_name = None
+        self.current_frag = None
+        if item is None:
+            self.dest_list.setCurrentRow(0)
+            self.name.setText(_('(Untitled)'))
+            self.dest_label.setText(self.base_msg + '\n' + _('None'))
+
+    def elem_clicked(self, tag, frac, elem_id, loc):
+        self.current_frag = elem_id or loc
+        frac = int(round(frac * 100))
+        base = _('Location: A <%s> tag inside the file')%tag
+        if frac == 0:
+            loctext = _('Top of the file')
+        else:
+            loctext =  _('Approximately %d%% from the top')%frac
+        loctext = base + ' [%s]'%loctext
+        self.dest_label.setText(self.base_msg + '\n' +
+                    _('File:') + ' ' + self.current_name + '\n' + loctext)
+
+    @property
+    def result(self):
+        return (self.current_item, self.current_where, self.current_name,
+                self.current_frag)
+
+

src/calibre/gui2/toc/main.py

@@ -0,0 +1,331 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import sys, os
+from threading import Thread
+
+from PyQt4.Qt import (QPushButton, QFrame,
+    QDialog, QVBoxLayout, QDialogButtonBox, QSize, QStackedWidget, QWidget,
+    QLabel, Qt, pyqtSignal, QIcon, QTreeWidget, QGridLayout, QTreeWidgetItem,
+    QToolButton, QItemSelectionModel)
+
+from calibre.ebooks.oeb.polish.container import get_container
+from calibre.ebooks.oeb.polish.toc import get_toc
+from calibre.gui2 import Application
+from calibre.gui2.progress_indicator import ProgressIndicator
+from calibre.gui2.toc.location import ItemEdit
+from calibre.utils.logging import GUILog
+
+ICON_SIZE = 24
+
+class ItemView(QFrame): # {{{
+
+    add_new_item = pyqtSignal(object, object)
+
+    def __init__(self, parent):
+        QFrame.__init__(self, parent)
+        self.setFrameShape(QFrame.StyledPanel)
+        self.setMinimumWidth(250)
+        self.stack = s = QStackedWidget(self)
+        self.l = l = QVBoxLayout()
+        self.setLayout(l)
+        l.addWidget(s)
+        self.root_pane = rp = QWidget(self)
+        self.item_pane = ip = QWidget(self)
+        s.addWidget(rp)
+        s.addWidget(ip)
+
+        self.l1 = la = QLabel('<p>'+_(
+            'You can edit existing entries in the Table of Contents by clicking them'
+            ' in the panel to the left.')+'<p>'+_(
+            'Entries with a green tick next to them point to a location that has '
+            'been verified to exist. Entries with a red dot are broken and may need'
+            ' to be fixed.'))
+        la.setStyleSheet('QLabel { margin-bottom: 20px }')
+        la.setWordWrap(True)
+        l = QVBoxLayout()
+        rp.setLayout(l)
+        l.addWidget(la)
+        self.add_new_to_root_button = b = QPushButton(_('Create a &new entry'))
+        b.clicked.connect(self.add_new_to_root)
+        l.addWidget(b)
+        l.addStretch()
+
+    def add_new_to_root(self):
+        self.add_new_item.emit(None, None)
+
+    def __call__(self, item):
+        if item is None:
+            self.stack.setCurrentIndex(0)
+        else:
+            self.stack.setCurrentIndex(1)
+
+# }}}
+
+class TOCView(QWidget): # {{{
+
+    add_new_item = pyqtSignal(object, object)
+
+    def __init__(self, parent):
+        QWidget.__init__(self, parent)
+        l = self.l = QGridLayout()
+        self.setLayout(l)
+        self.tocw = t = QTreeWidget(self)
+        t.setHeaderLabel(_('Table of Contents'))
+        t.setIconSize(QSize(ICON_SIZE, ICON_SIZE))
+        t.setDragEnabled(True)
+        t.setSelectionMode(t.ExtendedSelection)
+        t.viewport().setAcceptDrops(True)
+        t.setDropIndicatorShown(True)
+        t.setDragDropMode(t.InternalMove)
+        t.setAutoScroll(True)
+        t.setAutoScrollMargin(ICON_SIZE*2)
+        t.setDefaultDropAction(Qt.MoveAction)
+        t.setAutoExpandDelay(1000)
+        t.setAnimated(True)
+        t.setMouseTracking(True)
+        l.addWidget(t, 0, 0, 5, 3)
+        self.up_button = b = QToolButton(self)
+        b.setIcon(QIcon(I('arrow-up.png')))
+        b.setIconSize(QSize(ICON_SIZE, ICON_SIZE))
+        l.addWidget(b, 0, 3)
+        b.setToolTip(_('Move current entry up'))
+        b.clicked.connect(self.move_up)
+        self.del_button = b = QToolButton(self)
+        b.setIcon(QIcon(I('trash.png')))
+        b.setIconSize(QSize(ICON_SIZE, ICON_SIZE))
+        l.addWidget(b, 2, 3)
+        b.setToolTip(_('Remove all selected entries'))
+        b.clicked.connect(self.del_items)
+        self.down_button = b = QToolButton(self)
+        b.setIcon(QIcon(I('arrow-down.png')))
+        b.setIconSize(QSize(ICON_SIZE, ICON_SIZE))
+        l.addWidget(b, 4, 3)
+        b.setToolTip(_('Move current entry down'))
+        b.clicked.connect(self.move_down)
+        self.expand_all_button = b = QPushButton(_('&Expand all'))
+        col = 5
+        l.addWidget(b, col, 0)
+        b.clicked.connect(self.tocw.expandAll)
+        self.collapse_all_button = b = QPushButton(_('&Collapse all'))
+        b.clicked.connect(self.tocw.collapseAll)
+        l.addWidget(b, col, 1)
+        self.default_msg = _('Double click on an entry to change the text')
+        self.hl = hl = QLabel(self.default_msg)
+        l.addWidget(hl, col, 2, 1, -1)
+        self.item_view = i = ItemView(self)
+        i.add_new_item.connect(self.add_new_item)
+        l.addWidget(i, 0, 4, col, 1)
+
+        l.setColumnStretch(2, 10)
+
+    def event(self, e):
+        if e.type() == e.StatusTip:
+            txt = unicode(e.tip()) or self.default_msg
+            self.hl.setText(txt)
+        return super(TOCView, self).event(e)
+
+    def item_title(self, item):
+        return unicode(item.data(0, Qt.DisplayRole).toString())
+
+    def del_items(self):
+        for item in self.tocw.selectedItems():
+            p = item.parent() or self.root
+            p.removeChild(item)
+
+    def highlight_item(self, item):
+        self.tocw.setCurrentItem(item, 0, QItemSelectionModel.ClearAndSelect)
+        self.tocw.scrollToItem(item)
+
+    def move_down(self):
+        item = self.tocw.currentItem()
+        if item is None:
+            if self.root.childCount() == 0:
+                return
+            item = self.root.child(0)
+            self.highlight_item(item)
+            return
+        parent = item.parent() or self.root
+        idx = parent.indexOfChild(item)
+        if idx == parent.childCount() - 1:
+            # At end of parent, need to become sibling of parent
+            if parent is self.root:
+                return
+            gp = parent.parent() or self.root
+            parent.removeChild(item)
+            gp.insertChild(gp.indexOfChild(parent)+1, item)
+        else:
+            sibling = parent.child(idx+1)
+            parent.removeChild(item)
+            sibling.insertChild(0, item)
+        self.highlight_item(item)
+
+    def move_up(self):
+        item = self.tocw.currentItem()
+        if item is None:
+            if self.root.childCount() == 0:
+                return
+            item = self.root.child(self.root.childCount()-1)
+            self.highlight_item(item)
+            return
+        parent = item.parent() or self.root
+        idx = parent.indexOfChild(item)
+        if idx == 0:
+            # At end of parent, need to become sibling of parent
+            if parent is self.root:
+                return
+            gp = parent.parent() or self.root
+            parent.removeChild(item)
+            gp.insertChild(gp.indexOfChild(parent), item)
+        else:
+            sibling = parent.child(idx-1)
+            parent.removeChild(item)
+            sibling.addChild(item)
+        self.highlight_item(item)
+
+    def update_status_tip(self, item):
+        c = item.data(0, Qt.UserRole).toPyObject()
+        frag = c.frag or ''
+        if frag:
+            frag = '#'+frag
+        item.setStatusTip(0, _('<b>Title</b>: {0} <b>Dest</b>: {1}{2}').format(
+            c.title, c.dest, frag))
+
+    def data_changed(self, top_left, bottom_right):
+        for r in xrange(top_left.row(), bottom_right.row()+1):
+            idx = self.tocw.model().index(r, 0, top_left.parent())
+            new_title = unicode(idx.data(Qt.DisplayRole).toString()).strip()
+            toc = idx.data(Qt.UserRole).toPyObject()
+            toc.title = new_title or _('(Untitled)')
+            item = self.tocw.itemFromIndex(idx)
+            self.update_status_tip(item)
+
+    def __call__(self, ebook):
+        self.ebook = ebook
+        self.toc = get_toc(self.ebook)
+        blank = self.blank = QIcon(I('blank.png'))
+        ok = self.ok = QIcon(I('ok.png'))
+        err = self.err = QIcon(I('dot_red.png'))
+        icon_map = {None:blank, True:ok, False:err}
+
+        def process_item(node, parent):
+            for child in node:
+                c = QTreeWidgetItem(parent)
+                c.setData(0, Qt.DisplayRole, child.title or _('(Untitled)'))
+                c.setData(0, Qt.UserRole, child)
+                c.setFlags(Qt.ItemIsDragEnabled|Qt.ItemIsEditable|Qt.ItemIsEnabled|
+                           Qt.ItemIsSelectable|Qt.ItemIsDropEnabled)
+                c.setData(0, Qt.DecorationRole, icon_map[child.dest_exists])
+                if child.dest_exists is False:
+                    c.setData(0, Qt.ToolTipRole, _(
+                        'The location this entry point to does not exist:\n%s')
+                        %child.dest_error)
+
+                self.update_status_tip(c)
+                process_item(child, c)
+
+        root = self.root = self.tocw.invisibleRootItem()
+        root.setData(0, Qt.UserRole, self.toc)
+        process_item(self.toc, root)
+        self.tocw.model().dataChanged.connect(self.data_changed)
+        self.tocw.currentItemChanged.connect(self.current_item_changed)
+
+    def current_item_changed(self, current, previous):
+        self.item_view(current)
+
+# }}}
+
+class TOCEditor(QDialog): # {{{
+
+    explode_done = pyqtSignal()
+
+    def __init__(self, pathtobook, title=None, parent=None):
+        QDialog.__init__(self, parent)
+        self.pathtobook = pathtobook
+
+        t = title or os.path.basename(pathtobook)
+        self.setWindowTitle(_('Edit the ToC in %s')%t)
+        self.setWindowIcon(QIcon(I('highlight_only_on.png')))
+
+        l = self.l = QVBoxLayout()
+        self.setLayout(l)
+
+        self.stacks = s = QStackedWidget(self)
+        l.addWidget(s)
+        self.loading_widget = lw = QWidget(self)
+        s.addWidget(lw)
+        ll = self.ll = QVBoxLayout()
+        lw.setLayout(ll)
+        self.pi = pi = ProgressIndicator()
+        pi.setDisplaySize(200)
+        pi.startAnimation()
+        ll.addWidget(pi, alignment=Qt.AlignHCenter|Qt.AlignCenter)
+        la = self.la = QLabel(_('Loading %s, please wait...')%t)
+        la.setStyleSheet('QLabel { font-size: 20pt }')
+        ll.addWidget(la, alignment=Qt.AlignHCenter|Qt.AlignTop)
+        self.toc_view = TOCView(self)
+        self.toc_view.add_new_item.connect(self.add_new_item)
+        s.addWidget(self.toc_view)
+        self.item_edit = ItemEdit(self)
+        s.addWidget(self.item_edit)
+
+        bb = self.bb = QDialogButtonBox(QDialogButtonBox.Ok|QDialogButtonBox.Cancel)
+        l.addWidget(bb)
+        bb.accepted.connect(self.accept)
+        bb.rejected.connect(self.reject)
+
+        self.explode_done.connect(self.read_toc, type=Qt.QueuedConnection)
+
+        self.resize(950, 630)
+
+    def add_new_item(self, item, where):
+        self.item_edit(item, where)
+        self.stacks.setCurrentIndex(2)
+
+    def accept(self):
+        if self.stacks.currentIndex() == 2:
+            self.toc_view.update_item(self.item_edit.result)
+            self.stacks.setCurrentIndex(1)
+        else:
+            super(TOCEditor, self).accept()
+
+    def reject(self):
+        if self.stacks.currentIndex() == 2:
+            self.stacks.setCurrentIndex(1)
+        else:
+            super(TOCEditor, self).accept()
+
+    def start(self):
+        t = Thread(target=self.explode)
+        t.daemon = True
+        self.log = GUILog()
+        t.start()
+
+    def explode(self):
+        self.ebook = get_container(self.pathtobook, log=self.log)
+        if not self.isVisible():
+            return
+        self.explode_done.emit()
+
+    def read_toc(self):
+        self.pi.stopAnimation()
+        self.toc_view(self.ebook)
+        self.item_edit.load(self.ebook)
+        self.stacks.setCurrentIndex(1)
+
+# }}}
+
+if __name__ == '__main__':
+    app = Application([], force_calibre_style=True)
+    app
+    d = TOCEditor(sys.argv[-1])
+    d.start()
+    d.exec_()
+    del d # Needed to prevent sigsegv in exit cleanup
+

src/calibre/library/server/content.py

@@ -210,7 +210,9 @@ class ContentServer(object):
         fm = self.db.format_metadata(id, format, allow_cache=False)
         if not fm:
             raise cherrypy.HTTPError(404, 'book: %d does not have format: %s'%(id, format))
-        mi = newmi = self.db.get_metadata(id, index_is_id=True)
+        update_metadata = format in {'MOBI', 'EPUB', 'AZW3'}
+        mi = newmi = self.db.get_metadata(
+            id, index_is_id=True, cover_as_data=True, get_cover=update_metadata)
 
         cherrypy.response.headers['Last-Modified'] = \
             self.last_modified(max(fm['mtime'], mi.last_modified))
@@ -236,7 +238,7 @@ class ContentServer(object):
                 newmi = mi.deepcopy_metadata()
                 newmi.template_to_attribute(mi, cpb)
 
-        if format in {'MOBI', 'EPUB', 'AZW3'}:
+        if update_metadata:
             # Write the updated file
             from calibre.ebooks.metadata.meta import set_metadata
             set_metadata(fmt, newmi, format.lower())