Sync to trunk.

Changelog.yaml

@@ -19,6 +19,73 @@
 #  new recipes:
 #    - title: 
 
+- version: 0.7.50
+  date: 2011-03-18
+
+  new features:
+    - title: "Add 'Read a random book' to the view menu"
+
+    - title: "Add option to show composite columns in the tag browser."
+
+    - title: "Add a tweak in Preferences->Tweaks to control where news that is automatically uploaded to a reader is sent."
+      tickets: [9427]
+
+    - title: "Do not also show text in composite columns when showing an icon"
+
+    - title: "Add a menu item to clear the last viewed books history in the ebook viewer"
+
+    - title: "Kobo driver: Add support for the 'Closed' collection"
+
+    - title: "Add rename/delete saved search options to Tag browser context menu"
+
+    - title: "Make searches in the tag browser a possible hierarchical field"
+  
+    - title: "Allow using empty username and password when setting up an SMTP relay"
+      tickets: [9195]
+ 
+  bug fixes:
+    - title: "Fix regression in 0.7.49 that broke deleting of news downloads older than x days."
+      tickets: [9417]
+
+    - title: "Restore the ability to remove missing formats from metadata.db to the Check Library operation"
+      tickets: [9377]
+
+    - title: "EPUB metadata: Read ISBN from Penguin epubs that dont correctly specify it"
+
+    - title: "Conversion pipeline: Handle the case where the ncx file is incorrectly given an HTML mimetype"
+
+    - title: "Make numpad navigation keys work in viewer"
+      tickets: [9428]
+
+    - title: "Fix ratings not being downloaded from Amazon"
+
+    - title: "Content server: Add workaround for Internet Explorer not supporting the ' entity."
+      tickets: [9413]
+
+    - title: "Conversion pipeline: When detecting chapters/toc links from HTML normalize spaces and increase maximum TOC title length to 1000 characters from 100 characters."
+      tickets: [9363]
+
+    - title: "Fix regression that broke Search and Replace on custom fields"
+      tickets: [9397]
+
+    - title: "Fix regression that caused currently selected row to be unfocussed int he device view when updataing metadata"
+      tickets: [9395]
+
+    - title: "Coversion S&R: Do not strip leading and trailing whitespace from the search and replace expressions in the GUI"
+
+
+  improved recipes:
+    - Sports Illustrated
+    - Draw and Cook
+
+  new recipes:
+    - title: "Evangelizo.org  and pro-linux.de"
+      author: Bobus
+
+    - title: "Office Space and Modoros"
+      author: Zsolt Botykai
+
+
 - version: 0.7.49
   date: 2011-03-11
 

resources/default_tweaks.py

@@ -355,3 +355,11 @@ draw_hidden_section_indicators = True
 # large covers
 maximum_cover_size = (1200, 1600)
 
+#: Where to send downloaded news
+# When automatically sending downloaded news to a connected device, calibre
+# will by default send it to the main memory. By changing this tweak, you can
+# control where it is sent. Valid values are "main", "carda", "cardb". Note
+# that if there isn't enough free space available on the location you choose,
+# the files will be sent to the location with the most free space.
+send_news_to_device_location = "main"
+

resources/recipes/DrawAndCook.recipe

@@ -1,8 +1,11 @@
 from calibre.web.feeds.news import BasicNewsRecipe
+import re
 
 class DrawAndCook(BasicNewsRecipe):
     title               = 'DrawAndCook'
     __author__          = 'Starson17'
+    __version__         = 'v1.10'
+    __date__            = '13 March 2011'
     description         = 'Drawings of recipes!'
     language            = 'en'
     publisher           = 'Starson17'
@@ -13,6 +16,7 @@ class DrawAndCook(BasicNewsRecipe):
     remove_javascript   = True
     remove_empty_feeds    = True
     cover_url           = 'http://farm5.static.flickr.com/4043/4471139063_4dafced67f_o.jpg'
+    INDEX = 'http://www.theydrawandcook.com'
     max_articles_per_feed = 30
 
     remove_attributes = ['style', 'font']
@@ -34,20 +38,21 @@ class DrawAndCook(BasicNewsRecipe):
         date = ''
         current_articles = []
         soup = self.index_to_soup(url)
-        recipes = soup.findAll('div', attrs={'class': 'date-outer'})
+        featured_major_slider = soup.find(name='div', attrs={'id':'featured_major_slider'})
+        recipes = featured_major_slider.findAll('li', attrs={'data-id': re.compile(r'artwork_entry_\d+', re.DOTALL)})
         for recipe in recipes:
-            title = recipe.h3.a.string
+            page_url = self.INDEX + recipe.a['href']
-            page_url = recipe.h3.a['href']
+            print 'page_url is: ', page_url
+            title = recipe.find('strong').string
+            print 'title is: ', title
             current_articles.append({'title': title, 'url': page_url, 'description':'', 'date':date})
         return current_articles
 
-
+    keep_only_tags     = [dict(name='h1', attrs={'id':'page_title'})
-    keep_only_tags     = [dict(name='h3', attrs={'class':'post-title entry-title'})
+                         ,dict(name='section', attrs={'id':'artwork'})
-                         ,dict(name='div', attrs={'class':'post-body entry-content'})
                          ]
 
-    remove_tags = [dict(name='div', attrs={'class':['separator']})
+    remove_tags = [dict(name='article', attrs={'id':['recipe_actions', 'metadata']})
-                  ,dict(name='div', attrs={'class':['post-share-buttons']})
                   ]
 
     extra_css = '''

resources/recipes/evangelizo.recipe

@@ -0,0 +1,21 @@
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Evangelizo(BasicNewsRecipe):
+    title   = 'Evangelizo.org'
+    oldest_article         = 2
+    max_articles_per_feed  = 30
+    language               = 'de'
+    __author__ = 'Bobus'
+    feeds                  = [
+        ('EvangleliumTagfuerTag', 'http://www.evangeliumtagfuertag.org/rss/evangelizo_rss-de.xml'),
+        ]
+    use_embedded_content   = True
+    preprocess_regexps     = [
+        (re.compile(r'&lt;font size="-2"&gt;([(][0-9]*[)])&lt;/font&gt;'), r'\g<1>'),
+        (re.compile(r'([\.!]\n)'), r'\g<1><br />'),
+    ]
+
+    def populate_article_metadata(self, article, soup, first):
+       article.title = re.sub(r'<font size="-2">([(][0-9]*[)])</font>', r'\g<1>', article.title)
+       return

resources/recipes/instapaper.recipe

@@ -1,23 +1,12 @@
-__license__   = 'GPL v3'
-__copyright__ = '2009-2010, Darko Miletic <darko.miletic at gmail.com>'
-'''
-www.instapaper.com
-'''
-
-import urllib
 from calibre import strftime
 from calibre.web.feeds.news import BasicNewsRecipe
 
-class Instapaper(BasicNewsRecipe):
+class AdvancedUserRecipe1299694372(BasicNewsRecipe):
-    title                 = 'Instapaper.com'
+    title          = u'Instapaper'
     __author__            = 'Darko Miletic'
-    description           = '''Personalized news feeds. Go to instapaper.com to
-                               setup up your news. Fill in your instapaper
-                               username, and leave the password field
-                               below blank.'''
     publisher             = 'Instapaper.com'
-    category              = 'news, custom'
+    category              = 'info, custom, Instapaper'
-    oldest_article        = 7
+    oldest_article = 365
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False
@@ -25,16 +14,9 @@ class Instapaper(BasicNewsRecipe):
     INDEX                 = u'http://www.instapaper.com'
     LOGIN                 = INDEX + u'/user/login'
 
-    conversion_options = {
-                          'comment'   : description
-                        , 'tags'      : category
-                        , 'publisher' : publisher
-                        }
 
-    feeds = [
+
-              (u'Unread articles' , INDEX + u'/u'      )
+    feeds          = [(u'Instapaper Unread', u'http://www.instapaper.com/u'), (u'Instapaper Starred', u'http://www.instapaper.com/starred')]
-             ,(u'Starred articles', INDEX + u'/starred')
-            ]
 
     def get_browser(self):
         br = BasicNewsRecipe.get_browser()
@@ -70,7 +52,3 @@ class Instapaper(BasicNewsRecipe):
                                     })
             totalfeeds.append((feedtitle, articles))
         return totalfeeds
-
-    def print_version(self, url):
-        return self.INDEX + '/text?u=' + urllib.quote(url)
-

resources/recipes/modoros.recipe

@@ -0,0 +1,89 @@
+import re
+from calibre.web.feeds.recipes import BasicNewsRecipe
+from calibre.constants import config_dir, CONFIG_DIR_MODE
+import os, os.path, urllib
+from hashlib import md5
+
+class ModorosBlogHu(BasicNewsRecipe):
+    __author__              = 'Zsolt Botykai'
+    title                   = u'Modoros Blog'
+    description             = u"Modoros.blog.hu"
+    oldest_article          = 10000
+    max_articles_per_feed   = 10000
+    reverse_article_order   = True
+    language                = 'hu'
+    remove_javascript       = True
+    remove_empty_feeds      = True
+    no_stylesheets          = True
+    feeds                   = [(u'Modoros Blog', u'http://modoros.blog.hu/rss')]
+    remove_javascript       = True
+    use_embedded_content    = False
+    preprocess_regexps      = [
+        (re.compile(r'<!--megosztas -->.*?</body>', re.DOTALL|re.IGNORECASE),
+         lambda match: '</body>'),
+        (re.compile(r'<p align="left"'), lambda m: '<p'),
+        (re.compile(r'<noscript.+?noscript>', re.DOTALL|re.IGNORECASE), lambda m: ''),
+        (re.compile(r'<img style="position: absolute;top:-10px.+?>', re.DOTALL|re.IGNORECASE), lambda m: ''),
+        (re.compile(r'<p>( |&nbsp;)*?</p>', re.DOTALL|re.IGNORECASE), lambda match: ''),
+    ]
+    extra_css = '''
+                    body { background-color: white; color: black }
+                '''
+
+
+    remove_tags = [
+                       dict(name='div', attrs={'id':['csucs']}) ,
+                       dict(name='img', attrs={'style':['position: absolute;top:-10px;left:-10px;']}) ,
+                       dict(name='div', attrs={'class':['tovabb-is-van', \
+                                                        'page-break', \
+                                                        'clear']}) ,
+                       dict(name='span', attrs={'class':['hozzaszolas-szamlalo']})
+                  ]
+
+    masthead_url='http://modoros.blog.hu/media/skins/modoros-neon/img/modorosblog-felirat.png'
+
+    def get_cover_url(self):
+        return 'http://modoros.blog.hu/media/skins/modoros-neon/img/modorosblog-felirat.png'
+
+    # As seen here: http://www.mobileread.com/forums/showpost.php?p=1295505&postcount=10
+    def parse_feeds(self):
+        recipe_dir = os.path.join(config_dir,'recipes')
+        hash_dir = os.path.join(recipe_dir,'recipe_storage')
+        feed_dir = os.path.join(hash_dir,self.title.encode('utf-8').replace('/',':'))
+        if not os.path.isdir(feed_dir):
+            os.makedirs(feed_dir,mode=CONFIG_DIR_MODE)
+
+        feeds = BasicNewsRecipe.parse_feeds(self)
+
+        for feed in feeds:
+            feed_hash = urllib.quote(feed.title.encode('utf-8'),safe='')
+            feed_fn = os.path.join(feed_dir,feed_hash)
+
+            past_items = set()
+            if os.path.exists(feed_fn):
+               with file(feed_fn) as f:
+                   for h in f:
+                       past_items.add(h.strip())
+
+            cur_items = set()
+            for article in feed.articles[:]:
+                item_hash = md5()
+                if article.content: item_hash.update(article.content.encode('utf-8'))
+                if article.summary: item_hash.update(article.summary.encode('utf-8'))
+                item_hash = item_hash.hexdigest()
+                if article.url:
+                    item_hash = article.url + ':' + item_hash
+                cur_items.add(item_hash)
+                if item_hash in past_items:
+                    feed.articles.remove(article)
+            with file(feed_fn,'w') as f:
+                for h in cur_items:
+                    f.write(h+'\n')
+
+        remove = [f for f in feeds if len(f) == 0 and
+                self.remove_empty_feeds]
+        for f in remove:
+            feeds.remove(f)
+
+        return feeds
+

resources/recipes/office_space.recipe

@@ -0,0 +1,109 @@
+import re
+from calibre.web.feeds.recipes import BasicNewsRecipe
+from calibre.constants import config_dir, CONFIG_DIR_MODE
+import os, os.path, urllib
+from hashlib import md5
+
+class OfficeSpaceBlogHu(BasicNewsRecipe):
+    __author__              = 'Zsolt Botykai'
+    title                   = u'Office Space Blog'
+    description             = u"officespace.blog.hu"
+    oldest_article          = 10000
+    max_articles_per_feed   = 10000
+    reverse_article_order   = True
+    language                = 'hu'
+    remove_javascript       = True
+    remove_empty_feeds      = True
+    no_stylesheets          = True
+    feeds                   = [(u'Office Space Blog', u'http://officespace.blog.hu/rss')]
+    remove_javascript       = True
+    use_embedded_content    = False
+    title          = u'Irodai patkényok'
+    feeds          = [(u'Office Space', u'http://officespace.blog.hu/rss')]
+
+    masthead_url='http://m.blog.hu/of/officespace/ipfejlec7.jpg'
+
+    keep_only_tags = [
+                    dict(name='div', attrs={'id':['mainWrapper']})
+                    ]
+
+    #   1.: I like justified lines more
+    #   2.: remove empty paragraphs
+    #   3.: drop header and sidebar
+    #   4.: drop comments counter
+    #   5.: drop everything after article-tags
+    # 6-8.: drop audit images
+
+    preprocess_regexps = [
+        (re.compile(r'<p align="left"'), lambda m: '<p'),
+        (re.compile(r'<p>( |&nbsp;)*?</p>', re.DOTALL|re.IGNORECASE), lambda match: ''),
+        (re.compile(r'<body[^>]+>.*?<div id="mainIn"', re.DOTALL|re.IGNORECASE), lambda match: '<body><div id="mainIn"'),
+        (re.compile(r'<h3 class="comments">.*?</h3>', re.DOTALL|re.IGNORECASE), lambda match: ''),
+        (re.compile(r'<div class="related">.*?</body>', re.DOTALL|re.IGNORECASE), lambda match: '<body>'),
+        (re.compile(r'<img style="position: absolute;" src="[^"]+pixel\?uc.*?>', re.DOTALL|re.IGNORECASE), lambda match: ''),
+        (re.compile(r'<noscript.+?noscript>', re.DOTALL|re.IGNORECASE), lambda m: ''),
+        (re.compile(r'<img style="position: absolute;top:-10px.+?>', re.DOTALL|re.IGNORECASE), lambda m: ''),
+                         ]
+    extra_css = '''
+                    body { background-color: white; color: black }
+                '''
+
+    def get_cover_url(self):
+        return 'http://m.blog.hu/of/officespace/ipfejlec7.jpg'
+
+    def preprocess_html(self, soup):
+        for tagz in soup.findAll('h3', attrs={'class':'tags'}):
+            for taglink in tagz.findAll('a'):
+                if taglink.string is not None:
+                   tstr = taglink.string + ','
+                   taglink.replaceWith(tstr)
+
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+
+        return soup
+
+    # As seen here: http://www.mobileread.com/forums/showpost.php?p=1295505&postcount=10
+    def parse_feeds(self):
+        recipe_dir = os.path.join(config_dir,'recipes')
+        hash_dir = os.path.join(recipe_dir,'recipe_storage')
+        feed_dir = os.path.join(hash_dir,self.title.encode('utf-8').replace('/',':'))
+        if not os.path.isdir(feed_dir):
+            os.makedirs(feed_dir,mode=CONFIG_DIR_MODE)
+
+        feeds = BasicNewsRecipe.parse_feeds(self)
+
+        for feed in feeds:
+            feed_hash = urllib.quote(feed.title.encode('utf-8'),safe='')
+            feed_fn = os.path.join(feed_dir,feed_hash)
+
+            past_items = set()
+            if os.path.exists(feed_fn):
+               with file(feed_fn) as f:
+                   for h in f:
+                       past_items.add(h.strip())
+
+            cur_items = set()
+            for article in feed.articles[:]:
+                item_hash = md5()
+                if article.content: item_hash.update(article.content.encode('utf-8'))
+                if article.summary: item_hash.update(article.summary.encode('utf-8'))
+                item_hash = item_hash.hexdigest()
+                if article.url:
+                    item_hash = article.url + ':' + item_hash
+                cur_items.add(item_hash)
+                if item_hash in past_items:
+                    feed.articles.remove(article)
+            with file(feed_fn,'w') as f:
+                for h in cur_items:
+                    f.write(h+'\n')
+
+        remove = [f for f in feeds if len(f) == 0 and
+                self.remove_empty_feeds]
+        for f in remove:
+            feeds.remove(f)
+
+        return feeds
+

resources/recipes/pro_linux_de.recipe

@@ -0,0 +1,15 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1295265555(BasicNewsRecipe):
+    title          = u'Pro-Linux.de'
+    language = 'de'
+    __author__ = 'Bobus'
+    oldest_article = 3
+    max_articles_per_feed = 100
+
+    feeds          = [(u'Pro-Linux', u'http://www.pro-linux.de/backend/pro-linux.rdf')]
+
+    def print_version(self, url):
+        return url.replace('/news/1/', '/news/1/print/').replace('/artikel/2/', '/artikel/2/print/')
+
+    remove_tags_after = [dict(name='div', attrs={'class':'print_links'})]

resources/recipes/rbc_ru.recipe

@@ -1,24 +1,25 @@
+# -*- coding: utf-8 -*-
+
 from calibre.web.feeds.news import BasicNewsRecipe
 
-class AdvancedUserRecipe1286819935(BasicNewsRecipe):
+class RBC_ru(BasicNewsRecipe):
     title = u'RBC.ru'
     __author__ = 'A. Chewi'
-    oldest_article = 7
+    description = u'Российское информационное агентство «РосБизнесКонсалтинг» (РБК) - ленты новостей политики, экономики и финансов, аналитические материалы, комментарии и прогнозы, тематические статьи'
-    max_articles_per_feed = 100
+    needs_subscription = False
+    cover_url = 'http://pics.rbc.ru/img/fp_v4/skin/img/logo.gif'
+    cover_margins = (80, 160, '#ffffff')
+    oldest_article = 10
+    max_articles_per_feed = 50
+    summary_length = 200
+    remove_empty_feeds = True
     no_stylesheets = True
+    remove_javascript = True
     use_embedded_content  = False
     conversion_options = {'linearize_tables' : True}
-    remove_attributes = ['style']
     language = 'ru'
     timefmt = ' [%a, %d %b, %Y]'
 
-    keep_only_tags = [dict(name='h2', attrs={}),
-	dict(name='div', attrs={'class': 'box _ga1_on_'}),
-	dict(name='h1', attrs={'class': 'news_section'}),
-	dict(name='div', attrs={'class': 'news_body dotted_border_bottom'}),
-	dict(name='table', attrs={'class': 'newsBody'}),
-	dict(name='h2', attrs={'class': 'black'})]
-
     feeds = [(u'Главные новости', u'http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/rbc.ru/mainnews.rss'),
 	(u'Политика', u'http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/rbc.ru/politics.rss'),
 	(u'Экономика', u'http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/rbc.ru/economics.rss'),
@@ -26,6 +27,12 @@ class AdvancedUserRecipe1286819935(BasicNewsRecipe):
 	(u'Происшествия', u'http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/rbc.ru/incidents.rss'),
 	(u'Финансовые новости Quote.rbc.ru', u'http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/quote.ru/mainnews.rss')]
 
+    keep_only_tags = [dict(name='h2', attrs={}),
+	dict(name='div', attrs={'class': 'box _ga1_on_'}),
+	dict(name='h1', attrs={'class': 'news_section'}),
+	dict(name='div', attrs={'class': 'news_body dotted_border_bottom'}),
+	dict(name='table', attrs={'class': 'newsBody'}),
+	dict(name='h2', attrs={'class': 'black'})]
 
     remove_tags = [dict(name='div', attrs={'class': "video-frame"}),
 	dict(name='div', attrs={'class': "photo-container videoContainer videoSWFLinks videoPreviewSlideContainer notes"}),

resources/recipes/sportsillustrated.recipe

@@ -1,6 +1,7 @@
 from calibre.web.feeds.recipes import BasicNewsRecipe
 #from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from urllib import quote
+import re
 
 class SportsIllustratedRecipe(BasicNewsRecipe) :
     __author__  = 'kwetal'
@@ -16,64 +17,52 @@ class SportsIllustratedRecipe(BasicNewsRecipe) :
     use_embedded_content   = False
 
     INDEX = 'http://sportsillustrated.cnn.com/'
+    INDEX2 = 'http://sportsillustrated.cnn.com/vault/cover/home/index.htm'
 
     def parse_index(self):
         answer = []
-        soup = self.index_to_soup(self.INDEX)
+        soup = self.index_to_soup(self.INDEX2)
-        # Find the link to the current issue on the front page. SI Cover
-        cover = soup.find('img', attrs = {'alt' : 'Read All Articles', 'style' : 'vertical-align:bottom;'})
-        if cover:
-            currentIssue = cover.parent['href']
-            if currentIssue:
-                # Open the index of current issue
 
-                index = self.index_to_soup(currentIssue)
+        #Loop through all of the "latest" covers until we find one that actually has articles
-                self.log('\tLooking for current issue in: ' + currentIssue)
+        for item in soup.findAll('div', attrs={'id': re.compile("ecomthumb_latest_*")}):
-                # Now let us see if they updated their frontpage
+            regex = re.compile('ecomthumb_latest_(\d*)')
-                nav = index.find('div', attrs = {'class': 'siv_trav_top'})
+            result = regex.search(str(item))
-                if nav:
+            current_issue_number = str(result.group(1))
-                    img = nav.find('img', attrs = {'src': 'http://i.cdn.turner.com/sivault/.element/img/1.0/btn_next_v2.jpg'})
+            current_issue_link = 'http://sportsillustrated.cnn.com/vault/cover/toc/' + current_issue_number + '/index.htm'
-                    if img:
+            self.log('Checking this link for a TOC:  ', current_issue_link)
-                        parent = img.parent
-                        if parent.name == 'a':
-                            # They didn't update their frontpage; Load the next issue from here
-                            href = self.INDEX + parent['href']
-                            index = self.index_to_soup(href)
-                            self.log('\tLooking for current issue in: ' + href)
 
+            index = self.index_to_soup(current_issue_link)
+            if index:
                 if index.find('div', 'siv_noArticleMessage'):
-                    nav = index.find('div', attrs = {'class': 'siv_trav_top'})
+                    self.log('No TOC for this one.  Skipping...')
-                    if nav:
+                else:
-                    # Their frontpage points to an issue without any articles; Use the previous issue
+                    self.log('Found a TOC...  Using this link')
-                        img = nav.find('img', attrs = {'src': 'http://i.cdn.turner.com/sivault/.element/img/1.0/btn_previous_v2.jpg'})
+                    break
-                        if img:
-                            parent = img.parent
-                            if parent.name == 'a':
-                                href = self.INDEX + parent['href']
-                                index = self.index_to_soup(href)
-                                self.log('\tLooking for current issue in: ' + href)
 
+        # Find all articles.
+        list = index.find('div', attrs = {'class' : 'siv_artList'})
+        if list:
+            self.log ('found siv_artList')
+            articles = []
+            # Get all the artcles ready for calibre.
+            counter = 0
+            for headline in list.findAll('div', attrs = {'class' : 'headline'}):
+                counter = counter + 1
+                title = self.tag_to_string(headline.a) + '\n' + self.tag_to_string(headline.findNextSibling('div', attrs = {'class' : 'info'}))
+                url = self.INDEX + headline.a['href']
+                description = self.tag_to_string(headline.findNextSibling('a').div)
+                article = {'title' : title, 'date' : u'', 'url'  : url, 'description' : description}
+                articles.append(article)
+                #if counter > 5:
+                    #break
 
-                # Find all articles.
+            # See if we can find a meaningfull title
-                list = index.find('div', attrs = {'class' : 'siv_artList'})
+            feedTitle = 'Current Issue'
-                if list:
+            hasTitle = index.find('div', attrs = {'class' : 'siv_imageText_head'})
-                    articles = []
+            if hasTitle :
-                    # Get all the artcles ready for calibre.
+                feedTitle = self.tag_to_string(hasTitle.h1)
-                    for headline in list.findAll('div', attrs = {'class' : 'headline'}):
-                        title = self.tag_to_string(headline.a) + '\n' + self.tag_to_string(headline.findNextSibling('div', attrs = {'class' : 'info'}))
-                        url = self.INDEX + headline.a['href']
-                        description = self.tag_to_string(headline.findNextSibling('a').div)
-                        article = {'title' : title, 'date' : u'', 'url'  : url, 'description' : description}
 
-                        articles.append(article)
+            answer.append([feedTitle, articles])
-
-                    # See if we can find a meaningfull title
-                    feedTitle = 'Current Issue'
-                    hasTitle = index.find('div', attrs = {'class' : 'siv_imageText_head'})
-                    if hasTitle :
-                        feedTitle = self.tag_to_string(hasTitle.h1)
-
-                    answer.append([feedTitle, articles])
 
         return answer
 
@@ -82,7 +71,6 @@ class SportsIllustratedRecipe(BasicNewsRecipe) :
         # This is the url and the parameters that work to get the print version.
         printUrl = 'http://si.printthis.clickability.com/pt/printThis?clickMap=printThis'
         printUrl += '&fb=Y&partnerID=2356&url=' + quote(url)
-
         return printUrl
 
         # However the original javascript also uses the following parameters, but they can be left out:
@@ -116,4 +104,3 @@ class SportsIllustratedRecipe(BasicNewsRecipe) :
 
         return homeMadeSoup
         '''
-

src/calibre/__init__.py

@@ -3,7 +3,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import uuid, sys, os, re, logging, time, \
+import uuid, sys, os, re, logging, time, random, \
        __builtin__, warnings, multiprocessing
 from contextlib import closing
 from urllib import getproxies
@@ -273,6 +273,17 @@ def get_parsed_proxy(typ='http', debug=True):
 USER_AGENT = 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.13) Gecko/20101210 Gentoo Firefox/3.6.13'
 USER_AGENT_MOBILE = 'Mozilla/5.0 (Windows; U; Windows CE 5.1; rv:1.8.1a3) Gecko/20060610 Minimo/0.016'
 
+def random_user_agent():
+    choices = [
+        'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.2.11) Gecko/20101012 Firefox/3.6.11'
+        'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)'
+        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)'
+        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1)'
+        'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/0.2.153.1 Safari/525.19'
+        'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.2.11) Gecko/20101012 Firefox/3.6.11'
+    ]
+    return choices[random.randint(0, len(choices)-1)]
+
 def browser(honor_time=True, max_time=2, mobile_browser=False, user_agent=None):
     '''
     Create a mechanize browser for web scraping. The browser handles cookies,

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.7.49'
+__version__   = '0.7.50'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 
 import re

src/calibre/customize/builtins.py

@@ -1036,8 +1036,9 @@ plugins += [LookAndFeel, Behavior, Columns, Toolbar, Search, InputOptions,
 
 # New metadata download plugins {{{
 from calibre.ebooks.metadata.sources.google import GoogleBooks
+from calibre.ebooks.metadata.sources.amazon import Amazon
 
-plugins += [GoogleBooks]
+plugins += [GoogleBooks, Amazon]
 
 # }}}
 

src/calibre/devices/__init__.py

@@ -47,7 +47,7 @@ def get_connected_device():
 
     for d in connected_devices:
         try:
-            d.open()
+            d.open(None)
         except:
             continue
         else:
@@ -121,7 +121,7 @@ def debug(ioreg_to_tmp=False, buf=None):
             out('Trying to open', dev.name, '...', end=' ')
             try:
                 dev.reset(detected_device=det)
-                dev.open()
+                dev.open(None)
                 out('OK')
             except:
                 import traceback

src/calibre/devices/android/driver.py

@@ -48,6 +48,7 @@ class ANDROID(USBMS):
             0x04e8 : { 0x681d : [0x0222, 0x0223, 0x0224, 0x0400],
                        0x681c : [0x0222, 0x0224, 0x0400],
                        0x6640 : [0x0100],
+                       0x6877 : [0x0400],
                      },
 
             # Acer
@@ -97,7 +98,7 @@ class ANDROID(USBMS):
             'SCH-I500_CARD', 'SPH-D700_CARD', 'MB810', 'GT-P1000', 'DESIRE',
             'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H',
             'IDEOS_TABLET', 'MYTOUCH_4G', 'UMS_COMPOSITE', 'SCH-I800_CARD',
-            '7', 'A956']
+            '7', 'A956', 'A955']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT', '7']

src/calibre/devices/kobo/driver.py

@@ -115,6 +115,8 @@ class KOBO(USBMS):
                     playlist_map[lpath]= "Im_Reading"
                 elif readstatus == 2:
                     playlist_map[lpath]= "Read"
+                elif readstatus == 3:
+                    playlist_map[lpath]= "Closed"
 
                 path = self.normalize_path(path)
                 # print "Normalized FileName: " + path
@@ -599,11 +601,47 @@ class KOBO(USBMS):
                         try:
                             cursor.execute('update content set ReadStatus=2,FirstTimeReading=\'true\' where BookID is Null and ContentID = ?', t)
                         except:
-                            debug_print('Database Exception:  Unable set book as Rinished')
+                            debug_print('Database Exception:  Unable set book as Finished')
                             raise
                         else:
                             connection.commit()
 #                            debug_print('Database: Commit set ReadStatus as Finished')
+                if category == 'Closed':
+                    # Reset Im_Reading list in the database
+                    if oncard == 'carda':
+                        query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ReadStatus = 3 and ContentID like \'file:///mnt/sd/%\''
+                    elif oncard != 'carda' and oncard != 'cardb':
+                        query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ReadStatus = 3 and ContentID not like \'file:///mnt/sd/%\''
+
+                    try:
+                        cursor.execute (query)
+                    except:
+                        debug_print('Database Exception:  Unable to reset Closed list')
+                        raise
+                    else:
+#                       debug_print('Commit: Reset Closed list')
+                        connection.commit()
+
+                    for book in books:
+#                       debug_print('Title:', book.title, 'lpath:', book.path)
+                        book.device_collections = ['Closed']
+
+                        extension =  os.path.splitext(book.path)[1]
+                        ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path)
+
+                        ContentID = self.contentid_from_path(book.path, ContentType)
+#                        datelastread = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime())
+
+                        t = (ContentID,)
+
+                        try:
+                            cursor.execute('update content set ReadStatus=3,FirstTimeReading=\'true\' where BookID is Null and ContentID = ?', t)
+                        except:
+                            debug_print('Database Exception:  Unable set book as Closed')
+                            raise
+                        else:
+                            connection.commit()
+#                            debug_print('Database: Commit set ReadStatus as Closed')
         else: # No collections
             # Since no collections exist the ReadStatus needs to be reset to 0 (Unread)
             print "Reseting ReadStatus to 0"

src/calibre/devices/prs505/driver.py

@@ -221,7 +221,8 @@ class PRS505(USBMS):
                                           os.path.splitext(os.path.basename(p))[0],
                                           book, p)
                     except:
-                        debug_print('FAILED to upload cover', p)
+                        debug_print('FAILED to upload cover',
+                                prefix, book.lpath)
         else:
             debug_print('PRS505: NOT uploading covers in sync_booklists')
 

src/calibre/devices/usbms/driver.py

@@ -10,7 +10,7 @@ driver. It is intended to be subclassed with the relevant parts implemented
 for a particular device.
 '''
 
-import os, re, time, json, uuid
+import os, re, time, json, uuid, functools
 from itertools import cycle
 
 from calibre.constants import numeric_version
@@ -372,15 +372,21 @@ class USBMS(CLI, Device):
 
     @classmethod
     def build_template_regexp(cls):
-        def replfunc(match):
+        def replfunc(match, seen=None):
-            if match.group(1) in ['title', 'series', 'series_index', 'isbn']:
+            v = match.group(1)
-                return '(?P<' + match.group(1) + '>.+?)'
+            if v in ['title', 'series', 'series_index', 'isbn']:
-            elif match.group(1) in ['authors', 'author_sort']:
+                if v not in seen:
-                return '(?P<author>.+?)'
+                    seen |= set([v])
-            else:
+                    return '(?P<' + v + '>.+?)'
-                return '(.+?)'
+            elif v in ['authors', 'author_sort']:
+                if v not in seen:
+                    seen |= set([v])
+                    return '(?P<author>.+?)'
+            return '(.+?)'
+        s = set()
+        f = functools.partial(replfunc, seen=s)
         template = cls.save_template().rpartition('/')[2]
-        return re.compile(re.sub('{([^}]*)}', replfunc, template) + '([_\d]*$)')
+        return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)')
 
     @classmethod
     def path_to_unicode(cls, path):

src/calibre/ebooks/metadata/amazon.py

@@ -154,17 +154,16 @@ def get_metadata(br, asin, mi):
         return False
     if root.xpath('//*[@id="errorMessage"]'):
         return False
-    ratings = root.xpath('//form[@id="handleBuy"]/descendant::*[@class="asinReviewsSummary"]')
+
+    ratings = root.xpath('//div[@class="jumpBar"]/descendant::span[@class="asinReviewsSummary"]')
+    pat = re.compile(r'([0-9.]+) out of (\d+) stars')
     if ratings:
-        pat = re.compile(r'([0-9.]+) out of (\d+) stars')
+        for elem in ratings[0].xpath('descendant::*[@title]'):
-        r = ratings[0]
+            t = elem.get('title').strip()
-        for elem in r.xpath('descendant::*[@title]'):
-            t = elem.get('title')
             m = pat.match(t)
             if m is not None:
                 try:
                     mi.rating = float(m.group(1))/float(m.group(2)) * 5
-                    break
                 except:
                     pass
 
@@ -216,6 +215,7 @@ def main(args=sys.argv):
             print 'Failed to downlaod social metadata for', title
             return 1
         #print '\n\n', time.time() - st, '\n\n'
+        print mi
         print '\n'
 
     return 0

src/calibre/ebooks/metadata/book/base.py

@@ -127,6 +127,8 @@ class Metadata(object):
             field, val = self._clean_identifier(field, val)
             _data['identifiers'].update({field: val})
         elif field == 'identifiers':
+            if not val:
+                val = copy.copy(NULL_VALUES.get('identifiers', None))
             self.set_identifiers(val)
         elif field in STANDARD_METADATA_FIELDS:
             if val is None:
@@ -169,10 +171,13 @@ class Metadata(object):
                     pass
             return default
 
-    def get_extra(self, field):
+    def get_extra(self, field, default=None):
         _data = object.__getattribute__(self, '_data')
         if field in _data['user_metadata'].iterkeys():
-            return _data['user_metadata'][field]['#extra#']
+            try:
+                return _data['user_metadata'][field]['#extra#']
+            except:
+                return default
         raise AttributeError(
                 'Metadata object has no attribute named: '+ repr(field))
 
@@ -222,6 +227,11 @@ class Metadata(object):
         if val:
             identifiers[typ] = val
 
+    def has_identifier(self, typ):
+        identifiers = object.__getattribute__(self,
+            '_data')['identifiers']
+        return typ in identifiers
+
     # field-oriented interface. Intended to be the same as in LibraryDatabase
 
     def standard_field_keys(self):
@@ -628,10 +638,6 @@ class Metadata(object):
             fmt('Publisher', self.publisher)
         if getattr(self, 'book_producer', False):
             fmt('Book Producer', self.book_producer)
-        if self.comments:
-            fmt('Comments', self.comments)
-        if self.isbn:
-            fmt('ISBN', self.isbn)
         if self.tags:
             fmt('Tags', u', '.join([unicode(t) for t in self.tags]))
         if self.series:
@@ -646,6 +652,12 @@ class Metadata(object):
             fmt('Published', isoformat(self.pubdate))
         if self.rights is not None:
             fmt('Rights', unicode(self.rights))
+        if self.identifiers:
+            fmt('Identifiers', u', '.join(['%s:%s'%(k, v) for k, v in
+                self.identifiers.iteritems()]))
+        if self.comments:
+            fmt('Comments', self.comments)
+
         for key in self.custom_field_keys():
             val = self.get(key, None)
             if val:

src/calibre/ebooks/metadata/opf2.py

@@ -16,7 +16,7 @@ from lxml import etree
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.constants import __appname__, __version__, filesystem_encoding
 from calibre.ebooks.metadata.toc import TOC
-from calibre.ebooks.metadata import string_to_authors, MetaInformation
+from calibre.ebooks.metadata import string_to_authors, MetaInformation, check_isbn
 from calibre.ebooks.metadata.book.base import Metadata
 from calibre.utils.date import parse_date, isoformat
 from calibre.utils.localization import get_lang
@@ -863,6 +863,7 @@ class OPF(object): # {{{
         for x in self.XPath(
             'descendant::*[local-name() = "identifier" and text()]')(
                     self.metadata):
+            found_scheme = False
             for attr, val in x.attrib.iteritems():
                 if attr.endswith('scheme'):
                     typ = icu_lower(val)
@@ -870,7 +871,15 @@ class OPF(object): # {{{
                             method='text').strip()
                     if val and typ not in ('calibre', 'uuid'):
                         identifiers[typ] = val
+                    found_scheme = True
                     break
+            if not found_scheme:
+                val = etree.tostring(x, with_tail=False, encoding=unicode,
+                            method='text').strip()
+                if val.lower().startswith('urn:isbn:'):
+                    val = check_isbn(val.split(':')[-1])
+                    if val is not None:
+                        identifiers['isbn'] = val
         return identifiers
 
     @dynamic_property
@@ -1251,6 +1260,7 @@ def metadata_to_opf(mi, as_string=True):
     from lxml import etree
     import textwrap
     from calibre.ebooks.oeb.base import OPF, DC
+    from calibre.utils.cleantext import clean_ascii_chars
 
     if not mi.application_id:
         mi.application_id = str(uuid.uuid4())
@@ -1306,7 +1316,7 @@ def metadata_to_opf(mi, as_string=True):
     if hasattr(mi, 'category') and mi.category:
         factory(DC('type'), mi.category)
     if mi.comments:
-        factory(DC('description'), mi.comments)
+        factory(DC('description'), clean_ascii_chars(mi.comments))
     if mi.publisher:
         factory(DC('publisher'), mi.publisher)
     for key, val in mi.get_identifiers().iteritems():

src/calibre/ebooks/metadata/sources/amazon.py

@@ -7,16 +7,470 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
+import socket, time, re
+from urllib import urlencode
+from threading import Thread
 
+from lxml.html import soupparser, tostring
+
+from calibre import as_unicode
+from calibre.ebooks.metadata import check_isbn
 from calibre.ebooks.metadata.sources.base import Source
+from calibre.utils.cleantext import clean_ascii_chars
+from calibre.ebooks.chardet import xml_to_unicode
+from calibre.ebooks.metadata.book.base import Metadata
+from calibre.library.comments import sanitize_comments_html
+from calibre.utils.date import parse_date
+
+class Worker(Thread): # {{{
+
+    '''
+    Get book details from amazons book page in a separate thread
+    '''
+
+    def __init__(self, url, result_queue, browser, log, timeout=20):
+        Thread.__init__(self)
+        self.daemon = True
+        self.url, self.result_queue = url, result_queue
+        self.log, self.timeout = log, timeout
+        self.browser = browser.clone_browser()
+        self.cover_url = self.amazon_id = self.isbn = None
+
+    def run(self):
+        try:
+            self.get_details()
+        except:
+            self.log.error('get_details failed for url: %r'%self.url)
+
+    def get_details(self):
+        try:
+            raw = self.browser.open_novisit(self.url, timeout=self.timeout).read().strip()
+        except Exception, e:
+            if callable(getattr(e, 'getcode', None)) and \
+                    e.getcode() == 404:
+                self.log.error('URL malformed: %r'%self.url)
+                return
+            attr = getattr(e, 'args', [None])
+            attr = attr if attr else [None]
+            if isinstance(attr[0], socket.timeout):
+                msg = 'Amazon timed out. Try again later.'
+                self.log.error(msg)
+            else:
+                msg = 'Failed to make details query: %r'%self.url
+                self.log.exception(msg)
+            return
+
+        raw = xml_to_unicode(raw, strip_encoding_pats=True,
+                resolve_entities=True)[0]
+        # open('/t/t.html', 'wb').write(raw)
+
+        if '<title>404 - ' in raw:
+            self.log.error('URL malformed: %r'%self.url)
+            return
+
+        try:
+            root = soupparser.fromstring(clean_ascii_chars(raw))
+        except:
+            msg = 'Failed to parse amazon details page: %r'%self.url
+            self.log.exception(msg)
+            return
+
+        errmsg = root.xpath('//*[@id="errorMessage"]')
+        if errmsg:
+            msg = 'Failed to parse amazon details page: %r'%self.url
+            msg += tostring(errmsg, method='text', encoding=unicode).strip()
+            self.log.error(msg)
+            return
+
+        self.parse_details(root)
+
+    def parse_details(self, root):
+        try:
+            asin = self.parse_asin(root)
+        except:
+            self.log.exception('Error parsing asin for url: %r'%self.url)
+            asin = None
+
+        try:
+            title = self.parse_title(root)
+        except:
+            self.log.exception('Error parsing title for url: %r'%self.url)
+            title = None
+
+        try:
+            authors = self.parse_authors(root)
+        except:
+            self.log.exception('Error parsing authors for url: %r'%self.url)
+            authors = []
+
+
+        if not title or not authors or not asin:
+            self.log.error('Could not find title/authors/asin for %r'%self.url)
+            self.log.error('ASIN: %r Title: %r Authors: %r'%(asin, title,
+                authors))
+            return
+
+        mi = Metadata(title, authors)
+        mi.set_identifier('amazon', asin)
+        self.amazon_id = asin
+
+        try:
+            mi.rating = self.parse_rating(root)
+        except:
+            self.log.exception('Error parsing ratings for url: %r'%self.url)
+
+        try:
+            mi.comments = self.parse_comments(root)
+        except:
+            self.log.exception('Error parsing comments for url: %r'%self.url)
+
+        try:
+            self.cover_url = self.parse_cover(root)
+        except:
+            self.log.exception('Error parsing cover for url: %r'%self.url)
+        mi.has_cover = bool(self.cover_url)
+
+        pd = root.xpath('//h2[text()="Product Details"]/../div[@class="content"]')
+        if pd:
+            pd = pd[0]
+
+            try:
+                isbn = self.parse_isbn(pd)
+                if isbn:
+                    self.isbn = mi.isbn = isbn
+            except:
+                self.log.exception('Error parsing ISBN for url: %r'%self.url)
+
+            try:
+                mi.publisher = self.parse_publisher(pd)
+            except:
+                self.log.exception('Error parsing publisher for url: %r'%self.url)
+
+            try:
+                mi.pubdate = self.parse_pubdate(pd)
+            except:
+                self.log.exception('Error parsing publish date for url: %r'%self.url)
+
+            try:
+                lang = self.parse_language(pd)
+                if lang:
+                    mi.language = lang
+            except:
+                self.log.exception('Error parsing language for url: %r'%self.url)
+
+        else:
+            self.log.warning('Failed to find product description for url: %r'%self.url)
+
+        self.result_queue.put(mi)
+
+    def parse_asin(self, root):
+        link = root.xpath('//link[@rel="canonical" and @href]')
+        for l in link:
+            return l.get('href').rpartition('/')[-1]
+
+    def parse_title(self, root):
+        tdiv = root.xpath('//h1[@class="parseasinTitle"]')[0]
+        actual_title = tdiv.xpath('descendant::*[@id="btAsinTitle"]')
+        if actual_title:
+            title = tostring(actual_title[0], encoding=unicode,
+                    method='text').strip()
+        else:
+            title = tostring(tdiv, encoding=unicode, method='text').strip()
+        return re.sub(r'[(\[].*[)\]]', '', title).strip()
+
+    def parse_authors(self, root):
+        x = '//h1[@class="parseasinTitle"]/following-sibling::span/*[(name()="a" and @href) or (name()="span" and @class="contributorNameTrigger")]'
+        aname = root.xpath(x)
+        for x in aname:
+            x.tail = ''
+        authors = [tostring(x, encoding=unicode, method='text').strip() for x
+                in aname]
+        return authors
+
+    def parse_rating(self, root):
+        ratings = root.xpath('//div[@class="jumpBar"]/descendant::span[@class="asinReviewsSummary"]')
+        pat = re.compile(r'([0-9.]+) out of (\d+) stars')
+        if ratings:
+            for elem in ratings[0].xpath('descendant::*[@title]'):
+                t = elem.get('title').strip()
+                m = pat.match(t)
+                if m is not None:
+                    return float(m.group(1))/float(m.group(2)) * 5
+
+    def parse_comments(self, root):
+        desc = root.xpath('//div[@id="productDescription"]/*[@class="content"]')
+        if desc:
+            desc = desc[0]
+            for c in desc.xpath('descendant::*[@class="seeAll" or'
+                    ' @class="emptyClear" or @href]'):
+                c.getparent().remove(c)
+            desc = tostring(desc, method='html', encoding=unicode).strip()
+            # remove all attributes from tags
+            desc = re.sub(r'<([a-zA-Z0-9]+)\s[^>]+>', r'<\1>', desc)
+            # Collapse whitespace
+            #desc = re.sub('\n+', '\n', desc)
+            #desc = re.sub(' +', ' ', desc)
+            # Remove the notice about text referring to out of print editions
+            desc = re.sub(r'(?s)<em>--This text ref.*?</em>', '', desc)
+            # Remove comments
+            desc = re.sub(r'(?s)<!--.*?-->', '', desc)
+            return sanitize_comments_html(desc)
+
+    def parse_cover(self, root):
+        imgs = root.xpath('//img[@id="prodImage" and @src]')
+        if imgs:
+            src = imgs[0].get('src')
+            if '/no-image-avail' not in src:
+                parts = src.split('/')
+                if len(parts) > 3:
+                    bn = parts[-1]
+                    sparts = bn.split('_')
+                    if len(sparts) > 2:
+                        bn = sparts[0] + sparts[-1]
+                        return ('/'.join(parts[:-1]))+'/'+bn
+
+    def parse_isbn(self, pd):
+        for x in reversed(pd.xpath(
+            'descendant::*[starts-with(text(), "ISBN")]')):
+            if x.tail:
+                ans = check_isbn(x.tail.strip())
+                if ans:
+                    return ans
+
+    def parse_publisher(self, pd):
+        for x in reversed(pd.xpath(
+            'descendant::*[starts-with(text(), "Publisher:")]')):
+            if x.tail:
+                ans = x.tail.partition(';')[0]
+                return ans.partition('(')[0].strip()
+
+    def parse_pubdate(self, pd):
+        for x in reversed(pd.xpath(
+            'descendant::*[starts-with(text(), "Publisher:")]')):
+            if x.tail:
+                ans = x.tail
+                date = ans.partition('(')[-1].replace(')', '').strip()
+                return parse_date(date, assume_utc=True)
+
+    def parse_language(self, pd):
+        for x in reversed(pd.xpath(
+            'descendant::*[starts-with(text(), "Language:")]')):
+            if x.tail:
+                ans = x.tail.strip()
+                if ans == 'English':
+                    return 'en'
+# }}}
 
 class Amazon(Source):
 
     name = 'Amazon'
     description = _('Downloads metadata from Amazon')
 
-    capabilities = frozenset(['identify', 'cover'])
+    capabilities = frozenset(['identify'])
-    touched_fields = frozenset(['title', 'authors', 'isbn', 'pubdate',
+    touched_fields = frozenset(['title', 'authors', 'identifier:amazon',
-        'comments', 'cover_data'])
+        'identifier:isbn', 'rating', 'comments', 'publisher', 'pubdate'])
+
+    AMAZON_DOMAINS = {
+            'com': _('US'),
+            'fr' : _('France'),
+            'de' : _('Germany'),
+    }
+
+    def create_query(self, log, title=None, authors=None, identifiers={}): # {{{
+        domain = self.prefs.get('domain', 'com')
+
+        # See the amazon detailed search page to get all options
+        q = {   'search-alias' : 'aps',
+                'unfiltered' : '1',
+            }
+
+        if domain == 'com':
+            q['sort'] = 'relevanceexprank'
+        else:
+            q['sort'] = 'relevancerank'
+
+        asin = identifiers.get('amazon', None)
+        isbn = check_isbn(identifiers.get('isbn', None))
+
+        if asin is not None:
+            q['field-keywords'] = asin
+        elif isbn is not None:
+            q['field-isbn'] = isbn
+        else:
+            # Only return book results
+            q['search-alias'] = 'stripbooks'
+            if title:
+                title_tokens = list(self.get_title_tokens(title))
+                if title_tokens:
+                    q['field-title'] = ' '.join(title_tokens)
+            if authors:
+                author_tokens = self.get_author_tokens(authors,
+                        only_first_author=True)
+                if author_tokens:
+                    q['field-author'] = ' '.join(author_tokens)
+
+        if not ('field-keywords' in q or 'field-isbn' in q or
+                ('field-title' in q and 'field-author' in q)):
+            # Insufficient metadata to make an identify query
+            return None
+
+        utf8q = dict([(x.encode('utf-8'), y.encode('utf-8')) for x, y in
+            q.iteritems()])
+        url = 'http://www.amazon.%s/s/?'%domain + urlencode(utf8q)
+        return url
+
+    # }}}
+
+    def identify(self, log, result_queue, abort, title=None, authors=None, # {{{
+            identifiers={}, timeout=30):
+        '''
+        Note this method will retry without identifiers automatically if no
+        match is found with identifiers.
+        '''
+        query = self.create_query(log, title=title, authors=authors,
+                identifiers=identifiers)
+        if query is None:
+            log.error('Insufficient metadata to construct query')
+            return
+        br = self.browser
+        try:
+            raw = br.open_novisit(query, timeout=timeout).read().strip()
+        except Exception, e:
+            if callable(getattr(e, 'getcode', None)) and \
+                    e.getcode() == 404:
+                log.error('Query malformed: %r'%query)
+                return
+            attr = getattr(e, 'args', [None])
+            attr = attr if attr else [None]
+            if isinstance(attr[0], socket.timeout):
+                msg = _('Amazon timed out. Try again later.')
+                log.error(msg)
+            else:
+                msg = 'Failed to make identify query: %r'%query
+                log.exception(msg)
+            return as_unicode(msg)
+
+
+        raw = xml_to_unicode(raw, strip_encoding_pats=True,
+                resolve_entities=True)[0]
+
+        matches = []
+        found = '<title>404 - ' not in raw
+
+        if found:
+            try:
+                root = soupparser.fromstring(clean_ascii_chars(raw))
+            except:
+                msg = 'Failed to parse amazon page for query: %r'%query
+                log.exception(msg)
+                return msg
+
+                errmsg = root.xpath('//*[@id="errorMessage"]')
+                if errmsg:
+                    msg = tostring(errmsg, method='text', encoding=unicode).strip()
+                    log.error(msg)
+                    # The error is almost always a not found error
+                    found = False
+
+        if found:
+            for div in root.xpath(r'//div[starts-with(@id, "result_")]'):
+                for a in div.xpath(r'descendant::a[@class="title" and @href]'):
+                    title = tostring(a, method='text', encoding=unicode).lower()
+                    if 'bulk pack' not in title:
+                        matches.append(a.get('href'))
+                    break
+
+        # Keep only the top 5 matches as the matches are sorted by relevance by
+        # Amazon so lower matches are not likely to be very relevant
+        matches = matches[:5]
+
+        if abort.is_set():
+            return
+
+        if not matches:
+            if identifiers and title and authors:
+                log('No matches found with identifiers, retrying using only'
+                        ' title and authors')
+                return self.identify(log, result_queue, abort, title=title,
+                        authors=authors, timeout=timeout)
+            log.error('No matches found with query: %r'%query)
+            return
+
+        workers = [Worker(url, result_queue, br, log) for url in matches]
+
+        for w in workers:
+            w.start()
+            # Don't send all requests at the same time
+            time.sleep(0.1)
+
+        while not abort.is_set():
+            a_worker_is_alive = False
+            for w in workers:
+                w.join(0.2)
+                if abort.is_set():
+                    break
+                if w.is_alive():
+                    a_worker_is_alive = True
+            if not a_worker_is_alive:
+                break
+
+        for w in workers:
+            if w.amazon_id:
+                if w.isbn:
+                    self.cache_isbn_to_identifier(w.isbn, w.amazon_id)
+                if w.cover_url:
+                    self.cache_identifier_to_cover_url(w.amazon_id,
+                            w.cover_url)
+
+        return None
+    # }}}
+
+
+if __name__ == '__main__':
+    # To run these test use: calibre-debug -e
+    # src/calibre/ebooks/metadata/sources/amazon.py
+    from calibre.ebooks.metadata.sources.test import (test_identify_plugin,
+            title_test, authors_test)
+    test_identify_plugin(Amazon.name,
+        [
+
+            ( # An e-book ISBN not on Amazon, one of the authors is
+              # unknown to Amazon, so no popup wrapper
+                {'identifiers':{'isbn': '9780307459671'},
+                    'title':'Invisible Gorilla', 'authors':['Christopher Chabris']},
+                [title_test('The Invisible Gorilla: And Other Ways Our Intuitions Deceive Us',
+                    exact=True), authors_test(['Christopher Chabris', 'Daniel Simons'])]
+
+            ),
+
+            (  # This isbn not on amazon
+                {'identifiers':{'isbn': '8324616489'}, 'title':'Learning Python',
+                    'authors':['Lutz']},
+                [title_test('Learning Python: Powerful Object-Oriented Programming',
+                    exact=True), authors_test(['Mark Lutz'])
+                 ]
+
+            ),
+
+            ( # Sophisticated comment formatting
+                {'identifiers':{'isbn': '9781416580829'}},
+                [title_test('Angels & Demons - Movie Tie-In: A Novel',
+                    exact=True), authors_test(['Dan Brown'])]
+            ),
+
+            ( # No specific problems
+                {'identifiers':{'isbn': '0743273567'}},
+                [title_test('The great gatsby', exact=True),
+                    authors_test(['F. Scott Fitzgerald'])]
+            ),
+
+            (  # A newer book
+                {'identifiers':{'isbn': '9780316044981'}},
+                [title_test('The Heroes', exact=True),
+                    authors_test(['Joe Abercrombie'])]
+
+            ),
+
+        ])
 
 

src/calibre/ebooks/metadata/sources/base.py

@@ -9,8 +9,12 @@ __docformat__ = 'restructuredtext en'
 
 import re, threading
 
+from calibre import browser, random_user_agent
 from calibre.customize import Plugin
 from calibre.utils.logging import ThreadSafeLog, FileStream
+from calibre.utils.config import JSONConfig
+
+msprefs = JSONConfig('metadata_sources.json')
 
 def create_log(ostream=None):
     log = ThreadSafeLog(level=ThreadSafeLog.DEBUG)
@@ -24,8 +28,6 @@ class Source(Plugin):
 
     supported_platforms = ['windows', 'osx', 'linux']
 
-    result_of_identify_is_complete = True
-
     capabilities = frozenset()
 
     touched_fields = frozenset()
@@ -33,7 +35,29 @@ class Source(Plugin):
     def __init__(self, *args, **kwargs):
         Plugin.__init__(self, *args, **kwargs)
         self._isbn_to_identifier_cache = {}
+        self._identifier_to_cover_url_cache = {}
         self.cache_lock = threading.RLock()
+        self._config_obj = None
+        self._browser = None
+
+    # Configuration {{{
+
+    @property
+    def prefs(self):
+        if self._config_obj is None:
+            self._config_obj = JSONConfig('metadata_sources/%s.json'%self.name)
+        return self._config_obj
+    # }}}
+
+    # Browser {{{
+
+    @property
+    def browser(self):
+        if self._browser is None:
+            self._browser = browser(user_agent=random_user_agent())
+        return self._browser
+
+    # }}}
 
     # Utility functions {{{
 
@@ -45,6 +69,14 @@ class Source(Plugin):
         with self.cache_lock:
             return self._isbn_to_identifier_cache.get(isbn, None)
 
+    def cache_identifier_to_cover_url(self, id_, url):
+        with self.cache_lock:
+            self._identifier_to_cover_url_cache[id_] = url
+
+    def cached_identifier_to_cover_url(self, id_):
+        with self.cache_lock:
+            return self._identifier_to_cover_url_cache.get(id_, None)
+
     def get_author_tokens(self, authors, only_first_author=True):
         '''
         Take a list of authors and return a list of tokens useful for an
@@ -105,6 +137,16 @@ class Source(Plugin):
         '''
         Identify a book by its title/author/isbn/etc.
 
+        If identifiers(s) are specified and no match is found and this metadata
+        source does not store all related identifiers (for example, all ISBNs
+        of a book), this method should retry with just the title and author
+        (assuming they were specified).
+
+        If this metadata source also provides covers, the URL to the cover
+        should be cached so that a subsequent call to the get covers API with
+        the same ISBN/special identifier does not need to get the cover URL
+        again. Use the caching API for this.
+
         :param log: A log object, use it to output debugging information/errors
         :param result_queue: A result Queue, results should be put into it.
                             Each result is a Metadata object

src/calibre/ebooks/metadata/sources/google.py

@@ -19,7 +19,7 @@ from calibre.ebooks.metadata.book.base import Metadata
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.utils.date import parse_date, utcnow
 from calibre.utils.cleantext import clean_ascii_chars
-from calibre import browser, as_unicode
+from calibre import as_unicode
 
 NAMESPACES = {
               'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/',
@@ -42,7 +42,7 @@ subject        = XPath('descendant::dc:subject')
 description    = XPath('descendant::dc:description')
 language       = XPath('descendant::dc:language')
 
-def get_details(browser, url, timeout):
+def get_details(browser, url, timeout): # {{{
     try:
         raw = browser.open_novisit(url, timeout=timeout).read()
     except Exception as e:
@@ -50,12 +50,13 @@ def get_details(browser, url, timeout):
         if gc() != 403:
             raise
         # Google is throttling us, wait a little
-        time.sleep(1)
+        time.sleep(2)
         raw = browser.open_novisit(url, timeout=timeout).read()
 
     return raw
+# }}}
 
-def to_metadata(browser, log, entry_, timeout):
+def to_metadata(browser, log, entry_, timeout): # {{{
 
     def get_text(extra, x):
         try:
@@ -94,12 +95,6 @@ def to_metadata(browser, log, entry_, timeout):
     #mi.language = get_text(extra, language)
     mi.publisher = get_text(extra, publisher)
 
-    # Author sort
-    for x in creator(extra):
-        for key, val in x.attrib.items():
-            if key.endswith('file-as') and val and val.strip():
-                mi.author_sort = val
-                break
     # ISBN
     isbns = []
     for x in identifier(extra):
@@ -137,7 +132,7 @@ def to_metadata(browser, log, entry_, timeout):
 
 
     return mi
-
+# }}}
 
 class GoogleBooks(Source):
 
@@ -145,12 +140,13 @@ class GoogleBooks(Source):
     description = _('Downloads metadata from Google Books')
 
     capabilities = frozenset(['identify'])
-    touched_fields = frozenset(['title', 'authors', 'isbn', 'tags', 'pubdate',
+    touched_fields = frozenset(['title', 'authors', 'tags', 'pubdate',
-        'comments', 'publisher', 'author_sort']) # language currently disabled
+        'comments', 'publisher', 'identifier:isbn',
+        'identifier:google']) # language currently disabled
 
-    def create_query(self, log, title=None, authors=None, identifiers={}):
+    def create_query(self, log, title=None, authors=None, identifiers={}): # {{{
         BASE_URL = 'http://books.google.com/books/feeds/volumes?'
-        isbn = identifiers.get('isbn', None)
+        isbn = check_isbn(identifiers.get('isbn', None))
         q = ''
         if isbn is not None:
             q += 'isbn:'+isbn
@@ -176,6 +172,7 @@ class GoogleBooks(Source):
             'start-index':1,
             'min-viewability':'none',
             })
+    # }}}
 
     def cover_url_from_identifiers(self, identifiers):
         goog = identifiers.get('google', None)
@@ -198,7 +195,7 @@ class GoogleBooks(Source):
                 ans = to_metadata(br, log, i, timeout)
                 if isinstance(ans, Metadata):
                     result_queue.put(ans)
-                    for isbn in ans.all_isbns:
+                    for isbn in getattr(ans, 'all_isbns', []):
                         self.cache_isbn_to_identifier(isbn,
                                 ans.identifiers['google'])
             except:
@@ -208,11 +205,11 @@ class GoogleBooks(Source):
             if abort.is_set():
                 break
 
-    def identify(self, log, result_queue, abort, title=None, authors=None,
+    def identify(self, log, result_queue, abort, title=None, authors=None, # {{{
-            identifiers={}, timeout=5):
+            identifiers={}, timeout=30):
         query = self.create_query(log, title=title, authors=authors,
                 identifiers=identifiers)
-        br = browser()
+        br = self.browser
         try:
             raw = br.open_novisit(query, timeout=timeout).read()
         except Exception, e:
@@ -228,22 +225,31 @@ class GoogleBooks(Source):
             log.exception('Failed to parse identify results')
             return as_unicode(e)
 
+        if not entries and identifiers and title and authors and \
+                not abort.is_set():
+            return self.identify(log, result_queue, abort, title=title,
+                    authors=authors, timeout=timeout)
+
         # There is no point running these queries in threads as google
         # throttles requests returning 403 Forbidden errors
         self.get_all_details(br, log, entries, abort, result_queue, timeout)
 
         return None
+    # }}}
 
 if __name__ == '__main__':
     # To run these test use: calibre-debug -e src/calibre/ebooks/metadata/sources/google.py
     from calibre.ebooks.metadata.sources.test import (test_identify_plugin,
-            title_test)
+            title_test, authors_test)
     test_identify_plugin(GoogleBooks.name,
         [
 
+
             (
-                {'identifiers':{'isbn': '0743273567'}},
+                {'identifiers':{'isbn': '0743273567'}, 'title':'Great Gatsby',
-                [title_test('The great gatsby', exact=True)]
+                    'authors':['Fitzgerald']},
+                [title_test('The great gatsby', exact=True),
+                    authors_test(['Francis Scott Fitzgerald'])]
             ),
 
             #(

src/calibre/ebooks/metadata/sources/test.py

@@ -37,6 +37,25 @@ def title_test(title, exact=False):
 
     return test
 
+def authors_test(authors):
+    authors = set([x.lower() for x in authors])
+
+    def test(mi):
+        au = set([x.lower() for x in mi.authors])
+        return au == authors
+
+    return test
+
+def _test_fields(touched_fields, mi):
+    for key in touched_fields:
+        if key.startswith('identifier:'):
+            key = key.partition(':')[-1]
+            if not mi.has_identifier(key):
+                return 'identifier: ' + key
+        elif mi.is_null(key):
+            return key
+
+
 def test_identify_plugin(name, tests):
     '''
     :param name: Plugin name
@@ -86,7 +105,7 @@ def test_identify_plugin(name, tests):
             prints(mi)
             prints('\n\n')
 
-        match_found = None
+        possibles = []
         for mi in results:
             test_failed = False
             for tfunc in test_funcs:
@@ -94,16 +113,23 @@ def test_identify_plugin(name, tests):
                     test_failed = True
                     break
             if not test_failed:
-                match_found = mi
+                possibles.append(mi)
-                break
 
-        if match_found is None:
+        if not possibles:
             prints('ERROR: No results that passed all tests were found')
             prints('Log saved to', lf)
             raise SystemExit(1)
 
+        good = [x for x in possibles if _test_fields(plugin.touched_fields, x) is
+                None]
+        if not good:
+            prints('Failed to find', _test_fields(plugin.touched_fields,
+                possibles[0]))
+            raise SystemExit(1)
+
+
     prints('Average time per query', sum(times)/len(times))
 
     if os.stat(lf).st_size > 10:
-        prints('There were some errors, see log', lf)
+        prints('There were some errors/warnings, see log', lf)
 

src/calibre/ebooks/oeb/base.py

@@ -229,7 +229,11 @@ def rewrite_links(root, link_repl_func, resolve_base_href=False):
         if 'style' in el.attrib:
             text = el.attrib['style']
             if _css_url_re.search(text) is not None:
-                stext = parseStyle(text)
+                try:
+                    stext = parseStyle(text)
+                except:
+                    # Parsing errors are raised by cssutils
+                    continue
                 for p in stext.getProperties(all=True):
                     v = p.cssValue
                     if v.CSS_VALUE_LIST == v.cssValueType:
@@ -846,6 +850,7 @@ class Manifest(object):
             return data
 
         def _parse_xhtml(self, data):
+            orig_data = data
             self.oeb.log.debug('Parsing', self.href, '...')
             # Convert to Unicode and normalize line endings
             data = self.oeb.decode(data)
@@ -923,6 +928,8 @@ class Manifest(object):
 
             # Handle weird (non-HTML/fragment) files
             if barename(data.tag) != 'html':
+                if barename(data.tag) == 'ncx':
+                    return self._parse_xml(orig_data)
                 self.oeb.log.warn('File %r does not appear to be (X)HTML'%self.href)
                 nroot = etree.fromstring('<html></html>')
                 has_body = False

src/calibre/ebooks/oeb/output.py

@@ -38,6 +38,11 @@ class OEBOutput(OutputFormatPlugin):
                         except:
                             self.log.exception('Something went wrong while trying to'
                                     ' workaround Nook cover bug, ignoring')
+                        try:
+                            self.workaround_pocketbook_cover_bug(root)
+                        except:
+                            self.log.exception('Something went wrong while trying to'
+                                    ' workaround Pocketbook cover bug, ignoring')
                     raw = etree.tostring(root, pretty_print=True,
                             encoding='utf-8', xml_declaration=True)
                     if key == OPF_MIME:
@@ -90,3 +95,12 @@ class OEBOutput(OutputFormatPlugin):
                     cov.set('content', 'cover')
     # }}}
 
+    def workaround_pocketbook_cover_bug(self, root): # {{{
+        m = root.xpath('//*[local-name() = "manifest"]/*[local-name() = "item" '
+                ' and @id="cover"]')
+        if len(m) == 1:
+            m = m[0]
+            p = m.getparent()
+            p.remove(m)
+            p.insert(0, m)
+    # }}}

src/calibre/ebooks/oeb/transforms/structure.py

@@ -81,6 +81,7 @@ class DetectStructure(object):
             page_break_after = 'display: block; page-break-after: always'
             for item, elem in self.detected_chapters:
                 text = xml2text(elem).strip()
+                text = re.sub(r'\s+', ' ', text.strip())
                 self.log('\tDetected chapter:', text[:50])
                 if chapter_mark == 'none':
                     continue
@@ -137,7 +138,8 @@ class DetectStructure(object):
             text = elem.get('title', '')
         if not text:
             text = elem.get('alt', '')
-        text = text[:100].strip()
+        text = re.sub(r'\s+', ' ', text.strip())
+        text = text[:1000].strip()
         id = elem.get('id', 'calibre_toc_%d'%counter)
         elem.set('id', id)
         href = '#'.join((item.href, id))

src/calibre/ebooks/textile/functions.py

@@ -5,11 +5,13 @@ PyTextile
 A Humane Web Text Generator
 """
 
-__version__ = '2.1.4'
+# Last upstream version basis
-
+# __version__ = '2.1.4'
-__date__ = '2009/12/04'
+#__date__ = '2009/12/04'
 
 __copyright__ = """
+Copyright (c) 2011, Leigh Parry
+Copyright (c) 2011, John Schember <john@nachtimwald.com>
 Copyright (c) 2009, Jason Samsa, http://jsamsa.com/
 Copyright (c) 2004, Roberto A. F. De Almeida, http://dealmeida.net/
 Copyright (c) 2003, Mark Pilgrim, http://diveintomark.org/
@@ -120,6 +122,82 @@ class Textile(object):
     btag_lite = ('bq', 'bc', 'p')
 
     glyph_defaults = (
+        ('mac_cent',               '&#162;'),
+        ('mac_pound',              '&#163;'),
+        ('mac_yen',                '&#165;'),
+        ('mac_quarter',            '&#188;'),
+        ('mac_half',               '&#189;'),
+        ('mac_three-quarter',      '&#190;'),
+        ('mac_cA-grave',           '&#192;'),
+        ('mac_cA-acute',           '&#193;'),
+        ('mac_cA-circumflex',      '&#194;'),
+        ('mac_cA-tilde',           '&#195;'),
+        ('mac_cA-diaeresis',       '&#196;'),
+        ('mac_cA-ring',            '&#197;'),
+        ('mac_cAE',                '&#198;'),
+        ('mac_cC-cedilla',         '&#199;'),
+        ('mac_cE-grave',           '&#200;'),
+        ('mac_cE-acute',           '&#201;'),
+        ('mac_cE-circumflex',      '&#202;'),
+        ('mac_cE-diaeresis',       '&#203;'),
+        ('mac_cI-grave',           '&#204;'),
+        ('mac_cI-acute',           '&#205;'),
+        ('mac_cI-circumflex',      '&#206;'),
+        ('mac_cI-diaeresis',       '&#207;'),
+        ('mac_cEth',               '&#208;'),
+        ('mac_cN-tilde',           '&#209;'),
+        ('mac_cO-grave',           '&#210;'),
+        ('mac_cO-acute',           '&#211;'),
+        ('mac_cO-circumflex',      '&#212;'),
+        ('mac_cO-tilde',           '&#213;'),
+        ('mac_cO-diaeresis',       '&#214;'),
+        ('mac_cO-stroke',          '&#216;'),
+        ('mac_cU-grave',           '&#217;'),
+        ('mac_cU-acute',           '&#218;'),
+        ('mac_cU-circumflex',      '&#219;'),
+        ('mac_cU-diaeresis',       '&#220;'),
+        ('mac_cY-acute',           '&#221;'),
+        ('mac_sa-grave',           '&#224;'),
+        ('mac_sa-acute',           '&#225;'),
+        ('mac_sa-circumflex',      '&#226;'),
+        ('mac_sa-tilde',           '&#227;'),
+        ('mac_sa-diaeresis',       '&#228;'),
+        ('mac_sa-ring',            '&#229;'),
+        ('mac_sae',                '&#230;'),
+        ('mac_sc-cedilla',         '&#231;'),
+        ('mac_se-grave',           '&#232;'),
+        ('mac_se-acute',           '&#233;'),
+        ('mac_se-circumflex',      '&#234;'),
+        ('mac_se-diaeresis',       '&#235;'),
+        ('mac_si-grave',           '&#236;'),
+        ('mac_si-acute',           '&#237;'),
+        ('mac_si-circumflex',      '&#238;'),
+        ('mac_si-diaeresis',       '&#239;'),
+        ('mac_sn-tilde',           '&#241;'),
+        ('mac_so-grave',           '&#242;'),
+        ('mac_so-acute',           '&#243;'),
+        ('mac_so-circumflex',      '&#244;'),
+        ('mac_so-tilde',           '&#245;'),
+        ('mac_so-diaeresis',       '&#246;'),
+        ('mac_so-stroke',          '&#248;'),
+        ('mac_su-grave',           '&#249;'),
+        ('mac_su-acute',           '&#250;'),
+        ('mac_su-circumflex',      '&#251;'),
+        ('mac_su-diaeresis',       '&#252;'),
+        ('mac_sy-acute',           '&#253;'),
+        ('mac_sy-diaeresis',       '&#255;'),
+        ('mac_cOE',                '&#338;'),
+        ('mac_soe',                '&#339;'),
+        ('mac_bullet',             '&#8226;'),
+        ('mac_franc',              '&#8355;'),
+        ('mac_lira',               '&#8356;'),
+        ('mac_rupee',              '&#8360;'),
+        ('mac_euro',               '&#8364;'),
+        ('mac_spade',              '&#9824;'),
+        ('mac_club',               '&#9827;'),
+        ('mac_heart',              '&#9829;'),
+        ('mac_diamond',            '&#9830;'),
+        ('txt_dimension',          '&#215;'),
         ('txt_quote_single_open',  '&#8216;'),
         ('txt_quote_single_close', '&#8217;'),
         ('txt_quote_double_open',  '&#8220;'),
@@ -130,7 +208,6 @@ class Textile(object):
         ('txt_ellipsis',           '&#8230;'),
         ('txt_emdash',             '&#8212;'),
         ('txt_endash',             '&#8211;'),
-        ('txt_dimension',          '&#215;'),
         ('txt_trademark',          '&#8482;'),
         ('txt_registered',         '&#174;'),
         ('txt_copyright',          '&#169;'),
@@ -593,45 +670,210 @@ class Textile(object):
         '<p><cite>Cat&#8217;s Cradle</cite> by Vonnegut</p>'
 
         """
-         # fix: hackish
+        # fix: hackish
         text = re.sub(r'"\Z', '\" ', text)
 
         glyph_search = (
-            re.compile(r"(\w)\'(\w)"),                                      # apostrophe's
+            re.compile(r'(\d+\'?\"?)( ?)x( ?)(?=\d+)'),                     #  dimension sign
-            re.compile(r'(\s)\'(\d+\w?)\b(?!\')'),                          # back in '88
+            re.compile(r"(\w)\'(\w)"),                                      #  apostrophe's
-            re.compile(r'(\S)\'(?=\s|'+self.pnct+'|<|$)'),                       #  single closing
+            re.compile(r'(\s)\'(\d+\w?)\b(?!\')'),                          #  back in '88
+            re.compile(r'(\S)\'(?=\s|'+self.pnct+'|<|$)'),                  #  single closing
             re.compile(r'\'/'),                                             #  single opening
-            re.compile(r'(\S)\"(?=\s|'+self.pnct+'|<|$)'),                       #  double closing
+            re.compile(r'(\")\"'),                                          #  double closing - following another
+            re.compile(r'(\S)\"(?=\s|'+self.pnct+'|<|$)'),                  #  double closing
             re.compile(r'"'),                                               #  double opening
             re.compile(r'\b([A-Z][A-Z0-9]{2,})\b(?:[(]([^)]*)[)])'),        #  3+ uppercase acronym
             re.compile(r'\b([A-Z][A-Z\'\-]+[A-Z])(?=[\s.,\)>])'),           #  3+ uppercase
-            re.compile(r'\b(\s{0,1})?\.{3}'),                                     #  ellipsis
+            re.compile(r'\b(\s{0,1})?\.{3}'),                               #  ellipsis
             re.compile(r'(\s?)--(\s?)'),                                    #  em dash
             re.compile(r'\s-(?:\s|$)'),                                     #  en dash
-            re.compile(r'(\d+)( ?)x( ?)(?=\d+)'),                           #  dimension sign
+            re.compile(r'\b( ?)[([]TM[])]', re.I),                            #  trademark
-            re.compile(r'\b ?[([]TM[])]', re.I),                            #  trademark
+            re.compile(r'\b( ?)[([]R[])]', re.I),                             #  registered
-            re.compile(r'\b ?[([]R[])]', re.I),                             #  registered
+            re.compile(r'\b( ?)[([]C[])]', re.I)                              #  copyright
-            re.compile(r'\b ?[([]C[])]', re.I),                             #  copyright
          )
 
         glyph_replace = [x % dict(self.glyph_defaults) for x in (
-            r'\1%(txt_apostrophe)s\2',           # apostrophe's
+            r'\1\2%(txt_dimension)s\3',          #  dimension sign
-            r'\1%(txt_apostrophe)s\2',           # back in '88
+            r'\1%(txt_apostrophe)s\2',           #  apostrophe's
+            r'\1%(txt_apostrophe)s\2',           #  back in '88
             r'\1%(txt_quote_single_close)s',     #  single closing
-            r'%(txt_quote_single_open)s',         #  single opening
+            r'%(txt_quote_single_open)s',        #  single opening
-            r'\1%(txt_quote_double_close)s',        #  double closing
+            r'\1%(txt_quote_double_close)s',     #  double closing - following another
-            r'%(txt_quote_double_open)s',             #  double opening
+            r'\1%(txt_quote_double_close)s',     #  double closing
+            r'%(txt_quote_double_open)s',        #  double opening
             r'<acronym title="\2">\1</acronym>', #  3+ uppercase acronym
             r'<span class="caps">\1</span>',     #  3+ uppercase
-            r'\1%(txt_ellipsis)s',                  #  ellipsis
+            r'\1%(txt_ellipsis)s',               #  ellipsis
             r'\1%(txt_emdash)s\2',               #  em dash
             r' %(txt_endash)s ',                 #  en dash
-            r'\1\2%(txt_dimension)s\3',          #  dimension sign
+            r'\1%(txt_trademark)s',              #  trademark
-            r'%(txt_trademark)s',                #  trademark
+            r'\1%(txt_registered)s',             #  registered
-            r'%(txt_registered)s',                #  registered
+            r'\1%(txt_copyright)s'               #  copyright
-            r'%(txt_copyright)s',                #  copyright
         )]
 
+        if re.search(r'{.+?}', text):
+            glyph_search += (
+                re.compile(r'{(c\||\|c)}'),                               #  cent
+                re.compile(r'{(L-|-L)}'),                                 #  pound
+                re.compile(r'{(Y=|=Y)}'),                                 #  yen
+                re.compile(r'{\(c\)}'),                                   #  copyright
+                re.compile(r'{\(r\)}'),                                   #  registered
+                re.compile(r'{1/4}'),                                     #  quarter
+                re.compile(r'{1/2}'),                                     #  half
+                re.compile(r'{3/4}'),                                     #  three-quarter
+                re.compile(r'{(A`|`A)}'),                                 #  192;
+                re.compile(r'{(A\'|\'A)}'),                               #  193;
+                re.compile(r'{(A\^|\^A)}'),                               #  194;
+                re.compile(r'{(A~|~A)}'),                                 #  195;
+                re.compile(r'{(A\"|\"A)}'),                               #  196;
+                re.compile(r'{(Ao|oA)}'),                                 #  197;
+                re.compile(r'{(AE)}'),                                    #  198;
+                re.compile(r'{(C,|,C)}'),                                 #  199;
+                re.compile(r'{(E`|`E)}'),                                 #  200;
+                re.compile(r'{(E\'|\'E)}'),                               #  201;
+                re.compile(r'{(E\^|\^E)}'),                               #  202;
+                re.compile(r'{(E\"|\"E)}'),                               #  203;
+                re.compile(r'{(I`|`I)}'),                                 #  204;
+                re.compile(r'{(I\'|\'I)}'),                               #  205;
+                re.compile(r'{(I\^|\^I)}'),                               #  206;
+                re.compile(r'{(I\"|\"I)}'),                               #  207;
+                re.compile(r'{(D-|-D)}'),                                 #  208;
+                re.compile(r'{(N~|~N)}'),                                 #  209;
+                re.compile(r'{(O`|`O)}'),                                 #  210;
+                re.compile(r'{(O\'|\'O)}'),                               #  211;
+                re.compile(r'{(O\^|\^O)}'),                               #  212;
+                re.compile(r'{(O~|~O)}'),                                 #  213;
+                re.compile(r'{(O\"|\"O)}'),                               #  214;
+                re.compile(r'{(O\/|\/O)}'),                               #  215;
+                re.compile(r'{(U`|`U)}'),                                 #  216;
+                re.compile(r'{(U\'|\'U)}'),                               #  217;
+                re.compile(r'{(U\^|\^U)}'),                               #  218;
+                re.compile(r'{(U\"|\"U)}'),                               #  219;
+                re.compile(r'{(Y\'|\'Y)}'),                               #  220;
+                re.compile(r'{(a`|`a)}'),                                 #  a-grace
+                re.compile(r'{(a\'|\'a)}'),                               #  a-acute
+                re.compile(r'{(a\^|\^a)}'),                               #  a-circumflex
+                re.compile(r'{(a~|~a)}'),                                 #  a-tilde
+                re.compile(r'{(a\"|\"a)}'),                               #  a-diaeresis
+                re.compile(r'{(ao|oa)}'),                                 #  a-ring
+                re.compile(r'{ae}'),                                      #  ae
+                re.compile(r'{(c,|,c)}'),                                 #  c-cedilla
+                re.compile(r'{(e`|`e)}'),                                 #  e-grace
+                re.compile(r'{(e\'|\'e)}'),                               #  e-acute
+                re.compile(r'{(e\^|\^e)}'),                               #  e-circumflex
+                re.compile(r'{(e\"|\"e)}'),                               #  e-diaeresis
+                re.compile(r'{(i`|`i)}'),                                 #  i-grace
+                re.compile(r'{(i\'|\'i)}'),                               #  i-acute
+                re.compile(r'{(i\^|\^i)}'),                               #  i-circumflex
+                re.compile(r'{(i\"|\"i)}'),                               #  i-diaeresis
+                re.compile(r'{(n~|~n)}'),                                 #  n-tilde
+                re.compile(r'{(o`|`o)}'),                                 #  o-grace
+                re.compile(r'{(o\'|\'o)}'),                               #  o-acute
+                re.compile(r'{(o\^|\^o)}'),                               #  o-circumflex
+                re.compile(r'{(o~|~o)}'),                                 #  o-tilde
+                re.compile(r'{(o\"|\"o)}'),                               #  o-diaeresis
+                re.compile(r'{(o\/|\/o)}'),                               #  o-stroke
+                re.compile(r'{(u`|`u)}'),                                 #  u-grace
+                re.compile(r'{(u\'|\'u)}'),                               #  u-acute
+                re.compile(r'{(u\^|\^u)}'),                               #  u-circumflex
+                re.compile(r'{(u\"|\"u)}'),                               #  u-diaeresis
+                re.compile(r'{(y\'|\'y)}'),                               #  y-acute
+                re.compile(r'{(y\"|\"y)}'),                               #  y-diaeresis
+                re.compile(r'{OE}'),                                      #  y-diaeresis
+                re.compile(r'{oe}'),                                      #  y-diaeresis
+                re.compile(r'{\*}'),                                      #  bullet
+                re.compile(r'{Fr}'),                                      #  Franc
+                re.compile(r'{(L=|=L)}'),                                 #  Lira
+                re.compile(r'{Rs}'),                                      #  Rupee
+                re.compile(r'{(C=|=C)}'),                                 #  euro
+                re.compile(r'{tm}'),                                      #  euro
+                re.compile(r'{spade}'),                                   #  spade
+                re.compile(r'{club}'),                                    #  club
+                re.compile(r'{heart}'),                                   #  heart
+                re.compile(r'{diamond}')                                  #  diamond
+             )
+    
+            glyph_replace += [x % dict(self.glyph_defaults) for x in (
+                r'%(mac_cent)s',                     #  cent
+                r'%(mac_pound)s',                    #  pound
+                r'%(mac_yen)s',                      #  yen
+                r'%(txt_copyright)s',                #  copyright
+                r'%(txt_registered)s',               #  registered
+                r'%(mac_quarter)s',                  #  quarter
+                r'%(mac_half)s',                     #  half
+                r'%(mac_three-quarter)s',            #  three-quarter
+                r'%(mac_cA-grave)s',                 #  192;
+                r'%(mac_cA-acute)s',                 #  193;
+                r'%(mac_cA-circumflex)s',            #  194;
+                r'%(mac_cA-tilde)s',                 #  195;
+                r'%(mac_cA-diaeresis)s',             #  196;
+                r'%(mac_cA-ring)s',                  #  197;
+                r'%(mac_cAE)s',                      #  198;
+                r'%(mac_cC-cedilla)s',               #  199;
+                r'%(mac_cE-grave)s',                 #  200;
+                r'%(mac_cE-acute)s',                 #  201;
+                r'%(mac_cE-circumflex)s',            #  202;
+                r'%(mac_cE-diaeresis)s',             #  203;
+                r'%(mac_cI-grave)s',                 #  204;
+                r'%(mac_cI-acute)s',                 #  205;
+                r'%(mac_cI-circumflex)s',            #  206;
+                r'%(mac_cI-diaeresis)s',             #  207;
+                r'%(mac_cEth)s',                     #  208;
+                r'%(mac_cN-tilde)s',                 #  209;
+                r'%(mac_cO-grave)s',                 #  210;
+                r'%(mac_cO-acute)s',                 #  211;
+                r'%(mac_cO-circumflex)s',            #  212;
+                r'%(mac_cO-tilde)s',                 #  213;
+                r'%(mac_cO-diaeresis)s',             #  214;
+                r'%(mac_cO-stroke)s',                #  216;
+                r'%(mac_cU-grave)s',                 #  217;
+                r'%(mac_cU-acute)s',                 #  218;
+                r'%(mac_cU-circumflex)s',            #  219;
+                r'%(mac_cU-diaeresis)s',             #  220;
+                r'%(mac_cY-acute)s',                 #  221;
+                r'%(mac_sa-grave)s',                 #  224;
+                r'%(mac_sa-acute)s',                 #  225;
+                r'%(mac_sa-circumflex)s',            #  226;
+                r'%(mac_sa-tilde)s',                 #  227;
+                r'%(mac_sa-diaeresis)s',             #  228;
+                r'%(mac_sa-ring)s',                  #  229;
+                r'%(mac_sae)s',                      #  230;
+                r'%(mac_sc-cedilla)s',               #  231;
+                r'%(mac_se-grave)s',                 #  232;
+                r'%(mac_se-acute)s',                 #  233;
+                r'%(mac_se-circumflex)s',            #  234;
+                r'%(mac_se-diaeresis)s',             #  235;
+                r'%(mac_si-grave)s',                 #  236;
+                r'%(mac_si-acute)s',                 #  237;
+                r'%(mac_si-circumflex)s',            #  238;
+                r'%(mac_si-diaeresis)s',             #  239;
+                r'%(mac_sn-tilde)s',                 #  241;
+                r'%(mac_so-grave)s',                 #  242;
+                r'%(mac_so-acute)s',                 #  243;
+                r'%(mac_so-circumflex)s',            #  244;
+                r'%(mac_so-tilde)s',                 #  245;
+                r'%(mac_so-diaeresis)s',             #  246;
+                r'%(mac_so-stroke)s',                #  248;
+                r'%(mac_su-grave)s',                 #  249;
+                r'%(mac_su-acute)s',                 #  250;
+                r'%(mac_su-circumflex)s',            #  251;
+                r'%(mac_su-diaeresis)s',             #  252;
+                r'%(mac_sy-acute)s',                 #  253;
+                r'%(mac_sy-diaeresis)s',             #  255;
+                r'%(mac_cOE)s',                      #  338;
+                r'%(mac_soe)s',                      #  339;
+                r'%(mac_bullet)s',                   #  bullet
+                r'%(mac_franc)s',                    #  franc
+                r'%(mac_lira)s',                     #  lira
+                r'%(mac_rupee)s',                    #  rupee
+                r'%(mac_euro)s',                     #  euro
+                r'%(txt_trademark)s',                #  trademark
+                r'%(mac_spade)s',                    #  spade
+                r'%(mac_club)s',                     #  club
+                r'%(mac_heart)s',                    #  heart
+                r'%(mac_diamond)s'                   #  diamond
+            )]
+
         result = []
         for line in re.compile(r'(<.*?>)', re.U).split(text):
             if not re.search(r'<.*>', line):
@@ -807,7 +1049,7 @@ class Textile(object):
 
         for qtag in qtags:
             pattern = re.compile(r"""
-                (?:^|(?<=[\s>%(pnct)s])|([\]}]))
+                (?:^|(?<=[\s>%(pnct)s])|\[|([\]}]))
                 (%(qtag)s)(?!%(qtag)s)
                 (%(c)s)
                 (?::(\S+))?

src/calibre/gui2/actions/view.py

@@ -34,6 +34,13 @@ class ViewAction(InterfaceAction):
         self.qaction.setMenu(self.view_menu)
         ac.triggered.connect(self.view_specific_format, type=Qt.QueuedConnection)
 
+        self.view_menu.addSeparator()
+        ac = self.create_action(spec=(_('Read a random book'), 'catalog.png',
+            None, None), attr='action_pick_random')
+        ac.triggered.connect(self.view_random)
+        self.view_menu.addAction(ac)
+
+
     def location_selected(self, loc):
         enabled = loc == 'library'
         for action in list(self.view_menu.actions())[1:]:
@@ -151,6 +158,10 @@ class ViewAction(InterfaceAction):
     def view_specific_book(self, index):
         self._view_books([index])
 
+    def view_random(self, *args):
+        self.gui.iactions['Choose Library'].pick_random()
+        self._view_books([self.gui.library_view.currentIndex()])
+
     def _view_books(self, rows):
         if not rows or len(rows) == 0:
             self._launch_viewer()

src/calibre/gui2/convert/search_and_replace.py

@@ -6,6 +6,8 @@ __docformat__ = 'restructuredtext en'
 
 import re
 
+from PyQt4.Qt import QLineEdit, QTextEdit
+
 from calibre.gui2.convert.search_and_replace_ui import Ui_Form
 from calibre.gui2.convert import Widget
 from calibre.gui2 import error_dialog
@@ -72,3 +74,13 @@ class SearchAndReplaceWidget(Widget, Ui_Form):
                              _('Invalid regular expression: %s')%err, show=True)
                 return False
         return True
+    
+    def get_vaule(self, g):
+        if isinstance(g, (QLineEdit, QTextEdit)):
+            func = getattr(g, 'toPlainText', getattr(g, 'text', None))()
+            ans = unicode(func)
+            if not ans:
+                ans = None
+            return ans
+        else:
+            return Widget.get_value(self, g)

src/calibre/gui2/cover_flow.py

@@ -53,7 +53,7 @@ if pictureflow is not None:
         def __init__(self, model, buffer=20):
             pictureflow.FlowImages.__init__(self)
             self.model = model
-            self.model.modelReset.connect(self.reset)
+            self.model.modelReset.connect(self.reset, type=Qt.QueuedConnection)
 
         def count(self):
             return self.model.count()
@@ -83,6 +83,8 @@ if pictureflow is not None:
 
     class CoverFlow(pictureflow.PictureFlow):
 
+        dc_signal = pyqtSignal()
+
         def __init__(self, parent=None):
             pictureflow.PictureFlow.__init__(self, parent,
                                 config['cover_flow_queue_length']+1)
@@ -90,6 +92,8 @@ if pictureflow is not None:
             self.setFocusPolicy(Qt.WheelFocus)
             self.setSizePolicy(QSizePolicy(QSizePolicy.Expanding,
                 QSizePolicy.Expanding))
+            self.dc_signal.connect(self._data_changed,
+                    type=Qt.QueuedConnection)
 
         def sizeHint(self):
             return self.minimumSize()
@@ -101,6 +105,12 @@ if pictureflow is not None:
             elif ev.delta() > 0:
                 self.showPrevious()
 
+        def dataChanged(self):
+            self.dc_signal.emit()
+
+        def _data_changed(self):
+            pictureflow.PictureFlow.dataChanged(self)
+
 
 else:
     CoverFlow = None
@@ -135,8 +145,7 @@ class CoverFlowMixin(object):
         self.cover_flow = None
         if CoverFlow is not None:
             self.cf_last_updated_at = None
-            self.cover_flow_sync_timer = QTimer(self)
+            self.cover_flow_syncing_enabled = False
-            self.cover_flow_sync_timer.timeout.connect(self.cover_flow_do_sync)
             self.cover_flow_sync_flag = True
             self.cover_flow = CoverFlow(parent=self)
             self.cover_flow.currentChanged.connect(self.sync_listview_to_cf)
@@ -179,14 +188,15 @@ class CoverFlowMixin(object):
         self.cover_flow.setFocus(Qt.OtherFocusReason)
         if CoverFlow is not None:
             self.cover_flow.setCurrentSlide(self.library_view.currentIndex().row())
-            self.cover_flow_sync_timer.start(500)
+            self.cover_flow_syncing_enabled = True
+            QTimer.singleShot(500, self.cover_flow_do_sync)
         self.library_view.setCurrentIndex(
                 self.library_view.currentIndex())
         self.library_view.scroll_to_row(self.library_view.currentIndex().row())
 
     def cover_browser_hidden(self):
         if CoverFlow is not None:
-            self.cover_flow_sync_timer.stop()
+            self.cover_flow_syncing_enabled = False
             idx = self.library_view.model().index(self.cover_flow.currentSlide(), 0)
             if idx.isValid():
                 sm = self.library_view.selectionModel()
@@ -242,6 +252,8 @@ class CoverFlowMixin(object):
         except:
             import traceback
             traceback.print_exc()
+        if self.cover_flow_syncing_enabled:
+            QTimer.singleShot(500, self.cover_flow_do_sync)
 
     def sync_listview_to_cf(self, row):
         self.cf_last_updated_at = time.time()

src/calibre/gui2/device.py

@@ -1052,11 +1052,13 @@ class DeviceMixin(object): # {{{
                     except:
                         pass
                     total_size = self.location_manager.free[0]
-                if self.location_manager.free[0] > total_size + (1024**2):
+                loc = tweaks['send_news_to_device_location']
+                loc_index = {"carda": 1, "cardb": 2}.get(loc, 0)
+                if self.location_manager.free[loc_index] > total_size + (1024**2):
                     # Send news to main memory if enough space available
                     # as some devices like the Nook Color cannot handle
                     # periodicals on SD cards properly
-                    on_card = None
+                    on_card = loc if loc in ('carda', 'cardb') else None
                 self.upload_books(files, names, metadata,
                         on_card=on_card,
                         memory=[files, remove])

src/calibre/gui2/dialogs/check_library.py

@@ -202,13 +202,19 @@ class CheckLibraryDialog(QDialog):
         <p><i>Delete marked</i> is used to remove extra files/folders/covers that
         have no entries in the database. Check the box next to the item you want
         to delete. Use with caution.</p>
-        <p><i>Fix marked</i> is applicable only to covers (the two lines marked
+
-        'fixable'). In the case of missing cover files, checking the fixable
+        <p><i>Fix marked</i> is applicable only to covers and missing formats
-        box and pushing this button will remove the cover mark from the
+        (the three lines marked 'fixable'). In the case of missing cover files,
-        database for all the files in that category. In the case of extra
+        checking the fixable box and pushing this button will tell calibre that
-        cover files, checking the fixable box and pushing this button will
+        there is no cover for all of the books listed. Use this option if you
-        add the cover mark to the database for all the files in that
+        are not going to restore the covers from a backup. In the case of extra
-        category.</p>
+        cover files, checking the fixable box and pushing this button will tell
+        calibre that the cover files it found are correct for all the books
+        listed. Use this when you are not going to delete the file(s). In the
+        case of missing formats, checking the fixable box and pushing this
+        button will tell calibre that the formats are really gone. Use this if
+        you are not going to restore the formats from a backup.</p>
+
         '''))
 
         self.log = QTreeWidget(self)
@@ -381,6 +387,19 @@ class CheckLibraryDialog(QDialog):
                                 unicode(it.text(1))))
         self.run_the_check()
 
+    def fix_missing_formats(self):
+        tl = self.top_level_items['missing_formats']
+        child_count = tl.childCount()
+        for i in range(0, child_count):
+            item = tl.child(i);
+            id = item.data(0, Qt.UserRole).toInt()[0]
+            all = self.db.formats(id, index_is_id=True, verify_formats=False)
+            all = set([f.strip() for f in all.split(',')]) if all else set()
+            valid = self.db.formats(id, index_is_id=True, verify_formats=True)
+            valid = set([f.strip() for f in valid.split(',')]) if valid else set()
+            for fmt in all-valid:
+                self.db.remove_format(id, fmt, index_is_id=True, db_only=True)
+
     def fix_missing_covers(self):
         tl = self.top_level_items['missing_covers']
         child_count = tl.childCount()

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -783,6 +783,12 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
             books_to_refresh = self.db.set_custom(id, val, label=dfm['label'],
                                                   extra=extra, commit=False,
                                                   allow_case_change=True)
+        elif dest.startswith('#') and dest.endswith('_index'):
+            label = self.db.field_metadata[dest[:-6]]['label']
+            series = self.db.get_custom(id, label=label, index_is_id=True)
+            books_to_refresh = self.db.set_custom(id, series, label=label,
+                                                  extra=val, commit=False,
+                                                  allow_case_change=True)
         else:
             if dest == 'comments':
                 setter = self.db.set_comment

src/calibre/gui2/dialogs/saved_search_editor.py

@@ -9,12 +9,13 @@ from PyQt4.QtGui import QDialog
 from calibre.gui2.dialogs.saved_search_editor_ui import Ui_SavedSearchEditor
 from calibre.utils.search_query_parser import saved_searches
 from calibre.utils.icu import sort_key
+from calibre.gui2 import error_dialog
 from calibre.gui2.dialogs.confirm_delete import confirm
 
 class SavedSearchEditor(QDialog, Ui_SavedSearchEditor):
 
-    def __init__(self, window, initial_search=None):
+    def __init__(self, parent, initial_search=None):
-        QDialog.__init__(self, window)
+        QDialog.__init__(self, parent)
         Ui_SavedSearchEditor.__init__(self)
         self.setupUi(self)
 
@@ -22,12 +23,13 @@ class SavedSearchEditor(QDialog, Ui_SavedSearchEditor):
         self.connect(self.search_name_box, SIGNAL('currentIndexChanged(int)'),
                                     self.current_index_changed)
         self.connect(self.delete_search_button, SIGNAL('clicked()'), self.del_search)
+        self.rename_button.clicked.connect(self.rename_search)
 
         self.current_search_name = None
         self.searches = {}
-        self.searches_to_delete = []
         for name in saved_searches().names():
             self.searches[name] = saved_searches().lookup(name)
+        self.search_names = set([icu_lower(n) for n in saved_searches().names()])
 
         self.populate_search_list()
         if initial_search is not None and initial_search in self.searches:
@@ -42,6 +44,11 @@ class SavedSearchEditor(QDialog, Ui_SavedSearchEditor):
         search_name = unicode(self.input_box.text()).strip()
         if search_name == '':
             return False
+        if icu_lower(search_name) in self.search_names:
+            error_dialog(self, _('Saved search already exists'),
+                     _('The saved search %s already exists, perhaps with '
+                       'different case')%search_name).exec_()
+            return False
         if search_name not in self.searches:
             self.searches[search_name] = ''
             self.populate_search_list()
@@ -57,10 +64,25 @@ class SavedSearchEditor(QDialog, Ui_SavedSearchEditor):
                         +'</p>', 'saved_search_editor_delete', self):
                 return
             del self.searches[self.current_search_name]
-            self.searches_to_delete.append(self.current_search_name)
             self.current_search_name = None
             self.search_name_box.removeItem(self.search_name_box.currentIndex())
 
+    def rename_search(self):
+        new_search_name = unicode(self.input_box.text()).strip()
+        if new_search_name == '':
+            return False
+        if icu_lower(new_search_name) in self.search_names:
+            error_dialog(self, _('Saved search already exists'),
+                    _('The saved search %s already exists, perhaps with '
+                      'different case')%new_search_name).exec_()
+            return False
+        if self.current_search_name in self.searches:
+            self.searches[new_search_name] = self.searches[self.current_search_name]
+            del self.searches[self.current_search_name]
+            self.populate_search_list()
+            self.select_search(new_search_name)
+        return True
+
     def select_search(self, name):
         self.search_name_box.setCurrentIndex(self.search_name_box.findText(name))
 
@@ -78,7 +100,7 @@ class SavedSearchEditor(QDialog, Ui_SavedSearchEditor):
     def accept(self):
         if self.current_search_name:
             self.searches[self.current_search_name] = unicode(self.search_text.toPlainText())
-        for name in self.searches_to_delete:
+        for name in saved_searches().names():
             saved_searches().delete(name)
         for name in self.searches:
             saved_searches().add(name, self.searches[name])

src/calibre/gui2/dialogs/saved_search_editor.ui

@@ -134,6 +134,20 @@
        </property>
       </widget>
      </item>
+     <item row="0" column="6">
+      <widget class="QToolButton" name="rename_button">
+       <property name="toolTip">
+        <string>Rename the current search to what is in the box</string>
+       </property>
+       <property name="text">
+        <string>...</string>
+       </property>
+       <property name="icon">
+        <iconset>
+         <normaloff>:/images/edit-undo.png</normaloff>:/images/edit-undo.png</iconset>
+       </property>
+      </widget>
+     </item>
     </layout>
    </item>
    <item row="1" column="0">

src/calibre/gui2/dialogs/scheduler.py

@@ -442,7 +442,7 @@ class Scheduler(QObject):
         if self.oldest > 0:
             delta = timedelta(days=self.oldest)
             try:
-                ids = list(self.recipe_model.db.tags_older_than(_('News'),
+                ids = list(self.db.tags_older_than(_('News'),
                     delta))
             except:
                 # Happens if library is being switched

src/calibre/gui2/dialogs/scheduler.ui

@@ -6,7 +6,7 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>767</width>
+    <width>792</width>
     <height>575</height>
    </rect>
   </property>
@@ -44,7 +44,7 @@
        <rect>
         <x>0</x>
         <y>0</y>
-        <width>469</width>
+        <width>486</width>
         <height>504</height>
        </rect>
       </property>

src/calibre/gui2/dialogs/tag_categories.py

@@ -59,14 +59,24 @@ class TagCategories(QDialog, Ui_TagCategories):
                           ]
         category_names  = ['', _('Authors'), _('Series'), _('Publishers'), _('Tags')]
 
-        cc_map = self.db.custom_column_label_map
+        cvals = {}
-        for cc in cc_map:
+        for key,cc in self.db.custom_field_metadata().iteritems():
-            if cc_map[cc]['datatype'] in ['text', 'series']:
+            if cc['datatype'] in ['text', 'series', 'enumeration']:
-                self.category_labels.append(db.field_metadata.label_to_key(cc))
+                self.category_labels.append(key)
                 category_icons.append(cc_icon)
-                category_values.append(lambda col=cc: self.db.all_custom(label=col))
+                category_values.append(lambda col=cc['label']: self.db.all_custom(label=col))
-                category_names.append(cc_map[cc]['name'])
+                category_names.append(cc['name'])
-
+            elif cc['datatype'] == 'composite' and \
+                    cc['display'].get('make_category', False):
+                self.category_labels.append(key)
+                category_icons.append(cc_icon)
+                category_names.append(cc['name'])
+                dex = cc['rec_index']
+                cvals = set()
+                for book in db.data.iterall():
+                    if book[dex]:
+                        cvals.add(book[dex])
+                category_values.append(lambda s=list(cvals): s)
         self.all_items = []
         self.all_items_dict = {}
         for idx,label in enumerate(self.category_labels):
@@ -88,7 +98,8 @@ class TagCategories(QDialog, Ui_TagCategories):
                 if l[1] in self.category_labels:
                     if t is None:
                         t = Item(name=l[0], label=l[1], index=len(self.all_items),
-                                 icon=category_icons[self.category_labels.index(l[1])], exists=False)
+                                 icon=category_icons[self.category_labels.index(l[1])],
+                                 exists=False)
                         self.all_items.append(t)
                         self.all_items_dict[key] = t
                     l[2] = t.index
@@ -108,13 +119,16 @@ class TagCategories(QDialog, Ui_TagCategories):
         self.add_category_button.clicked.connect(self.add_category)
         self.rename_category_button.clicked.connect(self.rename_category)
         self.category_box.currentIndexChanged[int].connect(self.select_category)
-        self.category_filter_box.currentIndexChanged[int].connect(self.display_filtered_categories)
+        self.category_filter_box.currentIndexChanged[int].connect(
+                                                self.display_filtered_categories)
         self.delete_category_button.clicked.connect(self.del_category)
         if islinux:
             self.available_items_box.itemDoubleClicked.connect(self.apply_tags)
         else:
-            self.connect(self.available_items_box, SIGNAL('itemActivated(QListWidgetItem*)'), self.apply_tags)
+            self.connect(self.available_items_box,
-        self.connect(self.applied_items_box,   SIGNAL('itemActivated(QListWidgetItem*)'), self.unapply_tags)
+                         SIGNAL('itemActivated(QListWidgetItem*)'), self.apply_tags)
+        self.connect(self.applied_items_box,
+                     SIGNAL('itemActivated(QListWidgetItem*)'), self.unapply_tags)
 
         self.populate_category_list()
         if on_category is not None:
@@ -129,6 +143,7 @@ class TagCategories(QDialog, Ui_TagCategories):
         n = item.name if item.exists else item.name + _(' (not on any book)')
         w = QListWidgetItem(item.icon, n)
         w.setData(Qt.UserRole, item.index)
+        w.setToolTip(_('Category lookup name: ') + item.label)
         return w
 
     def display_filtered_categories(self, idx):

src/calibre/gui2/library/models.py

@@ -646,6 +646,14 @@ class BooksModel(QAbstractTableModel): # {{{
                 return QVariant(', '.join(sorted(text.split('|'),key=sort_key)))
             return QVariant(text)
 
+        def decorated_text_type(r, mult=False, idx=-1):
+            text = self.db.data[r][idx]
+            if force_to_bool(text) is not None:
+                return None
+            if text and mult:
+                return QVariant(', '.join(sorted(text.split('|'),key=sort_key)))
+            return QVariant(text)
+
         def number_type(r, idx=-1):
             return QVariant(self.db.data[r][idx])
 
@@ -687,6 +695,8 @@ class BooksModel(QAbstractTableModel): # {{{
                 self.dc[col] = functools.partial(text_type, idx=idx, mult=mult)
                 if datatype in ['text', 'composite', 'enumeration'] and not mult:
                     if self.custom_columns[col]['display'].get('use_decorations', False):
+                        self.dc[col] = functools.partial(decorated_text_type,
+                                                         idx=idx, mult=mult)
                         self.dc_decorator[col] = functools.partial(
                             bool_type_decorator, idx=idx,
                             bool_cols_are_tristate=

src/calibre/gui2/main_window.py

@@ -1,10 +1,14 @@
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
-import StringIO, traceback, sys
 
-from PyQt4.Qt import QMainWindow, QString, Qt, QFont, QCoreApplication, SIGNAL,\
+import StringIO, traceback, sys, gc
-                     QAction, QMenu, QMenuBar, QIcon, pyqtSignal
+
+from PyQt4.Qt import QMainWindow, QString, Qt, QFont, QTimer, \
+                     QAction, QMenu, QMenuBar, QIcon, pyqtSignal, QObject
 from calibre.gui2.dialogs.conversion_error import ConversionErrorDialog
 from calibre.utils.config import OptionParser
 from calibre.gui2 import error_dialog
@@ -16,7 +20,8 @@ Usage: %prog [options]
 Launch the Graphical User Interface
 '''):
     parser = OptionParser(usage)
-    parser.add_option('--redirect-console-output', default=False, action='store_true', dest='redirect',
+    # The b is required because of a regression in optparse.py in python 2.7.0
+    parser.add_option(b'--redirect-console-output', default=False, action='store_true', dest='redirect',
                       help=_('Redirect console output to a dialog window (both stdout and stderr). Useful on windows where GUI apps do not have a output streams.'))
     return parser
 
@@ -35,6 +40,53 @@ class DebugWindow(ConversionErrorDialog):
     def flush(self):
         pass
 
+class GarbageCollector(QObject):
+
+    '''
+    Disable automatic garbage collection and instead collect manually
+    every INTERVAL milliseconds.
+
+    This is done to ensure that garbage collection only happens in the GUI
+    thread, as otherwise Qt can crash.
+    '''
+
+    INTERVAL = 5000
+
+    def __init__(self, parent, debug=False):
+        QObject.__init__(self, parent)
+        self.debug = debug
+
+        self.timer = QTimer(self)
+        self.timer.timeout.connect(self.check)
+
+        self.threshold = gc.get_threshold()
+        gc.disable()
+        self.timer.start(self.INTERVAL)
+        #gc.set_debug(gc.DEBUG_SAVEALL)
+
+    def check(self):
+        #return self.debug_cycles()
+        l0, l1, l2 = gc.get_count()
+        if self.debug:
+            print ('gc_check called:', l0, l1, l2)
+        if l0 > self.threshold[0]:
+            num = gc.collect(0)
+            if self.debug:
+                print ('collecting gen 0, found:', num, 'unreachable')
+            if l1 > self.threshold[1]:
+                num = gc.collect(1)
+                if self.debug:
+                    print ('collecting gen 1, found:', num, 'unreachable')
+                if l2 > self.threshold[2]:
+                    num = gc.collect(2)
+                    if self.debug:
+                        print ('collecting gen 2, found:', num, 'unreachable')
+
+    def debug_cycles(self):
+        gc.collect()
+        for obj in gc.garbage:
+            print (obj, repr(obj), type(obj))
+
 class MainWindow(QMainWindow):
 
     ___menu_bar = None
@@ -64,19 +116,15 @@ class MainWindow(QMainWindow):
         quit_action.setMenuRole(QAction.QuitRole)
         return preferences_action, quit_action
 
-    def __init__(self, opts, parent=None):
+    def __init__(self, opts, parent=None, disable_automatic_gc=False):
         QMainWindow.__init__(self, parent)
-        app = QCoreApplication.instance()
+        if disable_automatic_gc:
-        if app is not None:
+            self._gc = GarbageCollector(self, debug=False)
-            self.connect(app, SIGNAL('unixSignal(int)'), self.unix_signal)
         if getattr(opts, 'redirect', False):
             self.__console_redirect = DebugWindow(self)
             sys.stdout = sys.stderr = self.__console_redirect
             self.__console_redirect.show()
 
-    def unix_signal(self, signal):
-        print 'Received signal:', repr(signal)
-
     def unhandled_exception(self, type, value, tb):
         if type == KeyboardInterrupt:
             self.keyboard_interrupt.emit()

src/calibre/gui2/pictureflow/pictureflow.cpp

@@ -439,7 +439,8 @@ void PictureFlowPrivate::setImages(FlowImages *images)
 	QObject::disconnect(slideImages, SIGNAL(dataChanged()), widget, SLOT(dataChanged()));
 	slideImages = images;
 	dataChanged();
-	QObject::connect(slideImages, SIGNAL(dataChanged()), widget, SLOT(dataChanged()));
+	QObject::connect(slideImages, SIGNAL(dataChanged()), widget, SLOT(dataChanged()),
+            Qt::QueuedConnection);
 }
 
 int PictureFlowPrivate::slideCount() const

src/calibre/gui2/preferences/create_custom_column.py

@@ -118,6 +118,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             else:
                 sb = 0
             self.composite_sort_by.setCurrentIndex(sb)
+            self.composite_make_category.setChecked(
+                                c['display'].get('make_category', False))
         elif ct == 'enumeration':
             self.enum_box.setText(','.join(c['display'].get('enum_values', [])))
         self.datatype_changed()
@@ -159,7 +161,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             col_type = None
         for x in ('box', 'default_label', 'label'):
             getattr(self, 'date_format_'+x).setVisible(col_type == 'datetime')
-        for x in ('box', 'default_label', 'label', 'sort_by', 'sort_by_label'):
+        for x in ('box', 'default_label', 'label', 'sort_by', 'sort_by_label',
+                  'make_category'):
             getattr(self, 'composite_'+x).setVisible(col_type == 'composite')
         for x in ('box', 'default_label', 'label'):
             getattr(self, 'enum_'+x).setVisible(col_type == 'enumeration')
@@ -222,7 +225,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
                     ' composite columns'))
             display_dict = {'composite_template':unicode(self.composite_box.text()).strip(),
                             'composite_sort': ['text', 'number', 'date', 'bool']
-                                    [self.composite_sort_by.currentIndex()]
+                                        [self.composite_sort_by.currentIndex()],
+                            'make_category': self.composite_make_category.isChecked(),
                         }
         elif col_type == 'enumeration':
             if not unicode(self.enum_box.text()).strip():

src/calibre/gui2/preferences/create_custom_column.ui

@@ -220,18 +220,18 @@ Everything else will show nothing.</string>
          </item>
         </layout>
        </item>
-       <item row="6" column="0">
-        <widget class="QLabel" name="composite_sort_by_label">
-         <property name="text">
-          <string>&amp;Sort/search column by</string>
-         </property>
-         <property name="buddy">
-          <cstring>composite_sort_by</cstring>
-         </property>
-        </widget>
-       </item>
        <item row="6" column="2">
-        <layout class="QHBoxLayout" name="horizontalLayout_4">
+        <layout class="QHBoxLayout" name="composite_layout">
+         <item>
+          <widget class="QLabel" name="composite_sort_by_label">
+           <property name="text">
+            <string>&amp;Sort/search column by</string>
+           </property>
+           <property name="buddy">
+            <cstring>composite_sort_by</cstring>
+           </property>
+          </widget>
+         </item>
          <item>
           <widget class="QComboBox" name="composite_sort_by">
            <property name="toolTip">
@@ -239,6 +239,16 @@ Everything else will show nothing.</string>
            </property>
           </widget>
          </item>
+         <item>
+          <widget class="QCheckBox" name="composite_make_category">
+           <property name="text">
+            <string>Show in tags browser</string>
+           </property>
+           <property name="toolTip">
+            <string>If checked, this column will appear in the tags browser as a category</string>
+           </property>
+          </widget>
+         </item>
          <item>
           <spacer name="horizontalSpacer_24">
            <property name="orientation">

src/calibre/gui2/preferences/look_feel.py

@@ -67,6 +67,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                     if db.field_metadata[k]['is_category'] and
                        db.field_metadata[k]['datatype'] in ['text', 'series', 'enumeration']])
         choices -= set(['authors', 'publisher', 'formats', 'news', 'identifiers'])
+        choices |= set(['search'])
         self.opt_categories_using_hierarchy.update_items_cache(choices)
         r('categories_using_hierarchy', db.prefs, setting=CommaSeparatedList,
           choices=sorted(list(choices), key=sort_key))

src/calibre/gui2/preferences/toolbar.py

@@ -55,6 +55,10 @@ class BaseModel(QAbstractListModel):
                 text = _('Choose library')
             return QVariant(text)
         if role == Qt.DecorationRole:
+            if hasattr(self._data[row], 'qaction'):
+                icon = self._data[row].qaction.icon()
+                if not icon.isNull():
+                    return QVariant(icon)
             ic = action[1]
             if ic is None:
                 ic = 'blank.png'

src/calibre/gui2/search_box.py

@@ -453,8 +453,11 @@ class SavedSearchBoxMixin(object): # {{{
         d = SavedSearchEditor(self, search)
         d.exec_()
         if d.result() == d.Accepted:
-            self.saved_searches_changed()
+            self.do_rebuild_saved_searches()
-            self.saved_search.clear()
+
+    def do_rebuild_saved_searches(self):
+        self.saved_searches_changed()
+        self.saved_search.clear()
 
     # }}}
 

src/calibre/gui2/shortcuts.py

@@ -71,7 +71,7 @@ class Customize(QFrame, Ui_Frame):
         button = getattr(self, 'button%d'%which)
         font = QFont()
         button.setFont(font)
-        sequence = QKeySequence(code|int(ev.modifiers()))
+        sequence = QKeySequence(code|(int(ev.modifiers())&~Qt.KeypadModifier))
         button.setText(sequence.toString())
         self.capture = 0
         setattr(self, 'shortcut%d'%which, sequence)
@@ -195,7 +195,7 @@ class Shortcuts(QAbstractListModel):
     def get_match(self, event_or_sequence, ignore=tuple()):
         q = event_or_sequence
         if isinstance(q, QKeyEvent):
-            q = QKeySequence(q.key()|int(q.modifiers()))
+            q = QKeySequence(q.key()|(int(q.modifiers())&~Qt.KeypadModifier))
         for key in self.order:
             if key not in ignore:
                 for seq in self.get_sequences(key):

src/calibre/gui2/tag_view.py

@@ -81,6 +81,7 @@ class TagsView(QTreeView): # {{{
     add_subcategory         = pyqtSignal(object)
     tag_list_edit           = pyqtSignal(object, object)
     saved_search_edit       = pyqtSignal(object)
+    rebuild_saved_searches  = pyqtSignal()
     author_sort_edit        = pyqtSignal(object, object)
     tag_item_renamed        = pyqtSignal()
     search_item_renamed     = pyqtSignal()
@@ -111,6 +112,8 @@ class TagsView(QTreeView): # {{{
             self.collapse_model = gprefs['tags_browser_partition_method']
         self.search_icon = QIcon(I('search.png'))
         self.user_category_icon = QIcon(I('tb_folder.png'))
+        self.delete_icon = QIcon(I('list_remove.png'))
+        self.rename_icon = QIcon(I('edit-undo.png'))
 
     def set_pane_is_visible(self, to_what):
         pv = self.pane_is_visible
@@ -251,6 +254,10 @@ class TagsView(QTreeView): # {{{
             if action == 'delete_user_category':
                 self.delete_user_category.emit(key)
                 return
+            if action == 'delete_search':
+                saved_searches().delete(key)
+                self.rebuild_saved_searches.emit()
+                return
             if action == 'delete_item_from_user_category':
                 tag = index.tag
                 if len(index.children) > 0:
@@ -284,6 +291,14 @@ class TagsView(QTreeView): # {{{
             return
 
     def show_context_menu(self, point):
+        def display_name( tag):
+            if tag.category == 'search':
+                n = tag.name
+                if len(n) > 45:
+                    n = n[:45] + '...'
+                return "'" + n + "'"
+            return tag.name
+
         index = self.indexAt(point)
         self.context_menu = QMenu(self)
 
@@ -313,18 +328,19 @@ class TagsView(QTreeView): # {{{
                     # the possibility of renaming that item.
                     if tag.is_editable:
                         # Add the 'rename' items
-                        self.context_menu.addAction(_('Rename %s')%tag.name,
+                        self.context_menu.addAction(self.rename_icon,
+                                                    _('Rename %s')%display_name(tag),
                             partial(self.context_menu_handler, action='edit_item',
                                     index=index))
                         if key == 'authors':
-                            self.context_menu.addAction(_('Edit sort for %s')%tag.name,
+                            self.context_menu.addAction(_('Edit sort for %s')%display_name(tag),
                                     partial(self.context_menu_handler,
                                             action='edit_author_sort', index=tag.id))
 
                         # is_editable is also overloaded to mean 'can be added
                         # to a user category'
                         m = self.context_menu.addMenu(self.user_category_icon,
-                                        _('Add %s to user category')%tag.name)
+                                        _('Add %s to user category')%display_name(tag))
                         nt = self.model().category_node_tree
                         def add_node_tree(tree_dict, m, path):
                             p = path[:]
@@ -341,28 +357,37 @@ class TagsView(QTreeView): # {{{
                                     add_node_tree(tree_dict[k], tm, p)
                                 p.pop()
                         add_node_tree(nt, m, [])
-
+                    elif key == 'search':
+                        self.context_menu.addAction(self.rename_icon,
+                                                    _('Rename %s')%display_name(tag),
+                            partial(self.context_menu_handler, action='edit_item',
+                                    index=index))
+                        self.context_menu.addAction(self.delete_icon,
+                                _('Delete search %s')%display_name(tag),
+                                partial(self.context_menu_handler,
+                                        action='delete_search', key=tag.name))
                     if key.startswith('@') and not item.is_gst:
                         self.context_menu.addAction(self.user_category_icon,
-                                _('Remove %s from category %s')%(tag.name, item.py_name),
+                                _('Remove %s from category %s')%
+                                            (display_name(tag), item.py_name),
                                 partial(self.context_menu_handler,
                                         action='delete_item_from_user_category',
                                         key = key, index = tag_item))
                     # Add the search for value items. All leaf nodes are searchable
                     self.context_menu.addAction(self.search_icon,
-                            _('Search for %s')%tag.name,
+                            _('Search for %s')%display_name(tag),
                             partial(self.context_menu_handler, action='search',
                                     search_state=TAG_SEARCH_STATES['mark_plus'],
                                     index=index))
                     self.context_menu.addAction(self.search_icon,
-                            _('Search for everything but %s')%tag.name,
+                            _('Search for everything but %s')%display_name(tag),
                             partial(self.context_menu_handler, action='search',
                                     search_state=TAG_SEARCH_STATES['mark_minus'],
                                     index=index))
                     self.context_menu.addSeparator()
                 elif key.startswith('@') and not item.is_gst:
                     if item.can_be_edited:
-                        self.context_menu.addAction(self.user_category_icon,
+                        self.context_menu.addAction(self.rename_icon,
                             _('Rename %s')%item.py_name,
                             partial(self.context_menu_handler, action='edit_item',
                                     index=index))
@@ -370,7 +395,7 @@ class TagsView(QTreeView): # {{{
                             _('Add sub-category to %s')%item.py_name,
                             partial(self.context_menu_handler,
                                     action='add_subcategory', key=key))
-                    self.context_menu.addAction(self.user_category_icon,
+                    self.context_menu.addAction(self.delete_icon,
                             _('Delete user category %s')%item.py_name,
                             partial(self.context_menu_handler,
                                     action='delete_user_category', key=key))
@@ -485,9 +510,11 @@ class TagsView(QTreeView): # {{{
                 if hasattr(md, 'column_name'):
                     fm_src = self.db.metadata_for_field(md.column_name)
                     if md.column_name in ['authors', 'publisher', 'series'] or \
-                            (fm_src['is_custom'] and
+                            (fm_src['is_custom'] and (
-                            fm_src['datatype'] in ['series', 'text'] and
+                             (fm_src['datatype'] in ['series', 'text', 'enumeration'] and
-                            not fm_src['is_multiple']):
+                              not fm_src['is_multiple']) or
+                             (fm_src['datatype'] == 'composite' and
+                              fm_src['display'].get('make_category', False)))):
                         self.setDropIndicatorShown(True)
 
     def clear(self):
@@ -533,7 +560,9 @@ class TagsView(QTreeView): # {{{
             self.setModel(self._model)
         except:
             # The DB must be gone. Set the model to None and hope that someone
-            # will call set_database later. I don't know if this in fact works
+            # will call set_database later. I don't know if this in fact works.
+            # But perhaps a Bad Thing Happened, so print the exception
+            traceback.print_exc()
             self._model = None
             self.setModel(None)
     # }}}
@@ -678,7 +707,8 @@ class TagTreeItem(object): # {{{
                         break
                 elif self.tag.state == TAG_SEARCH_STATES['mark_plusplus'] or\
                         self.tag.state == TAG_SEARCH_STATES['mark_minusminus']:
-                    if self.tag.is_hierarchical and len(self.children):
+                    if self.tag.is_searchable and self.tag.is_hierarchical \
+                                and len(self.children):
                         break
                 else:
                     break
@@ -948,8 +978,11 @@ class TagsModel(QAbstractItemModel): # {{{
                 fm = self.db.metadata_for_field(node.tag.category)
                 if node.tag.category in \
                     ('tags', 'series', 'authors', 'rating', 'publisher') or \
-                    (fm['is_custom'] and \
+                    (fm['is_custom'] and (
-                        fm['datatype'] in ['text', 'rating', 'series']):
+                            fm['datatype'] in ['text', 'rating', 'series',
+                                               'enumeration'] or
+                                (fm['datatype'] == 'composite' and
+                                 fm['display'].get('make_category', False)))):
                     mime = 'application/calibre+from_library'
                     ids = list(map(int, str(md.data(mime)).split()))
                     self.handle_drop(node, ids)
@@ -959,9 +992,11 @@ class TagsModel(QAbstractItemModel): # {{{
                 if fm_dest['kind'] == 'user':
                     fm_src = self.db.metadata_for_field(md.column_name)
                     if md.column_name in ['authors', 'publisher', 'series'] or \
-                            (fm_src['is_custom'] and
+                            (fm_src['is_custom'] and (
-                             fm_src['datatype'] in ['series', 'text'] and
+                             (fm_src['datatype'] in ['series', 'text', 'enumeration'] and
-                             not fm_src['is_multiple']):
+                              not fm_src['is_multiple']))or
+                             (fm_src['datatype'] == 'composite' and
+                              fm_src['display'].get('make_category', False))):
                         mime = 'application/calibre+from_library'
                         ids = list(map(int, str(md.data(mime)).split()))
                         self.handle_user_category_drop(node, ids, md.column_name)
@@ -975,7 +1010,6 @@ class TagsModel(QAbstractItemModel): # {{{
             return
         fm_src = self.db.metadata_for_field(column)
         for id in ids:
-            vmap = {}
             label = fm_src['label']
             if not fm_src['is_custom']:
                 if label == 'authors':
@@ -991,19 +1025,21 @@ class TagsModel(QAbstractItemModel): # {{{
                     value = self.db.series(id, index_is_id=True)
             else:
                 items = self.db.get_custom_items_with_ids(label=label)
-                value = self.db.get_custom(id, label=label, index_is_id=True)
+                if fm_src['datatype'] != 'composite':
+                    value = self.db.get_custom(id, label=label, index_is_id=True)
+                else:
+                    value = self.db.get_property(id, loc=fm_src['rec_index'],
+                                                 index_is_id=True)
             if value is None:
                 return
             if not isinstance(value, list):
                 value = [value]
-            for v in items:
-                vmap[v[1]] = v[0]
             for val in value:
                 for (v, c, id) in category:
                     if v == val and c == column:
                         break
                 else:
-                    category.append([val, column, vmap[val]])
+                    category.append([val, column, 0])
             categories[on_node.category_key[1:]] = category
             self.db.prefs.set('user_categories', categories)
             self.tags_view.recount()
@@ -1258,19 +1294,22 @@ class TagsModel(QAbstractItemModel): # {{{
                                             if t.type != TagTreeItem.CATEGORY])
                         if (comp,tag.category) in child_map:
                             node_parent = child_map[(comp,tag.category)]
-                            node_parent.tag.is_hierarchical = True
+                            node_parent.tag.is_hierarchical = key != 'search'
                         else:
                             if i < len(components)-1:
                                 t = copy.copy(tag)
                                 t.original_name = '.'.join(components[:i+1])
-                                # This 'manufactured' intermediate node can
+                                if key != 'search':
-                                # be searched, but cannot be edited.
+                                    # This 'manufactured' intermediate node can
-                                t.is_editable = False
+                                    # be searched, but cannot be edited.
+                                    t.is_editable = False
+                                else:
+                                    t.is_searchable = t.is_editable = False
                             else:
                                 t = tag
                                 if not in_uc:
                                     t.original_name = t.name
-                            t.is_hierarchical = True
+                            t.is_hierarchical = key != 'search'
                             t.name = comp
                             self.beginInsertRows(category_index, 999999, 1)
                             node_parent = TagTreeItem(parent=node_parent, data=t,
@@ -1762,6 +1801,7 @@ class TagBrowserMixin(object): # {{{
         self.tags_view.add_subcategory.connect(self.do_add_subcategory)
         self.tags_view.add_item_to_user_cat.connect(self.do_add_item_to_user_cat)
         self.tags_view.saved_search_edit.connect(self.do_saved_search_edit)
+        self.tags_view.rebuild_saved_searches.connect(self.do_rebuild_saved_searches)
         self.tags_view.author_sort_edit.connect(self.do_author_sort_edit)
         self.tags_view.tag_item_renamed.connect(self.do_tag_item_renamed)
         self.tags_view.search_item_renamed.connect(self.saved_searches_changed)

src/calibre/gui2/ui.py

@@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en'
 
 '''The main GUI'''
 
-import collections, os, sys, textwrap, time
+import collections, os, sys, textwrap, time, gc
 from Queue import Queue, Empty
 from threading import Thread
 from PyQt4.Qt import Qt, SIGNAL, QTimer, QHelpEvent, QAction, \
@@ -97,7 +97,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
 
 
     def __init__(self, opts, parent=None, gui_debug=None):
-        MainWindow.__init__(self, opts, parent)
+        MainWindow.__init__(self, opts, parent=parent, disable_automatic_gc=True)
         self.opts = opts
         self.device_connected = None
         self.gui_debug = gui_debug
@@ -334,6 +334,9 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
                     raise
         self.device_manager.set_current_library_uuid(db.library_id)
 
+        # Collect cycles now
+        gc.collect()
+
         if show_gui and self.gui_debug is not None:
             info_dialog(self, _('Debug mode'), '<p>' +
                     _('You have started calibre in debug mode. After you '
@@ -435,6 +438,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         elif msg.startswith('refreshdb:'):
             self.library_view.model().refresh()
             self.library_view.model().research()
+            self.tags_view.recount()
         else:
             print msg
 
@@ -499,6 +503,9 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
             self.card_a_view.reset()
             self.card_b_view.reset()
         self.device_manager.set_current_library_uuid(db.library_id)
+        # Run a garbage collection now so that it does not freeze the
+        # interface later
+        gc.collect()
 
 
     def set_window_title(self):
@@ -685,6 +692,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
             pass
         time.sleep(2)
         self.hide_windows()
+        # Do not report any errors that happen after the shutdown
+        sys.excepthook = sys.__excepthook__
         return True
 
     def run_wizard(self, *args):

src/calibre/gui2/viewer/main.py

@@ -225,6 +225,12 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         self.action_quit.setShortcuts(qs)
         self.connect(self.action_quit, SIGNAL('triggered(bool)'),
                      lambda x:QApplication.instance().quit())
+        self.action_focus_search = QAction(self)
+        self.addAction(self.action_focus_search)
+        self.action_focus_search.setShortcuts([Qt.Key_Slash,
+            QKeySequence(QKeySequence.Find)])
+        self.action_focus_search.triggered.connect(lambda x:
+                self.search.setFocus(Qt.OtherFocusReason))
         self.action_copy.setDisabled(True)
         self.action_metadata.setCheckable(True)
         self.action_metadata.setShortcut(Qt.CTRL+Qt.Key_I)
@@ -293,6 +299,9 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         ca.setShortcut(QKeySequence.Copy)
         self.addAction(ca)
         self.open_history_menu = QMenu()
+        self.clear_recent_history_action = QAction(
+                _('Clear list of recently opened books'), self)
+        self.clear_recent_history_action.triggered.connect(self.clear_recent_history)
         self.build_recent_menu()
         self.action_open_ebook.setMenu(self.open_history_menu)
         self.open_history_menu.triggered[QAction].connect(self.open_recent)
@@ -301,11 +310,19 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
 
         self.restore_state()
 
+    def clear_recent_history(self, *args):
+        vprefs.set('viewer_open_history', [])
+        self.build_recent_menu()
+
     def build_recent_menu(self):
         m = self.open_history_menu
         m.clear()
+        recent = vprefs.get('viewer_open_history', [])
+        if recent:
+            m.addAction(self.clear_recent_history_action)
+            m.addSeparator()
         count = 0
-        for path in vprefs.get('viewer_open_history', []):
+        for path in recent:
             if count > 9:
                 break
             if os.path.exists(path):
@@ -494,12 +511,6 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         if self.view.search(text, backwards=backwards):
             self.scrolled(self.view.scroll_fraction)
 
-    def keyPressEvent(self, event):
-        if event.key() == Qt.Key_Slash:
-            self.search.setFocus(Qt.OtherFocusReason)
-        else:
-            return MainWindow.keyPressEvent(self, event)
-
     def internal_link_clicked(self, frac):
         self.history.add(self.pos.value())
 

src/calibre/gui2/wizard/send_email.py

@@ -92,7 +92,8 @@ class SendEmail(QWidget, Ui_Form):
         pa = self.preferred_to_address()
         to_set = pa is not None
         if self.set_email_settings(to_set):
-            if question_dialog(self, _('OK to proceed?'),
+            opts = smtp_prefs().parse()
+            if not opts.relay_password or question_dialog(self, _('OK to proceed?'),
                     _('This will display your email password on the screen'
                     '. Is it OK to proceed?'), show_copy_button=False):
                 TestEmail(pa, self).exec_()
@@ -204,19 +205,32 @@ class SendEmail(QWidget, Ui_Form):
         username = unicode(self.relay_username.text()).strip()
         password = unicode(self.relay_password.text()).strip()
         host = unicode(self.relay_host.text()).strip()
-        if host and not (username and password):
+        enc_method = ('TLS' if self.relay_tls.isChecked() else 'SSL'
-            error_dialog(self, _('Bad configuration'),
+                if self.relay_ssl.isChecked() else 'NONE')
-                         _('You must set the username and password for '
+        if host:
-                           'the mail server.')).exec_()
+            # Validate input
-            return False
+            if ((username and not password) or (not username and password)):
+                error_dialog(self, _('Bad configuration'),
+                            _('You must either set both the username <b>and</b> password for '
+                            'the mail server or no username and no password at all.')).exec_()
+                return False
+            if not username and not password and enc_method != 'NONE':
+                error_dialog(self, _('Bad configuration'),
+                            _('Please enter a username and password or set'
+                               ' encryption to None ')).exec_()
+                return False
+            if not (username and password) and not question_dialog(self,
+                    _('Are you sure?'),
+                _('No username and password set for mailserver. Most '
+                    ' mailservers need a username and password. Are you sure?')):
+                    return False
         conf = smtp_prefs()
         conf.set('from_', from_)
         conf.set('relay_host', host if host else None)
         conf.set('relay_port', self.relay_port.value())
         conf.set('relay_username', username if username else None)
         conf.set('relay_password', hexlify(password))
-        conf.set('encryption', 'TLS' if self.relay_tls.isChecked() else 'SSL'
+        conf.set('encryption', enc_method)
-                if self.relay_ssl.isChecked() else 'NONE')
         return True
 
 

src/calibre/library/caches.py

@@ -123,14 +123,22 @@ REGEXP_MATCH   = 2
 def _match(query, value, matchkind):
     if query.startswith('..'):
         query = query[1:]
-        prefix_match_ok = False
+        sq = query[1:]
+        internal_match_ok = True
     else:
-        prefix_match_ok = True
+        internal_match_ok = False
     for t in value:
         t = icu_lower(t)
         try:     ### ignore regexp exceptions, required because search-ahead tries before typing is finished
             if (matchkind == EQUALS_MATCH):
-                if prefix_match_ok and query[0] == '.':
+                if internal_match_ok:
+                    if query == t:
+                        return True
+                    comps = [c.strip() for c in t.split('.') if c.strip()]
+                    for comp in comps:
+                        if sq == comp:
+                            return True
+                elif query[0] == '.':
                     if t.startswith(query[1:]):
                         ql = len(query) - 1
                         if (len(t) == ql) or (t[ql:ql+1] == '.'):
@@ -575,6 +583,8 @@ class ResultCache(SearchQueryParser): # {{{
             candidates = self.universal_set()
         if len(candidates) == 0:
             return matches
+        if location not in self.all_search_locations:
+            return matches
 
         if len(location) > 2 and location.startswith('@') and \
                     location[1:] in self.db_prefs['grouped_search_terms']:

src/calibre/library/check_library.py

@@ -27,7 +27,7 @@ CHECKS = [('invalid_titles',    _('Invalid titles'), True, False),
           ('extra_titles',      _('Extra titles'), True, False),
           ('invalid_authors',   _('Invalid authors'), True, False),
           ('extra_authors',     _('Extra authors'), True, False),
-          ('missing_formats',   _('Missing book formats'), False, False),
+          ('missing_formats',   _('Missing book formats'), False, True),
           ('extra_formats',     _('Extra book formats'), True, False),
           ('extra_files',       _('Unknown files in books'), True, False),
           ('missing_covers',    _('Missing covers files'), False, True),

src/calibre/library/database2.py

@@ -56,7 +56,7 @@ class Tag(object):
         self.is_hierarchical = False
         self.is_editable = is_editable
         self.is_searchable = is_searchable
-        self.id_set = id_set
+        self.id_set = id_set if id_set is not None else set([])
         self.avg_rating = avg/2.0 if avg is not None else 0
         self.sort = sort
         if self.avg_rating > 0:
@@ -1154,15 +1154,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if notify:
             self.notify('delete', [id])
 
-    def remove_format(self, index, format, index_is_id=False, notify=True, commit=True):
+    def remove_format(self, index, format, index_is_id=False, notify=True,
+                      commit=True, db_only=False):
         id = index if index_is_id else self.id(index)
         name = self.conn.get('SELECT name FROM data WHERE book=? AND format=?', (id, format), all=False)
         if name:
-            path = self.format_abspath(id, format, index_is_id=True)
+            if not db_only:
-            try:
+                try:
-                delete_file(path)
+                    path = self.format_abspath(id, format, index_is_id=True)
-            except:
+                    if path:
-                traceback.print_exc()
+                        delete_file(path)
+                except:
+                    traceback.print_exc()
             self.conn.execute('DELETE FROM data WHERE book=? AND format=?', (id, format.upper()))
             if commit:
                 self.conn.commit()
@@ -1207,6 +1210,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             return ans
 
         field = self.field_metadata[category]
+        if field['datatype'] == 'composite':
+            dex = field['rec_index']
+            for book in self.data.iterall():
+                if book[dex] == id_:
+                    ans.add(book[0])
+            return ans
+
         ans = self.conn.get(
                 'SELECT book FROM books_{tn}_link WHERE {col}=?'.format(
                     tn=field['table'], col=field['link_column']), (id_,))
@@ -1278,7 +1288,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
         # First, build the maps. We need a category->items map and an
         # item -> (item_id, sort_val) map to use in the books loop
-        for category in tb_cats.keys():
+        for category in tb_cats.iterkeys():
             cat = tb_cats[category]
             if not cat['is_category'] or cat['kind'] in ['user', 'search'] \
                     or category in ['news', 'formats'] or cat.get('is_csp',
@@ -1321,8 +1331,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             tcategories[category] = {}
             # create a list of category/field_index for the books scan to use.
             # This saves iterating through field_metadata for each book
-            md.append((category, cat['rec_index'], cat['is_multiple']))
+            md.append((category, cat['rec_index'], cat['is_multiple'], False))
 
+        for category in tb_cats.iterkeys():
+            cat = tb_cats[category]
+            if cat['datatype'] == 'composite' and \
+                                cat['display'].get('make_category', False):
+                tcategories[category] = {}
+                md.append((category, cat['rec_index'], cat['is_multiple'],
+                           cat['datatype'] == 'composite'))
         #print 'end phase "collection":', time.clock() - last, 'seconds'
         #last = time.clock()
 
@@ -1336,11 +1353,22 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 continue
             rating = book[rating_dex]
             # We kept track of all possible category field_map positions above
-            for (cat, dex, mult) in md:
+            for (cat, dex, mult, is_comp) in md:
-                if book[dex] is None:
+                if not book[dex]:
                     continue
                 if not mult:
                     val = book[dex]
+                    if is_comp:
+                        item = tcategories[cat].get(val, None)
+                        if not item:
+                            item = tag_class(val, val)
+                            tcategories[cat][val] = item
+                        item.c += 1
+                        item.id = val
+                        if rating > 0:
+                            item.rt += rating
+                            item.rc += 1
+                        continue
                     try:
                         (item_id, sort_val) = tids[cat][val] # let exceptions fly
                         item = tcategories[cat].get(val, None)
@@ -1402,7 +1430,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         # and building the Tag instances.
         categories = {}
         tag_class = Tag
-        for category in tb_cats.keys():
+        for category in tb_cats.iterkeys():
             if category not in tcategories:
                 continue
             cat = tb_cats[category]
@@ -1690,10 +1718,20 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.notify('metadata', [id])
         return books_to_refresh
 
-    def set_metadata(self, id, mi, ignore_errors=False,
+    def set_metadata(self, id, mi, ignore_errors=False, set_title=True,
-                     set_title=True, set_authors=True, commit=True):
+                     set_authors=True, commit=True, force_changes=False):
         '''
         Set metadata for the book `id` from the `Metadata` object `mi`
+
+        Setting force_changes=True will force set_metadata to update fields even
+        if mi contains empty values. In this case, 'None' is distinguished from
+        'empty'. If mi.XXX is None, the XXX is not replaced, otherwise it is.
+        The tags, identifiers, and cover attributes are special cases. Tags and
+        identifiers cannot be set to None so then will always be replaced if
+        force_changes is true. You must ensure that mi contains the values you
+        want the book to have. Covers are always changed if a new cover is
+        provided, but are never deleted. Also note that force_changes has no
+        effect on setting title or authors.
         '''
         if callable(getattr(mi, 'to_book_metadata', None)):
             # Handle code passing in a OPF object instead of a Metadata object
@@ -1707,6 +1745,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                     traceback.print_exc()
                 else:
                     raise
+
+        def should_replace_field(attr):
+            return (force_changes and (mi.get(attr, None) is not None)) or \
+                    not mi.is_null(attr)
+
         path_changed = False
         if set_title and mi.title:
             self._set_title(id, mi.title)
@@ -1721,16 +1764,21 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             path_changed = True
         if path_changed:
             self.set_path(id, index_is_id=True)
-        if mi.author_sort:
+
+        if should_replace_field('author_sort'):
             doit(self.set_author_sort, id, mi.author_sort, notify=False,
                     commit=False)
-        if mi.publisher:
+        if should_replace_field('publisher'):
             doit(self.set_publisher, id, mi.publisher, notify=False,
                     commit=False)
-        if mi.rating:
+
+        # Setting rating to zero is acceptable.
+        if mi.rating is not None:
             doit(self.set_rating, id, mi.rating, notify=False, commit=False)
-        if mi.series:
+        if should_replace_field('series'):
             doit(self.set_series, id, mi.series, notify=False, commit=False)
+
+        # force_changes has no effect on cover manipulation
         if mi.cover_data[1] is not None:
             doit(self.set_cover, id, mi.cover_data[1], commit=False)
         elif mi.cover is not None:
@@ -1739,21 +1787,30 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                     raw = f.read()
                 if raw:
                     doit(self.set_cover, id, raw, commit=False)
-        if mi.tags:
+
+        # if force_changes is true, tags are always replaced because the
+        # attribute cannot be set to None.
+        if should_replace_field('tags'):
             doit(self.set_tags, id, mi.tags, notify=False, commit=False)
-        if mi.comments:
+
+        if should_replace_field('comments'):
             doit(self.set_comment, id, mi.comments, notify=False, commit=False)
-        if mi.series_index:
+
+        # Setting series_index to zero is acceptable
+        if mi.series_index is not None:
             doit(self.set_series_index, id, mi.series_index, notify=False,
                     commit=False)
-        if mi.pubdate:
+        if should_replace_field('pubdate'):
             doit(self.set_pubdate, id, mi.pubdate, notify=False, commit=False)
         if getattr(mi, 'timestamp', None) is not None:
             doit(self.set_timestamp, id, mi.timestamp, notify=False,
                     commit=False)
 
+        # identifiers will always be replaced if force_changes is True
         mi_idents = mi.get_identifiers()
-        if mi_idents:
+        if force_changes:
+            self.set_identifiers(id, mi_idents, notify=False, commit=False)
+        elif mi_idents:
             identifiers = self.get_identifiers(id, index_is_id=True)
             for key, val in mi_idents.iteritems():
                 if val and val.strip(): # Don't delete an existing identifier
@@ -1765,10 +1822,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         for key in user_mi.iterkeys():
             if key in self.field_metadata and \
                     user_mi[key]['datatype'] == self.field_metadata[key]['datatype']:
-                doit(self.set_custom, id,
+                val = mi.get(key, None)
-                     val=mi.get(key),
+                if force_changes or val is not None:
-                     extra=mi.get_extra(key),
+                    doit(self.set_custom, id, val=val, extra=mi.get_extra(key),
-                     label=user_mi[key]['label'], commit=False)
+                         label=user_mi[key]['label'], commit=False)
         if commit:
             self.conn.commit()
         self.notify('metadata', [id])
@@ -2358,6 +2415,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         @param tags: list of strings
         @param append: If True existing tags are not removed
         '''
+        if not tags:
+            tags = []
         if not append:
             self.conn.execute('DELETE FROM books_tags_link WHERE book=?', (id,))
             self.conn.execute('''DELETE FROM tags WHERE (SELECT COUNT(id)
@@ -2508,6 +2567,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             self.notify('metadata', [id])
 
     def set_rating(self, id, rating, notify=True, commit=True):
+        if not rating:
+            rating = 0
         rating = int(rating)
         self.conn.execute('DELETE FROM books_ratings_link WHERE book=?',(id,))
         rat = self.conn.get('SELECT id FROM ratings WHERE rating=?', (rating,), all=False)
@@ -2522,7 +2583,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
     def set_comment(self, id, text, notify=True, commit=True):
         self.conn.execute('DELETE FROM comments WHERE book=?', (id,))
-        self.conn.execute('INSERT INTO comments(book,text) VALUES (?,?)', (id, text))
+        if text:
+            self.conn.execute('INSERT INTO comments(book,text) VALUES (?,?)', (id, text))
+        else:
+            text = ''
         if commit:
             self.conn.commit()
         self.data.set(id, self.FIELD_MAP['comments'], text, row_is_id=True)
@@ -2531,6 +2595,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             self.notify('metadata', [id])
 
     def set_author_sort(self, id, sort, notify=True, commit=True):
+        if not sort:
+            sort = ''
         self.conn.execute('UPDATE books SET author_sort=? WHERE id=?', (sort, id))
         self.dirtied([id], commit=False)
         if commit:
@@ -2602,6 +2668,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
     def set_identifiers(self, id_, identifiers, notify=True, commit=True):
         cleaned = {}
+        if not identifiers:
+            identifiers = {}
         for typ, val in identifiers.iteritems():
             typ, val = self._clean_identifier(typ, val)
             if val:

src/calibre/library/server/browse.py

@@ -12,7 +12,7 @@ import cherrypy
 
 from calibre.constants import filesystem_encoding
 from calibre import isbytestring, force_unicode, fit_image, \
-        prepare_string_for_xml as xml
+        prepare_string_for_xml
 from calibre.utils.ordered_dict import OrderedDict
 from calibre.utils.filenames import ascii_filename
 from calibre.utils.config import prefs, tweaks
@@ -23,6 +23,10 @@ from calibre.library.server import custom_fields_to_display
 from calibre.library.field_metadata import category_icon_map
 from calibre.library.server.utils import quote, unquote
 
+def xml(*args, **kwargs):
+    ans = prepare_string_for_xml(*args, **kwargs)
+    return ans.replace(''', ''')
+
 def render_book_list(ids, prefix, suffix=''): # {{{
     pages = []
     num = len(ids)
@@ -626,6 +630,8 @@ class BrowseServer(object):
             elif category == 'allbooks':
                 ids = all_ids
             else:
+                if fm.get(category, {'datatype':None})['datatype'] == 'composite':
+                    cid = cid.decode('utf-8')
                 q = category
                 if q == 'news':
                     q = 'tags'

src/calibre/manual/faq.rst

@@ -508,9 +508,9 @@ You have two choices:
  1. Create a patch by hacking on |app| and send it to me for review and inclusion. See `Development <http://calibre-ebook.com/get-involved>`_.
  2. `Open a ticket <http://bugs.calibre-ebook.com/newticket>`_ (you have to register and login first). Remember that |app| development is done by volunteers, so if you get no response to your feature request, it means no one feels like implementing it.
 
-Can I include |app| on a CD to be distributed with my product/magazine?
+How is |app| licensed?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-|app| is licensed under the GNU General Public License v3 (an open source license). This means that you are free to redistribute |app| as long as you make the source code available. So if you want to put |app| on a CD with your product, you must also put the |app| source code on the CD. The source code is available for download `from googlecode <http://code.google.com/p/calibre-ebook/downloads/list>`_.
+|app| is licensed under the GNU General Public License v3 (an open source license). This means that you are free to redistribute |app| as long as you make the source code available. So if you want to put |app| on a CD with your product, you must also put the |app| source code on the CD. The source code is available for download `from googlecode <http://code.google.com/p/calibre-ebook/downloads/list>`_. You are free to use the results of conversions from |app| however you want. You cannot use code, libraries from |app| in your software without maing your software open source. For details, see `The GNU GPL v3 http://www.gnu.org/licenses/gpl.html`_.
 
 How do I run calibre from my USB stick?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

src/calibre/manual/sub_groups.rst

@@ -70,7 +70,7 @@ Then after restarting |app|, you must tell |app| that the column is to be treate
 
 At the point there are no genres in the column. We are left with the last step: how to apply a genre to a book. A genre does not exist in |app| until it appears on at least one book. To learn how to apply a genre for the first time, we must go into some detail about what a genre looks like in the metadata for a book.
 
-A hierarchy of 'things' is built by creating an item consisting of phrases separated by periods. Continuing the genre example, these items would "History.Military", "Mysteries.Vampire", "Science Fiction.Space Opera", etc. Thus to create a new genre, you pick a book that should have that genre, edit its metadata, and enter the new genre into the column you created. Continuing our example, if you want to assign a new genre "Comics" with a sub-genre "Superheros" to a book, you would 'edit metadata' for that (comic) book, choose the Custom metadata tab, and then enter "Comics.Superheros" as shown in the following (ignore the other custom columns):
+A hierarchy of 'things' is built by creating an item consisting of phrases separated by periods. Continuing the genre example, these items would "History.Military", "Mysteries.Vampire", "Science Fiction.Space Opera", etc. Thus to create a new genre, you pick a book that should have that genre, edit its metadata, and enter the new genre into the column you created. Continuing our example, if you want to assign a new genre "Comics" with a sub-genre "Superheroes" to a book, you would 'edit metadata' for that (comic) book, choose the Custom metadata tab, and then enter "Comics.Superheroes" as shown in the following (ignore the other custom columns):
 
 .. image:: images/sg_genre.jpg
     :align: center
@@ -105,3 +105,13 @@ After creating the saved search, you can use it as a restriction.
 .. image:: images/sg_restrict2.jpg
     :align: center
 
+ Useful Template Functions
+ -------------------------
+ 
+ You might want to use the genre information in a template, such as with save to disk or send to device. The question might then be "How do I get the outermost genre name or names?" An |app| template function, subitems, is provided to make doing this easier.
+ 
+ For example, assume you want to add the outermost genre level to the save-to-disk template to make genre folders, as in "History/The Gathering Storm - Churchill, Winston". To do this, you must extract the first level of the hierarchy and add it to the front along with a slash to indicate that it should make a folder. The template below accomplishes this::
+ 
+    {#genre:subitems(0,1)||/}{title} - {authors}
+
+See :ref:`The |app| template language <templatelangcalibre>` for more information templates and the subitem function.

src/calibre/manual/template_lang.rst

@@ -129,7 +129,7 @@ The functions available are:
     * ``switch(pattern, value, pattern, value, ..., else_value)`` -- for each ``pattern, value`` pair, checks if the field matches the regular expression ``pattern`` and if so, returns that ``value``. If no ``pattern`` matches, then ``else_value`` is returned. You can have as many ``pattern, value`` pairs as you want.
     * ``lookup(pattern, field, pattern, field, ..., else_field)`` -- like switch, except the arguments are field (metadata) names, not text. The value of the appropriate field will be fetched and used. Note that because composite columns are fields, you can use this function in one composite field to use the value of some other composite field. This is extremely useful when constructing variable save paths (more later).
     * ``select(key)`` -- interpret the field as a comma-separated list of items, with the items being of the form "id:value". Find the pair with the id equal to key, and return the corresponding value. This function is particularly useful for extracting a value such as an isbn from the set of identifiers for a book.
-    * ``subitems(val, start_index, end_index)`` -- This function is used to break apart lists of tag-like hierarchical items such as genres. It interprets the value as a comma-separated list of tag-like items, where each item is a period-separated list. Returns a new list made by first finding all the period-separated tag-like items, then for each such item extracting the `start_index` th to the `end_index` th components, then combining the results back together. The first component in a period-separated list has an index of zero. If an index is negative, then it counts from the end of the list. As a special case, an end_index of zero is assumed to be the length of the list. Examples::
+    * ``subitems(val, start_index, end_index)`` -- This function is used to break apart lists of tag-like hierarchical items such as genres. It interprets the value as a comma-separated list of tag-like items, where each item is a period-separated list. Returns a new list made by first finding all the period-separated tag-like items, then for each such item extracting the components from `start_index` to `end_index`, then combining the results back together. The first component in a period-separated list has an index of zero. If an index is negative, then it counts from the end of the list. As a special case, an end_index of zero is assumed to be the length of the list. Examples::
 
         Assuming a #genre column containing "A.B.C":    
             {#genre:subitems(0,1)} returns "A"
@@ -139,7 +139,7 @@ The functions available are:
             {#genre:subitems(0,1)} returns "A, D"
             {#genre:subitems(0,2)} returns "A.B, D.E"
 
-    * ``sublist(val, start_index, end_index, separator)`` -- interpret the value as a list of items separated by `separator`, returning a new list made from the `start_index` th to the `end_index` th item. The first item is number zero. If an index is negative, then it counts from the end of the list. As a special case, an end_index of zero is assumed to be the length of the list. Examples assuming that the tags column (which is comma-separated) contains "A, B ,C"::
+    * ``sublist(val, start_index, end_index, separator)`` -- interpret the value as a list of items separated by `separator`, returning a new list made from the items from `start_index`to `end_index`. The first item is number zero. If an index is negative, then it counts from the end of the list. As a special case, an end_index of zero is assumed to be the length of the list. Examples assuming that the tags column (which is comma-separated) contains "A, B ,C"::
     
         {tags:sublist(0,1,\,)} returns "A"
         {tags:sublist(-1,0,\,)} returns "C"