Sync to trunk.

Changelog.yaml

@@ -19,6 +19,10 @@
 #  new recipes:
 #    - title: 
 
+#    - title: "Launch of a new website that catalogues DRM free books. http://drmfree.calibre-ebook.com"
+#      description: "A growing catalogue of DRM free books. Books that you actually own after buying instead of renting."
+#      type: major
+
 - version: 0.7.47
   date: 2011-02-25
 
@@ -88,8 +92,8 @@
     - title: "Various Romanian news sources"
       author: Silviu Coatara
 
-    - title: "Osnews.pl and SwiatKindle"
-      author: Mori
+    - title: "Osnews.pl and SwiatCzytnikow"
+      author: Tomasz Dlugosz
 
     - title: "Roger Ebert Journal"
       author: Shane Erstad

resources/recipes/bucataras.recipe

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = u'2011, Silviu Cotoar\u0103'
+'''
+bucataras.ro
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Bucataras(BasicNewsRecipe):
+    title                 = u'Bucataras'
+    __author__            = u'Silviu Cotoar\u0103'
+    description           = ''
+    publisher             = 'Bucataras'
+    oldest_article        = 5
+    language              = 'ro'
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    category              = 'Ziare,Bucatarie,Retete'
+    encoding              = 'utf-8'
+    cover_url             = 'http://www.bucataras.ro/templates/default/images/pink/logo.jpg'
+
+    conversion_options = {
+                             'comments'   : description
+                            ,'tags'       : category
+                            ,'language'   : language
+                            ,'publisher'  : publisher
+                         }
+
+    keep_only_tags = [
+              dict(name='h1', attrs={'class':'titlu'})
+            , dict(name='div', attrs={'class':'contentL'})
+            , dict(name='div', attrs={'class':'contentBottom'})
+
+                     ]
+
+    remove_tags = [
+             dict(name='div', attrs={'class':['sociale']})
+           , dict(name='div', attrs={'class':['contentR']})
+           , dict(name='a', attrs={'target':['_self']})
+           , dict(name='div', attrs={'class':['comentarii']})
+                  ]
+
+    remove_tags_after = [
+                            dict(name='div', attrs={'class':['comentarii']})
+                        ]
+
+    feeds          = [
+                        (u'Feeds', u'http://www.bucataras.ro/rss/retete/')
+                     ]
+
+    def preprocess_html(self, soup):
+        return self.adeify_images(soup)

resources/recipes/buffalo_news.recipe

@@ -0,0 +1,58 @@
+__license__   = 'GPL v3'
+__author__    = 'Todd Chapman'
+__copyright__ = 'Todd Chapman'
+__version__     = 'v0.2'
+__date__        = '2 March 2011'
+
+'''
+http://www.buffalonews.com/RSS/
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1298680852(BasicNewsRecipe):
+    title          = u'Buffalo News'
+    oldest_article = 2
+    language = 'en'
+    __author__ = 'ChappyOnIce'
+    max_articles_per_feed = 20
+    encoding = 'utf-8'
+    masthead_url = 'http://www.buffalonews.com/buffalonews/skins/buffalonews/images/masthead/the_buffalo_news_logo.png'
+    remove_javascript = True
+    extra_css = 'body {text-align: justify;}\n  \
+       p {text-indent: 20px;}'
+
+    keep_only_tags    = [
+                       dict(name='div', attrs={'class':['main-content-left']})
+                        ]
+
+    remove_tags = [
+                       dict(name='div', attrs={'id':['commentCount']}),
+       dict(name='div', attrs={'class':['story-list-links']})
+                        ]
+
+    remove_tags_after  = dict(name='div', attrs={'class':['body storyContent']})
+
+    feeds          = [(u'City of Buffalo', u'http://www.buffalonews.com/city/communities/buffalo/?widget=rssfeed&view=feed&contentId=77944'),
+         (u'Southern Erie County', u'http://www.buffalonews.com/city/communities/southern-erie/?widget=rssfeed&view=feed&contentId=77944'),
+         (u'Eastern Erie County', u'http://www.buffalonews.com/city/communities/eastern-erie/?widget=rssfeed&view=feed&contentId=77944'),
+         (u'Southern Tier', u'http://www.buffalonews.com/city/communities/southern-tier/?widget=rssfeed&view=feed&contentId=77944'),
+         (u'Niagara County', u'http://www.buffalonews.com/city/communities/niagara-county/?widget=rssfeed&view=feed&contentId=77944'),
+         (u'Business', u'http://www.buffalonews.com/business/?widget=rssfeed&view=feed&contentId=77944'),
+         (u'MoneySmart', u'http://www.buffalonews.com/business/moneysmart/?widget=rssfeed&view=feed&contentId=77944'),
+         (u'Bills & NFL', u'http://www.buffalonews.com/sports/bills-nfl/?widget=rssfeed&view=feed&contentId=77944'),
+         (u'Sabres & NHL', u'http://www.buffalonews.com/sports/sabres-nhl/?widget=rssfeed&view=feed&contentId=77944'),
+         (u'Bob DiCesare', u'http://www.buffalonews.com/sports/columns/bob-dicesare/?widget=rssfeed&view=feed&contentId=77944'),
+         (u'Bucky Gleason', u'http://www.buffalonews.com/sports/columns/bucky-gleason/?widget=rssfeed&view=feed&contentId=77944'),
+         (u'Mark Gaughan', u'http://www.buffalonews.com/sports/bills-nfl/inside-the-nfl/?widget=rssfeed&view=feed&contentId=77944'),
+         (u'Mike Harrington', u'http://www.buffalonews.com/sports/columns/mike-harrington/?widget=rssfeed&view=feed&contentId=77944'),
+         (u'Jerry Sullivan', u'http://www.buffalonews.com/sports/columns/jerry-sullivan/?widget=rssfeed&view=feed&contentId=77944'),
+         (u'Other Sports Columns', u'http://www.buffalonews.com/sports/columns/other-sports-columns/?widget=rssfeed&view=feed&contentId=77944'),
+         (u'Life', u'http://www.buffalonews.com/life/?widget=rssfeed&view=feed&contentId=77944'),
+         (u'Bruce Andriatch', u'http://www.buffalonews.com/city/columns/bruce-andriatch/?widget=rssfeed&view=feed&contentId=77944'),
+         (u'Donn Esmonde', u'http://www.buffalonews.com/city/columns/donn-esmonde/?widget=rssfeed&view=feed&contentId=77944'),
+         (u'Rod Watson', u'http://www.buffalonews.com/city/columns/rod-watson/?widget=rssfeed&view=feed&contentId=77944'),
+         (u'Entertainment', u'http://www.buffalonews.com/entertainment/?widget=rssfeed&view=feed&contentId=77944'),
+         (u'Off Main Street', u'http://www.buffalonews.com/city/columns/off-main-street/?widget=rssfeed&view=feed&contentId=77944'),
+         (u'Editorials', u'http://www.buffalonews.com/editorial-page/buffalo-news-editorials/?widget=rssfeed&view=feed&contentId=77944')
+         ]

resources/recipes/credit_slips.recipe

@@ -1,35 +1,44 @@
 #!/usr/bin/env  python
 __license__ = 'GPL 3'
-__copyright__ = 'zotzot'
+__copyright__ = 'zotzo'
 __docformat__ = 'restructuredtext en'
 
 from calibre.web.feeds.news import BasicNewsRecipe
 
 
 class CreditSlips(BasicNewsRecipe):
-    __license__ = 'GPL v3'
-    __author__ = 'zotzot'
     language = 'en'
-    version = 1
+    __author__ = 'zotzot'
+    version = 2
     title = u'Credit Slips.org'
     publisher = u'Bankr-L'
     category = u'Economic blog'
-    description = u'All things about credit.'
-    cover_url = 'http://bit.ly/hyZSTr'
-    oldest_article = 50
+    description = u'A discussion on credit and bankruptcy'
+    cover_url = 'http://bit.ly/eAKNCB'
+    oldest_article = 15
     max_articles_per_feed = 100
     use_embedded_content = True
+    no_stylesheets = True
+    remove_javascript = True
+
+    conversion_options =   {
+                               'comments': description,
+                               'tags': category,
+                               'language': 'en',
+                               'publisher': publisher,
+                           }
 
     feeds = [
-(u'Credit Slips', u'http://www.creditslips.org/creditslips/atom.xml')
-]
-    conversion_options = {
-'comments': description,
-'tags': category,
-'language': 'en',
-'publisher': publisher
-}
-    extra_css = '''
-                body{font-family:verdana,arial,helvetica,geneva,sans-serif;}
-                img {float: left; margin-right: 0.5em;}
-                '''
+               (u'Credit Slips', u'http://www.creditslips.org/creditslips/atom.xml')
+           ]
+
+    extra_css =    '''
+                       .author {font-family:Helvetica,sans-serif; font-weight:normal;font-size:small;}
+                       h1      {font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
+                       p       {font-family:Helvetica,Arial,sans-serif;font-size:small;}
+                       body    {font-family:Helvetica,Arial,sans-serif;font-size:small;}
+                   '''
+
+    def populate_article_metadata(self, article, soup, first):
+       h2 = soup.find('h2')
+       h2.replaceWith(h2.prettify() + '<p><em>Posted by ' + article.author + '</em></p>')

resources/recipes/dotpod.recipe

@@ -0,0 +1,27 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011-2011, Federico Escalada <fedeescalada at gmail.com>'
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Dotpod(BasicNewsRecipe):
+    __author__ = 'Federico Escalada'
+    description = 'Tecnologia y Comunicacion Audiovisual'
+    encoding = 'utf-8'
+    language = 'es'
+    max_articles_per_feed = 100
+    no_stylesheets = True
+    oldest_article = 7
+    publication_type = 'blog'
+    title = 'Dotpod'
+    authors = 'Federico Picone'
+
+    conversion_options = {
+                             'authors'         : authors
+                            ,'comments'        : description
+                            ,'language'        : language
+                         }
+
+    feeds = [('Dotpod', 'http://www.dotpod.com.ar/feed/')]
+
+    remove_tags = [dict(name='div', attrs={'class':'feedflare'})]
+

resources/recipes/epl_talk.recipe

@@ -1,6 +1,6 @@
 #!/usr/bin/env  python
 __license__ = 'GPL 3'
-__copyright__ = 'zotzot'
+__copyright__ = 'zotzo'
 __docformat__ = 'restructuredtext en'
 '''
 http://www.epltalk.com
@@ -9,10 +9,9 @@ from calibre.web.feeds.news import BasicNewsRecipe
 
 
 class EPLTalkRecipe(BasicNewsRecipe):
-    __license__ = 'GPL v3'
-    __author__ = u'The Gaffer'
     language = 'en'
-    version = 1
+    version = 2
+    __author__ = 'rylsfan'
 
     title = u'EPL Talk'
     publisher = u'The Gaffer'
@@ -21,17 +20,40 @@ class EPLTalkRecipe(BasicNewsRecipe):
     description = u'News and Analysis from the English Premier League'
     cover_url = 'http://bit.ly/hJxZPu'
 
-    oldest_article = 45
-    max_articles_per_feed = 150
+    oldest_article = 3
+    max_articles_per_feed = 100
     use_embedded_content = True
     remove_javascript = True
     encoding = 'utf8'
 
-    remove_tags_after = [dict(name='div', attrs={'class':'pd-rating'})]
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
 
-    feeds = [(u'EPL Talk', u'http://feeds.feedburner.com/EPLTalk')]
+    remove_tags =  [
+                       {'class': 'feedflare'},
+                       {'class': 'tweetmeme_button'},
+                       {'class': 'eplrelated'},
+                       {'p': 'Related posts:<ol>'},
+                   ]
+
+    def preprocess_html(self, soup):
+       return self.adeify_images(soup)
+
+    feeds =[
+               (u'EPL Talk', u'http://feeds.feedburner.com/EPLTalk'),
+               (u'MLS Talk', u'http://feeds.feedburner.com/majorleaguesoccertalksite'),
+               #(),
+               #(),
+               #(),
+           ]
 
     extra_css = '''
-                body{font-family:verdana,arial,helvetica,geneva,sans-serif;}
-                img {float: left; margin-right: 0.5em;}
-                '''
+           h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
+           h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
+           p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
+           body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
+               '''

resources/recipes/helsingin_sanomat.recipe

@@ -0,0 +1,31 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1298137661(BasicNewsRecipe):
+  title          = u'Helsingin Sanomat'
+  __author__ = 'oneillpt'
+  language = 'fi'
+  oldest_article = 7
+  max_articles_per_feed = 100
+  no_stylesheets = True
+  remove_javascript     = True
+  conversion_options = {
+                         'linearize_tables' : True
+                       }
+  remove_tags = [
+                  dict(name='a', attrs={'id':'articleCommentUrl'}),
+                  dict(name='p', attrs={'class':'newsSummary'}),
+                  dict(name='div', attrs={'class':'headerTools'})
+                ]
+
+  feeds          = [(u'Uutiset - HS.fi', u'http://www.hs.fi/uutiset/rss/'), (u'Politiikka - HS.fi', u'http://www.hs.fi/politiikka/rss/'),
+                     (u'Ulkomaat - HS.fi', u'http://www.hs.fi/ulkomaat/rss/'), (u'Kulttuuri - HS.fi', u'http://www.hs.fi/kulttuuri/rss/'),
+                     (u'Kirjat - HS.fi', u'http://www.hs.fi/kulttuuri/kirjat/rss/'), (u'Elokuvat - HS.fi', u'http://www.hs.fi/kulttuuri/elokuvat/rss/')
+                     ]
+
+  def print_version(self, url):
+    j = url.rfind("/")
+    s = url[j:]
+    i = s.rfind("?ref=rss")
+    if i > 0:
+      s = s[:i]
+    return "http://www.hs.fi/tulosta" + s

resources/recipes/historiaro.recipe

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = u'2011, Silviu Cotoar\u0103'
+'''
+historia.ro
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class HistoriaRo(BasicNewsRecipe):
+    title                        = u'Historia'
+    __author__            = u'Silviu Cotoar\u0103'
+    description             = ''
+    publisher                = 'Historia'
+    oldest_article        = 5
+    language              = 'ro'
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    category              = 'Ziare,Reviste,Istorie'
+    encoding              = 'utf-8'
+    cover_url   = 'http://www.historia.ro/sites/all/themes/historia/images/historia.png'
+
+    conversion_options = {
+                             'comments'   : description
+                            ,'tags'       : category
+                            ,'language'   : language
+                 ,'publisher'  : publisher
+                         }
+
+    keep_only_tags = [
+              dict(name='div', attrs={'class':'c_antet_title'})
+            , dict(name='a', attrs={'class':'overlaybox'})
+            , dict(name='div', attrs={'class':'art_content'})
+                              ]
+
+    remove_tags = [
+             dict(name='div', attrs={'class':['fl_left']})
+           , dict(name='div', attrs={'id':['article_toolbar']})
+           , dict(name='div', attrs={'class':['zoom_cont']})
+                          ]
+
+
+    feeds          = [
+                        (u'Feeds', u'http://www.historia.ro/rss.xml')
+                     ]
+
+    def preprocess_html(self, soup):
+        return self.adeify_images(soup)

resources/recipes/lwn_weekly.recipe

@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Davide Cavalca <davide125 at tiscali.it>'
+'''
+lwn.net
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+import re
+
+class WeeklyLWN(BasicNewsRecipe):
+    title = 'LWN.net Weekly Edition'
+    description = 'Weekly summary of what has happened in the free software world.'
+    __author__ = 'Davide Cavalca'
+    language = 'en'
+
+    cover_url = 'http://lwn.net/images/lcorner.png'
+    #masthead_url = 'http://lwn.net/images/lcorner.png'
+    publication_type = 'magazine'
+
+    remove_tags_before = dict(attrs={'class':'PageHeadline'})
+    remove_tags_after = dict(attrs={'class':'ArticleText'})
+    remove_tags = [dict(name=['h2', 'form'])]
+
+    conversion_options = { 'linearize_tables' : True }
+
+    oldest_article = 7.0
+    needs_subscription = 'optional'
+
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser()
+        if self.username is not None and self.password is not None:
+            br.open('https://lwn.net/login')
+            br.select_form(name='loginform')
+            br['Username'] = self.username
+            br['Password'] = self.password
+            br.submit()
+        return br
+
+    def parse_index(self):
+        if self.username is not None and self.password is not None:
+            index_url = 'http://lwn.net/current/bigpage'
+        else:
+            index_url = 'http://lwn.net/free/bigpage'
+        soup = self.index_to_soup(index_url)
+        body = soup.body
+
+        articles = {}
+        ans = []
+        url_re = re.compile('^http://lwn.net/Articles/')
+
+        while True:
+            tag_title = body.findNext(name='p', attrs={'class':'SummaryHL'})
+            if tag_title == None:
+                break
+
+            tag_section = tag_title.findPrevious(name='p', attrs={'class':'Cat1HL'})
+            if tag_section == None:
+                section = 'Front Page'
+            else:
+                section = tag_section.string
+
+            tag_section2 = tag_title.findPrevious(name='p', attrs={'class':'Cat2HL'})
+            if tag_section2 != None:
+                if tag_section2.findPrevious(name='p', attrs={'class':'Cat1HL'}) == tag_section:
+                    section = "%s: %s" %(section, tag_section2.string)
+
+            if section not in articles.keys():
+                articles[section] = []
+            if section not in ans:
+                ans.append(section)
+
+            body = tag_title
+            while True:
+                tag_url = body.findNext(name='a', attrs={'href':url_re})
+                if tag_url == None:
+                    break
+                body = tag_url
+                if tag_url.string == None:
+                    continue
+                elif tag_url.string == 'Full Story':
+                    break
+                elif tag_url.string.startswith('Comments ('):
+                    break
+                else:
+                    continue
+
+            if tag_url == None:
+                break
+
+            article = dict(
+                title=tag_title.string,
+                url=tag_url['href'].split('#')[0],
+                description='', content='', date='')
+            articles[section].append(article)
+
+        ans = [(key, articles[key]) for key in ans if articles.has_key(key)]
+        if not ans:
+            raise Exception('Could not find any articles.')
+
+        return ans
+
+# vim: expandtab:ts=4:sw=4

resources/recipes/mac_world.recipe

@@ -11,7 +11,6 @@ http://www.macworld.co.uk/
 '''
 
 from calibre.web.feeds.news import BasicNewsRecipe
-from calibre.ptempfile import PersistentTemporaryFile
 
 temp_files = []
 articles_are_obfuscated = True
@@ -36,26 +35,17 @@ class macWorld(BasicNewsRecipe):
     remove_javascript     = True
     no_stylesheets        = True
 
-    def get_obfuscated_article(self, url):
-        br = self.get_browser()
-        br.open(url+'&print')
-
-        response = br.follow_link(url, nr = 0)
-        html = response.read()
-
-        self.temp_files.append(PersistentTemporaryFile('_fa.html'))
-        self.temp_files[-1].write(html)
-        self.temp_files[-1].close()
-        return self.temp_files[-1].name
-
     keep_only_tags     = [
-                            dict(name='div', attrs={'id':'article'})
+                            dict(name='div', attrs={'id':'content'})
                         ]
 
     remove_tags        = [
-                            dict(name='div', attrs={'class':['toolBar','mac_tags','toolBar  btmTools','textAds']}),
+                            {'class':['toolBar','mac_tags','toolBar  btmTools','textAds']},
                             dict(name='p', attrs={'class':'breadcrumbs'}),
-                            dict(name='div', attrs={'id':['breadcrumb','sidebar','comments']})
+                            dict(id=['breadcrumb','sidebar','comments','topContentWrapper',
+                                    'rightColumn', 'aboveFootPromo', 'storyCarousel']),
+                            {'class':lambda x: x and ('tools' in x or 'toolBar'
+                                in x)}
 
                         ]
 

resources/recipes/nytimes_sports.recipe

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+from __future__ import with_statement
+__license__ = 'GPL 3'
+__copyright__ = 'zotzo'
+__docformat__ = 'restructuredtext en'
+"""
+http://fifthdown.blogs.nytimes.com/
+http://offthedribble.blogs.nytimes.com/
+http://thequad.blogs.nytimes.com/
+http://slapshot.blogs.nytimes.com/
+http://goal.blogs.nytimes.com/
+http://bats.blogs.nytimes.com/
+http://straightsets.blogs.nytimes.com/
+http://formulaone.blogs.nytimes.com/
+http://onpar.blogs.nytimes.com/
+"""
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+
+class NYTimesSports(BasicNewsRecipe):
+    title = 'New York Times Sports Beat'
+    language = 'en'
+    __author__ = 'rylsfan'
+    description = 'Indepth sports from the New York Times'
+    publisher = 'The New York Times'
+    category = 'Sports'
+    oldest_article = 3
+    max_articles_per_feed = 25
+    no_stylesheets = True
+    language = 'en'
+    #cover_url ='http://bit.ly/h8F4DO'
+    feeds = [
+               (u'The Fifth Down', u'http://fifthdown.blogs.nytimes.com/feed/'),
+               (u'Off The Dribble', u'http://offthedribble.blogs.nytimes.com/feed/'),
+               (u'The Quad', u'http://thequad.blogs.nytimes.com/feed/'),
+               (u'Slap Shot', u'http://slapshot.blogs.nytimes.com/feed/'),
+               (u'Goal', u'http://goal.blogs.nytimes.com/feed/'),
+               (u'Bats', u'http://bats.blogs.nytimes.com/feed/'),
+               (u'Straight Sets', u'http://straightsets.blogs.nytimes.com/feed/'),
+               (u'Formula One', u'http://formulaone.blogs.nytimes.com/feed/'),
+               (u'On Par', u'http://onpar.blogs.nytimes.com/feed/'),
+           ]
+    keep_only_tags = [dict(name='div', attrs={'id':'header'}),
+            dict(name='h1'),
+            dict(name='h2'),
+            dict(name='div', attrs={'class':'entry-content'})]
+    extra_css =    '''
+           h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
+           h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
+           p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
+           body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
+                   '''

resources/recipes/nytimes_tech.recipe

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+from __future__ import with_statement
+__license__ = 'GPL 3'
+__copyright__ = 'zotzo'
+__docformat__ = 'restructuredtext en'
+"""
+http://pogue.blogs.nytimes.com/
+"""
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+
+class NYTimesTechnology(BasicNewsRecipe):
+    title = 'New York Times Technology Beat'
+    language = 'en'
+    __author__ = 'David Pogue'
+    description = 'The latest in technology from David Pogue'
+    publisher = 'The New York Times'
+    category = 'Technology'
+    oldest_article = 14
+    max_articles_per_feed = 25
+    no_stylesheets = True
+    language = 'en'
+    cover_url ='http://bit.ly/g0SKJT'
+    feeds = [
+               (u'Pogues Posts', u'http://pogue.blogs.nytimes.com/feed/'),
+               (u'Bits', u'http://bits.blogs.nytimes.com/feed/'),
+               (u'Gadgetwise', u'http://gadgetwise.blogs.nytimes.com/feed/'),
+               (u'Open', u'http://open.blogs.nytimes.com/feed/')
+                   ]
+    keep_only_tags = [dict(name='div', attrs={'id':'header'}),
+            dict(name='h1'),
+            dict(name='h2'),
+            dict(name='div', attrs={'class':'entry-content'})]
+    extra_css = '''
+           h1{font-family:Arial,Helvetica,sans-serif;
+               font-weight:bold;font-size:large;}
+
+           h2{font-family:Arial,Helvetica,sans-serif;
+               font-weight:normal;font-size:small;}
+
+           p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
+           body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
+               '''

resources/recipes/post_today.recipe

@@ -0,0 +1,21 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1299061355(BasicNewsRecipe):
+    title          = u'Post Today'
+    language       = 'th'
+    __author__     = "Chotechai"
+    oldest_article = 7
+    max_articles_per_feed = 100
+    cover_url = 'http://upload.wikimedia.org/wikipedia/th/2/2e/Posttoday_Logo.png'
+    feeds          = [(u'Breaking News', u'http://www.posttoday.com/rss/src/breakingnews.xml'), (u'\u0e02\u0e48\u0e32\u0e27', u'http://www.posttoday.com/rss/src/news.xml'), (u'\u0e27\u0e34\u0e40\u0e04\u0e23\u0e32\u0e30\u0e2b\u0e4c', u'http://www.posttoday.com/rss/src/analyse.xml'), (u'\u0e40\u0e21\u0e32\u0e17\u0e4c\u0e01\u0e31\u0e19\u0e43\u0e2b\u0e49 z', u'http://www.posttoday.com/rss/src/mouth.xml'), (u'\u0e44\u0e17\u0e22\u0e42\u0e0b\u0e44\u0e0b\u0e15\u0e35\u0e49', u'http://www.posttoday.com/rss/src/thaisociety.xml'), (u'\u0e44\u0e25\u0e1f\u0e4c\u0e2a\u0e44\u0e15\u0e25\u0e4c', u'http://www.posttoday.com/rss/src/lifestyle.xml'), (u'\u0e0a\u0e35\u0e49\u0e0a\u0e48\u0e2d\u0e07\u0e23\u0e27\u0e22', u'http://www.posttoday.com/rss/src/moneyguide.xml'), (u'\u0e1a\u0e49\u0e32\u0e19-\u0e04\u0e2d\u0e19\u0e42\u0e14', u'http://www.posttoday.com/rss/src/homecondo.xml'), (u'\u0e22\u0e32\u0e19\u0e22\u0e19\u0e15\u0e4c', u'http://www.posttoday.com/rss/src/motor.xml'), (u'\u0e14\u0e34\u0e08\u0e34\u0e15\u0e2d\u0e25\u0e44\u0e25\u0e1f\u0e4c', u'http://www.posttoday.com/rss/src/digitallife.xml'), (u'\u0e01\u0e35\u0e2c\u0e32', u'http://www.posttoday.com/rss/src/sport.xml'), (u'\u0e23\u0e2d\u0e1a\u0e42\u0e25\u0e01', u'http://www.posttoday.com/rss/src/world.xml'), (u'\u0e01\u0e34\u0e19-\u0e40\u0e17\u0e35\u0e48\u0e22\u0e27', u'http://www.posttoday.com/rss/src/eattravel.xml'), (u'Mind & Soul', u'http://www.posttoday.com/rss/src/mindsoul.xml'), (u'\u0e1a\u0e25\u0e47\u0e2d\u0e01 \u0e1a\u0e01.', u'http://www.posttoday.com/rss/src/blogs.xml')]
+    keep_only_tags = []
+    keep_only_tags.append(dict(name = 'div', attrs = {'class' :
+'articleContents'}))
+
+    remove_tags = []
+    remove_tags.append(dict(name = 'label'))
+    remove_tags.append(dict(name = 'span'))
+    remove_tags.append(dict(name = 'div', attrs = {'class' :
+'socialBookmark'}))
+    remove_tags.append(dict(name = 'div', attrs = {'class' :
+'misc'}))

resources/recipes/rbc_ru.recipe

@@ -0,0 +1,49 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1286819935(BasicNewsRecipe):
+    title = u'RBC.ru'
+    __author__ = 'A. Chewi'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets = True
+    use_embedded_content  = False
+    conversion_options = {'linearize_tables' : True}
+    remove_attributes = ['style']
+    language = 'ru'
+    timefmt = ' [%a, %d %b, %Y]'
+
+    keep_only_tags = [dict(name='h2', attrs={}),
+	dict(name='div', attrs={'class': 'box _ga1_on_'}),
+	dict(name='h1', attrs={'class': 'news_section'}),
+	dict(name='div', attrs={'class': 'news_body dotted_border_bottom'}),
+	dict(name='table', attrs={'class': 'newsBody'}),
+	dict(name='h2', attrs={'class': 'black'})]
+
+    feeds = [(u'Главные новости', u'http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/rbc.ru/mainnews.rss'),
+	(u'Политика', u'http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/rbc.ru/politics.rss'),
+	(u'Экономика', u'http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/rbc.ru/economics.rss'),
+	(u'Общество', u'http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/rbc.ru/society.rss'),
+	(u'Происшествия', u'http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/rbc.ru/incidents.rss'),
+	(u'Финансовые новости Quote.rbc.ru', u'http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/quote.ru/mainnews.rss')]
+
+
+    remove_tags = [dict(name='div', attrs={'class': "video-frame"}),
+	dict(name='div', attrs={'class': "photo-container videoContainer videoSWFLinks videoPreviewSlideContainer notes"}),
+	dict(name='div', attrs={'class': "notes"}),
+	dict(name='div', attrs={'class': "publinks"}),
+	dict(name='a', attrs={'class': "print"}),
+	dict(name='div', attrs={'class': "photo-report_new notes newslider"}),
+	dict(name='div', attrs={'class': "videoContainer"}),
+	dict(name='div', attrs={'class': "videoPreviewSlideContainer"}),
+	dict(name='a', attrs={'class': "videoPreviewContainer"}),
+	dict(name='a', attrs={'class': "red"}),]
+
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
+    def print_version(self, url):
+        return url + '?print=true'

resources/recipes/seattle_times.recipe

@@ -69,12 +69,16 @@ class SeattleTimes(BasicNewsRecipe):
                               u'http://seattletimes.nwsource.com/rss/mostreadarticles.xml'),
                          ]
 
+    keep_only_tags = [dict(id='content')]
     remove_tags        = [
-                             dict(name=['object','link','script'])
-                            ,dict(name='p', attrs={'class':'permission'})
+                             dict(name=['object','link','script']),
+                            {'class':['permission', 'note', 'bottomtools',
+                                'homedelivery']},
+                            dict(id=["rightcolumn", 'footer', 'adbottom']),
                          ]
 
     def print_version(self, url):
+        return url
         start_url, sep, rest_url = url.rpartition('_')
         rurl, rsep, article_id = start_url.rpartition('/')
         return u'http://seattletimes.nwsource.com/cgi-bin/PrintStory.pl?document_id=' + article_id

resources/recipes/statesman.recipe

@@ -10,12 +10,14 @@ class AdvancedUserRecipe1278049615(BasicNewsRecipe):
 
     max_articles_per_feed = 100
 
-    feeds          = [(u'News', u'http://www.statesman.com/section-rss.do?source=news&includeSubSections=true'),
-	(u'Business', u'http://www.statesman.com/section-rss.do?source=business&includeSubSections=true'),
-	(u'Life', u'http://www.statesman.com/section-rss.do?source=life&includesubsection=true'),
-	(u'Editorial', u'http://www.statesman.com/section-rss.do?source=opinion&includesubsections=true'),
-	(u'Sports', u'http://www.statesman.com/section-rss.do?source=sports&includeSubSections=true')
-	]
+    feeds          = [(u'News',
+        u'http://www.statesman.com/section-rss.do?source=news&includeSubSections=true'),
+    (u'Local', u'http://www.statesman.com/section-rss.do?source=local&includeSubSections=true'),
+    (u'Business', u'http://www.statesman.com/section-rss.do?source=business&includeSubSections=true'),
+    (u'Life', u'http://www.statesman.com/section-rss.do?source=life&includesubsection=true'),
+    (u'Editorial', u'http://www.statesman.com/section-rss.do?source=opinion&includesubsections=true'),
+    (u'Sports', u'http://www.statesman.com/section-rss.do?source=sports&includeSubSections=true')
+    ]
     masthead_url = "http://www.statesman.com/images/cmg-logo.gif"
     #temp_files = []
     #articles_are_obfuscated = True
@@ -28,8 +30,11 @@ class AdvancedUserRecipe1278049615(BasicNewsRecipe):
     conversion_options = {'linearize_tables':True}
     remove_tags = [
                     dict(name='div', attrs={'id':'cxArticleOptions'}),
+                    {'class':['perma', 'comments', 'trail', 'share-buttons',
+                        'toggle_show_on']},
                         ]
     keep_only_tags = [
-	 dict(name='div', attrs={'class':'cxArticleHeader'}),
-                     dict(name='div', attrs={'id':'cxArticleBodyText'}),
+     dict(name='div', attrs={'class':'cxArticleHeader'}),
+                     dict(name='div', attrs={'id':['cxArticleBodyText',
+                         'content']}),
                                ]

resources/recipes/swiatkindle.recipe

@@ -7,6 +7,7 @@ swiatczytnikow.pl
 '''
 
 import re
+from calibre.web.feeds.news import BasicNewsRecipe
 
 class swiatczytnikow(BasicNewsRecipe):
     title          = u'Swiat Czytnikow'

resources/recipes/thai_post_daily.recipe

@@ -0,0 +1,18 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1299054026(BasicNewsRecipe):
+    title          = u'Thai Post Daily'
+    __author__ = 'Chotechai P.'
+    language = 'th'
+    oldest_article = 7
+    max_articles_per_feed = 100
+
+    feeds          = [(u'\u0e02\u0e48\u0e32\u0e27\u0e2b\u0e19\u0e49\u0e32\u0e2b\u0e19\u0e36\u0e48\u0e07', u'http://thaipost.net/taxonomy/term/1/all/feed'), (u'\u0e1a\u0e17\u0e1a\u0e23\u0e23\u0e13\u0e32\u0e18\u0e34\u0e01\u0e32\u0e23', u'http://thaipost.net/taxonomy/term/11/all/feed'), (u'\u0e40\u0e1b\u0e25\u0e27 \u0e2a\u0e35\u0e40\u0e07\u0e34\u0e19', u'http://thaipost.net/taxonomy/term/2/all/feed'), (u'\u0e2a\u0e20\u0e32\u0e1b\u0e23\u0e30\u0e0a\u0e32\u0e0a\u0e19', u'http://thaipost.net/taxonomy/term/3/all/feed'), (u'\u0e16\u0e39\u0e01\u0e17\u0e38\u0e01\u0e02\u0e49\u0e2d', u'http://thaipost.net/taxonomy/term/4/all/feed'), (u'\u0e01\u0e32\u0e23\u0e40\u0e21\u0e37\u0e2d\u0e07', u'http://thaipost.net/taxonomy/term/5/all/feed'), (u'\u0e17\u0e48\u0e32\u0e19\u0e02\u0e38\u0e19\u0e19\u0e49\u0e2d\u0e22', u'http://thaipost.net/taxonomy/term/12/all/feed'), (u'\u0e1a\u0e17\u0e04\u0e27\u0e32\u0e21\u0e1e\u0e34\u0e40\u0e28\u0e29', u'http://thaipost.net/taxonomy/term/66/all/feed'), (u'\u0e23\u0e32\u0e22\u0e07\u0e32\u0e19\u0e1e\u0e34\u0e40\u0e28\u0e29', u'http://thaipost.net/taxonomy/term/67/all/feed'), (u'\u0e1a\u0e31\u0e19\u0e17\u0e36\u0e01\u0e2b\u0e19\u0e49\u0e32 4', u'http://thaipost.net/taxonomy/term/13/all/feed'), (u'\u0e40\u0e2a\u0e35\u0e22\u0e1a\u0e0b\u0e36\u0e48\u0e07\u0e2b\u0e19\u0e49\u0e32', u'http://thaipost.net/taxonomy/term/64/all/feed'), (u'\u0e04\u0e31\u0e19\u0e1b\u0e32\u0e01\u0e2d\u0e22\u0e32\u0e01\u0e40\u0e25\u0e48\u0e32', u'http://thaipost.net/taxonomy/term/65/all/feed'), (u'\u0e40\u0e28\u0e23\u0e29\u0e10\u0e01\u0e34\u0e08', u'http://thaipost.net/taxonomy/term/6/all/feed'), (u'\u0e01\u0e23\u0e30\u0e08\u0e01\u0e44\u0e23\u0e49\u0e40\u0e07\u0e32', u'http://thaipost.net/taxonomy/term/14/all/feed'), (u'\u0e01\u0e23\u0e30\u0e08\u0e01\u0e2b\u0e31\u0e01\u0e21\u0e38\u0e21', u'http://thaipost.net/taxonomy/term/71/all/feed'), (u'\u0e04\u0e34\u0e14\u0e40\u0e2b\u0e19\u0e37\u0e2d\u0e01\u0e23\u0e30\u0e41\u0e2a', u'http://thaipost.net/taxonomy/term/69/all/feed'), (u'\u0e23\u0e32\u0e22\u0e07\u0e32\u0e19', u'http://thaipost.net/taxonomy/term/68/all/feed'), (u'\u0e2d\u0e34\u0e42\u0e04\u0e42\u0e1f\u0e01\u0e31\u0e2a', u'http://thaipost.net/taxonomy/term/10/all/feed'), (u'\u0e01\u0e32\u0e23\u0e28\u0e36\u0e01\u0e29\u0e32-\u0e2a\u0e32\u0e18\u0e32\u0e23\u0e13\u0e2a\u0e38\u0e02', u'http://thaipost.net/taxonomy/term/7/all/feed'), (u'\u0e15\u0e48\u0e32\u0e07\u0e1b\u0e23\u0e30\u0e40\u0e17\u0e28', u'http://thaipost.net/taxonomy/term/8/all/feed'), (u'\u0e01\u0e35\u0e2c\u0e32', u'http://thaipost.net/taxonomy/term/9/all/feed')]
+
+    def print_version(self, url):
+      return url.replace(url, 'http://www.thaipost.net/print/' + url [32:])
+
+    remove_tags = []
+    remove_tags.append(dict(name = 'div', attrs = {'class' : 'print-logo'}))
+    remove_tags.append(dict(name = 'div', attrs = {'class' : 'print-site_name'}))
+    remove_tags.append(dict(name = 'div', attrs = {'class' : 'print-breadcrumb'}))

resources/template-functions.json

@@ -5,8 +5,9 @@
     "strcat": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    res = ''\n    for i in range(0, len(args)):\n        res += args[i]\n    return res\n", 
     "substr": "def evaluate(self, formatter, kwargs, mi, locals, str_, start_, end_):\n    return str_[int(start_): len(str_) if int(end_) == 0 else int(end_)]\n", 
     "ifempty": "def evaluate(self, formatter, kwargs, mi, locals, val, value_if_empty):\n    if val:\n        return val\n    else:\n        return value_if_empty\n", 
+    "select": "def evaluate(self, formatter, kwargs, mi, locals, val, key):\n    if not val:\n        return ''\n    vals = [v.strip() for v in val.split(',')]\n    for v in vals:\n        if v.startswith(key+':'):\n            return v[len(key)+1:]\n    return ''\n", 
     "field": "def evaluate(self, formatter, kwargs, mi, locals, name):\n    return formatter.get_value(name, [], kwargs)\n", 
-    "capitalize": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return capitalize(val)\n", 
+    "subtract": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x - y)\n", 
     "list_item": "def evaluate(self, formatter, kwargs, mi, locals, val, index, sep):\n    if not val:\n        return ''\n    index = int(index)\n    val = val.split(sep)\n    try:\n        return val[index]\n    except:\n        return ''\n", 
     "shorten": "def evaluate(self, formatter, kwargs, mi, locals,\n             val, leading, center_string, trailing):\n    l = max(0, int(leading))\n    t = max(0, int(trailing))\n    if len(val) > l + len(center_string) + t:\n        return val[0:l] + center_string + ('' if t == 0 else val[-t:])\n    else:\n        return val\n", 
     "re": "def evaluate(self, formatter, kwargs, mi, locals, val, pattern, replacement):\n    return re.sub(pattern, replacement, val)\n", 
@@ -19,11 +20,13 @@
     "test": "def evaluate(self, formatter, kwargs, mi, locals, val, value_if_set, value_not_set):\n    if val:\n        return value_if_set\n    else:\n        return value_not_set\n", 
     "eval": "def evaluate(self, formatter, kwargs, mi, locals, template):\n    from formatter import eval_formatter\n    template = template.replace('[[', '{').replace(']]', '}')\n    return eval_formatter.safe_format(template, locals, 'EVAL', None)\n", 
     "multiply": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x * y)\n", 
-    "subtract": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x - y)\n", 
+    "format_date": "def evaluate(self, formatter, kwargs, mi, locals, val, format_string):\n    print val\n    if not val:\n        return ''\n    try:\n        dt = parse_date(val)\n        s = format_date(dt, format_string)\n    except:\n        s = 'BAD DATE'\n    return s\n", 
+    "capitalize": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return capitalize(val)\n", 
     "count": "def evaluate(self, formatter, kwargs, mi, locals, val, sep):\n    return unicode(len(val.split(sep)))\n", 
     "lowercase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return val.lower()\n", 
     "assign": "def evaluate(self, formatter, kwargs, mi, locals, target, value):\n    locals[target] = value\n    return value\n", 
     "switch": "def evaluate(self, formatter, kwargs, mi, locals, val, *args):\n    if (len(args) % 2) != 1:\n        raise ValueError(_('switch requires an odd number of arguments'))\n    i = 0\n    while i < len(args):\n        if i + 1 >= len(args):\n            return args[i]\n        if re.search(args[i], val):\n            return args[i+1]\n        i += 2\n", 
     "strcmp": "def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):\n    v = strcmp(x, y)\n    if v < 0:\n        return lt\n    if v == 0:\n        return eq\n    return gt\n", 
+    "raw_field": "def evaluate(self, formatter, kwargs, mi, locals, name):\n    return unicode(getattr(mi, name, None))\n", 
     "cmp": "def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    if x < y:\n        return lt\n    if x == y:\n        return eq\n    return gt\n"
 }

src/calibre/devices/apple/driver.py

@@ -701,7 +701,7 @@ class ITUNES(DriverBase):
             self.log.info("ITUNES.get_file(): exporting '%s'" % path)
         outfile.write(open(self.cached_books[path]['lib_book'].location().path).read())
 
-    def open(self):
+    def open(self, library_uuid):
         '''
         Perform any device specific initialization. Called after the device is
         detected but before any other functions that communicate with the device.

src/calibre/devices/bambook/driver.py

@@ -61,7 +61,7 @@ class BAMBOOK(DeviceConfig, DevicePlugin):
             detected_device=None) :
         self.open()
 
-    def open(self):
+    def open(self, library_uuid):
         # Make sure the Bambook library is ready
         if not is_bambook_lib_ready():
             raise OpenFeedback(_("Unable to connect to Bambook, you need to install Bambook library first."))

src/calibre/devices/folder_device/driver.py

@@ -47,6 +47,7 @@ class FOLDER_DEVICE(USBMS):
     #: Icon for this device
     icon = I('devices/folder.png')
     METADATA_CACHE = '.metadata.calibre'
+    DRIVEINFO = '.driveinfo.calibre'
 
     _main_prefix = ''
     _card_a_prefix = None
@@ -77,7 +78,8 @@ class FOLDER_DEVICE(USBMS):
             only_presence=False):
         return self.is_connected, self
 
-    def open(self):
+    def open(self, library_uuid):
+        self.current_library_uuid = library_uuid
         if not self._main_prefix:
             return False
         return True

src/calibre/devices/hanlin/driver.py

@@ -116,6 +116,7 @@ class BOOX(HANLINV3):
     author         = 'Jesus Manuel Marinho Valcarce'
     supported_platforms = ['windows', 'osx', 'linux']
     METADATA_CACHE = '.metadata.calibre'
+    DRIVEINFO = '.driveinfo.calibre'
 
     # Ordered list of supported formats
     FORMATS     = ['epub', 'fb2', 'djvu', 'pdf', 'html', 'txt', 'rtf', 'mobi',

src/calibre/devices/interface.py

@@ -215,7 +215,7 @@ class DevicePlugin(Plugin):
 
         return True
 
-    def open(self):
+    def open(self, library_uuid):
         '''
         Perform any device specific initialization. Called after the device is
         detected but before any other functions that communicate with the device.
@@ -260,6 +260,8 @@ class DevicePlugin(Plugin):
         Ask device for device information. See L{DeviceInfoQuery}.
 
         :return: (device name, device version, software version on device, mime type)
+                 The tuple can optionally have a fifth element, which is a
+                 drive information diction. See usbms.driver for an example.
 
         """
         raise NotImplementedError()
@@ -447,6 +449,15 @@ class DevicePlugin(Plugin):
         '''
         pass
 
+    def set_driveinfo_name(self, location_code, name):
+        '''
+        Set the device name in the driveinfo file to 'name'. This setting will
+        persist until the file is re-created or the name is changed again.
+
+        Non-disk devices will ignore this request.
+        '''
+        pass
+
 class BookList(list):
     '''
     A list of books. Each Book object must have the fields

src/calibre/devices/misc.py

@@ -272,6 +272,7 @@ class NEXTBOOK(USBMS):
     VENDOR_NAME = 'NEXT2'
     WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = '1.0.14'
     SUPPORTS_SUB_DIRS = True
+    THUMBNAIL_HEIGHT = 120
 
     '''
     def upload_cover(self, path, filename, metadata, filepath):

src/calibre/devices/prs500/driver.py

@@ -240,7 +240,7 @@ class PRS500(DeviceConfig, DevicePlugin):
     def set_progress_reporter(self, report_progress):
         self.report_progress = report_progress
 
-    def open(self) :
+    def open(self, library_uuid) :
         """
         Claim an interface on the device for communication.
         Requires write privileges to the device file.

src/calibre/devices/prs505/driver.py

@@ -153,9 +153,6 @@ class PRS505(USBMS):
         # updated on every connect
         self.WANTS_UPDATED_THUMBNAILS = self.settings().extra_customization[2]
 
-    def get_device_information(self, end_session=True):
-        return (self.gui_name, '', '', '')
-
     def filename_callback(self, fname, mi):
         if getattr(mi, 'application_id', None) is not None:
             base = fname.rpartition('.')[0]

src/calibre/devices/usbms/device.py

@@ -700,7 +700,7 @@ class Device(DeviceConfig, DevicePlugin):
 
 
 
-    def open(self):
+    def open(self, library_uuid):
         time.sleep(5)
         self._main_prefix = self._card_a_prefix = self._card_b_prefix = None
         if islinux:
@@ -722,6 +722,7 @@ class Device(DeviceConfig, DevicePlugin):
                 time.sleep(7)
                 self.open_osx()
 
+        self.current_library_uuid = library_uuid
         self.post_open_callback()
 
     def post_open_callback(self):

src/calibre/devices/usbms/driver.py

@@ -10,17 +10,18 @@ driver. It is intended to be subclassed with the relevant parts implemented
 for a particular device.
 '''
 
-import os
-import re
-import time
+import os, re, time, json, uuid
 from itertools import cycle
 
+from calibre.constants import numeric_version
 from calibre import prints, isbytestring
 from calibre.constants import filesystem_encoding, DEBUG
 from calibre.devices.usbms.cli import CLI
 from calibre.devices.usbms.device import Device
 from calibre.devices.usbms.books import BookList, Book
 from calibre.ebooks.metadata.book.json_codec import JsonCodec
+from calibre.utils.config import from_json, to_json
+from calibre.utils.date import now, isoformat
 
 BASE_TIME = None
 def debug_print(*args):
@@ -52,10 +53,59 @@ class USBMS(CLI, Device):
     FORMATS = []
     CAN_SET_METADATA = []
     METADATA_CACHE = 'metadata.calibre'
+    DRIVEINFO = 'driveinfo.calibre'
+
+    def _update_driveinfo_record(self, dinfo, prefix, location_code, name=None):
+        if not isinstance(dinfo, dict):
+            dinfo = {}
+        if dinfo.get('device_store_uuid', None) is None:
+            dinfo['device_store_uuid'] = unicode(uuid.uuid4())
+        if dinfo.get('device_name') is None:
+            dinfo['device_name'] = self.get_gui_name()
+        if name is not None:
+            dinfo['device_name'] = name
+        dinfo['location_code'] = location_code
+        dinfo['last_library_uuid'] = getattr(self, 'current_library_uuid', None)
+        dinfo['calibre_version'] = '.'.join([unicode(i) for i in numeric_version])
+        dinfo['date_last_connected'] = isoformat(now())
+        dinfo['prefix'] = prefix.replace('\\', '/')
+        return dinfo
+
+    def _update_driveinfo_file(self, prefix, location_code, name=None):
+        if os.path.exists(os.path.join(prefix, self.DRIVEINFO)):
+            with open(os.path.join(prefix, self.DRIVEINFO), 'rb') as f:
+                try:
+                    driveinfo = json.loads(f.read(), object_hook=from_json)
+                except:
+                    driveinfo = None
+                driveinfo = self._update_driveinfo_record(driveinfo, prefix,
+                                                          location_code, name)
+            with open(os.path.join(prefix, self.DRIVEINFO), 'wb') as f:
+                f.write(json.dumps(driveinfo, default=to_json))
+        else:
+            driveinfo = self._update_driveinfo_record({}, prefix, location_code, name)
+            with open(os.path.join(prefix, self.DRIVEINFO), 'wb') as f:
+                f.write(json.dumps(driveinfo, default=to_json))
+        return driveinfo
 
     def get_device_information(self, end_session=True):
         self.report_progress(1.0, _('Get device information...'))
-        return (self.get_gui_name(), '', '', '')
+        self.driveinfo = {}
+        if self._main_prefix is not None:
+            self.driveinfo['main'] = self._update_driveinfo_file(self._main_prefix, 'main')
+        if self._card_a_prefix is not None:
+            self.driveinfo['A'] = self._update_driveinfo_file(self._card_a_prefix, 'A')
+        if self._card_b_prefix is not None:
+            self.driveinfo['B'] = self._update_driveinfo_file(self._card_b_prefix, 'B')
+        return (self.get_gui_name(), '', '', '', self.driveinfo)
+
+    def set_driveinfo_name(self, location_code, name):
+        if location_code == 'main':
+            self._update_driveinfo_file(self._main_prefix, location_code, name)
+        elif location_code == 'A':
+            self._update_driveinfo_file(self._card_a_prefix, location_code, name)
+        elif location_code == 'B':
+            self._update_driveinfo_file(self._card_b_prefix, location_code, name)
 
     def books(self, oncard=None, end_session=True):
         from calibre.ebooks.metadata.meta import path_to_ext

src/calibre/ebooks/__init__.py

@@ -28,7 +28,7 @@ class ParserError(ValueError):
 BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'txtz', 'htm', 'xhtm',
                    'html', 'xhtml', 'pdf', 'pdb', 'pdr', 'prc', 'mobi', 'azw', 'doc',
                    'epub', 'fb2', 'djvu', 'lrx', 'cbr', 'cbz', 'cbc', 'oebzip',
-                   'rb', 'imp', 'odt', 'chm', 'tpz', 'azw1', 'pml', 'mbp', 'tan', 'snb']
+                   'rb', 'imp', 'odt', 'chm', 'tpz', 'azw1', 'pml', 'pmlz', 'mbp', 'tan', 'snb']
 
 class HTMLRenderer(object):
 

src/calibre/ebooks/conversion/preprocess.py

@@ -265,16 +265,28 @@ class CSSPreProcessor(object):
 
     PAGE_PAT   = re.compile(r'@page[^{]*?{[^}]*?}')
     # Remove some of the broken CSS Microsoft products
-    # create, slightly dangerous as it removes to end of line
-    # rather than semi-colon
-    MS_PAT     = re.compile(r'^\s*(mso-|panose-).+?$',
-            re.MULTILINE|re.IGNORECASE)
+    # create
+    MS_PAT     = re.compile(r'''
+        (?P<start>^|;|\{)\s*    # The end of the previous rule or block start
+        (%s).+?                 # The invalid selectors
+        (?P<end>$|;|\})         # The end of the declaration
+        '''%'mso-|panose-|text-underline|tab-interval',
+        re.MULTILINE|re.IGNORECASE|re.VERBOSE)
+
+    def ms_sub(self, match):
+        end = match.group('end')
+        try:
+            start = match.group('start')
+        except:
+            start = ''
+        if end == ';':
+            end = ''
+        return start + end
 
     def __call__(self, data, add_namespace=False):
         from calibre.ebooks.oeb.base import XHTML_CSS_NAMESPACE
         data = self.PAGE_PAT.sub('', data)
-        if '\n' in data:
-            data = self.MS_PAT.sub('', data)
+        data = self.MS_PAT.sub(self.ms_sub, data)
         if not add_namespace:
             return data
         ans, namespaced = [], False

src/calibre/ebooks/metadata/book/__init__.py

@@ -18,14 +18,14 @@ SOCIAL_METADATA_FIELDS = frozenset([
     'series_index',     # A floating point number
     # Of the form { scheme1:value1, scheme2:value2}
     # For example: {'isbn':'123456789', 'doi':'xxxx', ... }
-    'classifiers',
+    'identifiers',
 ])
 
 '''
-The list of names that convert to classifiers when in get and set.
+The list of names that convert to identifiers when in get and set.
 '''
 
-TOP_LEVEL_CLASSIFIERS = frozenset([
+TOP_LEVEL_IDENTIFIERS = frozenset([
     'isbn',
 ])
 
@@ -108,7 +108,7 @@ STANDARD_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union(
 SC_FIELDS_NOT_COPIED =     frozenset(['title', 'title_sort', 'authors',
                                       'author_sort', 'author_sort_map',
                                       'cover_data', 'tags', 'language',
-                                      'classifiers'])
+                                      'identifiers'])
 
 # Metadata fields that smart update should copy only if the source is not None
 SC_FIELDS_COPY_NOT_NULL =  frozenset(['lpath', 'size', 'comments', 'thumbnail'])

src/calibre/ebooks/metadata/book/base.py

@@ -12,7 +12,7 @@ from calibre.constants import DEBUG
 from calibre.ebooks.metadata.book import SC_COPYABLE_FIELDS
 from calibre.ebooks.metadata.book import SC_FIELDS_COPY_NOT_NULL
 from calibre.ebooks.metadata.book import STANDARD_METADATA_FIELDS
-from calibre.ebooks.metadata.book import TOP_LEVEL_CLASSIFIERS
+from calibre.ebooks.metadata.book import TOP_LEVEL_IDENTIFIERS
 from calibre.ebooks.metadata.book import ALL_METADATA_FIELDS
 from calibre.library.field_metadata import FieldMetadata
 from calibre.utils.date import isoformat, format_date
@@ -24,7 +24,7 @@ NULL_VALUES = {
                 'user_metadata': {},
                 'cover_data'   : (None, None),
                 'tags'         : [],
-                'classifiers'  : {},
+                'identifiers'  : {},
                 'languages'    : [],
                 'device_collections': [],
                 'author_sort_map': {},
@@ -96,8 +96,8 @@ class Metadata(object):
 
     def __getattribute__(self, field):
         _data = object.__getattribute__(self, '_data')
-        if field in TOP_LEVEL_CLASSIFIERS:
-            return _data.get('classifiers').get(field, None)
+        if field in TOP_LEVEL_IDENTIFIERS:
+            return _data.get('identifiers').get(field, None)
         if field in STANDARD_METADATA_FIELDS:
             return _data.get(field, None)
         try:
@@ -123,8 +123,11 @@ class Metadata(object):
 
     def __setattr__(self, field, val, extra=None):
         _data = object.__getattribute__(self, '_data')
-        if field in TOP_LEVEL_CLASSIFIERS:
-            _data['classifiers'].update({field: val})
+        if field in TOP_LEVEL_IDENTIFIERS:
+            field, val = self._clean_identifier(field, val)
+            _data['identifiers'].update({field: val})
+        elif field == 'identifiers':
+            self.set_identifiers(val)
         elif field in STANDARD_METADATA_FIELDS:
             if val is None:
                 val = NULL_VALUES.get(field, None)
@@ -176,17 +179,48 @@ class Metadata(object):
     def set(self, field, val, extra=None):
         self.__setattr__(field, val, extra)
 
-    def get_classifiers(self):
+    def get_identifiers(self):
         '''
-        Return a copy of the classifiers dictionary.
+        Return a copy of the identifiers dictionary.
         The dict is small, and the penalty for using a reference where a copy is
         needed is large. Also, we don't want any manipulations of the returned
         dict to show up in the book.
         '''
-        return copy.deepcopy(object.__getattribute__(self, '_data')['classifiers'])
+        ans = object.__getattribute__(self,
+            '_data')['identifiers']
+        if not ans:
+            ans = {}
+        return copy.deepcopy(ans)
 
-    def set_classifiers(self, classifiers):
-        object.__getattribute__(self, '_data')['classifiers'] = classifiers
+    def _clean_identifier(self, typ, val):
+        typ = icu_lower(typ).strip().replace(':', '').replace(',', '')
+        val = val.strip().replace(',', '|').replace(':', '|')
+        return typ, val
+
+    def set_identifiers(self, identifiers):
+        '''
+        Set all identifiers. Note that if you previously set ISBN, calling
+        this method will delete it.
+        '''
+        cleaned = {}
+        for key, val in identifiers.iteritems():
+            key, val = self._clean_identifier(key, val)
+            if key and val:
+                cleaned[key] = val
+        object.__getattribute__(self, '_data')['identifiers'] = cleaned
+
+    def set_identifier(self, typ, val):
+        'If val is empty, deletes identifier of type typ'
+        typ, val = self._clean_identifier(typ, val)
+        if not typ:
+            return
+        identifiers = object.__getattribute__(self,
+            '_data')['identifiers']
+
+        if not val and typ in identifiers:
+            identifiers.pop(typ)
+        if val:
+            identifiers[typ] = val
 
     # field-oriented interface. Intended to be the same as in LibraryDatabase
 
@@ -229,7 +263,7 @@ class Metadata(object):
             if v is not None:
                 result[attr] = v
         # separate these because it uses the self.get(), not _data.get()
-        for attr in TOP_LEVEL_CLASSIFIERS:
+        for attr in TOP_LEVEL_IDENTIFIERS:
             v = self.get(attr, None)
             if v is not None:
                 result[attr] = v
@@ -400,8 +434,8 @@ class Metadata(object):
             self.set_all_user_metadata(other.get_all_user_metadata(make_copy=True))
             for x in SC_FIELDS_COPY_NOT_NULL:
                 copy_not_none(self, other, x)
-            if callable(getattr(other, 'get_classifiers', None)):
-                self.set_classifiers(other.get_classifiers())
+            if callable(getattr(other, 'get_identifiers', None)):
+                self.set_identifiers(other.get_identifiers())
             # language is handled below
         else:
             for attr in SC_COPYABLE_FIELDS:
@@ -456,15 +490,15 @@ class Metadata(object):
             if len(other_comments.strip()) > len(my_comments.strip()):
                 self.comments = other_comments
 
-            # Copy all the non-none classifiers
-            if callable(getattr(other, 'get_classifiers', None)):
-                d = self.get_classifiers()
-                s = other.get_classifiers()
+            # Copy all the non-none identifiers
+            if callable(getattr(other, 'get_identifiers', None)):
+                d = self.get_identifiers()
+                s = other.get_identifiers()
                 d.update([v for v in s.iteritems() if v[1] is not None])
-                self.set_classifiers(d)
+                self.set_identifiers(d)
             else:
-                # other structure not Metadata. Copy the top-level classifiers
-                for attr in TOP_LEVEL_CLASSIFIERS:
+                # other structure not Metadata. Copy the top-level identifiers
+                for attr in TOP_LEVEL_IDENTIFIERS:
                     copy_not_none(self, other, attr)
 
         other_lang = getattr(other, 'language', None)
@@ -561,6 +595,8 @@ class Metadata(object):
             elif key == 'series_index':
                 res = self.format_series_index(res)
             elif datatype == 'text' and fmeta['is_multiple']:
+                if isinstance(res, dict):
+                    res = [k + ':' + v for k,v in res.items()]
                 res = u', '.join(sorted(res, key=sort_key))
             elif datatype == 'series' and series_with_index:
                 res = res + ' [%s]'%self.format_series_index()

src/calibre/ebooks/metadata/book/json_codec.py

@@ -119,6 +119,8 @@ class JsonCodec(object):
             for item in js:
                 book = book_class(prefix, item.get('lpath', None))
                 for key in item.keys():
+                    if key == 'classifiers':
+                        key = 'identifiers'
                     meta = self.decode_metadata(key, item[key])
                     if key == 'user_metadata':
                         book.set_all_user_metadata(meta)

src/calibre/ebooks/metadata/opf2.py

@@ -596,6 +596,9 @@ class OPF(object): # {{{
         ans = MetaInformation(self)
         for n, v in self._user_metadata_.items():
             ans.set_user_metadata(n, v)
+
+        ans.set_identifiers(self.get_identifiers())
+
         return ans
 
     def write_user_metadata(self):
@@ -855,6 +858,21 @@ class OPF(object): # {{{
 
         return property(fget=fget, fset=fset)
 
+    def get_identifiers(self):
+        identifiers = {}
+        for x in self.XPath(
+            'descendant::*[local-name() = "identifier" and text()]')(
+                    self.metadata):
+            for attr, val in x.attrib.iteritems():
+                if attr.endswith('scheme'):
+                    typ = icu_lower(val)
+                    val = etree.tostring(x, with_tail=False, encoding=unicode,
+                            method='text').strip()
+                    if val and typ not in ('calibre', 'uuid'):
+                        identifiers[typ] = val
+                    break
+        return identifiers
+
     @dynamic_property
     def application_id(self):
 
@@ -1166,8 +1184,8 @@ class OPFCreator(Metadata):
             a(DC_ELEM('description', self.comments))
         if self.publisher:
             a(DC_ELEM('publisher', self.publisher))
-        if self.isbn:
-            a(DC_ELEM('identifier', self.isbn, opf_attrs={'scheme':'ISBN'}))
+        for key, val in self.get_identifiers().iteritems():
+            a(DC_ELEM('identifier', val, opf_attrs={'scheme':icu_upper(key)}))
         if self.rights:
             a(DC_ELEM('rights', self.rights))
         if self.tags:
@@ -1291,8 +1309,8 @@ def metadata_to_opf(mi, as_string=True):
         factory(DC('description'), mi.comments)
     if mi.publisher:
         factory(DC('publisher'), mi.publisher)
-    if mi.isbn:
-        factory(DC('identifier'), mi.isbn, scheme='ISBN')
+    for key, val in mi.get_identifiers().iteritems():
+        factory(DC('identifier'), val, scheme=icu_upper(key))
     if mi.rights:
         factory(DC('rights'), mi.rights)
     factory(DC('language'), mi.language if mi.language and mi.language.lower()
@@ -1342,7 +1360,7 @@ def test_m2o():
     mi.language = 'en'
     mi.comments = 'what a fun book\n\n'
     mi.publisher = 'publisher'
-    mi.isbn = 'boooo'
+    mi.set_identifiers({'isbn':'booo', 'dummy':'dummy'})
     mi.tags = ['a', 'b']
     mi.series = 's"c\'l&<>'
     mi.series_index = 3.34
@@ -1350,7 +1368,7 @@ def test_m2o():
     mi.timestamp = nowf()
     mi.publication_type = 'ooooo'
     mi.rights = 'yes'
-    mi.cover = 'asd.jpg'
+    mi.cover = os.path.abspath('asd.jpg')
     opf = metadata_to_opf(mi)
     print opf
     newmi = MetaInformation(OPF(StringIO(opf)))
@@ -1363,6 +1381,9 @@ def test_m2o():
         o, n = getattr(mi, attr), getattr(newmi, attr)
         if o != n and o.strip() != n.strip():
             print 'FAILED:', attr, getattr(mi, attr), '!=', getattr(newmi, attr)
+    if mi.get_identifiers() != newmi.get_identifiers():
+        print 'FAILED:', 'identifiers', mi.get_identifiers(),
+        print '!=', newmi.get_identifiers()
 
 
 class OPFTest(unittest.TestCase):
@@ -1378,6 +1399,7 @@ class OPFTest(unittest.TestCase):
     <creator opf:role="aut">Next</creator>
     <dc:subject>One</dc:subject><dc:subject>Two</dc:subject>
     <dc:identifier scheme="ISBN">123456789</dc:identifier>
+    <dc:identifier scheme="dummy">dummy</dc:identifier>
     <meta name="calibre:series" content="A one book series" />
     <meta name="calibre:rating" content="4"/>
     <meta name="calibre:publication_type" content="test"/>
@@ -1405,6 +1427,8 @@ class OPFTest(unittest.TestCase):
         self.assertEqual(opf.rating, 4)
         self.assertEqual(opf.publication_type, 'test')
         self.assertEqual(list(opf.itermanifest())[0].get('href'), 'a ~ b')
+        self.assertEqual(opf.get_identifiers(), {'isbn':'123456789',
+            'dummy':'dummy'})
 
     def testWriting(self):
         for test in [('title', 'New & Title'), ('authors', ['One', 'Two']),
@@ -1461,5 +1485,5 @@ def test_user_metadata():
 
 if __name__ == '__main__':
     #test_user_metadata()
-    #test_m2o()
+    test_m2o()
     test()

src/calibre/ebooks/mobi/reader.py

@@ -18,6 +18,7 @@ from calibre import xml_entity_to_unicode, CurrentDir, entity_to_unicode, \
     replace_entities
 from calibre.utils.filenames import ascii_filename
 from calibre.utils.date import parse_date
+from calibre.utils.cleantext import clean_ascii_chars
 from calibre.ptempfile import TemporaryDirectory
 from calibre.ebooks import DRMError
 from calibre.ebooks.chardet import ENCODING_PATS
@@ -323,6 +324,7 @@ class MobiReader(object):
         self.cleanup_html()
 
         self.log.debug('Parsing HTML...')
+        self.processed_html = clean_ascii_chars(self.processed_html)
         try:
             root = html.fromstring(self.processed_html)
             if len(root.xpath('//html')) > 5:

src/calibre/ebooks/oeb/base.py

@@ -827,6 +827,24 @@ class Manifest(object):
                 return None
             return etree.fromstring(data, parser=RECOVER_PARSER)
 
+        def clean_word_doc(self, data):
+            prefixes = []
+            for match in re.finditer(r'xmlns:(\S+?)=".*?microsoft.*?"', data):
+                prefixes.append(match.group(1))
+            if prefixes:
+                self.oeb.log.warn('Found microsoft markup, cleaning...')
+                # Remove empty tags as they are not rendered by browsers
+                # but can become renderable HTML tags like <p/> if the
+                # document is parsed by an HTML parser
+                pat = re.compile(
+                        r'<(%s):([a-zA-Z0-9]+)[^>/]*?></\1:\2>'%('|'.join(prefixes)),
+                        re.DOTALL)
+                data = pat.sub('', data)
+                pat = re.compile(
+                        r'<(%s):([a-zA-Z0-9]+)[^>/]*?/>'%('|'.join(prefixes)))
+                data = pat.sub('', data)
+            return data
+
         def _parse_xhtml(self, data):
             self.oeb.log.debug('Parsing', self.href, '...')
             # Convert to Unicode and normalize line endings
@@ -884,6 +902,10 @@ class Manifest(object):
                         except etree.XMLSyntaxError:
                             data = etree.fromstring(data, parser=RECOVER_PARSER)
                 return data
+            try:
+                data = self.clean_word_doc(data)
+            except:
+                pass
             data = first_pass(data)
 
             # Handle weird (non-HTML/fragment) files
@@ -907,6 +929,7 @@ class Manifest(object):
                     parent.append(child)
                 data = nroot
 
+
             # Force into the XHTML namespace
             if not namespace(data.tag):
                 self.oeb.log.warn('Forcing', self.href, 'into XHTML namespace')

src/calibre/ebooks/oeb/stylizer.py

@@ -423,6 +423,7 @@ class Stylizer(object):
 
 class Style(object):
     UNIT_RE = re.compile(r'^(-*[0-9]*[.]?[0-9]*)\s*(%|em|ex|en|px|mm|cm|in|pt|pc)$')
+    MS_PAT = re.compile(r'^\s*(mso-|panose-|text-underline|tab-interval)')
 
     def __init__(self, element, stylizer):
         self._element = element
@@ -447,6 +448,8 @@ class Style(object):
             return
         css = attrib['style'].split(';')
         css = filter(None, (x.strip() for x in css))
+        css = [x.strip() for x in css]
+        css = [x for x in css if self.MS_PAT.match(x) is None]
         try:
             style = CSSStyleDeclaration('; '.join(css))
         except CSSSyntaxError:

src/calibre/gui2/actions/add.py

@@ -20,9 +20,26 @@ from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.utils.filenames import ascii_filename
 from calibre.constants import preferred_encoding, filesystem_encoding
 from calibre.gui2.actions import InterfaceAction
-from calibre.gui2 import config
+from calibre.gui2 import config, question_dialog
 from calibre.ebooks.metadata import MetaInformation
 
+def get_filters():
+    return [
+            (_('Books'), BOOK_EXTENSIONS),
+            (_('EPUB Books'), ['epub']),
+            (_('LRF Books'), ['lrf']),
+            (_('HTML Books'), ['htm', 'html', 'xhtm', 'xhtml']),
+            (_('LIT Books'), ['lit']),
+            (_('MOBI Books'), ['mobi', 'prc', 'azw']),
+            (_('Topaz books'), ['tpz','azw1']),
+            (_('Text books'), ['txt', 'rtf']),
+            (_('PDF Books'), ['pdf']),
+            (_('SNB Books'), ['snb']),
+            (_('Comics'), ['cbz', 'cbr', 'cbc']),
+            (_('Archives'), ['zip', 'rar']),
+    ]
+
+
 class AddAction(InterfaceAction):
 
     name = 'Add Books'
@@ -47,6 +64,10 @@ class AddAction(InterfaceAction):
         self.add_menu.addAction(_('Add Empty book. (Book entry with no '
             'formats)'), self.add_empty, _('Shift+Ctrl+E'))
         self.add_menu.addAction(_('Add from ISBN'), self.add_from_isbn)
+        self.add_menu.addSeparator()
+        self.add_menu.addAction(_('Add files to selected book records'),
+                self.add_formats, _('Shift+A'))
+
         self.qaction.setMenu(self.add_menu)
         self.qaction.triggered.connect(self.add_books)
 
@@ -55,6 +76,39 @@ class AddAction(InterfaceAction):
         for action in list(self.add_menu.actions())[1:]:
             action.setEnabled(enabled)
 
+    def add_formats(self, *args):
+        if self.gui.stack.currentIndex() != 0:
+            return
+        view = self.gui.library_view
+        rows = view.selectionModel().selectedRows()
+        if not rows:
+            return
+        ids = [view.model().id(r) for r in rows]
+
+        if len(ids) > 1 and not question_dialog(self.gui,
+                _('Are you sure'),
+            _('Are you sure you want to add the same'
+                ' files to all %d books? If the format'
+                'already exists for a book, it will be replaced.')%len(ids)):
+                return
+
+        books = choose_files(self.gui, 'add formats dialog dir',
+                _('Select book files'), filters=get_filters())
+        if not books:
+            return
+
+        db = view.model().db
+        for id_ in ids:
+            for fpath in books:
+                fmt = os.path.splitext(fpath)[1][1:].upper()
+                if fmt:
+                    db.add_format_with_hooks(id_, fmt, fpath, index_is_id=True,
+                        notify=True)
+        current_idx = self.gui.library_view.currentIndex()
+        if current_idx.isValid():
+            view.model().current_changed(current_idx, current_idx)
+
+
     def add_recursive(self, single):
         root = choose_dir(self.gui, 'recursive book import root dir dialog',
                           'Select root folder')
@@ -207,27 +261,14 @@ class AddAction(InterfaceAction):
         '''
         Add books from the local filesystem to either the library or the device.
         '''
-        filters = [
-                        (_('Books'), BOOK_EXTENSIONS),
-                        (_('EPUB Books'), ['epub']),
-                        (_('LRF Books'), ['lrf']),
-                        (_('HTML Books'), ['htm', 'html', 'xhtm', 'xhtml']),
-                        (_('LIT Books'), ['lit']),
-                        (_('MOBI Books'), ['mobi', 'prc', 'azw']),
-                        (_('Topaz books'), ['tpz','azw1']),
-                        (_('Text books'), ['txt', 'rtf']),
-                        (_('PDF Books'), ['pdf']),
-                        (_('SNB Books'), ['snb']),
-                        (_('Comics'), ['cbz', 'cbr', 'cbc']),
-                        (_('Archives'), ['zip', 'rar']),
-                        ]
+        filters = get_filters()
         to_device = self.gui.stack.currentIndex() != 0
         if to_device:
             fmts = self.gui.device_manager.device.settings().format_map
             filters = [(_('Supported books'), fmts)]
 
-        books = choose_files(self.gui, 'add books dialog dir', 'Select books',
-                             filters=filters)
+        books = choose_files(self.gui, 'add books dialog dir',
+                _('Select books'), filters=filters)
         if not books:
             return
         self._add_books(books, to_device)

src/calibre/gui2/actions/choose_library.py

@@ -355,6 +355,7 @@ class ChooseLibraryAction(InterfaceAction):
                 print
         print 'before:', self.before_mem
         print 'after:', memory()/1024**2
+        print
         self.dbref = self.before_mem = None
 
 

src/calibre/gui2/actions/delete.py

@@ -19,11 +19,11 @@ single_shot = partial(QTimer.singleShot, 10)
 
 class MultiDeleter(QObject):
 
-    def __init__(self, gui, rows, callback):
+    def __init__(self, gui, ids, callback):
         from calibre.gui2.dialogs.progress import ProgressDialog
         QObject.__init__(self, gui)
         self.model = gui.library_view.model()
-        self.ids = list(map(self.model.id, rows))
+        self.ids = ids
         self.gui = gui
         self.failures = []
         self.deleted_ids = []
@@ -231,6 +231,7 @@ class DeleteAction(InterfaceAction):
             return
         # Library view is visible.
         if self.gui.stack.currentIndex() == 0:
+            to_delete_ids = [view.model().id(r) for r in rows]
             # Ask the user if they want to delete the book from the library or device if it is in both.
             if self.gui.device_manager.is_device_connected:
                 on_device = False
@@ -264,10 +265,10 @@ class DeleteAction(InterfaceAction):
             if ci.isValid():
                 row = ci.row()
             if len(rows) < 5:
-                ids_deleted = view.model().delete_books(rows)
-                self.library_ids_deleted(ids_deleted, row)
+                view.model().delete_books_by_id(to_delete_ids)
+                self.library_ids_deleted(to_delete_ids, row)
             else:
-                self.__md = MultiDeleter(self.gui, rows,
+                self.__md = MultiDeleter(self.gui, to_delete_ids,
                         partial(self.library_ids_deleted, current_row=row))
         # Device view is visible.
         else:

src/calibre/gui2/actions/fetch_news.py

@@ -58,6 +58,21 @@ class FetchNewsAction(InterfaceAction):
             self.scheduler.recipe_download_failed(arg)
             return self.gui.job_exception(job)
         id = self.gui.library_view.model().add_news(pt.name, arg)
+
+        # Arg may contain a "keep_issues" variable. If it is non-zero,
+        # delete all but newest x issues.
+        try:
+            keep_issues = int(arg['keep_issues'])
+        except:
+            keep_issues = 0
+        if keep_issues > 0:
+            ids_with_tag = list(sorted(self.gui.library_view.model().
+                db.tags_older_than(arg['title'],
+                    None, must_have_tag=_('News')), reverse=True))
+            ids_to_delete = ids_with_tag[keep_issues:]
+            if ids_to_delete:
+                self.gui.library_view.model().delete_books_by_id(ids_to_delete)
+
         self.gui.library_view.model().reset()
         sync = self.gui.news_to_be_synced
         sync.add(id)

src/calibre/gui2/device.py

@@ -140,6 +140,8 @@ class DeviceManager(Thread): # {{{
         self.mount_connection_requests = Queue.Queue(0)
         self.open_feedback_slot = open_feedback_slot
         self.open_feedback_msg = open_feedback_msg
+        self._device_information = None
+        self.current_library_uuid = None
 
     def report_progress(self, *args):
         pass
@@ -159,7 +161,7 @@ class DeviceManager(Thread): # {{{
             try:
                 dev.reset(detected_device=detected_device,
                     report_progress=self.report_progress)
-                dev.open()
+                dev.open(self.current_library_uuid)
             except OpenFeedback, e:
                 if dev not in self.ejected_devices:
                     self.open_feedback_msg(dev.get_gui_name(), e.feedback_msg)
@@ -194,6 +196,7 @@ class DeviceManager(Thread): # {{{
         else:
             self.connected_slot(False, self.connected_device_kind)
         self.connected_device = None
+        self._device_information = None
 
     def detect_device(self):
         self.scanner.scan()
@@ -292,9 +295,13 @@ class DeviceManager(Thread): # {{{
 
     def _get_device_information(self):
         info = self.device.get_device_information(end_session=False)
-        info = [i.replace('\x00', '').replace('\x01', '') for i in info]
+        if len(info) < 5:
+            info = tuple(list(info) + [{}])
+        info = [i.replace('\x00', '').replace('\x01', '') if isinstance(i, basestring) else i
+                 for i in info]
         cp = self.device.card_prefix(end_session=False)
         fs = self.device.free_space()
+        self._device_information = {'info': info, 'prefixes': cp, 'freespace': fs}
         return info, cp, fs
 
     def get_device_information(self, done):
@@ -302,6 +309,9 @@ class DeviceManager(Thread): # {{{
         return self.create_job(self._get_device_information, done,
                     description=_('Get device information'))
 
+    def get_current_device_information(self):
+        return self._device_information
+
     def _books(self):
         '''Get metadata from device'''
         mainlist = self.device.books(oncard=None, end_session=False)
@@ -417,6 +427,13 @@ class DeviceManager(Thread): # {{{
         return self.create_job(self._view_book, done, args=[path, target],
                         description=_('View book on device'))
 
+    def set_current_library_uuid(self, uuid):
+        self.current_library_uuid = uuid
+
+    def set_driveinfo_name(self, location_code, name):
+        if self.connected_device:
+            self.connected_device.set_driveinfo_name(location_code, name)
+
     # }}}
 
 class DeviceAction(QAction): # {{{

src/calibre/gui2/dialogs/book_info.py

@@ -6,7 +6,7 @@ __docformat__ = 'restructuredtext en'
 import textwrap, os, re
 
 from PyQt4.Qt import QCoreApplication, SIGNAL, QModelIndex, QTimer, Qt, \
-    QDialog, QPixmap, QGraphicsScene, QIcon, QSize
+    QDialog, QPixmap, QIcon, QSize
 
 from calibre.gui2.dialogs.book_info_ui import Ui_BookInfo
 from calibre.gui2 import dynamic, open_local_file, open_url
@@ -14,12 +14,14 @@ from calibre import fit_image
 from calibre.library.comments import comments_to_html
 from calibre.utils.icu import sort_key
 
+
 class BookInfo(QDialog, Ui_BookInfo):
 
     def __init__(self, parent, view, row, view_func):
         QDialog.__init__(self, parent)
         Ui_BookInfo.__init__(self)
         self.setupUi(self)
+        self.gui = parent
         self.cover_pixmap = None
         self.comments.sizeHint = self.comments_size_hint
         self.comments.page().setLinkDelegationPolicy(self.comments.page().DelegateAllLinks)
@@ -38,11 +40,26 @@ class BookInfo(QDialog, Ui_BookInfo):
         self.connect(self.text, SIGNAL('linkActivated(QString)'), self.open_book_path)
         self.fit_cover.stateChanged.connect(self.toggle_cover_fit)
         self.cover.resizeEvent = self.cover_view_resized
+        self.cover.cover_changed.connect(self.cover_changed)
 
         desktop = QCoreApplication.instance().desktop()
         screen_height = desktop.availableGeometry().height() - 100
         self.resize(self.size().width(), screen_height)
 
+    def cover_changed(self, data):
+        if self.current_row is not None:
+            id_ = self.view.model().id(self.current_row)
+            self.view.model().db.set_cover(id_, data)
+        if self.gui.cover_flow:
+            self.gui.cover_flow.dataChanged()
+        ci = self.view.currentIndex()
+        if ci.isValid():
+            self.view.model().current_changed(ci, ci)
+        self.cover_pixmap = QPixmap()
+        self.cover_pixmap.loadFromData(data)
+        if self.fit_cover.isChecked():
+            self.resize_cover()
+
     def link_clicked(self, url):
         open_url(url)
 
@@ -83,7 +100,6 @@ class BookInfo(QDialog, Ui_BookInfo):
         if self.cover_pixmap is None:
             return
         self.setWindowIcon(QIcon(self.cover_pixmap))
-        self.scene = QGraphicsScene()
         pixmap = self.cover_pixmap
         if self.fit_cover.isChecked():
             scaled, new_width, new_height = fit_image(pixmap.width(),
@@ -92,8 +108,7 @@ class BookInfo(QDialog, Ui_BookInfo):
             if scaled:
                 pixmap = pixmap.scaled(new_width, new_height,
                         Qt.KeepAspectRatio, Qt.SmoothTransformation)
-        self.scene.addPixmap(pixmap)
-        self.cover.setScene(self.scene)
+        self.cover.set_pixmap(pixmap)
 
     def refresh(self, row):
         if isinstance(row, QModelIndex):

src/calibre/gui2/dialogs/book_info.ui

@@ -25,7 +25,7 @@
     </widget>
    </item>
    <item row="1" column="0">
-    <widget class="QGraphicsView" name="cover"/>
+    <widget class="CoverView" name="cover"/>
    </item>
    <item row="1" column="1">
     <layout class="QVBoxLayout" name="verticalLayout">
@@ -115,6 +115,11 @@
    <extends>QWidget</extends>
    <header>QtWebKit/QWebView</header>
   </customwidget>
+  <customwidget>
+   <class>CoverView</class>
+   <extends>QGraphicsView</extends>
+   <header>calibre/gui2/widgets.h</header>
+  </customwidget>
  </customwidgets>
  <resources>
   <include location="../../../../resources/images.qrc"/>

src/calibre/gui2/dialogs/metadata_bulk.ui

@@ -341,7 +341,7 @@ from the value in the box</string>
                 <number>1</number>
                </property>
                <property name="maximum">
-                <number>990000</number>
+                <number>99000000</number>
                </property>
                <property name="value">
                 <number>1</number>

src/calibre/gui2/dialogs/metadata_single.ui

@@ -419,7 +419,7 @@ If the box is colored green, then text matches the individual author's sort stri
                      <string>Book </string>
                     </property>
                     <property name="maximum">
-                     <double>9999.989999999999782</double>
+                     <double>99999999.989999994635582</double>
                     </property>
                    </widget>
                   </item>

src/calibre/gui2/dialogs/scheduler.py

@@ -10,11 +10,9 @@ Scheduler for automated recipe downloads
 from datetime import timedelta
 
 from PyQt4.Qt import QDialog, SIGNAL, Qt, QTime, QObject, QMenu, \
-        QAction, QIcon, QMutex, QTimer, pyqtSignal, QWidget, QHBoxLayout, \
-        QLabel
+        QAction, QIcon, QMutex, QTimer, pyqtSignal
 
 from calibre.gui2.dialogs.scheduler_ui import Ui_Dialog
-from calibre.gui2.search_box import SearchBox2
 from calibre.gui2 import config as gconf, error_dialog
 from calibre.web.feeds.recipes.model import RecipeModel
 from calibre.ptempfile import PersistentTemporaryFile
@@ -28,18 +26,12 @@ class SchedulerDialog(QDialog, Ui_Dialog):
         self.setupUi(self)
         self.recipe_model = recipe_model
         self.recipe_model.do_refresh()
+        self.count_label.setText(
+            _('%s news sources') %
+                self.recipe_model.showing_count)
 
-        self._cont = QWidget(self)
-        self._cont.l = QHBoxLayout()
-        self._cont.setLayout(self._cont.l)
-        self._cont.la = QLabel(_('&Search:'))
-        self._cont.l.addWidget(self._cont.la, 1)
-        self.search = SearchBox2(self)
-        self._cont.l.addWidget(self.search, 100)
-        self._cont.la.setBuddy(self.search)
-        self.search.setMinimumContentsLength(25)
         self.search.initialize('scheduler_search_history')
-        self.recipe_box.layout().insertWidget(0, self._cont)
+        self.search.setMinimumContentsLength(15)
         self.search.search.connect(self.recipe_model.search)
         self.recipe_model.searched.connect(self.search.search_done,
                 type=Qt.QueuedConnection)
@@ -153,9 +145,12 @@ class SchedulerDialog(QDialog, Ui_Dialog):
             self.recipe_model.un_schedule_recipe(urn)
 
         add_title_tag = self.add_title_tag.isChecked()
+        keep_issues = u'0'
+        if self.keep_issues.isEnabled():
+            keep_issues = unicode(self.keep_issues.value())
         custom_tags = unicode(self.custom_tags.text()).strip()
         custom_tags = [x.strip() for x in custom_tags.split(',')]
-        self.recipe_model.customize_recipe(urn, add_title_tag, custom_tags)
+        self.recipe_model.customize_recipe(urn, add_title_tag, custom_tags, keep_issues)
         return True
 
     def initialize_detail_box(self, urn):
@@ -215,9 +210,16 @@ class SchedulerDialog(QDialog, Ui_Dialog):
             if d < timedelta(days=366):
                 self.last_downloaded.setText(_('Last downloaded')+': '+tm)
 
-        add_title_tag, custom_tags = customize_info
+        add_title_tag, custom_tags, keep_issues = customize_info
         self.add_title_tag.setChecked(add_title_tag)
         self.custom_tags.setText(u', '.join(custom_tags))
+        try:
+            keep_issues = int(keep_issues)
+        except:
+            keep_issues = 0
+        self.keep_issues.setValue(keep_issues)
+        self.keep_issues.setEnabled(self.add_title_tag.isChecked())
+
 
 
 class Scheduler(QObject):
@@ -299,7 +301,7 @@ class Scheduler(QObject):
             un = pw = None
             if account_info is not None:
                 un, pw = account_info
-            add_title_tag, custom_tags = customize_info
+            add_title_tag, custom_tags, keep_issues = customize_info
             script = self.recipe_model.get_recipe(urn)
             pt = PersistentTemporaryFile('_builtin.recipe')
             pt.write(script)
@@ -312,6 +314,7 @@ class Scheduler(QObject):
                     'recipe':pt.name,
                     'title':recipe.get('title',''),
                     'urn':urn,
+                    'keep_issues':keep_issues
                    }
             self.download_queue.add(urn)
             self.start_recipe_fetch.emit(arg)

src/calibre/gui2/dialogs/scheduler.ui

@@ -14,358 +14,403 @@
    <string>Schedule news download</string>
   </property>
   <property name="windowIcon">
-   <iconset>
+   <iconset resource="../../../../resources/images.qrc">
     <normaloff>:/images/scheduler.png</normaloff>:/images/scheduler.png</iconset>
   </property>
-  <layout class="QGridLayout" name="gridLayout">
-   <item row="0" column="0" rowspan="3">
-    <widget class="QGroupBox" name="recipe_box">
-     <property name="title">
-      <string>Recipes</string>
+  <layout class="QGridLayout" name="gridLayout" columnstretch="0,1,2">
+   <item row="0" column="0">
+    <widget class="QLabel" name="label_8">
+     <property name="text">
+      <string>&amp;Search:</string>
+     </property>
+     <property name="buddy">
+      <cstring>search</cstring>
      </property>
-     <layout class="QVBoxLayout" name="verticalLayout">
-      <item>
-       <widget class="QTreeView" name="recipes">
-        <property name="showDropIndicator" stdset="0">
-         <bool>false</bool>
-        </property>
-        <property name="iconSize">
-         <size>
-          <width>16</width>
-          <height>16</height>
-         </size>
-        </property>
-        <property name="animated">
-         <bool>true</bool>
-        </property>
-        <property name="headerHidden">
-         <bool>true</bool>
-        </property>
-       </widget>
-      </item>
-      <item>
-       <widget class="QPushButton" name="download_all_button">
-        <property name="toolTip">
-         <string>Download all scheduled recipes at once</string>
-        </property>
-        <property name="text">
-         <string>Download &amp;all scheduled</string>
-        </property>
-       </widget>
-      </item>
-      <item>
-       <widget class="QLabel" name="rnumber">
-        <property name="text">
-         <string/>
-        </property>
-       </widget>
-      </item>
-     </layout>
     </widget>
    </item>
    <item row="0" column="1">
-    <layout class="QVBoxLayout" name="verticalLayout_3">
-     <item>
-      <widget class="QScrollArea" name="scrollArea">
-       <property name="frameShape">
-        <enum>QFrame::NoFrame</enum>
+    <widget class="SearchBox2" name="search"/>
+   </item>
+   <item row="0" column="2" rowspan="3">
+    <widget class="QScrollArea" name="scrollArea">
+     <property name="frameShape">
+      <enum>QFrame::NoFrame</enum>
+     </property>
+     <property name="widgetResizable">
+      <bool>true</bool>
+     </property>
+     <widget class="QWidget" name="scrollAreaWidgetContents">
+      <property name="geometry">
+       <rect>
+        <x>0</x>
+        <y>0</y>
+        <width>469</width>
+        <height>504</height>
+       </rect>
+      </property>
+      <layout class="QVBoxLayout" name="verticalLayout_5">
+       <property name="margin">
+        <number>0</number>
        </property>
-       <property name="widgetResizable">
-        <bool>true</bool>
-       </property>
-       <widget class="QWidget" name="scrollAreaWidgetContents">
-        <property name="geometry">
-         <rect>
-          <x>0</x>
-          <y>0</y>
-          <width>375</width>
-          <height>502</height>
-         </rect>
-        </property>
-        <layout class="QVBoxLayout" name="verticalLayout_5">
-         <property name="margin">
+       <item>
+        <widget class="QTabWidget" name="detail_box">
+         <property name="currentIndex">
           <number>0</number>
          </property>
-         <item>
-          <widget class="QTabWidget" name="detail_box">
-           <property name="sizePolicy">
-            <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
-             <horstretch>0</horstretch>
-             <verstretch>100</verstretch>
-            </sizepolicy>
-           </property>
-           <property name="currentIndex">
-            <number>0</number>
-           </property>
-           <widget class="QWidget" name="tab">
-            <attribute name="title">
-             <string>&amp;Schedule</string>
-            </attribute>
-            <layout class="QVBoxLayout" name="verticalLayout_2">
+         <widget class="QWidget" name="tab">
+          <attribute name="title">
+           <string>&amp;Schedule</string>
+          </attribute>
+          <layout class="QVBoxLayout" name="verticalLayout_2">
+           <item>
+            <widget class="QLabel" name="blurb">
+             <property name="text">
+              <string>blurb</string>
+             </property>
+             <property name="wordWrap">
+              <bool>true</bool>
+             </property>
+             <property name="openExternalLinks">
+              <bool>true</bool>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QCheckBox" name="schedule">
+             <property name="text">
+              <string>&amp;Schedule for download:</string>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <layout class="QHBoxLayout" name="horizontalLayout_2">
              <item>
-              <widget class="QLabel" name="blurb">
+              <widget class="QRadioButton" name="daily_button">
                <property name="text">
-                <string>blurb</string>
-               </property>
-               <property name="textFormat">
-                <enum>Qt::RichText</enum>
-               </property>
-               <property name="wordWrap">
-                <bool>true</bool>
-               </property>
-               <property name="openExternalLinks">
-                <bool>true</bool>
+                <string>Every </string>
                </property>
               </widget>
              </item>
              <item>
-              <widget class="QCheckBox" name="schedule">
+              <widget class="QComboBox" name="day">
+               <item>
+                <property name="text">
+                 <string>day</string>
+                </property>
+               </item>
+               <item>
+                <property name="text">
+                 <string>Monday</string>
+                </property>
+               </item>
+               <item>
+                <property name="text">
+                 <string>Tuesday</string>
+                </property>
+               </item>
+               <item>
+                <property name="text">
+                 <string>Wednesday</string>
+                </property>
+               </item>
+               <item>
+                <property name="text">
+                 <string>Thursday</string>
+                </property>
+               </item>
+               <item>
+                <property name="text">
+                 <string>Friday</string>
+                </property>
+               </item>
+               <item>
+                <property name="text">
+                 <string>Saturday</string>
+                </property>
+               </item>
+               <item>
+                <property name="text">
+                 <string>Sunday</string>
+                </property>
+               </item>
+              </widget>
+             </item>
+             <item>
+              <widget class="QLabel" name="label_4">
                <property name="text">
-                <string>&amp;Schedule for download:</string>
+                <string>at</string>
                </property>
               </widget>
              </item>
              <item>
-              <layout class="QHBoxLayout" name="horizontalLayout_2">
-               <item>
-                <widget class="QRadioButton" name="daily_button">
-                 <property name="text">
-                  <string>Every </string>
-                 </property>
-                </widget>
-               </item>
-               <item>
-                <widget class="QComboBox" name="day">
-                 <item>
-                  <property name="text">
-                   <string>day</string>
-                  </property>
-                 </item>
-                 <item>
-                  <property name="text">
-                   <string>Monday</string>
-                  </property>
-                 </item>
-                 <item>
-                  <property name="text">
-                   <string>Tuesday</string>
-                  </property>
-                 </item>
-                 <item>
-                  <property name="text">
-                   <string>Wednesday</string>
-                  </property>
-                 </item>
-                 <item>
-                  <property name="text">
-                   <string>Thursday</string>
-                  </property>
-                 </item>
-                 <item>
-                  <property name="text">
-                   <string>Friday</string>
-                  </property>
-                 </item>
-                 <item>
-                  <property name="text">
-                   <string>Saturday</string>
-                  </property>
-                 </item>
-                 <item>
-                  <property name="text">
-                   <string>Sunday</string>
-                  </property>
-                 </item>
-                </widget>
-               </item>
-               <item>
-                <widget class="QLabel" name="label_4">
-                 <property name="text">
-                  <string>at</string>
-                 </property>
-                </widget>
-               </item>
-               <item>
-                <widget class="QTimeEdit" name="time"/>
-               </item>
-               <item>
-                <spacer name="horizontalSpacer">
-                 <property name="orientation">
-                  <enum>Qt::Horizontal</enum>
-                 </property>
-                 <property name="sizeHint" stdset="0">
-                  <size>
-                   <width>40</width>
-                   <height>20</height>
-                  </size>
-                 </property>
-                </spacer>
-               </item>
-              </layout>
+              <widget class="QTimeEdit" name="time"/>
              </item>
              <item>
-              <layout class="QHBoxLayout" name="horizontalLayout">
-               <item>
-                <widget class="QRadioButton" name="interval_button">
-                 <property name="text">
-                  <string>Every </string>
-                 </property>
-                </widget>
-               </item>
-               <item>
-                <widget class="QDoubleSpinBox" name="interval">
-                 <property name="sizePolicy">
-                  <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
-                   <horstretch>0</horstretch>
-                   <verstretch>0</verstretch>
-                  </sizepolicy>
-                 </property>
-                 <property name="toolTip">
-                  <string>Interval at which to download this recipe. A value of zero means that the recipe will be downloaded every hour.</string>
-                 </property>
-                 <property name="suffix">
-                  <string> days</string>
-                 </property>
-                 <property name="decimals">
-                  <number>1</number>
-                 </property>
-                 <property name="minimum">
-                  <double>0.000000000000000</double>
-                 </property>
-                 <property name="maximum">
-                  <double>365.100000000000023</double>
-                 </property>
-                 <property name="singleStep">
-                  <double>1.000000000000000</double>
-                 </property>
-                 <property name="value">
-                  <double>1.000000000000000</double>
-                 </property>
-                </widget>
-               </item>
-              </layout>
-             </item>
-             <item>
-              <widget class="QLabel" name="last_downloaded">
-               <property name="text">
-                <string/>
-               </property>
-              </widget>
-             </item>
-             <item>
-              <widget class="QGroupBox" name="account">
-               <property name="title">
-                <string>&amp;Account</string>
-               </property>
-               <layout class="QGridLayout" name="gridLayout_2">
-                <item row="0" column="1">
-                 <widget class="QLineEdit" name="username"/>
-                </item>
-                <item row="0" column="0">
-                 <widget class="QLabel" name="label_2">
-                  <property name="text">
-                   <string>&amp;Username:</string>
-                  </property>
-                  <property name="buddy">
-                   <cstring>username</cstring>
-                  </property>
-                 </widget>
-                </item>
-                <item row="1" column="0">
-                 <widget class="QLabel" name="label_3">
-                  <property name="text">
-                   <string>&amp;Password:</string>
-                  </property>
-                  <property name="buddy">
-                   <cstring>password</cstring>
-                  </property>
-                 </widget>
-                </item>
-                <item row="1" column="1">
-                 <widget class="QLineEdit" name="password">
-                  <property name="echoMode">
-                   <enum>QLineEdit::Password</enum>
-                  </property>
-                 </widget>
-                </item>
-                <item row="2" column="0">
-                 <widget class="QCheckBox" name="show_password">
-                  <property name="text">
-                   <string>&amp;Show password</string>
-                  </property>
-                 </widget>
-                </item>
-               </layout>
-              </widget>
-             </item>
-             <item>
-              <widget class="QLabel" name="label">
-               <property name="text">
-                <string>For the scheduling to work, you must leave calibre running.</string>
-               </property>
-               <property name="wordWrap">
-                <bool>true</bool>
-               </property>
-              </widget>
-             </item>
-            </layout>
-           </widget>
-           <widget class="QWidget" name="tab_2">
-            <attribute name="title">
-             <string>&amp;Advanced</string>
-            </attribute>
-            <layout class="QVBoxLayout" name="verticalLayout_4">
-             <item>
-              <widget class="QCheckBox" name="add_title_tag">
-               <property name="text">
-                <string>Add &amp;title as tag</string>
-               </property>
-              </widget>
-             </item>
-             <item>
-              <widget class="QLabel" name="label_5">
-               <property name="text">
-                <string>&amp;Extra  tags:</string>
-               </property>
-               <property name="buddy">
-                <cstring>custom_tags</cstring>
-               </property>
-              </widget>
-             </item>
-             <item>
-              <widget class="QLineEdit" name="custom_tags"/>
-             </item>
-             <item>
-              <spacer name="verticalSpacer">
+              <spacer name="horizontalSpacer">
                <property name="orientation">
-                <enum>Qt::Vertical</enum>
+                <enum>Qt::Horizontal</enum>
                </property>
                <property name="sizeHint" stdset="0">
                 <size>
-                 <width>20</width>
-                 <height>40</height>
+                 <width>40</width>
+                 <height>20</height>
                 </size>
                </property>
               </spacer>
              </item>
             </layout>
-           </widget>
-          </widget>
-         </item>
-         <item>
-          <widget class="QPushButton" name="download_button">
-           <property name="text">
-            <string>&amp;Download now</string>
-           </property>
-          </widget>
-         </item>
-        </layout>
-       </widget>
+           </item>
+           <item>
+            <layout class="QHBoxLayout" name="horizontalLayout">
+             <item>
+              <widget class="QRadioButton" name="interval_button">
+               <property name="text">
+                <string>Every </string>
+               </property>
+              </widget>
+             </item>
+             <item>
+              <widget class="QDoubleSpinBox" name="interval">
+               <property name="sizePolicy">
+                <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
+                 <horstretch>0</horstretch>
+                 <verstretch>0</verstretch>
+                </sizepolicy>
+               </property>
+               <property name="toolTip">
+                <string>Interval at which to download this recipe. A value of zero means that the recipe will be downloaded every hour.</string>
+               </property>
+               <property name="suffix">
+                <string> days</string>
+               </property>
+               <property name="decimals">
+                <number>1</number>
+               </property>
+               <property name="minimum">
+                <double>0.000000000000000</double>
+               </property>
+               <property name="maximum">
+                <double>365.100000000000023</double>
+               </property>
+               <property name="singleStep">
+                <double>1.000000000000000</double>
+               </property>
+               <property name="value">
+                <double>1.000000000000000</double>
+               </property>
+              </widget>
+             </item>
+            </layout>
+           </item>
+           <item>
+            <widget class="QLabel" name="last_downloaded">
+             <property name="text">
+              <string/>
+             </property>
+             <property name="wordWrap">
+              <bool>true</bool>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QGroupBox" name="account">
+             <property name="title">
+              <string>&amp;Account</string>
+             </property>
+             <layout class="QGridLayout" name="gridLayout_2">
+              <item row="0" column="1">
+               <widget class="QLineEdit" name="username"/>
+              </item>
+              <item row="0" column="0">
+               <widget class="QLabel" name="label_2">
+                <property name="text">
+                 <string>&amp;Username:</string>
+                </property>
+                <property name="buddy">
+                 <cstring>username</cstring>
+                </property>
+               </widget>
+              </item>
+              <item row="1" column="0">
+               <widget class="QLabel" name="label_3">
+                <property name="text">
+                 <string>&amp;Password:</string>
+                </property>
+                <property name="buddy">
+                 <cstring>password</cstring>
+                </property>
+               </widget>
+              </item>
+              <item row="1" column="1">
+               <widget class="QLineEdit" name="password">
+                <property name="echoMode">
+                 <enum>QLineEdit::Password</enum>
+                </property>
+               </widget>
+              </item>
+              <item row="2" column="0">
+               <widget class="QCheckBox" name="show_password">
+                <property name="text">
+                 <string>&amp;Show password</string>
+                </property>
+               </widget>
+              </item>
+             </layout>
+            </widget>
+           </item>
+           <item>
+            <widget class="QLabel" name="label">
+             <property name="text">
+              <string>For the scheduling to work, you must leave calibre running.</string>
+             </property>
+             <property name="wordWrap">
+              <bool>true</bool>
+             </property>
+            </widget>
+           </item>
+          </layout>
+         </widget>
+         <widget class="QWidget" name="tab_2">
+          <attribute name="title">
+           <string>&amp;Advanced</string>
+          </attribute>
+          <layout class="QGridLayout" name="gridLayout_3">
+           <item row="0" column="0">
+            <widget class="QCheckBox" name="add_title_tag">
+             <property name="text">
+              <string>Add &amp;title as tag</string>
+             </property>
+            </widget>
+           </item>
+           <item row="1" column="0">
+            <widget class="QLabel" name="label_5">
+             <property name="text">
+              <string>&amp;Extra  tags:</string>
+             </property>
+             <property name="buddy">
+              <cstring>custom_tags</cstring>
+             </property>
+            </widget>
+           </item>
+           <item row="3" column="0" colspan="2">
+            <widget class="QLabel" name="label_6">
+             <property name="toolTip">
+              <string>Maximum number of copies (issues) of this recipe to keep.  Set to 0 to keep all (disable).</string>
+             </property>
+             <property name="text">
+              <string>&amp;Keep at most:</string>
+             </property>
+             <property name="buddy">
+              <cstring>keep_issues</cstring>
+             </property>
+            </widget>
+           </item>
+           <item row="3" column="2">
+            <widget class="QSpinBox" name="keep_issues">
+             <property name="toolTip">
+              <string>&lt;p&gt;When set, this option will cause calibre to keep, at most, the specified number of issues of this periodical. Every time a new issue is downloaded, the oldest one is deleted, if the total is larger than this number.
+&lt;p&gt;Note that this feature only works if you have the option to add the title as tag checked, above.
+&lt;p&gt;Also, the setting for deleting periodicals older than a number of days, below, takes priority over this setting.</string>
+             </property>
+             <property name="specialValueText">
+              <string>all issues</string>
+             </property>
+             <property name="suffix">
+              <string> issues</string>
+             </property>
+             <property name="maximum">
+              <number>100000</number>
+             </property>
+            </widget>
+           </item>
+           <item row="4" column="1">
+            <spacer name="verticalSpacer">
+             <property name="orientation">
+              <enum>Qt::Vertical</enum>
+             </property>
+             <property name="sizeHint" stdset="0">
+              <size>
+               <width>20</width>
+               <height>40</height>
+              </size>
+             </property>
+            </spacer>
+           </item>
+           <item row="1" column="2">
+            <widget class="QLineEdit" name="custom_tags"/>
+           </item>
+          </layout>
+         </widget>
+        </widget>
+       </item>
+       <item>
+        <widget class="QPushButton" name="download_button">
+         <property name="text">
+          <string>&amp;Download now</string>
+         </property>
+        </widget>
+       </item>
+      </layout>
+     </widget>
+    </widget>
+   </item>
+   <item row="1" column="0" colspan="2">
+    <widget class="QTreeView" name="recipes">
+     <property name="sizePolicy">
+      <sizepolicy hsizetype="Minimum" vsizetype="Expanding">
+       <horstretch>0</horstretch>
+       <verstretch>0</verstretch>
+      </sizepolicy>
+     </property>
+     <property name="showDropIndicator" stdset="0">
+      <bool>false</bool>
+     </property>
+     <property name="iconSize">
+      <size>
+       <width>16</width>
+       <height>16</height>
+      </size>
+     </property>
+     <property name="animated">
+      <bool>true</bool>
+     </property>
+     <property name="headerHidden">
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
+   <item row="3" column="2">
+    <layout class="QHBoxLayout" name="horizontalLayout_3">
+     <item>
+      <widget class="QLabel" name="label_7">
+       <property name="text">
+        <string>&amp;Delete downloaded news older than:</string>
+       </property>
+       <property name="buddy">
+        <cstring>old_news</cstring>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QSpinBox" name="old_news">
+       <property name="toolTip">
+        <string>&lt;p&gt;Delete downloaded news older than the specified number of days. Set to zero to disable.
+&lt;p&gt;You can also control the maximum number of issues of a specific periodical that are kept by clicking the Advanced tab for that periodical above.</string>
+       </property>
+       <property name="specialValueText">
+        <string>never delete</string>
+       </property>
+       <property name="suffix">
+        <string> days</string>
+       </property>
+       <property name="maximum">
+        <number>1000</number>
+       </property>
       </widget>
      </item>
     </layout>
    </item>
-   <item row="2" column="1">
+   <item row="4" column="2">
     <widget class="QDialogButtonBox" name="buttonBox">
      <property name="orientation">
       <enum>Qt::Horizontal</enum>
@@ -375,24 +420,35 @@
      </property>
     </widget>
    </item>
-   <item row="1" column="1">
-    <widget class="QSpinBox" name="old_news">
+   <item row="4" column="0" colspan="2">
+    <widget class="QPushButton" name="download_all_button">
      <property name="toolTip">
-      <string>Delete downloaded news older than the specified number of days. Set to zero to disable.</string>
+      <string>Download all scheduled news sources at once</string>
      </property>
-     <property name="suffix">
-      <string> days</string>
+     <property name="text">
+      <string>Download &amp;all scheduled</string>
      </property>
-     <property name="prefix">
-      <string>Delete downloaded news older than </string>
+    </widget>
+   </item>
+   <item row="3" column="0" colspan="2">
+    <widget class="QLabel" name="count_label">
+     <property name="text">
+      <string/>
      </property>
-     <property name="maximum">
-      <number>1000</number>
+     <property name="alignment">
+      <set>Qt::AlignCenter</set>
      </property>
     </widget>
    </item>
   </layout>
  </widget>
+ <customwidgets>
+  <customwidget>
+   <class>SearchBox2</class>
+   <extends>QComboBox</extends>
+   <header>calibre/gui2/search_box.h</header>
+  </customwidget>
+ </customwidgets>
  <resources>
   <include location="../../../../resources/images.qrc"/>
  </resources>
@@ -436,12 +492,12 @@
    <slot>setEnabled(bool)</slot>
    <hints>
     <hint type="sourcelabel">
-     <x>456</x>
-     <y>173</y>
+     <x>458</x>
+     <y>155</y>
     </hint>
     <hint type="destinationlabel">
-     <x>537</x>
-     <y>176</y>
+     <x>573</x>
+     <y>158</y>
     </hint>
    </hints>
   </connection>
@@ -452,12 +508,12 @@
    <slot>setEnabled(bool)</slot>
    <hints>
     <hint type="sourcelabel">
-     <x>456</x>
-     <y>173</y>
+     <x>458</x>
+     <y>155</y>
     </hint>
     <hint type="destinationlabel">
-     <x>647</x>
-     <y>176</y>
+     <x>684</x>
+     <y>157</y>
     </hint>
    </hints>
   </connection>
@@ -468,12 +524,28 @@
    <slot>setEnabled(bool)</slot>
    <hints>
     <hint type="sourcelabel">
-     <x>456</x>
-     <y>239</y>
+     <x>458</x>
+     <y>212</y>
     </hint>
     <hint type="destinationlabel">
-     <x>495</x>
-     <y>218</y>
+     <x>752</x>
+     <y>215</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>add_title_tag</sender>
+   <signal>toggled(bool)</signal>
+   <receiver>keep_issues</receiver>
+   <slot>setEnabled(bool)</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>508</x>
+     <y>42</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>577</x>
+     <y>108</y>
     </hint>
    </hints>
   </connection>

src/calibre/gui2/dialogs/tag_categories.py

@@ -178,8 +178,10 @@ class TagCategories(QDialog, Ui_TagCategories):
                       'multiple periods in a row or spaces before '
                       'or after periods.')).exec_()
             return False
-        for c in self.categories:
-            if strcmp(c, cat_name) == 0:
+        for c in sorted(self.categories.keys(), key=sort_key):
+            if strcmp(c, cat_name) == 0 or \
+                    (icu_lower(cat_name).startswith(icu_lower(c) + '.') and\
+                     not cat_name.startswith(c + '.')):
                 error_dialog(self, _('Name already used'),
                         _('That name is already used, perhaps with different case.')).exec_()
                 return False

src/calibre/gui2/init.py

@@ -141,6 +141,15 @@ class Stack(QStackedWidget): # {{{
 
 # }}}
 
+class UpdateLabel(QLabel): # {{{
+
+    def __init__(self, *args, **kwargs):
+        QLabel.__init__(self, *args, **kwargs)
+
+    def contextMenuEvent(self, e):
+        pass
+# }}}
+
 class StatusBar(QStatusBar): # {{{
 
     def __init__(self, parent=None):
@@ -148,7 +157,7 @@ class StatusBar(QStatusBar): # {{{
         self.default_message = __appname__ + ' ' + _('version') + ' ' + \
                 self.get_version() + ' ' + _('created by Kovid Goyal')
         self.device_string = ''
-        self.update_label = QLabel('')
+        self.update_label = UpdateLabel('')
         self.addPermanentWidget(self.update_label)
         self.update_label.setVisible(False)
         self._font = QFont()

src/calibre/gui2/metadata/basic_widgets.py

@@ -351,7 +351,7 @@ class SeriesIndexEdit(QDoubleSpinBox):
         QDoubleSpinBox.__init__(self, parent)
         self.dialog = parent
         self.db = self.original_series_name = None
-        self.setMaximum(1000000)
+        self.setMaximum(10000000)
         self.series_edit = series_edit
         series_edit.currentIndexChanged.connect(self.enable)
         series_edit.editTextChanged.connect(self.enable)

src/calibre/gui2/preferences/__init__.py

@@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
 import textwrap
 
 from PyQt4.Qt import QWidget, pyqtSignal, QCheckBox, QAbstractSpinBox, \
-    QLineEdit, QComboBox, QVariant
+    QLineEdit, QComboBox, QVariant, Qt
 
 from calibre.customize.ui import preferences_plugins
 from calibre.utils.config import ConfigProxy
@@ -82,6 +82,8 @@ class ConfigWidgetInterface(object):
 
 class Setting(object):
 
+    CHOICES_SEARCH_FLAGS = Qt.MatchExactly | Qt.MatchCaseSensitive
+
     def __init__(self, name, config_obj, widget, gui_name=None,
             empty_string_is_None=True, choices=None, restart_required=False):
         self.name, self.gui_name = name, gui_name
@@ -168,7 +170,8 @@ class Setting(object):
         elif self.datatype == 'string':
             self.gui_obj.setText(val if val else '')
         elif self.datatype == 'choice':
-            idx = self.gui_obj.findData(QVariant(val))
+            idx = self.gui_obj.findData(QVariant(val), role=Qt.UserRole,
+                    flags=self.CHOICES_SEARCH_FLAGS)
             if idx == -1:
                 idx = 0
             self.gui_obj.setCurrentIndex(idx)

src/calibre/gui2/preferences/behavior.py

@@ -9,7 +9,7 @@ import re
 
 from PyQt4.Qt import Qt, QVariant, QListWidgetItem
 
-from calibre.gui2.preferences import ConfigWidgetBase, test_widget
+from calibre.gui2.preferences import ConfigWidgetBase, test_widget, Setting
 from calibre.gui2.preferences.behavior_ui import Ui_Form
 from calibre.gui2 import config, info_dialog, dynamic
 from calibre.utils.config import prefs
@@ -20,6 +20,10 @@ from calibre.ebooks.oeb.iterator import is_supported
 from calibre.constants import iswindows
 from calibre.utils.icu import sort_key
 
+class OutputFormatSetting(Setting):
+
+    CHOICES_SEARCH_FLAGS = Qt.MatchFixedString
+
 class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
     def genesis(self, gui):
@@ -43,7 +47,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         output_formats = list(sorted(available_output_formats()))
         output_formats.remove('oeb')
         choices = [(x.upper(), x) for x in output_formats]
-        r('output_format', prefs, choices=choices)
+        r('output_format', prefs, choices=choices, setting=OutputFormatSetting)
 
         restrictions = sorted(saved_searches().names(), key=sort_key)
         choices = [('', '')] + [(x, x) for x in restrictions]

src/calibre/gui2/preferences/columns.ui

@@ -38,6 +38,9 @@
     <layout class="QVBoxLayout" name="verticalLayout_3">
      <item>
       <widget class="QToolButton" name="column_up">
+       <property name="toolTip">
+        <string>Move column up</string>
+       </property>
        <property name="text">
         <string>...</string>
        </property>
@@ -45,6 +48,12 @@
         <iconset resource="../../../../resources/images.qrc">
          <normaloff>:/images/arrow-up.png</normaloff>:/images/arrow-up.png</iconset>
        </property>
+       <property name="iconSize">
+        <size>
+         <width>32</width>
+         <height>32</height>
+        </size>
+       </property>
       </widget>
      </item>
      <item>
@@ -72,6 +81,12 @@
         <iconset resource="../../../../resources/images.qrc">
          <normaloff>:/images/minus.png</normaloff>:/images/minus.png</iconset>
        </property>
+       <property name="iconSize">
+        <size>
+         <width>32</width>
+         <height>32</height>
+        </size>
+       </property>
       </widget>
      </item>
      <item>
@@ -99,6 +114,12 @@
         <iconset resource="../../../../resources/images.qrc">
          <normaloff>:/images/plus.png</normaloff>:/images/plus.png</iconset>
        </property>
+       <property name="iconSize">
+        <size>
+         <width>32</width>
+         <height>32</height>
+        </size>
+       </property>
       </widget>
      </item>
      <item>
@@ -126,6 +147,12 @@
         <iconset resource="../../../../resources/images.qrc">
          <normaloff>:/images/edit_input.png</normaloff>:/images/edit_input.png</iconset>
        </property>
+       <property name="iconSize">
+        <size>
+         <width>32</width>
+         <height>32</height>
+        </size>
+       </property>
       </widget>
      </item>
      <item>
@@ -143,6 +170,9 @@
      </item>
      <item>
       <widget class="QToolButton" name="column_down">
+       <property name="toolTip">
+        <string>Move column down</string>
+       </property>
        <property name="text">
         <string>...</string>
        </property>
@@ -150,6 +180,12 @@
         <iconset resource="../../../../resources/images.qrc">
          <normaloff>:/images/arrow-down.png</normaloff>:/images/arrow-down.png</iconset>
        </property>
+       <property name="iconSize">
+        <size>
+         <width>32</width>
+         <height>32</height>
+        </size>
+       </property>
       </widget>
      </item>
     </layout>

src/calibre/gui2/preferences/create_custom_column.py

@@ -6,7 +6,6 @@ __copyright__ = '2010, Kovid Goyal <kovid at kovidgoyal.net>'
 import re
 from functools import partial
 
-from PyQt4.QtCore import SIGNAL
 from PyQt4.Qt import QDialog, Qt, QListWidgetItem, QVariant
 
 from calibre.gui2.preferences.create_custom_column_ui import Ui_QCreateCustomColumn
@@ -48,6 +47,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
         QDialog.__init__(self, parent)
         Ui_QCreateCustomColumn.__init__(self)
         self.setupUi(self)
+        self.setWindowTitle(_('Create a custom column'))
+        self.heading_label.setText(_('Create a custom column'))
         # Remove help icon on title bar
         icon = self.windowIcon()
         self.setWindowFlags(self.windowFlags()&(~Qt.WindowContextHelpButtonHint))
@@ -55,8 +56,18 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
 
         self.simple_error = partial(error_dialog, self, show=True,
             show_copy_button=False)
-        self.connect(self.button_box, SIGNAL("accepted()"), self.accept)
-        self.connect(self.button_box, SIGNAL("rejected()"), self.reject)
+        self.button_box.accepted.connect(self.accept)
+        self.button_box.rejected.connect(self.reject)
+        self.shortcuts.linkActivated.connect(self.shortcut_activated)
+        text = '<p>'+_('Quick create:')
+        for col, name in [('isbn', _('ISBN')), ('formats', _('Formats')),
+                ('last_modified', _('Modified Date')), ('yesno', _('Yes/No')),
+                ('tags', _('Tags')), ('series', _('Series')), ('rating',
+                    _('Rating'))]:
+            text += ' <a href="col:%s">%s</a>,'%(col, name)
+        text = text[:-1]
+        self.shortcuts.setText(text)
+
         self.parent = parent
         self.editing_col = editing
         self.standard_colheads = standard_colheads
@@ -69,6 +80,9 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             self.datatype_changed()
             self.exec_()
             return
+        self.setWindowTitle(_('Edit a custom column'))
+        self.heading_label.setText(_('Edit a custom column'))
+        self.shortcuts.setVisible(False)
         idx = parent.opt_columns.currentRow()
         if idx < 0:
             self.simple_error(_('No column selected'),
@@ -99,6 +113,32 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
         self.datatype_changed()
         self.exec_()
 
+    def shortcut_activated(self, url):
+        which = unicode(url).split(':')[-1]
+        self.column_type_box.setCurrentIndex({
+            'yesno': 9,
+            'tags' : 1,
+            'series': 3,
+            'rating': 8,
+            }.get(which, 10))
+        self.column_name_box.setText(which)
+        self.column_heading_box.setText({
+            'isbn':'ISBN',
+            'formats':_('Formats'),
+            'yesno':_('Yes/No'),
+            'tags': _('My Tags'),
+            'series': _('My Series'),
+            'rating': _('My Rating'),
+            'last_modified':_('Modified Date')}[which])
+        if self.composite_box.isVisible():
+            self.composite_box.setText(
+                {
+                    'isbn': '{identifiers:select(isbn)}',
+                    'formats': '{formats}',
+                    'last_modified':'''{last_modified:'format_date($, "%d %m, %Y")'}'''
+                    }[which])
+
+
     def datatype_changed(self, *args):
         try:
             col_type = self.column_types[self.column_type_box.currentIndex()]['datatype']

src/calibre/gui2/preferences/create_custom_column.ui

@@ -9,8 +9,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>528</width>
-    <height>212</height>
+    <width>603</width>
+    <height>344</height>
    </rect>
   </property>
   <property name="sizePolicy">
@@ -19,19 +19,20 @@
     <verstretch>0</verstretch>
    </sizepolicy>
   </property>
-  <property name="windowTitle">
-   <string>Create or edit custom columns</string>
+  <property name="windowIcon">
+   <iconset resource="../../../../resources/images.qrc">
+    <normaloff>:/images/column.png</normaloff>:/images/column.png</iconset>
   </property>
   <layout class="QVBoxLayout" name="verticalLayout">
    <item>
-    <layout class="QGridLayout" name="gridLayout_2" rowstretch="0,0,0,0,0,0,0,0,0,0,0,0">
+    <layout class="QGridLayout" name="gridLayout_2" rowstretch="0,0,0,0,0,0,0,0,0,0,0,0,0,0,0">
      <property name="sizeConstraint">
       <enum>QLayout::SetDefaultConstraint</enum>
      </property>
      <property name="margin">
       <number>5</number>
      </property>
-     <item row="2" column="0">
+     <item row="5" column="0">
       <layout class="QGridLayout" name="gridLayout">
        <property name="margin">
         <number>0</number>
@@ -238,7 +239,7 @@ four values, the first of them being the empty value.</string>
        </item>
       </layout>
      </item>
-     <item row="11" column="0">
+     <item row="14" column="0">
       <widget class="QDialogButtonBox" name="button_box">
        <property name="orientation">
         <enum>Qt::Horizontal</enum>
@@ -252,7 +253,7 @@ four values, the first of them being the empty value.</string>
       </widget>
      </item>
      <item row="1" column="0">
-      <widget class="QLabel" name="label_6">
+      <widget class="QLabel" name="heading_label">
        <property name="font">
         <font>
          <weight>75</weight>
@@ -260,7 +261,31 @@ four values, the first of them being the empty value.</string>
         </font>
        </property>
        <property name="text">
-        <string>Create or edit custom columns</string>
+        <string/>
+       </property>
+      </widget>
+     </item>
+     <item row="3" column="0">
+      <widget class="QLabel" name="shortcuts">
+       <property name="text">
+        <string/>
+       </property>
+       <property name="wordWrap">
+        <bool>true</bool>
+       </property>
+      </widget>
+     </item>
+     <item row="2" column="0">
+      <widget class="Line" name="line">
+       <property name="orientation">
+        <enum>Qt::Vertical</enum>
+       </property>
+      </widget>
+     </item>
+     <item row="4" column="0">
+      <widget class="Line" name="line_2">
+       <property name="orientation">
+        <enum>Qt::Vertical</enum>
        </property>
       </widget>
      </item>
@@ -276,6 +301,8 @@ four values, the first of them being the empty value.</string>
   <tabstop>composite_box</tabstop>
   <tabstop>button_box</tabstop>
  </tabstops>
- <resources/>
+ <resources>
+  <include location="../../../../resources/images.qrc"/>
+ </resources>
  <connections/>
 </ui>

src/calibre/gui2/preferences/look_feel.py

@@ -66,7 +66,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         choices = set([k for k in db.field_metadata.all_field_keys()
                     if db.field_metadata[k]['is_category'] and
                        db.field_metadata[k]['datatype'] in ['text', 'series', 'enumeration']])
-        choices -= set(['authors', 'publisher', 'formats', 'news'])
+        choices -= set(['authors', 'publisher', 'formats', 'news', 'identifiers'])
         self.opt_categories_using_hierarchy.update_items_cache(choices)
         r('categories_using_hierarchy', db.prefs, setting=CommaSeparatedList,
           choices=sorted(list(choices), key=sort_key))

src/calibre/gui2/search_box.py

@@ -217,11 +217,15 @@ class SearchBox2(QComboBox): # {{{
                 self.clear()
             else:
                 self.normalize_state()
+                self.lineEdit().setCompleter(None)
                 self.setEditText(txt)
                 self.line_edit.end(False)
                 if emit_changed:
                     self.changed.emit()
                 self._do_search(store_in_history=store_in_history)
+                c = QCompleter()
+                self.lineEdit().setCompleter(c)
+                c.setCompletionMode(c.PopupCompletion)
             self.focus_to_library.emit()
         finally:
             if not store_in_history:

src/calibre/gui2/tag_view.py

@@ -21,6 +21,7 @@ from PyQt4.Qt import Qt, QTreeView, QApplication, pyqtSignal, QFont, QSize, \
 from calibre.ebooks.metadata import title_sort
 from calibre.gui2 import config, NONE, gprefs
 from calibre.library.field_metadata import TagsIcons, category_icon_map
+from calibre.library.database2 import Tag
 from calibre.utils.config import tweaks
 from calibre.utils.icu import sort_key, lower, strcmp
 from calibre.utils.search_query_parser import saved_searches
@@ -69,7 +70,8 @@ class TagDelegate(QItemDelegate): # {{{
 
     # }}}
 
-TAG_SEARCH_STATES = {'clear': 0, 'mark_plus': 1, 'mark_minus': 2}
+TAG_SEARCH_STATES = {'clear': 0, 'mark_plus': 1, 'mark_plusplus': 2,
+                     'mark_minus': 3, 'mark_minusminus': 4}
 
 class TagsView(QTreeView): # {{{
 
@@ -127,13 +129,17 @@ class TagsView(QTreeView): # {{{
         self.set_new_model(self._model.get_filter_categories_by())
 
     def set_database(self, db, tag_match, sort_by):
-        self.hidden_categories = db.prefs.get('tag_browser_hidden_categories', None)
+        hidden_cats = db.prefs.get('tag_browser_hidden_categories', None)
+        self.hidden_categories = []
         # migrate from config to db prefs
-        if self.hidden_categories is None:
-            self.hidden_categories = config['tag_browser_hidden_categories']
-            db.prefs.set('tag_browser_hidden_categories', list(self.hidden_categories))
-        else:
-            self.hidden_categories = set(self.hidden_categories)
+        if hidden_cats is None:
+            hidden_cats = config['tag_browser_hidden_categories']
+        # strip out any non-existence field keys
+        for cat in hidden_cats:
+            if cat in db.field_metadata:
+                self.hidden_categories.append(cat)
+        db.prefs.set('tag_browser_hidden_categories', list(self.hidden_categories))
+        self.hidden_categories = set(self.hidden_categories)
 
         old = getattr(self, '_model', None)
         if old is not None:
@@ -370,14 +376,15 @@ class TagsView(QTreeView): # {{{
                                     action='delete_user_category', key=key))
                     self.context_menu.addSeparator()
                 # Hide/Show/Restore categories
-                if not key.startswith('@') or key.find('.') < 0:
-                    self.context_menu.addAction(_('Hide category %s') % category,
-                        partial(self.context_menu_handler, action='hide',
-                                category=category))
+                #if not key.startswith('@') or key.find('.') < 0:
+                self.context_menu.addAction(_('Hide category %s') % category,
+                    partial(self.context_menu_handler, action='hide',
+                            category=key))
                 if self.hidden_categories:
                     m = self.context_menu.addMenu(_('Show category'))
-                    for col in sorted(self.hidden_categories, key=sort_key):
-                        m.addAction(col,
+                    for col in sorted(self.hidden_categories,
+                            key=lambda x: sort_key(self.db.field_metadata[x]['name'])):
+                        m.addAction(self.db.field_metadata[col]['name'],
                             partial(self.context_menu_handler, action='show', category=col))
 
                 # search by category
@@ -540,6 +547,7 @@ class TagTreeItem(object): # {{{
         self.id_set = set()
         self.is_gst = False
         self.boxed = False
+        self.icon_state_map = list(map(QVariant, icon_map))
         if self.parent is not None:
             self.parent.append(self)
         if data is None:
@@ -554,14 +562,25 @@ class TagTreeItem(object): # {{{
             self.bold_font = QVariant(self.bold_font)
             self.category_key = category_key
             self.temporary = temporary
+            self.tag = Tag(data)
+            self.tag.is_hierarchical = category_key.startswith('@')
         elif self.type == self.TAG:
-            icon_map[0] = data.icon
-            self.tag, self.icon_state_map = data, list(map(QVariant, icon_map))
+            self.icon_state_map[0] = QVariant(data.icon)
+            self.tag = data
         if tooltip:
             self.tooltip = tooltip + ' '
         else:
             self.tooltip = ''
 
+    def break_cycles(self):
+        for x in self.children:
+            try:
+                x.break_cycles()
+            except:
+                pass
+        self.parent = self.icon_state_map = self.bold_font = self.tag = \
+                self.icon = self.children = None
+
     def __str__(self):
         if self.type == self.ROOT:
             return 'ROOT'
@@ -593,6 +612,8 @@ class TagTreeItem(object): # {{{
         if role == Qt.EditRole:
             return QVariant(self.py_name)
         if role == Qt.DecorationRole:
+            if self.tag.state:
+                return self.icon_state_map[self.tag.state]
             return self.icon
         if role == Qt.FontRole:
             return self.bold_font
@@ -642,11 +663,23 @@ class TagTreeItem(object): # {{{
         '''
         set_to: None => advance the state, otherwise a value from TAG_SEARCH_STATES
         '''
-        if self.type == self.TAG:
-            if set_to is None:
-                self.tag.state = (self.tag.state + 1)%3
-            else:
-                self.tag.state = set_to
+        basic_search_ok = self.tag.is_editable or \
+                self.tag.category == 'formats' or self.tag.category == 'rating'
+        if set_to is None:
+            while True:
+                self.tag.state = (self.tag.state + 1)%5
+                if self.tag.state == TAG_SEARCH_STATES['mark_plus'] or \
+                        self.tag.state == TAG_SEARCH_STATES['mark_minus']:
+                    if basic_search_ok:
+                        break
+                elif self.tag.state == TAG_SEARCH_STATES['mark_plusplus'] or\
+                        self.tag.state == TAG_SEARCH_STATES['mark_minusminus']:
+                    if self.tag.is_hierarchical and len(self.children):
+                        break
+                else:
+                    break
+        else:
+            self.tag.state = set_to
 
     def child_tags(self):
         res = []
@@ -677,7 +710,8 @@ class TagsModel(QAbstractItemModel): # {{{
         self.categories_with_ratings = ['authors', 'series', 'publisher', 'tags']
         self.drag_drop_finished = drag_drop_finished
 
-        self.icon_state_map = [None, QIcon(I('plus.png')), QIcon(I('minus.png'))]
+        self.icon_state_map = [None, QIcon(I('plus.png')), QIcon(I('plusplus.png')),
+                               QIcon(I('minus.png')), QIcon(I('minusminus.png'))]
         self.db = db
         self.tags_view = parent
         self.hidden_categories = hidden_categories
@@ -691,26 +725,33 @@ class TagsModel(QAbstractItemModel): # {{{
 
         data = self.get_node_tree(config['sort_tags_by'])
         gst = db.prefs.get('grouped_search_terms', {})
-        self.root_item = TagTreeItem()
+        self.root_item = TagTreeItem(icon_map=self.icon_state_map)
         self.category_nodes = []
 
         last_category_node = None
         category_node_map = {}
         self.category_node_tree = {}
-        for i, r in enumerate(self.row_map):
-            if self.hidden_categories and self.categories[i] in self.hidden_categories:
-                continue
+        for i, key in enumerate(self.row_map):
+            if self.hidden_categories:
+                if key in self.hidden_categories:
+                    continue
+                found = False
+                for cat in self.hidden_categories:
+                    if cat.startswith('@') and key.startswith(cat + '.'):
+                        found = True
+                if found:
+                    continue
             is_gst = False
-            if r.startswith('@') and r[1:] in gst:
-                tt = _(u'The grouped search term name is "{0}"').format(r[1:])
+            if key.startswith('@') and key[1:] in gst:
+                tt = _(u'The grouped search term name is "{0}"').format(key[1:])
                 is_gst = True
-            elif r == 'news':
+            elif key == 'news':
                 tt = ''
             else:
-                tt = _(u'The lookup/search name is "{0}"').format(r)
+                tt = _(u'The lookup/search name is "{0}"').format(key)
 
-            if r.startswith('@'):
-                path_parts = [p for p in r.split('.')]
+            if key.startswith('@'):
+                path_parts = [p for p in key.split('.')]
                 path = ''
                 last_category_node = self.root_item
                 tree_root = self.category_node_tree
@@ -719,9 +760,10 @@ class TagsModel(QAbstractItemModel): # {{{
                     if path not in category_node_map:
                         node = TagTreeItem(parent=last_category_node,
                                            data=p[1:] if i == 0 else p,
-                                           category_icon=self.category_icon_map[r],
-                                           tooltip=tt if path == r else path,
-                                           category_key=path)
+                                           category_icon=self.category_icon_map[key],
+                                           tooltip=tt if path == key else path,
+                                           category_key=path,
+                                           icon_map=self.icon_state_map)
                         last_category_node = node
                         category_node_map[path] = node
                         self.category_nodes.append(node)
@@ -736,16 +778,18 @@ class TagsModel(QAbstractItemModel): # {{{
                     path += '.'
             else:
                 node = TagTreeItem(parent=self.root_item,
-                                   data=self.categories[i],
-                                   category_icon=self.category_icon_map[r],
-                                   tooltip=tt, category_key=r)
+                                   data=self.categories[key],
+                                   category_icon=self.category_icon_map[key],
+                                   tooltip=tt, category_key=key,
+                                   icon_map=self.icon_state_map)
                 node.is_gst = False
-                category_node_map[r] = node
+                category_node_map[key] = node
                 last_category_node = node
                 self.category_nodes.append(node)
         self.refresh(data=data)
 
     def break_cycles(self):
+        self.root_item.break_cycles()
         self.db = self.root_item = None
 
     def mimeTypes(self):
@@ -1015,7 +1059,7 @@ class TagsModel(QAbstractItemModel): # {{{
     def get_node_tree(self, sort):
         old_row_map = self.row_map[:]
         self.row_map = []
-        self.categories = []
+        self.categories = {}
 
         # Get the categories
         if self.search_restriction:
@@ -1062,7 +1106,7 @@ class TagsModel(QAbstractItemModel): # {{{
         for category in tb_categories:
             if category in data: # The search category can come and go
                 self.row_map.append(category)
-                self.categories.append(tb_categories[category]['name'])
+                self.categories[category] = tb_categories[category]['name']
 
         if len(old_row_map) != 0 and len(old_row_map) != len(self.row_map):
             # A category has been added or removed. We must force a rebuild of
@@ -1163,7 +1207,8 @@ class TagsModel(QAbstractItemModel): # {{{
                             sub_cat = TagTreeItem(parent=category, data = name,
                                      tooltip = None, temporary=True,
                                      category_icon = category_node.icon,
-                                     category_key=category_node.category_key)
+                                     category_key=category_node.category_key,
+                                     icon_map=self.icon_state_map)
                             self.endInsertRows()
                     else: # by 'first letter'
                         cl = cl_list[idx]
@@ -1173,7 +1218,8 @@ class TagsModel(QAbstractItemModel): # {{{
                                      data = collapse_letter,
                                      category_icon = category_node.icon,
                                      tooltip = None, temporary=True,
-                                     category_key=category_node.category_key)
+                                     category_key=category_node.category_key,
+                                     icon_map=self.icon_state_map)
                     node_parent = sub_cat
                 else:
                     node_parent = category
@@ -1284,16 +1330,19 @@ class TagsModel(QAbstractItemModel): # {{{
                 return False
 
             user_cats = self.db.prefs.get('user_categories', {})
+            user_cat_keys_lower = [icu_lower(k) for k in user_cats]
             ckey = item.category_key[1:]
+            ckey_lower = icu_lower(ckey)
             dotpos = ckey.rfind('.')
             if dotpos < 0:
                 nkey = val
             else:
                 nkey = ckey[:dotpos+1] + val
-            for c in user_cats:
-                if c.startswith(ckey):
+            nkey_lower = icu_lower(nkey)
+            for c in sorted(user_cats.keys(), key=sort_key):
+                if icu_lower(c).startswith(ckey_lower):
                     if len(c) == len(ckey):
-                        if nkey in user_cats:
+                        if nkey_lower in user_cat_keys_lower:
                             error_dialog(self.tags_view, _('Rename user category'),
                                 _('The name %s is already used')%nkey, show=True)
                             return False
@@ -1301,7 +1350,7 @@ class TagsModel(QAbstractItemModel): # {{{
                         del user_cats[ckey]
                     elif c[len(ckey)] == '.':
                         rest = c[len(ckey):]
-                        if (nkey + rest) in user_cats:
+                        if icu_lower(nkey + rest) in user_cat_keys_lower:
                             error_dialog(self.tags_view, _('Rename user category'),
                                 _('The name %s is already used')%(nkey+rest), show=True)
                             return False
@@ -1477,16 +1526,15 @@ class TagsModel(QAbstractItemModel): # {{{
     def reset_all_states(self, except_=None):
         update_list = []
         def process_tag(tag_item):
-            if tag_item.type != TagTreeItem.CATEGORY:
-                tag = tag_item.tag
-                if tag is except_:
-                    tag_index = self.createIndex(tag_item.row(), 0, tag_item)
-                    self.dataChanged.emit(tag_index, tag_index)
-                elif tag.state != 0 or tag in update_list:
-                    tag_index = self.createIndex(tag_item.row(), 0, tag_item)
-                    tag.state = 0
-                    update_list.append(tag)
-                    self.dataChanged.emit(tag_index, tag_index)
+            tag = tag_item.tag
+            if tag is except_:
+                tag_index = self.createIndex(tag_item.row(), 0, tag_item)
+                self.dataChanged.emit(tag_index, tag_index)
+            elif tag.state != 0 or tag in update_list:
+                tag_index = self.createIndex(tag_item.row(), 0, tag_item)
+                tag.state = 0
+                update_list.append(tag)
+                self.dataChanged.emit(tag_index, tag_index)
             for t in tag_item.children:
                 process_tag(t)
 
@@ -1503,13 +1551,11 @@ class TagsModel(QAbstractItemModel): # {{{
         '''
         if not index.isValid(): return False
         item = index.internalPointer()
-        if item.type == TagTreeItem.TAG:
-            item.toggle(set_to=set_to)
-            if exclusive:
-                self.reset_all_states(except_=item.tag)
-            self.dataChanged.emit(index, index)
-            return True
-        return False
+        item.toggle(set_to=set_to)
+        if exclusive:
+            self.reset_all_states(except_=item.tag)
+        self.dataChanged.emit(index, index)
+        return True
 
     def tokens(self):
         ans = []
@@ -1523,19 +1569,31 @@ class TagsModel(QAbstractItemModel): # {{{
         # into the search string only once. The nodes_seen set helps us do that
         nodes_seen = set()
 
+        node_searches = {TAG_SEARCH_STATES['mark_plus']       : 'true',
+                         TAG_SEARCH_STATES['mark_plusplus']   : '.true',
+                         TAG_SEARCH_STATES['mark_minus']      : 'false',
+                         TAG_SEARCH_STATES['mark_minusminus'] : '.false'}
+
         for node in self.category_nodes:
+            if node.tag.state:
+                ans.append('%s:%s'%(node.category_key, node_searches[node.tag.state]))
+
             key = node.category_key
             for tag_item in node.child_tags():
                 tag = tag_item.tag
                 if tag.state != TAG_SEARCH_STATES['clear']:
-                    prefix = ' not ' if tag.state == TAG_SEARCH_STATES['mark_minus'] \
-                                     else ''
+                    if tag.state == TAG_SEARCH_STATES['mark_minus'] or \
+                            tag.state == TAG_SEARCH_STATES['mark_minusminus']:
+                        prefix = ' not '
+                    else:
+                        prefix = ''
                     category = tag.category if key != 'news' else 'tag'
                     if tag.name and tag.name[0] == u'\u2605': # char is a star. Assume rating
                         ans.append('%s%s:%s'%(prefix, category, len(tag.name)))
                     else:
                         name = original_name(tag)
-                        use_prefix = tag.is_hierarchical
+                        use_prefix = tag.state in [TAG_SEARCH_STATES['mark_plusplus'],
+                                                   TAG_SEARCH_STATES['mark_minusminus']]
                         if category == 'tags':
                             if name in tags_seen:
                                 continue
@@ -1543,9 +1601,11 @@ class TagsModel(QAbstractItemModel): # {{{
                         if tag in nodes_seen:
                             continue
                         nodes_seen.add(tag)
+                        n = name.replace(r'"', r'\"')
+                        if name.startswith('.'):
+                            n = '.' + n
                         ans.append('%s%s:"=%s%s"'%(prefix, category,
-                                                '.' if use_prefix else '',
-                                                 name.replace(r'"', r'\"')))
+                                                '.' if use_prefix else '', n))
         return ans
 
     def find_item_node(self, key, txt, start_path, equals_match=False):

src/calibre/gui2/ui.py

@@ -330,6 +330,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
                 traceback.print_exc()
                 if ac.plugin_path is None:
                     raise
+        self.device_manager.set_current_library_uuid(db.library_id)
 
         if show_gui and self.gui_debug is not None:
             info_dialog(self, _('Debug mode'), '<p>' +
@@ -495,6 +496,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
             self.memory_view.reset()
             self.card_a_view.reset()
             self.card_b_view.reset()
+        self.device_manager.set_current_library_uuid(db.library_id)
 
 
     def set_window_title(self):

src/calibre/gui2/widgets.py

@@ -11,9 +11,9 @@ from PyQt4.Qt import QIcon, QFont, QLabel, QListWidget, QAction, \
                         QPixmap, QSplitterHandle, QToolButton, \
                         QAbstractListModel, QVariant, Qt, SIGNAL, pyqtSignal, \
                         QRegExp, QSettings, QSize, QSplitter, \
-                        QPainter, QLineEdit, QComboBox, QPen, \
+                        QPainter, QLineEdit, QComboBox, QPen, QGraphicsScene, \
                         QMenu, QStringListModel, QCompleter, QStringList, \
-                        QTimer, QRect
+                        QTimer, QRect, QFontDatabase, QGraphicsView
 
 from calibre.gui2 import NONE, error_dialog, pixmap_to_data, gprefs
 from calibre.gui2.filename_pattern_ui import Ui_Form
@@ -181,22 +181,16 @@ class FormatList(QListWidget):
         else:
             return QListWidget.keyPressEvent(self, event)
 
-
-class ImageView(QWidget):
-
-    BORDER_WIDTH = 1
-    cover_changed = pyqtSignal(object)
-
-    def __init__(self, parent=None):
-        QWidget.__init__(self, parent)
-        self._pixmap = QPixmap(self)
-        self.setMinimumSize(QSize(150, 200))
-        self.setAcceptDrops(True)
-        self.draw_border = True
-
-    # Drag 'n drop {{{
+class ImageDropMixin(object): # {{{
+    '''
+    Adds support for dropping images onto widgets and a contect menu for
+    copy/pasting images.
+    '''
     DROPABBLE_EXTENSIONS = IMAGE_EXTENSIONS
 
+    def __init__(self):
+        self.setAcceptDrops(True)
+
     @classmethod
     def paths_from_event(cls, event):
         '''
@@ -223,14 +217,58 @@ class ImageView(QWidget):
             pmap = QPixmap()
             pmap.load(path)
             if not pmap.isNull():
-                self.setPixmap(pmap)
+                self.handle_image_drop(path, pmap)
                 event.accept()
-                self.cover_changed.emit(open(path, 'rb').read())
                 break
 
+    def handle_image_drop(self, path, pmap):
+        self.set_pixmap(pmap)
+        self.cover_changed.emit(open(path, 'rb').read())
+
     def dragMoveEvent(self, event):
         event.acceptProposedAction()
-    # }}}
+
+    def get_pixmap(self):
+        return self.pixmap()
+
+    def set_pixmap(self, pmap):
+        self.setPixmap(pmap)
+
+    def contextMenuEvent(self, ev):
+        cm = QMenu(self)
+        copy = cm.addAction(_('Copy Image'))
+        paste = cm.addAction(_('Paste Image'))
+        if not QApplication.instance().clipboard().mimeData().hasImage():
+            paste.setEnabled(False)
+        copy.triggered.connect(self.copy_to_clipboard)
+        paste.triggered.connect(self.paste_from_clipboard)
+        cm.exec_(ev.globalPos())
+
+    def copy_to_clipboard(self):
+        QApplication.instance().clipboard().setPixmap(self.get_pixmap())
+
+    def paste_from_clipboard(self):
+        cb = QApplication.instance().clipboard()
+        pmap = cb.pixmap()
+        if pmap.isNull() and cb.supportsSelection():
+            pmap = cb.pixmap(cb.Selection)
+        if not pmap.isNull():
+            self.set_pixmap(pmap)
+            self.cover_changed.emit(
+                    pixmap_to_data(pmap))
+# }}}
+
+class ImageView(QWidget, ImageDropMixin):
+
+    BORDER_WIDTH = 1
+    cover_changed = pyqtSignal(object)
+
+    def __init__(self, parent=None):
+        QWidget.__init__(self, parent)
+        self._pixmap = QPixmap(self)
+        self.setMinimumSize(QSize(150, 200))
+        ImageDropMixin.__init__(self)
+        self.draw_border = True
 
     def setPixmap(self, pixmap):
         if not isinstance(pixmap, QPixmap):
@@ -272,34 +310,23 @@ class ImageView(QWidget):
             p.drawRect(target)
         p.end()
 
+class CoverView(QGraphicsView, ImageDropMixin):
 
-    # Clipboard copy/paste # {{{
-    def contextMenuEvent(self, ev):
-        cm = QMenu(self)
-        copy = cm.addAction(_('Copy Image'))
-        paste = cm.addAction(_('Paste Image'))
-        if not QApplication.instance().clipboard().mimeData().hasImage():
-            paste.setEnabled(False)
-        copy.triggered.connect(self.copy_to_clipboard)
-        paste.triggered.connect(self.paste_from_clipboard)
-        cm.exec_(ev.globalPos())
-
-    def copy_to_clipboard(self):
-        QApplication.instance().clipboard().setPixmap(self.pixmap())
-
-    def paste_from_clipboard(self):
-        cb = QApplication.instance().clipboard()
-        pmap = cb.pixmap()
-        if pmap.isNull() and cb.supportsSelection():
-            pmap = cb.pixmap(cb.Selection)
-        if not pmap.isNull():
-            self.setPixmap(pmap)
-            self.cover_changed.emit(
-                    pixmap_to_data(pmap))
-    # }}}
+    cover_changed = pyqtSignal(object)
 
+    def __init__(self, *args, **kwargs):
+        QGraphicsView.__init__(self, *args, **kwargs)
+        ImageDropMixin.__init__(self)
 
+    def get_pixmap(self):
+        for item in self.scene().items():
+            if hasattr(item, 'pixmap'):
+                return item.pixmap()
 
+    def set_pixmap(self, pmap):
+        self.scene = QGraphicsScene()
+        self.scene.addPixmap(pmap)
+        self.setScene(self.scene)
 
 class FontFamilyModel(QAbstractListModel):
 
@@ -312,6 +339,9 @@ class FontFamilyModel(QAbstractListModel):
             self.families = []
             print 'WARNING: Could not load fonts'
             traceback.print_exc()
+        # Restrict to Qt families as Qt tends to crash
+        qt_families = set([unicode(x) for x in QFontDatabase().families()])
+        self.families = list(qt_families.intersection(set(self.families)))
         self.families.sort()
         self.families[:0] = [_('None')]
 

src/calibre/gui2/wizard/__init__.py

@@ -51,7 +51,7 @@ class Device(object):
     @classmethod
     def set_output_format(cls):
         if cls.output_format:
-            prefs.set('output_format', cls.output_format)
+            prefs.set('output_format', cls.output_format.lower())
 
     @classmethod
     def commit(cls):

src/calibre/library/caches.py

@@ -121,11 +121,16 @@ CONTAINS_MATCH = 0
 EQUALS_MATCH   = 1
 REGEXP_MATCH   = 2
 def _match(query, value, matchkind):
+    if query.startswith('..'):
+        query = query[1:]
+        prefix_match_ok = False
+    else:
+        prefix_match_ok = True
     for t in value:
         t = icu_lower(t)
         try:     ### ignore regexp exceptions, required because search-ahead tries before typing is finished
             if (matchkind == EQUALS_MATCH):
-                if query[0] == '.':
+                if prefix_match_ok and query[0] == '.':
                     if t.startswith(query[1:]):
                         ql = len(query) - 1
                         if (len(t) == ql) or (t[ql:ql+1] == '.'):
@@ -418,32 +423,91 @@ class ResultCache(SearchQueryParser): # {{{
         return matches
 
     def get_user_category_matches(self, location, query, candidates):
-        res = set([])
-        if self.db_prefs is None:
-            return  res
+        matches = set([])
+        if self.db_prefs is None or len(query) < 2:
+            return  matches
         user_cats = self.db_prefs.get('user_categories', [])
         c = set(candidates)
-        l = location.rfind('.')
-        if l > 0:
-            alt_loc = location[0:l]
-            alt_item = location[l+1:]
+
+        if query.startswith('.'):
+            check_subcats = True
+            query = query[1:]
         else:
-            alt_loc = None
+            check_subcats = False
+
         for key in user_cats:
-            if key == location or key.startswith(location + '.'):
+            if key == location or (check_subcats and key.startswith(location + '.')):
                 for (item, category, ign) in user_cats[key]:
                     s = self.get_matches(category, '=' + item, candidates=c)
                     c -= s
-                    res |= s
-            elif key == alt_loc:
-                for (item, category, ign) in user_cats[key]:
-                    if item == alt_item:
-                        s = self.get_matches(category, '=' + item, candidates=c)
-                        c -= s
-                        res |= s
+                    matches |= s
         if query == 'false':
-            return candidates - res
-        return res
+            return candidates - matches
+        return matches
+
+    def get_keypair_matches(self, location, query, candidates):
+        matches = set([])
+        if query.find(':') >= 0:
+            q = [q.strip() for q in query.split(':')]
+            if len(q) != 2:
+                raise ParseException(query, len(query),
+                        'Invalid query format for colon-separated search', self)
+            (keyq, valq) = q
+            keyq_mkind, keyq = self._matchkind(keyq)
+            valq_mkind, valq = self._matchkind(valq)
+        else:
+            keyq = keyq_mkind = ''
+            valq_mkind, valq = self._matchkind(query)
+
+        loc = self.field_metadata[location]['rec_index']
+        split_char = self.field_metadata[location]['is_multiple']
+        for id_ in candidates:
+            item = self._data[id_]
+            if item is None:
+                continue
+
+            if item[loc] is None:
+                if valq == 'false':
+                    matches.add(id_)
+                continue
+
+            pairs = [p.strip() for p in item[loc].split(split_char)]
+            for pair in pairs:
+                parts = pair.split(':')
+                if len(parts) != 2:
+                    continue
+                k = parts[:1]
+                v = parts[1:]
+                if keyq and not _match(keyq, k, keyq_mkind):
+                    continue
+                if valq:
+                    if valq == 'true':
+                        if not v:
+                            continue
+                    elif valq == 'false':
+                        if v:
+                            continue
+                    elif not _match(valq, v, valq_mkind):
+                        continue
+                matches.add(id_)
+        return matches
+
+    def _matchkind(self, query):
+        matchkind = CONTAINS_MATCH
+        if (len(query) > 1):
+            if query.startswith('\\'):
+                query = query[1:]
+            elif query.startswith('='):
+                matchkind = EQUALS_MATCH
+                query = query[1:]
+            elif query.startswith('~'):
+                matchkind = REGEXP_MATCH
+                query = query[1:]
+
+        if matchkind != REGEXP_MATCH:
+            # leave case in regexps because it can be significant e.g. \S \W \D
+            query = icu_lower(query)
+        return matchkind, query
 
     def get_matches(self, location, query, candidates=None,
             allow_recursion=True):
@@ -460,6 +524,7 @@ class ResultCache(SearchQueryParser): # {{{
         if query and query.strip():
             # get metadata key associated with the search term. Eliminates
             # dealing with plurals and other aliases
+            original_location = location
             location = self.field_metadata.search_term_to_field_key(icu_lower(location.strip()))
             # grouped search terms
             if isinstance(location, list):
@@ -510,24 +575,20 @@ class ResultCache(SearchQueryParser): # {{{
                     return self.get_numeric_matches(location, query[1:],
                                                     candidates, val_func=vf)
 
+                # special case: colon-separated fields such as identifiers. isbn
+                # is a special case within the case
+                if fm.get('is_csp', False):
+                    if location == 'identifiers' and original_location == 'isbn':
+                        return self.get_keypair_matches('identifiers',
+                                                   '=isbn:'+query, candidates)
+                    return self.get_keypair_matches(location, query, candidates)
+
             # check for user categories
             if len(location) >= 2 and location.startswith('@'):
                 return self.get_user_category_matches(location[1:], query.lower(),
                                                       candidates)
             # everything else, or 'all' matches
-            matchkind = CONTAINS_MATCH
-            if (len(query) > 1):
-                if query.startswith('\\'):
-                    query = query[1:]
-                elif query.startswith('='):
-                    matchkind = EQUALS_MATCH
-                    query = query[1:]
-                elif query.startswith('~'):
-                    matchkind = REGEXP_MATCH
-                    query = query[1:]
-            if matchkind != REGEXP_MATCH:
-                # leave case in regexps because it can be significant e.g. \S \W \D
-                query = icu_lower(query)
+            matchkind, query = self._matchkind(query)
 
             if not isinstance(query, unicode):
                 query = query.decode('utf-8')

src/calibre/library/cli.py

@@ -20,7 +20,8 @@ from calibre.utils.date import isoformat
 
 FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating',
     'timestamp', 'size', 'tags', 'comments', 'series', 'series_index',
-    'formats', 'isbn', 'uuid', 'pubdate', 'cover'])
+    'formats', 'isbn', 'uuid', 'pubdate', 'cover', 'last_modified',
+    'identifiers'])
 
 def send_message(msg=''):
     prints('Notifying calibre of the change')

src/calibre/library/custom_columns.py

@@ -188,7 +188,7 @@ class CustomColumns(object):
                     table=tn, column='value', datatype=v['datatype'],
                     colnum=v['num'], name=v['name'], display=v['display'],
                     is_multiple=is_m, is_category=is_category,
-                    is_editable=v['editable'])
+                    is_editable=v['editable'], is_csp=False)
 
     def get_custom(self, idx, label=None, num=None, index_is_id=False):
         if label is not None:

src/calibre/library/database2.py

@@ -6,7 +6,8 @@ __docformat__ = 'restructuredtext en'
 '''
 The database used to store ebook metadata
 '''
-import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, json
+import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, \
+        json, uuid
 import threading, random
 from itertools import repeat
 from math import ceil
@@ -94,6 +95,31 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
         return property(doc=doc, fget=fget, fset=fset)
 
+    @dynamic_property
+    def library_id(self):
+        doc = ('The UUID for this library. As long as the user only operates'
+                ' on libraries with calibre, it will be unique')
+
+        def fget(self):
+            if self._library_id_ is None:
+                ans = self.conn.get('SELECT uuid FROM library_id', all=False)
+                if ans is None:
+                    ans = str(uuid.uuid4())
+                    self.library_id = ans
+                else:
+                    self._library_id_ = ans
+            return self._library_id_
+
+        def fset(self, val):
+            self._library_id_ = unicode(val)
+            self.conn.executescript('''
+                    DELETE FROM library_id;
+                    INSERT INTO library_id (uuid) VALUES ("%s");
+                    '''%self._library_id_)
+            self.conn.commit()
+
+        return property(doc=doc, fget=fget, fset=fset)
+
     def connect(self):
         if 'win32' in sys.platform and len(self.library_path) + 4*self.PATH_LIMIT + 10 > 259:
             raise ValueError('Path to library too long. Must be less than %d characters.'%(259-4*self.PATH_LIMIT-10))
@@ -120,6 +146,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
     def __init__(self, library_path, row_factory=False, default_prefs=None,
             read_only=False):
         self.field_metadata = FieldMetadata()
+        self._library_id_ = None
         # Create the lock to be used to guard access to the metadata writer
         # queues. This must be an RLock, not a Lock
         self.dirtied_lock = threading.RLock()
@@ -148,6 +175,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.is_case_sensitive = not iswindows and not isosx and \
             not os.path.exists(self.dbpath.replace('metadata.db', 'MeTAdAtA.dB'))
         SchemaUpgrade.__init__(self)
+        # Guarantee that the library_id is set
+        self.library_id
 
         # if we are to copy the prefs and structure from some other DB, then
         # we need to do it before we call initialize_dynamic
@@ -293,14 +322,14 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
              'sort',
              'author_sort',
              '(SELECT group_concat(format) FROM data WHERE data.book=books.id) formats',
-             'isbn',
              'path',
-             'lccn',
              'pubdate',
-             'flags',
              'uuid',
              'has_cover',
-            ('au_map', 'authors', 'author', 'aum_sortconcat(link.id, authors.name, authors.sort)')
+            ('au_map', 'authors', 'author',
+                'aum_sortconcat(link.id, authors.name, authors.sort)'),
+            'last_modified',
+            '(SELECT identifiers_concat(type, val) FROM identifiers WHERE identifiers.book=books.id) identifiers',
             ]
         lines = []
         for col in columns:
@@ -318,8 +347,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.FIELD_MAP = {'id':0, 'title':1, 'authors':2, 'timestamp':3,
              'size':4, 'rating':5, 'tags':6, 'comments':7, 'series':8,
              'publisher':9, 'series_index':10, 'sort':11, 'author_sort':12,
-             'formats':13, 'isbn':14, 'path':15, 'lccn':16, 'pubdate':17,
-             'flags':18, 'uuid':19, 'cover':20, 'au_map':21}
+             'formats':13, 'path':14, 'pubdate':15, 'uuid':16, 'cover':17,
+             'au_map':18, 'last_modified':19, 'identifiers':20}
 
         for k,v in self.FIELD_MAP.iteritems():
             self.field_metadata.set_field_record_index(k, v, prefer_custom=False)
@@ -391,11 +420,16 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.has_id  = self.data.has_id
         self.count   = self.data.count
 
-        for prop in ('author_sort', 'authors', 'comment', 'comments', 'isbn',
-                     'publisher', 'rating', 'series', 'series_index', 'tags',
-                     'title', 'timestamp', 'uuid', 'pubdate', 'ondevice'):
+        for prop in (
+                'author_sort', 'authors', 'comment', 'comments',
+                'publisher', 'rating', 'series', 'series_index', 'tags',
+                'title', 'timestamp', 'uuid', 'pubdate', 'ondevice',
+                'metadata_last_modified',
+                ):
+            fm = {'comment':'comments', 'metadata_last_modified':
+                    'last_modified'}.get(prop, prop)
             setattr(self, prop, functools.partial(self.get_property,
-                    loc=self.FIELD_MAP['comments' if prop == 'comment' else prop]))
+                    loc=self.FIELD_MAP[fm]))
         setattr(self, 'title_sort', functools.partial(self.get_property,
                 loc=self.FIELD_MAP['sort']))
 
@@ -681,8 +715,20 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if commit:
             self.conn.commit()
 
+    def update_last_modified(self, book_ids, commit=False, now=None):
+        if now is None:
+            now = nowf()
+        if book_ids:
+            self.conn.executemany(
+                'UPDATE books SET last_modified=? WHERE id=?',
+                [(now, book) for book in book_ids])
+            for book_id in book_ids:
+                self.data.set(book_id, self.FIELD_MAP['last_modified'], now, row_is_id=True)
+            if commit:
+                self.conn.commit()
+
     def dirtied(self, book_ids, commit=True):
-        changed = False
+        self.update_last_modified(book_ids)
         for book in book_ids:
             with self.dirtied_lock:
                 # print 'dirtied: check id', book
@@ -691,21 +737,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                     self.dirtied_sequence += 1
                     continue
                 # print 'book not already dirty'
-                try:
-                    self.conn.execute(
-                        'INSERT INTO metadata_dirtied (book) VALUES (?)',
-                            (book,))
-                    changed = True
-                except IntegrityError:
-                    # Already in table
-                    pass
+
+                self.conn.execute(
+                    'INSERT OR IGNORE INTO metadata_dirtied (book) VALUES (?)',
+                    (book,))
                 self.dirtied_cache[book] = self.dirtied_sequence
                 self.dirtied_sequence += 1
+
         # If the commit doesn't happen, then the DB table will be wrong. This
         # could lead to a problem because on restart, we won't put the book back
         # into the dirtied_cache. We deal with this by writing the dirtied_cache
         # back to the table on GUI exit. Not perfect, but probably OK
-        if commit and changed:
+        if book_ids and commit:
             self.conn.commit()
 
     def get_a_dirtied_book(self):
@@ -790,6 +833,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         mi.pubdate     = row[fm['pubdate']]
         mi.uuid        = row[fm['uuid']]
         mi.title_sort  = row[fm['sort']]
+        mi.metadata_last_modified = row[fm['last_modified']]
         formats = row[fm['formats']]
         if not formats:
             formats = None
@@ -803,8 +847,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if mi.series:
             mi.series_index = row[fm['series_index']]
         mi.rating = row[fm['rating']]
-        mi.isbn = row[fm['isbn']]
         id = idx if index_is_id else self.id(idx)
+        mi.set_identifiers(self.get_identifiers(id, index_is_id=True))
         mi.application_id = id
         mi.id = id
         for key, meta in self.field_metadata.custom_iteritems():
@@ -911,10 +955,14 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             except (IOError, OSError):
                 time.sleep(0.2)
                 save_cover_data_to(data, path)
-        self.conn.execute('UPDATE books SET has_cover=1 WHERE id=?', (id,))
+        now = nowf()
+        self.conn.execute(
+            'UPDATE books SET has_cover=1,last_modified=? WHERE id=?',
+              (now, id))
         if commit:
             self.conn.commit()
         self.data.set(id, self.FIELD_MAP['cover'], True, row_is_id=True)
+        self.data.set(id, self.FIELD_MAP['last_modified'], now, row_is_id=True)
         if notify:
             self.notify('cover', [id])
 
@@ -923,8 +971,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
     def set_has_cover(self, id, val):
         dval = 1 if val else 0
-        self.conn.execute('UPDATE books SET has_cover=? WHERE id=?', (dval, id,))
+        now = nowf()
+        self.conn.execute(
+                'UPDATE books SET has_cover=?,last_modified=? WHERE id=?',
+                (dval, now, id))
         self.data.set(id, self.FIELD_MAP['cover'], val, row_is_id=True)
+        self.data.set(id, self.FIELD_MAP['last_modified'], now, row_is_id=True)
 
     def book_on_device(self, id):
         if callable(self.book_on_device_func):
@@ -1195,7 +1247,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                     i += 1
             else:
                 new_cats['.'.join(comps)] = user_cats[k]
-        self.prefs.set('user_categories', new_cats)
+        try:
+            if new_cats != user_cats:
+                self.prefs.set('user_categories', new_cats)
+        except:
+            pass
         return new_cats
 
     def get_categories(self, sort='name', ids=None, icon_map=None):
@@ -1218,7 +1274,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         for category in tb_cats.keys():
             cat = tb_cats[category]
             if not cat['is_category'] or cat['kind'] in ['user', 'search'] \
-                    or category in ['news', 'formats']:
+                    or category in ['news', 'formats'] or cat.get('is_csp',
+                            False):
                 continue
             # Get the ids for the item values
             if not cat['is_custom']:
@@ -1500,18 +1557,30 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
     ############# End get_categories
 
-    def tags_older_than(self, tag, delta):
+    def tags_older_than(self, tag, delta, must_have_tag=None):
+        '''
+        Return the ids of all books having the tag ``tag`` that are older than
+        than the specified time. tag comparison is case insensitive.
+
+        :param delta: A timedelta object or None. If None, then all ids with
+        the tag are returned.
+        :param must_have_tag: If not None the list of matches will be
+        restricted to books that have this tag
+        '''
         tag = tag.lower().strip()
+        mht = must_have_tag.lower().strip() if must_have_tag else None
         now = nowf()
         tindex = self.FIELD_MAP['timestamp']
         gindex = self.FIELD_MAP['tags']
+        iindex = self.FIELD_MAP['id']
         for r in self.data._data:
             if r is not None:
-                if (now - r[tindex]) > delta:
+                if delta is None or (now - r[tindex]) > delta:
                     tags = r[gindex]
-                    if tags and tag in [x.strip() for x in
-                            tags.lower().split(',')]:
-                        yield r[self.FIELD_MAP['id']]
+                    if tags:
+                        tags = [x.strip() for x in tags.lower().split(',')]
+                        if tag in tags and (mht is None or mht in tags):
+                            yield r[iindex]
 
     def get_next_series_num_for(self, series):
         series_id = self.conn.get('SELECT id from series WHERE name=?',
@@ -1636,8 +1705,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             doit(self.set_tags, id, mi.tags, notify=False, commit=False)
         if mi.comments:
             doit(self.set_comment, id, mi.comments, notify=False, commit=False)
-        if mi.isbn and mi.isbn.strip():
-            doit(self.set_isbn, id, mi.isbn, notify=False, commit=False)
         if mi.series_index:
             doit(self.set_series_index, id, mi.series_index, notify=False,
                     commit=False)
@@ -1647,6 +1714,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             doit(self.set_timestamp, id, mi.timestamp, notify=False,
                     commit=False)
 
+        mi_idents = mi.get_identifiers()
+        if mi_idents:
+            identifiers = self.get_identifiers(id, index_is_id=True)
+            for key, val in mi_idents.iteritems():
+                if val and val.strip(): # Don't delete an existing identifier
+                    identifiers[icu_lower(key)] = val
+            self.set_identifiers(id, identifiers, notify=False, commit=False)
+
+
         user_mi = mi.get_all_user_metadata(make_copy=False)
         for key in user_mi.iterkeys():
             if key in self.field_metadata and \
@@ -2425,14 +2501,84 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if notify:
             self.notify('metadata', [id])
 
-    def set_isbn(self, id, isbn, notify=True, commit=True):
-        self.conn.execute('UPDATE books SET isbn=? WHERE id=?', (isbn, id))
-        self.dirtied([id], commit=False)
+    def isbn(self, idx, index_is_id=False):
+        row = self.data._data[idx] if index_is_id else self.data[idx]
+        if row is not None:
+            raw = row[self.FIELD_MAP['identifiers']]
+            if raw:
+                for x in raw.split(','):
+                    if x.startswith('isbn:'):
+                        return x[5:].strip()
+
+    def get_identifiers(self, idx, index_is_id=False):
+        ans = {}
+        row = self.data._data[idx] if index_is_id else self.data[idx]
+        if row is not None:
+            raw = row[self.FIELD_MAP['identifiers']]
+            if raw:
+                for x in raw.split(','):
+                    key, _, val = x.partition(':')
+                    key, val = key.strip(), val.strip()
+                    if key and val:
+                        ans[key] = val
+
+        return ans
+
+    def _clean_identifier(self, typ, val):
+        typ = icu_lower(typ).strip().replace(':', '').replace(',', '')
+        val = val.strip().replace(',', '|').replace(':', '|')
+        return typ, val
+
+    def set_identifier(self, id_, typ, val, notify=True, commit=True):
+        'If val is empty, deletes identifier of type typ'
+        typ, val = self._clean_identifier(typ, val)
+        identifiers = self.get_identifiers(id_, index_is_id=True)
+        if not typ:
+            return
+        changed = False
+        if not val and typ in identifiers:
+            identifiers.pop(typ)
+            changed = True
+            self.conn.execute(
+                'DELETE from identifiers WHERE book=? AND type=?',
+                    (id_, typ))
+        if val and identifiers.get(typ, None) != val:
+            changed = True
+            identifiers[typ] = val
+            self.conn.execute(
+                'INSERT OR REPLACE INTO identifiers (book, type, val) VALUES (?, ?, ?)',
+                    (id_, typ, val))
+        if changed:
+            raw = ','.join(['%s:%s'%(k, v) for k, v in
+                identifiers.iteritems()])
+            self.data.set(id_, self.FIELD_MAP['identifiers'], raw,
+                    row_is_id=True)
+            if commit:
+                self.conn.commit()
+            if notify:
+                self.notify('metadata', [id_])
+
+    def set_identifiers(self, id_, identifiers, notify=True, commit=True):
+        cleaned = {}
+        for typ, val in identifiers.iteritems():
+            typ, val = self._clean_identifier(typ, val)
+            if val:
+                cleaned[typ] = val
+        self.conn.execute('DELETE FROM identifiers WHERE book=?', (id_,))
+        self.conn.executemany(
+            'INSERT INTO identifiers (book, type, val) VALUES (?, ?, ?)',
+            [(id_, k, v) for k, v in cleaned.iteritems()])
+        raw = ','.join(['%s:%s'%(k, v) for k, v in
+                cleaned.iteritems()])
+        self.data.set(id_, self.FIELD_MAP['identifiers'], raw,
+                    row_is_id=True)
         if commit:
             self.conn.commit()
-        self.data.set(id, self.FIELD_MAP['isbn'], isbn, row_is_id=True)
         if notify:
-            self.notify('metadata', [id])
+            self.notify('metadata', [id_])
+
+    def set_isbn(self, id_, isbn, notify=True, commit=True):
+        self.set_identifier(id_, 'isbn', isbn, notify=notify, commit=commit)
 
     def add_catalog(self, path, title):
         format = os.path.splitext(path)[1][1:].lower()
@@ -2730,7 +2876,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             prefix = self.library_path
         FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating',
             'timestamp', 'size', 'tags', 'comments', 'series', 'series_index',
-            'isbn', 'uuid', 'pubdate'])
+            'uuid', 'pubdate', 'last_modified', 'identifiers'])
         for x in self.custom_column_num_map:
             FIELDS.add(x)
         data = []
@@ -2745,6 +2891,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             data.append(x)
             x['id'] = db_id
             x['formats'] = []
+            isbn = self.isbn(db_id, index_is_id=True)
+            x['isbn'] = isbn if isbn else ''
             if not x['authors']:
                 x['authors'] = _('Unknown')
             x['authors'] = [i.replace('|', ',') for i in x['authors'].split(',')]

src/calibre/library/field_metadata.py

@@ -80,6 +80,8 @@ class FieldMetadata(dict):
 
     rec_index: the index of the field in the db metadata record.
 
+    is_csp: field contains colon-separated pairs. Must also be text, is_multiple
+
     '''
 
     VALID_DATA_TYPES = frozenset([None, 'rating', 'text', 'comments', 'datetime',
@@ -98,7 +100,8 @@ class FieldMetadata(dict):
                            'name':_('Authors'),
                            'search_terms':['authors', 'author'],
                            'is_custom':False,
-                           'is_category':True}),
+                           'is_category':True,
+                           'is_csp': False}),
             ('series',    {'table':'series',
                            'column':'name',
                            'link_column':'series',
@@ -109,7 +112,8 @@ class FieldMetadata(dict):
                            'name':_('Series'),
                            'search_terms':['series'],
                            'is_custom':False,
-                           'is_category':True}),
+                           'is_category':True,
+                           'is_csp': False}),
             ('formats',   {'table':None,
                            'column':None,
                            'datatype':'text',
@@ -118,7 +122,8 @@ class FieldMetadata(dict):
                            'name':_('Formats'),
                            'search_terms':['formats', 'format'],
                            'is_custom':False,
-                           'is_category':True}),
+                           'is_category':True,
+                           'is_csp': False}),
             ('publisher', {'table':'publishers',
                            'column':'name',
                            'link_column':'publisher',
@@ -129,7 +134,8 @@ class FieldMetadata(dict):
                            'name':_('Publishers'),
                            'search_terms':['publisher'],
                            'is_custom':False,
-                           'is_category':True}),
+                           'is_category':True,
+                           'is_csp': False}),
             ('rating',    {'table':'ratings',
                            'column':'rating',
                            'link_column':'rating',
@@ -140,7 +146,8 @@ class FieldMetadata(dict):
                            'name':_('Ratings'),
                            'search_terms':['rating'],
                            'is_custom':False,
-                           'is_category':True}),
+                           'is_category':True,
+                           'is_csp': False}),
             ('news',      {'table':'news',
                            'column':'name',
                            'category_sort':'name',
@@ -150,7 +157,8 @@ class FieldMetadata(dict):
                            'name':_('News'),
                            'search_terms':[],
                            'is_custom':False,
-                           'is_category':True}),
+                           'is_category':True,
+                           'is_csp': False}),
             ('tags',      {'table':'tags',
                            'column':'name',
                            'link_column': 'tag',
@@ -161,7 +169,18 @@ class FieldMetadata(dict):
                            'name':_('Tags'),
                            'search_terms':['tags', 'tag'],
                            'is_custom':False,
-                           'is_category':True}),
+                           'is_category':True,
+                           'is_csp': False}),
+            ('identifiers',   {'table':None,
+                           'column':None,
+                           'datatype':'text',
+                           'is_multiple':',',
+                           'kind':'field',
+                           'name':_('Identifiers'),
+                           'search_terms':['identifiers', 'identifier', 'isbn'],
+                           'is_custom':False,
+                           'is_category':True,
+                           'is_csp': True}),
             ('author_sort',{'table':None,
                             'column':None,
                             'datatype':'text',
@@ -170,7 +189,8 @@ class FieldMetadata(dict):
                            'name':None,
                            'search_terms':['author_sort'],
                            'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
             ('au_map',    {'table':None,
                            'column':None,
                            'datatype':'text',
@@ -179,7 +199,8 @@ class FieldMetadata(dict):
                            'name':None,
                            'search_terms':[],
                            'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
             ('comments',  {'table':None,
                            'column':None,
                            'datatype':'text',
@@ -187,7 +208,9 @@ class FieldMetadata(dict):
                            'kind':'field',
                            'name':_('Comments'),
                            'search_terms':['comments', 'comment'],
-                           'is_custom':False, 'is_category':False}),
+                           'is_custom':False,
+                           'is_category':False,
+                           'is_csp': False}),
             ('cover',     {'table':None,
                            'column':None,
                            'datatype':'int',
@@ -196,16 +219,8 @@ class FieldMetadata(dict):
                            'name':None,
                            'search_terms':['cover'],
                            'is_custom':False,
-                           'is_category':False}),
-            ('flags',     {'table':None,
-                           'column':None,
-                           'datatype':'text',
-                           'is_multiple':None,
-                           'kind':'field',
-                           'name':None,
-                           'search_terms':[],
-                           'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
             ('id',        {'table':None,
                            'column':None,
                            'datatype':'int',
@@ -214,25 +229,18 @@ class FieldMetadata(dict):
                            'name':None,
                            'search_terms':[],
                            'is_custom':False,
-                           'is_category':False}),
-            ('isbn',      {'table':None,
+                           'is_category':False,
+                           'is_csp': False}),
+            ('last_modified', {'table':None,
                            'column':None,
-                           'datatype':'text',
+                           'datatype':'datetime',
                            'is_multiple':None,
                            'kind':'field',
-                           'name':None,
-                           'search_terms':['isbn'],
+                           'name':_('Date'),
+                           'search_terms':['last_modified'],
                            'is_custom':False,
-                           'is_category':False}),
-            ('lccn',      {'table':None,
-                           'column':None,
-                           'datatype':'text',
-                           'is_multiple':None,
-                           'kind':'field',
-                           'name':None,
-                           'search_terms':[],
-                           'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
             ('ondevice',  {'table':None,
                            'column':None,
                            'datatype':'text',
@@ -241,7 +249,8 @@ class FieldMetadata(dict):
                            'name':_('On Device'),
                            'search_terms':['ondevice'],
                            'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
             ('path',      {'table':None,
                            'column':None,
                            'datatype':'text',
@@ -250,7 +259,8 @@ class FieldMetadata(dict):
                            'name':None,
                            'search_terms':[],
                            'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
             ('pubdate',   {'table':None,
                            'column':None,
                            'datatype':'datetime',
@@ -259,7 +269,8 @@ class FieldMetadata(dict):
                            'name':_('Published'),
                            'search_terms':['pubdate'],
                            'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
             ('series_index',{'table':None,
                              'column':None,
                              'datatype':'float',
@@ -268,7 +279,8 @@ class FieldMetadata(dict):
                              'name':None,
                              'search_terms':['series_index'],
                              'is_custom':False,
-                             'is_category':False}),
+                             'is_category':False,
+                           'is_csp': False}),
             ('sort',      {'table':None,
                            'column':None,
                            'datatype':'text',
@@ -277,7 +289,8 @@ class FieldMetadata(dict):
                            'name':_('Title Sort'),
                            'search_terms':['title_sort'],
                            'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
             ('size',      {'table':None,
                            'column':None,
                            'datatype':'float',
@@ -286,7 +299,8 @@ class FieldMetadata(dict):
                            'name':_('Size (MB)'),
                            'search_terms':['size'],
                            'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
             ('timestamp', {'table':None,
                            'column':None,
                            'datatype':'datetime',
@@ -295,7 +309,8 @@ class FieldMetadata(dict):
                            'name':_('Date'),
                            'search_terms':['date'],
                            'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
             ('title',     {'table':None,
                            'column':None,
                            'datatype':'text',
@@ -304,7 +319,8 @@ class FieldMetadata(dict):
                            'name':_('Title'),
                            'search_terms':['title'],
                            'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
             ('uuid',      {'table':None,
                            'column':None,
                            'datatype':'text',
@@ -313,7 +329,8 @@ class FieldMetadata(dict):
                            'name':None,
                            'search_terms':[],
                            'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
         ]
     # }}}
 
@@ -335,7 +352,8 @@ class FieldMetadata(dict):
             self._tb_cats[k]['display'] = {}
             self._tb_cats[k]['is_editable'] = True
             self._add_search_terms_to_map(k, v['search_terms'])
-        self._tb_cats['timestamp']['display'] = {
+        for x in ('timestamp', 'last_modified'):
+            self._tb_cats[x]['display'] = {
                         'date_format': tweaks['gui_timestamp_display_format']}
         self._tb_cats['pubdate']['display'] = {
                         'date_format': tweaks['gui_pubdate_display_format']}
@@ -441,7 +459,8 @@ class FieldMetadata(dict):
         return l
 
     def add_custom_field(self, label, table, column, datatype, colnum, name,
-                               display, is_editable, is_multiple, is_category):
+                         display, is_editable, is_multiple, is_category,
+                         is_csp=False):
         key = self.custom_field_prefix + label
         if key in self._tb_cats:
             raise ValueError('Duplicate custom field [%s]'%(label))
@@ -454,7 +473,7 @@ class FieldMetadata(dict):
                              'colnum':colnum,      'display':display,
                              'is_custom':True,     'is_category':is_category,
                              'link_column':'value','category_sort':'value',
-                             'is_editable': is_editable,}
+                             'is_csp' : is_csp,     'is_editable': is_editable,}
         self._add_search_terms_to_map(key, [key])
         self.custom_label_to_key_map[label] = key
         if datatype == 'series':
@@ -466,7 +485,7 @@ class FieldMetadata(dict):
                                  'colnum':None,        'display':{},
                                  'is_custom':False,    'is_category':False,
                                  'link_column':None,   'category_sort':None,
-                                 'is_editable': False,}
+                                 'is_editable': False, 'is_csp': False}
             self._add_search_terms_to_map(key, [key])
             self.custom_label_to_key_map[label+'_index'] = key
 
@@ -515,7 +534,7 @@ class FieldMetadata(dict):
                                 'datatype':None,       'is_multiple':None,
                                 'kind':'user',         'name':name,
                                 'search_terms':st,     'is_custom':False,
-                                'is_category':True}
+                                'is_category':True,    'is_csp': False}
         self._add_search_terms_to_map(label, st)
 
     def add_search_category(self, label, name):
@@ -524,8 +543,8 @@ class FieldMetadata(dict):
         self._tb_cats[label] = {'table':None,        'column':None,
                                 'datatype':None,     'is_multiple':None,
                                 'kind':'search',     'name':name,
-                                'search_terms':[],    'is_custom':False,
-                                'is_category':True}
+                                'search_terms':[],   'is_custom':False,
+                                'is_category':True,  'is_csp': False}
 
     def set_field_record_index(self, label, index, prefer_custom=False):
         if prefer_custom:

src/calibre/library/prefs.py

@@ -49,8 +49,7 @@ class DBPrefs(dict):
         if self.disable_setting:
             return
         raw = self.to_raw(val)
-        self.db.conn.execute('DELETE FROM preferences WHERE key=?', (key,))
-        self.db.conn.execute('INSERT INTO preferences (key,val) VALUES (?,?)', (key,
+        self.db.conn.execute('INSERT OR REPLACE INTO preferences (key,val) VALUES (?,?)', (key,
             raw))
         self.db.conn.commit()
         dict.__setitem__(self, key, val)

src/calibre/library/restore.py

@@ -13,6 +13,7 @@ from calibre.ptempfile import TemporaryDirectory
 from calibre.ebooks.metadata.opf2 import OPF
 from calibre.library.database2 import LibraryDatabase2
 from calibre.constants import filesystem_encoding
+from calibre.utils.date import utcfromtimestamp
 from calibre import isbytestring
 
 NON_EBOOK_EXTENSIONS = frozenset([
@@ -211,8 +212,8 @@ class Restore(Thread):
                 force_id=book['id'])
         if book['mi'].uuid:
             db.set_uuid(book['id'], book['mi'].uuid, commit=False, notify=False)
-        db.conn.execute('UPDATE books SET path=? WHERE id=?', (book['path'],
-            book['id']))
+        db.conn.execute('UPDATE books SET path=?,last_modified=? WHERE id=?', (book['path'],
+            utcfromtimestamp(book['timestamp']), book['id']))
 
         for fmt, size, name in book['formats']:
             db.conn.execute('''

src/calibre/library/schema_upgrades.py

@@ -8,6 +8,8 @@ __docformat__ = 'restructuredtext en'
 
 import os
 
+from calibre.utils.date import isoformat, DEFAULT_DATE
+
 class SchemaUpgrade(object):
 
     def __init__(self):
@@ -468,4 +470,116 @@ class SchemaUpgrade(object):
         '''
         self.conn.executescript(script)
 
+    def upgrade_version_18(self):
+        '''
+        Add a library UUID.
+        Add an identifiers table.
+        Add a languages table.
+        Add a last_modified column.
+        NOTE: You cannot downgrade after this update, if you do
+        any changes you make to book isbns will be lost.
+        '''
+        script = '''
+        DROP TABLE IF EXISTS library_id;
+        CREATE TABLE library_id ( id   INTEGER PRIMARY KEY,
+                                  uuid TEXT NOT NULL,
+                                  UNIQUE(uuid)
+        );
+
+        DROP TABLE IF EXISTS identifiers;
+        CREATE TABLE identifiers  ( id     INTEGER PRIMARY KEY,
+                                    book   INTEGER NON NULL,
+                                    type   TEXT NON NULL DEFAULT "isbn" COLLATE NOCASE,
+                                    val    TEXT NON NULL COLLATE NOCASE,
+                                    UNIQUE(book, type)
+        );
+
+        DROP TABLE IF EXISTS languages;
+        CREATE TABLE languages    ( id        INTEGER PRIMARY KEY,
+                                    lang_code TEXT NON NULL COLLATE NOCASE,
+                                    UNIQUE(lang_code)
+        );
+
+        DROP TABLE IF EXISTS books_languages_link;
+        CREATE TABLE books_languages_link ( id INTEGER PRIMARY KEY,
+                                            book INTEGER NOT NULL,
+                                            lang_code INTEGER NOT NULL,
+                                            item_order INTEGER NOT NULL DEFAULT 0,
+                                            UNIQUE(book, lang_code)
+        );
+
+        DROP TRIGGER IF EXISTS fkc_delete_on_languages;
+        CREATE TRIGGER fkc_delete_on_languages
+        BEFORE DELETE ON languages
+        BEGIN
+            SELECT CASE
+                WHEN (SELECT COUNT(id) FROM books_languages_link WHERE lang_code=OLD.id) > 0
+                THEN RAISE(ABORT, 'Foreign key violation: language is still referenced')
+            END;
+        END;
+
+        DROP TRIGGER IF EXISTS fkc_delete_on_languages_link;
+        CREATE TRIGGER fkc_delete_on_languages_link
+        BEFORE INSERT ON books_languages_link
+        BEGIN
+          SELECT CASE
+              WHEN (SELECT id from books WHERE id=NEW.book) IS NULL
+              THEN RAISE(ABORT, 'Foreign key violation: book not in books')
+              WHEN (SELECT id from languages WHERE id=NEW.lang_code) IS NULL
+              THEN RAISE(ABORT, 'Foreign key violation: lang_code not in languages')
+          END;
+        END;
+
+        DROP TRIGGER IF EXISTS fkc_update_books_languages_link_a;
+        CREATE TRIGGER fkc_update_books_languages_link_a
+        BEFORE UPDATE OF book ON books_languages_link
+        BEGIN
+            SELECT CASE
+                WHEN (SELECT id from books WHERE id=NEW.book) IS NULL
+                THEN RAISE(ABORT, 'Foreign key violation: book not in books')
+            END;
+        END;
+        DROP TRIGGER IF EXISTS fkc_update_books_languages_link_b;
+        CREATE TRIGGER fkc_update_books_languages_link_b
+        BEFORE UPDATE OF lang_code ON books_languages_link
+        BEGIN
+            SELECT CASE
+                WHEN (SELECT id from languages WHERE id=NEW.lang_code) IS NULL
+                THEN RAISE(ABORT, 'Foreign key violation: lang_code not in languages')
+            END;
+        END;
+
+        DROP INDEX IF EXISTS books_languages_link_aidx;
+        CREATE INDEX books_languages_link_aidx ON books_languages_link (lang_code);
+        DROP INDEX IF EXISTS books_languages_link_bidx;
+        CREATE INDEX books_languages_link_bidx ON books_languages_link (book);
+        DROP INDEX IF EXISTS languages_idx;
+        CREATE INDEX languages_idx ON languages (lang_code COLLATE NOCASE);
+
+        DROP TRIGGER IF EXISTS books_delete_trg;
+        CREATE TRIGGER books_delete_trg
+            AFTER DELETE ON books
+            BEGIN
+                DELETE FROM books_authors_link WHERE book=OLD.id;
+                DELETE FROM books_publishers_link WHERE book=OLD.id;
+                DELETE FROM books_ratings_link WHERE book=OLD.id;
+                DELETE FROM books_series_link WHERE book=OLD.id;
+                DELETE FROM books_tags_link WHERE book=OLD.id;
+                DELETE FROM books_languages_link WHERE book=OLD.id;
+                DELETE FROM data WHERE book=OLD.id;
+                DELETE FROM comments WHERE book=OLD.id;
+                DELETE FROM conversion_options WHERE book=OLD.id;
+                DELETE FROM books_plugin_data WHERE book=OLD.id;
+                DELETE FROM identifiers WHERE book=OLD.id;
+        END;
+
+        INSERT INTO identifiers (book, val) SELECT id,isbn FROM books WHERE isbn;
+
+        ALTER TABLE books ADD COLUMN last_modified TIMESTAMP NOT NULL DEFAULT "%s";
+
+        '''%isoformat(DEFAULT_DATE, sep=' ')
+        # Sqlite does not support non constant default values in alter
+        # statements
+        self.conn.executescript(script)
+
 

src/calibre/library/server/browse.py

@@ -673,6 +673,8 @@ class BrowseServer(object):
                         categories = [categories]
                     dbtags = []
                     for category in categories:
+                        if category not in ccache:
+                            continue
                         dbtag = None
                         for tag in ccache[key]:
                             if tag.name == category:

src/calibre/library/server/xml.py

@@ -89,13 +89,16 @@ class XMLServer(object):
             for x in ('id', 'title', 'sort', 'author_sort', 'rating', 'size'):
                 kwargs[x] = serialize(record[FM[x]])
 
-            for x in ('isbn', 'formats', 'series', 'tags', 'publisher',
-                    'comments'):
+            for x in ('formats', 'series', 'tags', 'publisher',
+                    'comments', 'identifiers'):
                 y = record[FM[x]]
                 if x == 'tags':
                     y = format_tag_string(y, ',', ignore_max=True)
                 kwargs[x] = serialize(y) if y else ''
 
+            isbn = self.db.isbn(record[FM['id']], index_is_id=True)
+            kwargs['isbn'] = serialize(isbn if isbn else '')
+
             kwargs['safe_title'] = ascii_filename(kwargs['title'])
 
             c = kwargs.pop('comments')

src/calibre/library/sqlite.py

@@ -8,6 +8,7 @@ Wrapper for multi-threaded access to a single sqlite database connection. Serial
 all calls.
 '''
 import sqlite3 as sqlite, traceback, time, uuid, sys, os
+import repr as reprlib
 from sqlite3 import IntegrityError, OperationalError
 from threading import Thread
 from Queue import Queue
@@ -20,6 +21,7 @@ from calibre.utils.date import parse_date, isoformat
 from calibre import isbytestring, force_unicode
 from calibre.constants import iswindows, DEBUG
 from calibre.utils.icu import strcmp
+from calibre import prints
 
 global_lock = RLock()
 
@@ -87,6 +89,18 @@ class SortedConcatenate(object):
 class SafeSortedConcatenate(SortedConcatenate):
     sep = '|'
 
+class IdentifiersConcat(object):
+    '''String concatenation aggregator for the identifiers map'''
+    def __init__(self):
+        self.ans = []
+
+    def step(self, key, val):
+        self.ans.append(u'%s:%s'%(key, val))
+
+    def finalize(self):
+        return ','.join(self.ans)
+
+
 class AumSortedConcatenate(object):
     '''String concatenation aggregator for the author sort map'''
     def __init__(self):
@@ -170,13 +184,13 @@ class DBThread(Thread):
                                    detect_types=sqlite.PARSE_DECLTYPES|sqlite.PARSE_COLNAMES)
         self.conn.execute('pragma cache_size=5000')
         encoding = self.conn.execute('pragma encoding').fetchone()[0]
-        c_ext_loaded = load_c_extensions(self.conn)
+        self.conn.create_aggregate('sortconcat', 2, SortedConcatenate)
+        self.conn.create_aggregate('sort_concat', 2, SafeSortedConcatenate)
+        self.conn.create_aggregate('identifiers_concat', 2, IdentifiersConcat)
+        load_c_extensions(self.conn)
         self.conn.row_factory = sqlite.Row if self.row_factory else  lambda cursor, row : list(row)
         self.conn.create_aggregate('concat', 1, Concatenate)
         self.conn.create_aggregate('aum_sortconcat', 3, AumSortedConcatenate)
-        if not c_ext_loaded:
-            self.conn.create_aggregate('sortconcat', 2, SortedConcatenate)
-            self.conn.create_aggregate('sort_concat', 2, SafeSortedConcatenate)
         self.conn.create_collation('PYNOCASE', partial(pynocase,
             encoding=encoding))
         self.conn.create_function('title_sort', 1, title_sort)
@@ -208,17 +222,21 @@ class DBThread(Thread):
                     except Exception, err:
                         ok, res = False, (err, traceback.format_exc())
                 else:
-                    func = getattr(self.conn, func)
+                    bfunc = getattr(self.conn, func)
                     try:
                         for i in range(3):
                             try:
-                                ok, res = True, func(*args, **kwargs)
+                                ok, res = True, bfunc(*args, **kwargs)
                                 break
                             except OperationalError, err:
                                 # Retry if unable to open db file
-                                if 'unable to open' not in str(err) or i == 2:
+                                e = str(err)
+                                if 'unable to open' not in e or i == 2:
+                                    if 'unable to open' in e:
+                                        prints('Unable to open database for func',
+                                            func, reprlib.repr(args),
+                                            reprlib.repr(kwargs))
                                     raise
-                                traceback.print_exc()
                             time.sleep(0.5)
                     except Exception, err:
                         ok, res = False, (err, traceback.format_exc())

src/calibre/library/sqlite_custom.c

@@ -77,6 +77,7 @@ static void sort_concat_free(SortConcatList *list) {
         free(list->vals[i]->val);
         free(list->vals[i]);
     }
+    free(list->vals);
 }
 
 static int sort_concat_cmp(const void *a_, const void *b_) {
@@ -142,11 +143,102 @@ static void sort_concat_finalize2(sqlite3_context *context) {
 
 // }}}
 
+// identifiers_concat {{{
+
+typedef struct {
+    char *val;
+    size_t length;
+} IdentifiersConcatItem;
+
+typedef struct {
+    IdentifiersConcatItem **vals;
+    size_t count;
+    size_t length;
+} IdentifiersConcatList;
+
+static void identifiers_concat_step(sqlite3_context *context, int argc, sqlite3_value **argv) {
+    const char *key, *val;
+    size_t len = 0;
+    IdentifiersConcatList *list;
+
+    assert(argc == 2);
+
+    list = (IdentifiersConcatList*) sqlite3_aggregate_context(context, sizeof(*list));
+    if (list == NULL) return;
+
+    if (list->vals == NULL) {
+        list->vals = (IdentifiersConcatItem**)calloc(100, sizeof(IdentifiersConcatItem*));
+        if (list->vals == NULL) return;
+        list->length = 100;
+        list->count = 0;
+    }
+
+    if (list->count == list->length) {
+        list->vals = (IdentifiersConcatItem**)realloc(list->vals, list->length + 100);
+        if (list->vals == NULL) return;
+        list->length = list->length + 100;
+    }
+
+    list->vals[list->count] = (IdentifiersConcatItem*)calloc(1, sizeof(IdentifiersConcatItem));
+    if (list->vals[list->count] == NULL) return;
+
+    key = (char*) sqlite3_value_text(argv[0]);
+    val = (char*) sqlite3_value_text(argv[1]);
+    if (key == NULL || val == NULL) {return;}
+    len = strlen(key) + strlen(val) + 1;
+
+    list->vals[list->count]->val = (char*)calloc(len+1, sizeof(char));
+    if (list->vals[list->count]->val == NULL) return;
+    snprintf(list->vals[list->count]->val, len+1, "%s:%s", key, val);
+    list->vals[list->count]->length = len;
+
+    list->count = list->count + 1;
+
+}
+
+
+static void identifiers_concat_finalize(sqlite3_context *context) {
+    IdentifiersConcatList *list;
+    IdentifiersConcatItem *item;
+    char *ans, *pos;
+    size_t sz = 0, i;
+
+    list = (IdentifiersConcatList*) sqlite3_aggregate_context(context, sizeof(*list));
+    if (list == NULL || list->vals == NULL || list->count < 1) return;
+
+    for (i = 0; i < list->count; i++) { 
+        sz += list->vals[i]->length;
+    }
+    sz += list->count; // Space for commas
+    ans = (char*)calloc(sz+2, sizeof(char));
+    if (ans == NULL) return;
+
+    pos = ans;
+
+    for (i = 0; i < list->count; i++) {
+        item = list->vals[i];
+        if (item == NULL || item->val == NULL) continue;
+        memcpy(pos, item->val, item->length);
+        pos += item->length;
+        *pos = ',';
+        pos += 1;
+        free(item->val);
+        free(item);
+    }
+    *(pos-1) = 0; // Remove trailing comma
+    sqlite3_result_text(context, ans, -1, SQLITE_TRANSIENT);
+    free(ans);
+    free(list->vals);
+}
+
+// }}}
+
 MYEXPORT int sqlite3_extension_init(
     sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi){
   SQLITE_EXTENSION_INIT2(pApi);
   sqlite3_create_function(db, "sortconcat", 2, SQLITE_UTF8, NULL, NULL, sort_concat_step, sort_concat_finalize);
   sqlite3_create_function(db, "sort_concat", 2, SQLITE_UTF8, NULL, NULL, sort_concat_step, sort_concat_finalize2);
+  sqlite3_create_function(db, "identifiers_concat", 2, SQLITE_UTF8, NULL, NULL, identifiers_concat_step, identifiers_concat_finalize);
   return 0;
 }
 

src/calibre/manual/faq.rst

@@ -81,7 +81,7 @@ Device Integration
 
 What devices does |app| support?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-At the moment |app| has full support for the SONY PRS line, Barnes & Noble Nook, Cybook Gen 3/Opus, Amazon Kindle line, Entourage Edge, Longshine ShineBook, Ectaco Jetbook, BeBook/BeBook Mini, Irex Illiad/DR1000, Foxit eSlick, PocketBook 360, Italica, eClicto, Iriver Story, Airis dBook, Hanvon N515, Binatone Readme, Teclast K3, SpringDesign Alex, Kobo Reader, various Android phones and the iPhone/iPad. In addition, using the :guilabel:`Save to disk` function you can use it with any ebook reader that exports itself as a USB disk.
+At the moment |app| has full support for the SONY PRS line, Barnes & Noble Nook line, Cybook Gen 3/Opus, Amazon Kindle line, Entourage Edge, Longshine ShineBook, Ectaco Jetbook, BeBook/BeBook Mini, Irex Illiad/DR1000, Foxit eSlick, PocketBook line, Italica, eClicto, Iriver Story, Airis dBook, Hanvon N515, Binatone Readme, Teclast K3 and clones, SpringDesign Alex, Kobo Reader, various Android phones and the iPhone/iPad. In addition, using the :guilabel:`Connect to folder` function you can use it with any ebook reader that exports itself as a USB disk.
 
 How can I help get my device supported in |app|?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -350,7 +350,7 @@ Why doesn't |app| have a column for foo?
 |app| is designed to have columns for the most frequently and widely used fields. In addition, you can add any columns you like. Columns can be added via :guilabel:`Preferences->Interface->Add your own columns`.
 Watch the tutorial `UI Power tips <http://calibre-ebook.com/demo#tutorials>`_ to learn how to create your own columns.
 
-You can also create "virtual columns" that contain combinations of the metadata from other columns. In the add column dialog choose the option "Column from other columns" and in the template enter the other column names. For example to create a virtual column containing formats or ISBN, enter ``{formats}`` for formats or ``{isbn}`` for ISBN. For more details, see :ref:`templatelangcalibre`.
+You can also create "virtual columns" that contain combinations of the metadata from other columns. In the add column dialog  use the :guilabel:`Quick create` links to easily create columns to show the book ISBN, formats or the time the book was last modified. For more details, see :ref:`templatelangcalibre`.
 
 
 Can I have a column showing the formats or the ISBN?

src/calibre/manual/gui.rst

@@ -338,7 +338,7 @@ You can build advanced search queries easily using the :guilabel:`Advanced Searc
 clicking the button |sbi|.
 
 Available fields for searching are: ``tag, title, author, publisher, series, series_index, rating, cover,
-comments, format, isbn, date, pubdate, search, size`` and custom columns. If a device is plugged in, the
+comments, format, identifiers, date, pubdate, search, size`` and custom columns. If a device is plugged in, the
 ``ondevice`` field becomes available. To find the search name for a custom column, hover your mouse over the
 column header.
 
@@ -385,6 +385,21 @@ with undefined values in the column. Searching for ``true`` will find all books
 values in the column. Searching for ``yes`` or ``checked`` will find all books with ``Yes`` in the column.
 Searching for ``no`` or ``unchecked`` will find all books with ``No`` in the column.
 
+Hierarchical items (e.g. A.B.C) use an extended syntax to match initial parts of the hierarchy. This is done by adding a period between the exact match indicator (=) and the text. For example, the query ``tags:=.A`` will find the tags `A` and `A.B`, but will not find the tags `AA` or `AA.B`. The query ``tags:=.A.B`` will find the tags `A.B` and `A.C`, but not the tag `A`.
+
+Identifiers (e.g., isbn, doi, lccn etc) also use an extended syntax. First, note that an identifier has the form ``key:value``, as in ``isbn:123456789``. The extended syntax permits you to specify independently which key and value to search for. Both the key and the value parts of the query can use `equality`, `contains`, or `regular expression` matches. Examples:
+
+    * ``identifiers:true`` will find books with any identifier.
+    * ``identifiers:false`` will find books with no identifier.
+    * ``identifiers:123`` will search for books with any key having a value containing `123`.
+    * ``identifiers:=123456789`` will search for books with any key having a value equal to `123456789`.
+    * ``identifiers:=isbn:`` and ``identifiers:isbn:true`` will find books with a key equal to isbn having any value
+    * ``identifiers:=isbn:false`` will find books with no key equal to isbn.
+    * ``identifiers:=isbn:123`` will find books with a key equal to isbn having a value containing `123`.
+    * ``identifiers:=isbn:=123456789`` will find books with a key equal to isbn having a value equal to `123456789`.
+    * ``identifiers:i:1`` will find books with a key containing an `i` having a value containing a `1`.
+    
+
 .. |sbi| image:: images/search_button.png
     :align: middle
 
@@ -436,25 +451,26 @@ Tag Browser
 .. image:: images/tag_browser.png
    :class: float-left-img
 
-The Tag Browser allows you to easily browse your collection by Author/Tags/Series/etc. If you click on any Item in the Tag Browser, for example, the Author name, Isaac Asimov, then the list of books to the right is restricted to books by that author. Clicking once again on Isaac Asimov will restrict the list of books to books not by Isaac Asimov. A third click will remove the restriction. If you hold down the Ctrl or Shift keys and click on multiple items, then restrictions based on multiple items are created. For example you could Hold Ctrl and click on the tags History and Europe for find books on European history. The Tag Browser works by constructing search expressions that are automatically entered into the Search bar. It is a good way to learn how to construct basic search expressions. 
+The Tag Browser allows you to easily browse your collection by Author/Tags/Series/etc. If you click on any item in the Tag Browser, for example the author name Isaac Asimov, then the list of books to the right is restricted to showing books by that author. You can click on category names as well. For example, clicking on "Series" will show you all books in any series.
 
-There is a search bar at the top of the Tag Browser that allows you to easily find any item in the Tag Browser. In addition, you can right click on any item and choose to hide it or rename it or open a "Manage x" dialog that allows you to manage items of that kind. For example the "Manage Authors" dialog allows you to rename authors and control how their names are sorted. 
+The first click on an item will restrict the list of books to those that contain/match the item. Continuing the above example, clicking on Isaac Asimov will show books by that author. Clicking again on the item will change what is shown, depending on whether the item has children (see sub-categories and hierarchical items below). Continuing the Isaac Asimov example, clicking again on Isaac Asimov will restrict the list of books to those not by Isaac Asimov. A third click will remove the restriction, showing all books. If you hold down the Ctrl or Shift keys and click on multiple items, then restrictions based on multiple items are created. For example you could hold Ctrl and click on the tags History and Europe for find books on European history. The Tag Browser works by constructing search expressions that are automatically entered into the Search bar. Looking at what the Tag Browser generates is a good way to learn how to construct basic search expressions. 
 
 Items in the Tag browser have their icons partially colored. The amount of color depends on the average rating of the books in that category. So for example if the books by Isaac Asimov have an average of four stars, the icon for Isaac Asimov in the Tag Browser will be 4/5th colored. You can hover your mouse over the icon to see the average rating.
 
-For convenience, you can drag and drop books from the book list to items in the Tag Browser and that item will be automatically applied to the dropped books. For example, dragging a book to Isaac Asimov will set the author of that book to Isaac Asimov or dragging it to the tag History will add the tag History to its tags.
+The outer-level items in the tag browser such as Authors and Series are called categories. You can create your own categories, called User Categories, which are useful for organizing items. For example, you can use the User Categories Editor (push the Manage User Categories button) to create a user category called Favorite Authors, then put the items for your favorites into the category. User categories can have sub-categories. For example, the user category Favorites.Authors is a sub-category of Favorites. You might also have Favorites.Series, in which case there will be two sub-categories under Favorites. Sub-categories can be created by right-clicking on a user category, choosing "Add sub-category to ...", and entering the sub-category name; or by using the User Categories Editor by entering names like the Favorites example above.
 
-The outer-level items in the tag browser such as Authors and Series are called categories. You can create your own categories, called User Categories, which are useful for organizing items. For example, you can use the user categories editor (push the Manage User Categories button) to create a user category called Favorite Authors, then put the items for your favorites into the category. User categories act like built-in categories; you can click on items to search for them. You can search for all items in a category by right-clicking on the category name and choosing "Search for books in ...".
+You can search user categories in the same way as built-in categories, by clicking on them. There are four different searches cycled through by clicking: "everything matching an item in the category" indicated by a single green plus sign, "everything matching an item in the category or its sub-categories" indicated by two green plus signs, "everything not matching an item in the category" shown by a single red minus sign, and "everything not matching an item in the category or its sub-categories" shown by two red minus signs.
 
-User categories can have sub-categories. For example, the user category Favorites.Authors is a sub-category of Favorites. You might also have Favorites.Series, in which case there will be two sub-categories under Favorites. Sub-categories can be created using Manage User Categories by entering names like the Favorites example. They can also be created by right-clicking on a user category, choosing "Add sub-category to ...", and entering the category name.
+It is also possible to create hierarchies inside some of the text categories such as tags, series, and custom columns. These hierarchies show with the small triangle, permitting the sub-items to be hidden. To use hierarchies of items in a category, you must first go to Preferences / Look & Feel and enter the category name(s) into the "Categories with hierarchical items" box. Once this is done, items in that category that contain periods will be shown using the small triangle. For example, assume you create a custom column called "Genre" and indicate that it contains hierarchical items. Once done, items such as Mystery.Thriller and Mystery.English will display as Mystery with the small triangle next to it. Clicking on the triangle will show Thriller and English as sub-items. 
 
-It is also possible to create hierarchies inside some of the built-in categories (the text categories). These hierarchies show with the small triangle permitting the sub-items to be hidden. To use hierarchies in a category, you must first go to Preferences / Look & Feel and enter the category name(s) into the "Categories with hierarchical items" box. Once this is done, items in that category that contain periods will be shown using the small triangle. For example, assume you create a custom column called "Genre" and indicate that it contains hierarchical items. Once done, items such as Mystery.Thriller and Mystery.English will display as Mystery with the small triangle next to it. Clicking on the triangle will show Thriller and English as sub-items.
+Hierarchical items (items with children) use the same four 'click-on' searches as user categories. Items that do not have children use two of the searches: "everything matching" and "everything not matching".
 
-You can drag and drop items in the Tag browser onto user categories to add them to that category.
+You can drag and drop items in the Tag browser onto user categories to add them to that category. If the source is a user category, holding the shift key while dragging will move the item to the new category. You can also drag and drop books from the book list onto items in the Tag Browser; dropping a book on an item causes that item to be automatically applied to the dropped books. For example, dragging a book onto Isaac Asimov will set the author of that book to Isaac Asimov. Dropping it onto the tag History will add the tag History to the book's tags.
+
+There is a search bar at the top of the Tag Browser that allows you to easily find any item in the Tag Browser. In addition, you can right click on any item and choose one of several operations. Some examples are to hide the it, rename it, or open a "Manage x" dialog that allows you to manage items of that kind. For example, the "Manage Authors" dialog allows you to rename authors and control how their names are sorted. 
 
 You can control how items are sorted in the Tag browser via the box at the bottom of the Tag Browser. You can choose to sort by name, average rating or popularity (popularity is the number of books with an item in your library; for example; the popularity of Isaac Asimov is the number of book sin your library by Isaac Asimov).
 
-
 Jobs
 -----
 .. image:: images/jobs.png
@@ -481,6 +497,8 @@ Calibre has several keyboard shortcuts to save you time and mouse movement. Thes
       - Edit the metadata of the currently selected field in the book list.
     * - :kbd:`A` 
       - Add Books
+    * - :kbd:`Shift+A` 
+      - Add Formats to the selected books
     * - :kbd:`C` 
       - Convert selected Books
     * - :kbd:`D` 

src/calibre/manual/template_lang.rst

@@ -30,7 +30,7 @@ You can use all the various metadata fields available in calibre in a template,
 In addition to the column based fields, you also can use::
 
     {formats} - A list of formats available in the calibre library for a book
-    {isbn}    - The ISBN number of the book
+    {identifiers:select(isbn)} - The ISBN number of the book
 
 If a particular book does not have a particular piece of metadata, the field in the template is automatically removed for that book. Consider, for example::
 
@@ -95,7 +95,7 @@ Advanced features
 Using templates in custom columns
 ----------------------------------
 
-There are sometimes cases where you want to display metadata that |app| does not normally display, or to display data in a way different from how |app| normally does. For example, you might want to display the ISBN, a field that |app| does not display. You can use custom columns for this by creating a column with the type 'column built from other columns' (hereafter called composite columns), and entering a template. Result: |app| will display a column showing the result of evaluating that template. To display the ISBN, create the column and enter ``{isbn}`` into the template box. To display a column containing the values of two series custom columns separated by a comma, use ``{#series1:||,}{#series2}``.
+There are sometimes cases where you want to display metadata that |app| does not normally display, or to display data in a way different from how |app| normally does. For example, you might want to display the ISBN, a field that |app| does not display. You can use custom columns for this by creating a column with the type 'column built from other columns' (hereafter called composite columns), and entering a template. Result: |app| will display a column showing the result of evaluating that template. To display the ISBN, create the column and enter ``{identifiers:select(isbn)}`` into the template box. To display a column containing the values of two series custom columns separated by a comma, use ``{#series1:||,}{#series2}``.
 
 Composite columns can use any template option, including formatting.
 
@@ -122,10 +122,11 @@ The functions available are:
     * ``count(separator)`` -- interprets the value as a list of items separated by `separator`, returning the number of items in the list. Most lists use a comma as the separator, but authors uses an ampersand. Examples: `{tags:count(,)}`, `{authors:count(&)}`
     * ``ifempty(text)``	-- if the field is not empty, return the value of the field. Otherwise return `text`.
     * ``list_item(index, separator)`` -- interpret the value as a list of items separated by `separator`, returning the `index`th item. The first item is number zero. The last item can be returned using `list_item(-1,separator)`. If the item is not in the list, then the empty value is returned. The separator has the same meaning as in the `count` function.
-    * ``lookup(pattern, field, pattern, field, ..., else_field)`` -- like switch, except the arguments are field (metadata) names, not text. The value of the appropriate field will be fetched and used. Note that because composite columns are fields, you can use this function in one composite field to use the value of some other composite field. This is extremely useful when constructing variable save paths (more later).
     * ``re(pattern, replacement)`` -- return the field after applying the regular expression. All instances of `pattern` are replaced with `replacement`. As in all of |app|, these are python-compatible regular expressions.
     * ``shorten(left chars, middle text, right chars)`` -- Return a shortened version of the field, consisting of `left chars` characters from the beginning of the field, followed by `middle text`, followed by `right chars` characters from the end of the string. `Left chars` and `right chars` must be integers. For example, assume the title of the book is `Ancient English Laws in the Times of Ivanhoe`, and you want it to fit in a space of at most 15 characters. If you use ``{title:shorten(9,-,5)}``, the result will be `Ancient E-nhoe`. If the field's length is less than ``left chars`` + ``right chars`` + the length of ``middle text``, then the field will be used intact. For example, the title `The Dome` would not be changed.
     * ``switch(pattern, value, pattern, value, ..., else_value)`` -- for each ``pattern, value`` pair, checks if the field matches the regular expression ``pattern`` and if so, returns that ``value``. If no ``pattern`` matches, then ``else_value`` is returned. You can have as many ``pattern, value`` pairs as you want.
+    * ``lookup(pattern, field, pattern, field, ..., else_field)`` -- like switch, except the arguments are field (metadata) names, not text. The value of the appropriate field will be fetched and used. Note that because composite columns are fields, you can use this function in one composite field to use the value of some other composite field. This is extremely useful when constructing variable save paths (more later).
+    * ``select(key)`` -- interpret the field as a comma-separated list of items, with the items being of the form "id:value". Find the pair with the id equal to key, and return the corresponding value. This function is particularly useful for extracting a value such as an isbn from the set of identifiers for a book.
     * ``test(text if not empty, text if empty)`` -- return `text if not empty` if the field is not empty, otherwise return `text if empty`.
 
 
@@ -141,7 +142,9 @@ Note that you can use the prefix and suffix as well. If you want the number to a
 Using functions in templates - template program mode
 ----------------------------------------------------
 
-The template language program mode differs from single-function mode in that it permits you to write template expressions that refer to other metadata fields, modify values, and do arithmetic. It is a reasonably complete programming language.
+The template language program mode differs from single-function mode in that it permits you to write template expressions that refer to other metadata fields, modify values, and do arithmetic. It is a reasonably complete programming language. 
+
+You can use the functions documented above in template program mode. See below for details.
 
 Beginning with an example, assume that you want your template to show the series for a book if it has one, otherwise show the value of a custom field #genre. You cannot do this in the basic language because you cannot make reference to another metadata field within a template expression. In program mode, you can. The following expression works::
 
@@ -203,7 +206,7 @@ For various values of series_index, the program returns:
     * series_index == 2, result = ``prefix 2->eq suffix``
     * series_index == 3, result = ``prefix 3->gt suffix``
 
-All the functions listed under single-function mode can be used in program mode, noting that unlike the functions described below you must supply a first parameter providing the value the function is to act upon. 
+**All the functions listed under single-function mode can be used in program mode**. To do so, you must supply the value that the function is to act upon as the first parameter, in addition to the parameters documented above. For example, in program mode the parameters of the `test` function are ``test(x, text_if_not_empty, text_if_empty)``. The `x` parameter, which is the value to be tested, will almost always be a variable or a function call, often `field()`.
 
 The following functions are available in addition to those described in single-function mode. Remember from the example above that the single-function mode functions require an additional first parameter specifying the field to operate on. With the exception of the ``id`` parameter of assign, all parameters can be statements (sequences of expressions):
 
@@ -212,9 +215,23 @@ The following functions are available in addition to those described in single-f
     * ``cmp(x, y, lt, eq, gt)`` -- compares x and y after converting both to numbers. Returns ``lt`` if x < y. Returns ``eq`` if x == y. Otherwise returns ``gt``.
     * ``divide(x, y)`` -- returns x / y. Throws an exception if either x or y are not numbers.
     * ``field(name)`` -- returns the metadata field named by ``name``.
+    * ``format_date(x, date_format)`` -- format_date(val, format_string) -- format the value, which must be a date field, using the format_string, returning a string. The formatting codes are::
+    
+        d    : the day as number without a leading zero (1 to 31)
+        dd   : the day as number with a leading zero (01 to 31) '
+        ddd  : the abbreviated localized day name (e.g. "Mon" to "Sun"). '
+        dddd : the long localized day name (e.g. "Monday" to "Sunday"). '
+        M    : the month as number without a leading zero (1 to 12). '
+        MM   : the month as number with a leading zero (01 to 12) '
+        MMM  : the abbreviated localized month name (e.g. "Jan" to "Dec"). '
+        MMMM : the long localized month name (e.g. "January" to "December"). '
+        yy   : the year as two digit number (00 to 99). '
+        yyyy : the year as four digit number.'
+    
     * ``eval(string)`` -- evaluates the string as a program, passing the local variables (those ``assign`` ed to). This permits using the template processor to construct complex results from local variables.
     * ``multiply(x, y)`` -- returns x * y. Throws an exception if either x or y are not numbers.
     * ``print(a, b, ...)`` -- prints the arguments to standard output. Unless you start calibre from the command line (``calibre-debug -g``), the output will go to a black hole.
+    * ``raw_field(name)`` -- returns the metadata field named by name without applying any formatting.
     * ``strcat(a, b, ...)`` -- can take any number of arguments. Returns a string formed by concatenating all the arguments.
     * ``strcmp(x, y, lt, eq, gt)`` -- does a case-insensitive comparison x and y as strings. Returns ``lt`` if x < y. Returns ``eq`` if x == y. Otherwise returns ``gt``.
     * ``substr(str, start, end)`` -- returns the ``start``'th through the ``end``'th characters of ``str``. The first character in ``str`` is the zero'th character. If end is negative, then it indicates that many characters counting from the right. If end is zero, then it indicates the last character. For example, ``substr('12345', 1, 0)`` returns ``'2345'``, and ``substr('12345', 1, -1)`` returns ``'234'``.

src/calibre/manual/templates/layout.html

@@ -1,23 +1,6 @@
 {% extends "!layout.html" %}
 
 {% block extrahead %}
-    {% if not embedded %}
-    <script type="text/javascript">
-
-        var _gaq = _gaq || [];
-        _gaq.push(['_setAccount', 'UA-20736318-1']);
-        _gaq.push(['_setDomainName', '.calibre-ebook.com']);
-        _gaq.push(['_trackPageview']);
-
-        (function() {
-            var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
-            ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
-            var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
-        })();
-
-    </script>
-    {% endif %}
-
     <style type="text/css">
         .float-left-img { float: left; margin-right: 1em; margin-bottom: 1em }
         .float-right-img { float: right; margin-left: 1em; margin-bottom: 1em }
@@ -52,6 +35,23 @@
         </div>
       {%- endif %}
       </div>
+        {% if not embedded %}
+            <script type="text/javascript">
+
+                var _gaq = _gaq || [];
+                _gaq.push(['_setAccount', 'UA-20736318-1']);
+                _gaq.push(['_setDomainName', '.calibre-ebook.com']);
+                _gaq.push(['_trackPageview']);
+
+                (function() {
+                    var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+                    ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+                    var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+                })();
+
+            </script>
+        {%- endif %}
+
 {% endblock %}
 
 {% block sidebarlogo %}

src/calibre/utils/date.py

@@ -45,6 +45,7 @@ utc_tz = _utc_tz = tzutc()
 local_tz = _local_tz = SafeLocalTimeZone()
 
 UNDEFINED_DATE = datetime(101,1,1, tzinfo=utc_tz)
+DEFAULT_DATE = datetime(2000,1,1, tzinfo=utc_tz)
 
 def is_date_undefined(qt_or_dt):
     d = qt_or_dt

src/calibre/utils/formatter_functions.py

@@ -12,6 +12,7 @@ import inspect, re, traceback, sys
 
 from calibre.utils.titlecase import titlecase
 from calibre.utils.icu import capitalize, strcmp
+from calibre.utils.date import parse_date, format_date
 
 
 class FormatterFunctions(object):
@@ -230,6 +231,15 @@ class BuiltinField(BuiltinFormatterFunction):
     def evaluate(self, formatter, kwargs, mi, locals, name):
         return formatter.get_value(name, [], kwargs)
 
+class BuiltinRaw_field(BuiltinFormatterFunction):
+    name = 'raw_field'
+    arg_count = 1
+    doc = _('raw_field(name) -- returns the metadata field named by name '
+            'without applying any formatting.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, name):
+        return unicode(getattr(mi, name, None))
+
 class BuiltinSubstr(BuiltinFormatterFunction):
     name = 'substr'
     arg_count = 3
@@ -396,6 +406,23 @@ class BuiltinListitem(BuiltinFormatterFunction):
         except:
             return ''
 
+class BuiltinSelect(BuiltinFormatterFunction):
+    name = 'select'
+    arg_count = 2
+    doc = _('select(val, key) -- interpret the value as a comma-separated list '
+            'of items, with the items being "id:value". Find the pair with the'
+            'id equal to key, and return the corresponding value.'
+            )
+
+    def evaluate(self, formatter, kwargs, mi, locals, val, key):
+        if not val:
+            return ''
+        vals = [v.strip() for v in val.split(',')]
+        for v in vals:
+            if v.startswith(key+':'):
+                return v[len(key)+1:]
+        return ''
+
 class BuiltinSublist(BuiltinFormatterFunction):
     name = 'sublist'
     arg_count = 4
@@ -424,6 +451,34 @@ class BuiltinSublist(BuiltinFormatterFunction):
         except:
             return ''
 
+class BuiltinFormat_date(BuiltinFormatterFunction):
+    name = 'format_date'
+    arg_count = 2
+    doc = _('format_date(val, format_string) -- format the value, which must '
+            'be a date field, using the format_string, returning a string. '
+            'The formatting codes are: '
+            'd    : the day as number without a leading zero (1 to 31) '
+            'dd   : the day as number with a leading zero (01 to 31) '
+            'ddd  : the abbreviated localized day name (e.g. "Mon" to "Sun"). '
+            'dddd : the long localized day name (e.g. "Monday" to "Sunday"). '
+            'M    : the month as number without a leading zero (1 to 12). '
+            'MM   : the month as number with a leading zero (01 to 12) '
+            'MMM  : the abbreviated localized month name (e.g. "Jan" to "Dec"). '
+            'MMMM : the long localized month name (e.g. "January" to "December"). '
+            'yy   : the year as two digit number (00 to 99). '
+            'yyyy : the year as four digit number.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val, format_string):
+        print val
+        if not val:
+            return ''
+        try:
+            dt = parse_date(val)
+            s = format_date(dt, format_string)
+        except:
+            s = 'BAD DATE'
+        return s
+
 class BuiltinUppercase(BuiltinFormatterFunction):
     name = 'uppercase'
     arg_count = 1
@@ -464,14 +519,17 @@ builtin_contains    = BuiltinContains()
 builtin_count       = BuiltinCount()
 builtin_divide      = BuiltinDivide()
 builtin_eval        = BuiltinEval()
-builtin_ifempty     = BuiltinIfempty()
+builtin_format_date = BuiltinFormat_date()
 builtin_field       = BuiltinField()
+builtin_ifempty     = BuiltinIfempty()
 builtin_list_item   = BuiltinListitem()
 builtin_lookup      = BuiltinLookup()
 builtin_lowercase   = BuiltinLowercase()
 builtin_multiply    = BuiltinMultiply()
 builtin_print       = BuiltinPrint()
+builtin_raw_field   = BuiltinRaw_field()
 builtin_re          = BuiltinRe()
+builtin_select      = BuiltinSelect()
 builtin_shorten     = BuiltinShorten()
 builtin_strcat      = BuiltinStrcat()
 builtin_strcmp      = BuiltinStrcmp()

src/calibre/utils/mem.py

@@ -5,53 +5,99 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import gc
+'''
+Measure memory usage of the current process.
 
-## {{{ http://code.activestate.com/recipes/286222/ (r1)
-import os
+The key function is memory() which returns the current memory usage in bytes.
+You can pass a number to memory and it will be subtracted from the returned
+value.
+'''
 
-_proc_status = '/proc/%d/status' % os.getpid()
+import gc, os
 
-_scale = {'kB': 1024.0, 'mB': 1024.0*1024.0,
-          'KB': 1024.0, 'MB': 1024.0*1024.0}
+from calibre.constants import iswindows, islinux
 
-def _VmB(VmKey):
-    '''Private.
-    '''
-    global _proc_status, _scale
-     # get pseudo file  /proc/<pid>/status
-    try:
-        t = open(_proc_status)
-        v = t.read()
-        t.close()
-    except:
-        return 0.0  # non-Linux?
-     # get VmKey line e.g. 'VmRSS:  9999  kB\n ...'
-    i = v.index(VmKey)
-    v = v[i:].split(None, 3)  # whitespace
-    if len(v) < 3:
-        return 0.0  # invalid format?
-     # convert Vm value to bytes
-    return float(v[1]) * _scale[v[2]]
+if islinux:
+    ## {{{ http://code.activestate.com/recipes/286222/ (r1)
+
+    _proc_status = '/proc/%d/status' % os.getpid()
+
+    _scale = {'kB': 1024.0, 'mB': 1024.0*1024.0,
+            'KB': 1024.0, 'MB': 1024.0*1024.0}
+
+    def _VmB(VmKey):
+        '''Private.
+        '''
+        global _proc_status, _scale
+        # get pseudo file  /proc/<pid>/status
+        try:
+            t = open(_proc_status)
+            v = t.read()
+            t.close()
+        except:
+            return 0.0  # non-Linux?
+        # get VmKey line e.g. 'VmRSS:  9999  kB\n ...'
+        i = v.index(VmKey)
+        v = v[i:].split(None, 3)  # whitespace
+        if len(v) < 3:
+            return 0.0  # invalid format?
+        # convert Vm value to bytes
+        return float(v[1]) * _scale[v[2]]
 
 
-def memory(since=0.0):
-    '''Return memory usage in bytes.
-    '''
-    return _VmB('VmSize:') - since
+    def linux_memory(since=0.0):
+        '''Return memory usage in bytes.
+        '''
+        return _VmB('VmSize:') - since
 
 
-def resident(since=0.0):
-    '''Return resident memory usage in bytes.
-    '''
-    return _VmB('VmRSS:') - since
+    def resident(since=0.0):
+        '''Return resident memory usage in bytes.
+        '''
+        return _VmB('VmRSS:') - since
 
 
-def stacksize(since=0.0):
-    '''Return stack size in bytes.
-    '''
-    return _VmB('VmStk:') - since
-## end of http://code.activestate.com/recipes/286222/ }}}
+    def stacksize(since=0.0):
+        '''Return stack size in bytes.
+        '''
+        return _VmB('VmStk:') - since
+    ## end of http://code.activestate.com/recipes/286222/ }}}
+    memory = linux_memory
+elif iswindows:
+    import win32process
+    import win32con
+    import win32api
+
+    # See http://msdn.microsoft.com/en-us/library/ms684877.aspx
+    # for details on the info returned by get_meminfo
+
+    def get_handle(pid):
+        return win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, 0,
+                pid)
+
+    def listprocesses(self):
+        for process in win32process.EnumProcesses():
+            try:
+                han = get_handle(process)
+                procmeminfo = meminfo(han)
+                procmemusage = procmeminfo["WorkingSetSize"]
+                yield process, procmemusage
+            except:
+                pass
+
+    def get_meminfo(pid):
+        han = win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, 0,
+                pid)
+        return meminfo(han)
+
+    def meminfo(handle):
+        return win32process.GetProcessMemoryInfo(handle)
+
+    def win_memory(since=0.0):
+        info = meminfo(get_handle(os.getpid()))
+        return info['WorkingSetSize'] - since
+
+    memory = win_memory
 
 
 def gc_histogram():

src/calibre/web/feeds/recipes/collection.py

@@ -201,12 +201,14 @@ class SchedulerConfig(object):
             self.root.append(sr)
             self.write_scheduler_file()
 
-    def customize_recipe(self, urn, add_title_tag, custom_tags):
+    # 'keep_issues' argument for recipe-specific number of copies to keep
+    def customize_recipe(self, urn, add_title_tag, custom_tags, keep_issues):
         with self.lock:
             for x in list(self.iter_customization()):
                 if x.get('id') == urn:
                     self.root.remove(x)
             cs = E.recipe_customization({
+                'keep_issues' : keep_issues,
                 'id' : urn,
                 'add_title_tag' : 'yes' if add_title_tag else 'no',
                 'custom_tags' : ','.join(custom_tags),
@@ -317,16 +319,18 @@ class SchedulerConfig(object):
                     return x.get('username', ''), x.get('password', '')
 
     def get_customize_info(self, urn):
+        keep_issues = 0
         add_title_tag = True
         custom_tags = []
         with self.lock:
             for x in self.iter_customization():
                 if x.get('id', False) == urn:
+                    keep_issues = x.get('keep_issues', '0')
                     add_title_tag = x.get('add_title_tag', 'yes') == 'yes'
                     custom_tags = [i.strip() for i in x.get('custom_tags',
                         '').split(',')]
                     break
-        return add_title_tag, custom_tags
+        return add_title_tag, custom_tags, keep_issues
 
     def get_schedule_info(self, urn):
         with self.lock:

src/calibre/web/feeds/recipes/model.py

@@ -196,6 +196,7 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser):
         lang_map = {}
         self.all_urns = set([])
         self.showing_count = 0
+        self.builtin_count = 0
         for x in self.custom_recipe_collection:
             urn = x.get('id')
             self.all_urns.add(urn)
@@ -211,6 +212,7 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser):
                     lang_map[lang] = factory(NewsCategory, new_root, lang)
                 factory(NewsItem, lang_map[lang], urn, x.get('title'))
                 self.showing_count += 1
+                self.builtin_count += 1
         for x in self.scheduler_config.iter_recipes():
             urn = x.get('id')
             if urn not in self.all_urns:
@@ -354,9 +356,9 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser):
         self.scheduler_config.schedule_recipe(self.recipe_from_urn(urn),
                 sched_type, schedule)
 
-    def customize_recipe(self, urn, add_title_tag, custom_tags):
+    def customize_recipe(self, urn, add_title_tag, custom_tags, keep_issues):
         self.scheduler_config.customize_recipe(urn, add_title_tag,
-                custom_tags)
+                custom_tags, keep_issues)
 
     def get_to_be_downloaded_recipes(self):
         ans = self.scheduler_config.get_to_be_downloaded_recipes()