merge from trunk

Changelog.yaml

@@ -19,9 +19,93 @@
 #  new recipes:
 #    - title: 
 
-#    - title: "Launch of a new website that catalogues DRM free books. http://drmfree.calibre-ebook.com"
-#      description: "A growing catalogue of DRM free books. Books that you actually own after buying instead of renting."
-#      type: major
+- version: 0.7.48
+  date: 2011-03-04
+
+  new features:
+    - title: "Changes to the internal database structure used by calibre"
+      description: >
+        "These changes will allow calibre, in the future, to support book language, arbitrary book identifiers and keep track of when the metadata for a book was last modified. WARNING: Because of these changes, if you downgrade calibre versions after upgrading to 0.7.48, you will lose any changes you make to the ISBN of book entries in your calibre database, so do not downgrade unless you really have to. Also note that the first time you start calibre after this update, the startup will be slow as the database structure is being changed."
+
+    - title: "Launch of a new website that catalogues DRM free ebooks. http://drmfree.calibre-ebook.com"
+      description: "A growing catalogue of DRM free ebooks. Ebooks that you actually own after paying, instead of just renting."
+      type: major
+
+    - title: "News download: Add an option to keep at most x issues of a particular periodical in the calibre library. Use the Advanced tab in the Fetch news dialog for your news source to set this option."
+      tickets: [9168]
+
+    - title: "You can now right click on the cover in the book details panel to copy/paste a new cover."
+      tickets: [9255]
+
+    - title: "Add an entry to the add books drop down menu to easily add formats to an existing book record"
+
+    - title: "Tag browser: Clicking on a nested category now searches for the category alone. Clicking twice searches for the category and all its descendants and so on."
+      tickets: [9166, 9169]
+
+    - title: "Add a button to the Manage authors dialog to copy author sort values to author"
+
+    - title: "Decrease startup times on large libraries by using a faster algorithm to parse stored dates"
+
+    - title: "Add quick create links to easily create custom columns of commonly used types to the add custom column dialog"
+
+    - title: "Allow drag drop of images to change cover in book details window."
+      tickets: [9226]
+
+    - title: "Device susbsytem: Create a drive info file named driveinfo.calibre in the root of each device drive for USB connected devices. This file contains various useful data. API Change: The open method of the device plugins now accepts an extra parameter library_uuid which is the id of the calibre library connected tot eh device"
+
+  bug fixes:
+    - title: "Conversion pipeline: Fix regression in 0.7.46 that caused loss of some CSS information when converting HTML produced by Microsoft Word. Also remove empty tags from microsoft namespaces when parsing HTML"
+
+    - title: "Try harder to ensure that the worker log temporary files are deleted in windows"
+
+    - title: "CHM Input: Handle CHM files that dont specify a topics file."
+      tickets: [9253]
+
+    - title: "Fix regression that caused memory leak in Tag Browser. This would show up as the memory usage of calibre increasing when switching libraries."
+      tickets: [9246]
+
+    - title: "Fix bug that caused preferences->behavior to not show the output format set by the welcome wizard, and instead default to showing EPUB"
+
+    - title: "Fix bug that caused wrong books to be deleted from library if you choose 'delete from library and device' while the library is sorted by the On device column"
+
+    - title: "MOBI Input: Ignore all ASCII control codes except CR, NL and Tab."
+      tickets: [9219]
+
+  improved recipes:
+    - Credit Slips
+    - Seattle Times
+    - MacWorld
+    - Austin Statesman
+    - EPL Talk
+    - Gawker
+    - Deadspin
+
+  new recipes:
+    - title: "Thai Post Today and Daily Post"
+      author: "Chotechai P."
+
+    - title: "RBC.ru"
+      author: Chewi
+
+    - title: Helsingin Sanomat
+      author: oneillpt
+
+    - title: "LWN Weekly"
+      author: David Cavalca
+
+    - title: "New York Times Sports and Technology Blogs"
+      author: rylsfan
+
+    - title: "Historia and Buctaras"
+      author: Silviu Cotoara
+
+    - title: "Buffalo News"
+      author: ChappyOnIce
+
+    - title: "Dotpod"
+      author: Federico Escalada
+
+
 
 - version: 0.7.47
   date: 2011-02-25
@@ -90,7 +174,7 @@
       author: Ricardo Jurado
 
     - title: "Various Romanian news sources"
-      author: Silviu Coatara
+      author: Silviu Cotoara
 
     - title: "Osnews.pl and SwiatCzytnikow"
       author: Tomasz Dlugosz

resources/default_tweaks.py

@@ -349,3 +349,9 @@ public_smtp_relay_delay = 301
 # after a restart of calibre.
 draw_hidden_section_indicators = True
 
+#: The maximum width and height for covers saved in the calibre library
+# All covers in the calibre library will be resized, preserving aspect ratio,
+# to fit within this size. This is to prevent slowdowns caused by extremely
+# large covers
+maximum_cover_size = (1200, 1600)
+

resources/recipes/buffalo_news.recipe

@@ -1,8 +1,8 @@
 __license__   = 'GPL v3'
 __author__    = 'Todd Chapman'
 __copyright__ = 'Todd Chapman'
-__version__     = 'v0.1'
-__date__        = '26 February 2011'
+__version__     = 'v0.2'
+__date__        = '2 March 2011'
 
 '''
 http://www.buffalonews.com/RSS/
@@ -12,12 +12,16 @@ from calibre.web.feeds.news import BasicNewsRecipe
 
 class AdvancedUserRecipe1298680852(BasicNewsRecipe):
     title          = u'Buffalo News'
-    __author__     = 'ChappyOnIce'
-    language = 'en'
     oldest_article = 2
+    language = 'en'
+    __author__ = 'ChappyOnIce'
     max_articles_per_feed = 20
     encoding = 'utf-8'
+    masthead_url = 'http://www.buffalonews.com/buffalonews/skins/buffalonews/images/masthead/the_buffalo_news_logo.png'
     remove_javascript = True
+    extra_css = 'body {text-align: justify;}\n  \
+       p {text-indent: 20px;}'
+
     keep_only_tags    = [
                        dict(name='div', attrs={'class':['main-content-left']})
                         ]
@@ -28,9 +32,7 @@ class AdvancedUserRecipe1298680852(BasicNewsRecipe):
                         ]
 
     remove_tags_after  = dict(name='div', attrs={'class':['body storyContent']})
-    conversion_options = {
-      'base_font_size'   : 14,
-    }
+
     feeds          = [(u'City of Buffalo', u'http://www.buffalonews.com/city/communities/buffalo/?widget=rssfeed&view=feed&contentId=77944'),
          (u'Southern Erie County', u'http://www.buffalonews.com/city/communities/southern-erie/?widget=rssfeed&view=feed&contentId=77944'),
          (u'Eastern Erie County', u'http://www.buffalonews.com/city/communities/eastern-erie/?widget=rssfeed&view=feed&contentId=77944'),

resources/recipes/credit_slips.recipe

@@ -1,35 +1,44 @@
 #!/usr/bin/env  python
 __license__ = 'GPL 3'
-__copyright__ = 'zotzot'
+__copyright__ = 'zotzo'
 __docformat__ = 'restructuredtext en'
 
 from calibre.web.feeds.news import BasicNewsRecipe
 
 
 class CreditSlips(BasicNewsRecipe):
-    __license__ = 'GPL v3'
-    __author__ = 'zotzot'
     language = 'en'
-    version = 1
+    __author__ = 'zotzot'
+    version = 2
     title = u'Credit Slips.org'
     publisher = u'Bankr-L'
     category = u'Economic blog'
-    description = u'All things about credit.'
-    cover_url = 'http://bit.ly/hyZSTr'
-    oldest_article = 50
+    description = u'A discussion on credit and bankruptcy'
+    cover_url = 'http://bit.ly/eAKNCB'
+    oldest_article = 15
     max_articles_per_feed = 100
     use_embedded_content = True
+    no_stylesheets = True
+    remove_javascript = True
 
-    feeds = [
-(u'Credit Slips', u'http://www.creditslips.org/creditslips/atom.xml')
-]
     conversion_options =   {
                                'comments': description,
                                'tags': category,
                                'language': 'en',
-'publisher': publisher
+                               'publisher': publisher,
                            }
+
+    feeds = [
+               (u'Credit Slips', u'http://www.creditslips.org/creditslips/atom.xml')
+           ]
+
     extra_css =    '''
-                body{font-family:verdana,arial,helvetica,geneva,sans-serif;}
-                img {float: left; margin-right: 0.5em;}
+                       .author {font-family:Helvetica,sans-serif; font-weight:normal;font-size:small;}
+                       h1      {font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
+                       p       {font-family:Helvetica,Arial,sans-serif;font-size:small;}
+                       body    {font-family:Helvetica,Arial,sans-serif;font-size:small;}
                    '''
+
+    def populate_article_metadata(self, article, soup, first):
+       h2 = soup.find('h2')
+       h2.replaceWith(h2.prettify() + '<p><em>Posted by ' + article.author + '</em></p>')

resources/recipes/epl_talk.recipe

@@ -1,6 +1,6 @@
 #!/usr/bin/env  python
 __license__ = 'GPL 3'
-__copyright__ = 'zotzot'
+__copyright__ = 'zotzo'
 __docformat__ = 'restructuredtext en'
 '''
 http://www.epltalk.com
@@ -9,10 +9,9 @@ from calibre.web.feeds.news import BasicNewsRecipe
 
 
 class EPLTalkRecipe(BasicNewsRecipe):
-    __license__ = 'GPL v3'
-    __author__ = u'The Gaffer'
     language = 'en'
-    version = 1
+    version = 2
+    __author__ = 'rylsfan'
 
     title = u'EPL Talk'
     publisher = u'The Gaffer'
@@ -21,17 +20,40 @@ class EPLTalkRecipe(BasicNewsRecipe):
     description = u'News and Analysis from the English Premier League'
     cover_url = 'http://bit.ly/hJxZPu'
 
-    oldest_article = 45
-    max_articles_per_feed = 150
+    oldest_article = 3
+    max_articles_per_feed = 100
     use_embedded_content = True
     remove_javascript = True
     encoding = 'utf8'
 
-    remove_tags_after = [dict(name='div', attrs={'class':'pd-rating'})]
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
 
-    feeds = [(u'EPL Talk', u'http://feeds.feedburner.com/EPLTalk')]
+    remove_tags =  [
+                       {'class': 'feedflare'},
+                       {'class': 'tweetmeme_button'},
+                       {'class': 'eplrelated'},
+                       {'p': 'Related posts:<ol>'},
+                   ]
+
+    def preprocess_html(self, soup):
+       return self.adeify_images(soup)
+
+    feeds =[
+               (u'EPL Talk', u'http://feeds.feedburner.com/EPLTalk'),
+               (u'MLS Talk', u'http://feeds.feedburner.com/majorleaguesoccertalksite'),
+               #(),
+               #(),
+               #(),
+           ]
 
     extra_css = '''
-                body{font-family:verdana,arial,helvetica,geneva,sans-serif;}
-                img {float: left; margin-right: 0.5em;}
+           h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
+           h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
+           p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
+           body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
                '''

resources/recipes/helsingin_sanomat.recipe

@@ -0,0 +1,31 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1298137661(BasicNewsRecipe):
+  title          = u'Helsingin Sanomat'
+  __author__ = 'oneillpt'
+  language = 'fi'
+  oldest_article = 7
+  max_articles_per_feed = 100
+  no_stylesheets = True
+  remove_javascript     = True
+  conversion_options = {
+                         'linearize_tables' : True
+                       }
+  remove_tags = [
+                  dict(name='a', attrs={'id':'articleCommentUrl'}),
+                  dict(name='p', attrs={'class':'newsSummary'}),
+                  dict(name='div', attrs={'class':'headerTools'})
+                ]
+
+  feeds          = [(u'Uutiset - HS.fi', u'http://www.hs.fi/uutiset/rss/'), (u'Politiikka - HS.fi', u'http://www.hs.fi/politiikka/rss/'),
+                     (u'Ulkomaat - HS.fi', u'http://www.hs.fi/ulkomaat/rss/'), (u'Kulttuuri - HS.fi', u'http://www.hs.fi/kulttuuri/rss/'),
+                     (u'Kirjat - HS.fi', u'http://www.hs.fi/kulttuuri/kirjat/rss/'), (u'Elokuvat - HS.fi', u'http://www.hs.fi/kulttuuri/elokuvat/rss/')
+                     ]
+
+  def print_version(self, url):
+    j = url.rfind("/")
+    s = url[j:]
+    i = s.rfind("?ref=rss")
+    if i > 0:
+      s = s[:i]
+    return "http://www.hs.fi/tulosta" + s

resources/recipes/leduc.recipe

@@ -3,8 +3,13 @@ from calibre.web.feeds.news import BasicNewsRecipe
 class AdvancedUserRecipe1292550626(BasicNewsRecipe):
      title = 'Leduc - Wetaskiwin Pipestone Flyer'
      __author__ = 'Brian Hahn'
-   description = 'News from Alberta, Canada'
-   oldest_article = 56
+     description = '''Provides news from central Alberta, Canada.  This is a
+ weekly publication that provides coverage from the Cities of Leduc and
+ Wetaskiwin, including news from two complete counties, plus the towns and
+ villages within.  The counties of Leduc and Wetaskiwin provide news
+ coverage of agriculture, sports, government, family, events and opinion.
+ This publication updated weekly every Thursday.'''
+     oldest_article = 13
      max_articles_per_feed = 100
      no_stylesheets = True
      #delay = 1
@@ -16,25 +21,32 @@ class AdvancedUserRecipe1292550626(BasicNewsRecipe):
      cover_url = 'http://www.pipestoneflyer.ca/images/calibre-cover.jpg'
      remove_tags_before = dict(id='ContentPanel')
      remove_tags_after  = dict(id='ContentPanel')
-   remove_tags = [dict(name='div', attrs={'id':'StoryNav'}),dict(name='div', attrs={'id':'BottomAds'}),dict(name='div', attrs={'id':'MoreStoryLinks'})]
+     remove_tags = [dict(name='div',
+ attrs={'id':'StoryNav'}),dict(name='div',
+ attrs={'id':'BottomAds'}),dict(name='div', attrs={'id':'MoreStoryLinks'})]
      extra_css = 'img { margin:5px }'
      feeds = [
  ('Feature', 'http://www.pipestoneflyer.ca/Feature.rss'),
  ('Editors Desk', 'http://www.pipestoneflyer.ca/Editor%27s%20Desk.rss'),
  ('Letters', 'http://www.pipestoneflyer.ca/Letters.rss'),
-('A Loco Viewpoint', 'http://www.pipestoneflyer.ca/A%20Loco%20Viewpoint.rss'),
+ ('A Loco Viewpoint',
+ 'http://www.pipestoneflyer.ca/A%20Loco%20Viewpoint.rss'),
  ('Lifes Doorway', 'http://www.pipestoneflyer.ca/Life%27s%20Doorway.rss'),
-('From the Otherside', 'http://www.pipestoneflyer.ca/From%20the%20Otherside.rss'),
+ ('From the Otherside',
+ 'http://www.pipestoneflyer.ca/From%20the%20Otherside.rss'),
  ('Opinion', 'http://www.pipestoneflyer.ca/Opinion.rss'),
  ('Community', 'http://www.pipestoneflyer.ca/Community.rss'),
  ('Sports', 'http://www.pipestoneflyer.ca/Sports.rss'),
  ('Chambers', 'http://www.pipestoneflyer.ca/Chambers.rss'),
  ('Government', 'http://www.pipestoneflyer.ca/Government.rss'),
+ ('Travel ', 'http://www.pipestoneflyer.ca/Travel%20.rss'),
  ('Environment', 'http://www.pipestoneflyer.ca/Environment.rss'),
  ('Health', 'http://www.pipestoneflyer.ca/Health.rss'),
  ('Funnies', 'http://www.pipestoneflyer.ca/Funnies.rss'),
+ ('Events', 'http://www.pipestoneflyer.ca/Events.rss'),
  ('Faith', 'http://www.pipestoneflyer.ca/Faith.rss'),
  ('News and Views', 'http://www.pipestoneflyer.ca/News%20and%20Views.rss'),
  ('Obituaries', 'http://www.pipestoneflyer.ca/Obituaries.rss'),
  ('Police Blotter', 'http://www.pipestoneflyer.ca/Police%20Blotter.rss'),
+ ('Careers', 'http://www.pipestoneflyer.ca/Careers.rss'),
  ]

resources/recipes/lwn_weekly.recipe

@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Davide Cavalca <davide125 at tiscali.it>'
+'''
+lwn.net
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+import re
+
+class WeeklyLWN(BasicNewsRecipe):
+    title = 'LWN.net Weekly Edition'
+    description = 'Weekly summary of what has happened in the free software world.'
+    __author__ = 'Davide Cavalca'
+    language = 'en'
+
+    cover_url = 'http://lwn.net/images/lcorner.png'
+    #masthead_url = 'http://lwn.net/images/lcorner.png'
+    publication_type = 'magazine'
+
+    remove_tags_before = dict(attrs={'class':'PageHeadline'})
+    remove_tags_after = dict(attrs={'class':'ArticleText'})
+    remove_tags = [dict(name=['h2', 'form'])]
+
+    conversion_options = { 'linearize_tables' : True }
+
+    oldest_article = 7.0
+    needs_subscription = 'optional'
+
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser()
+        if self.username is not None and self.password is not None:
+            br.open('https://lwn.net/login')
+            br.select_form(name='loginform')
+            br['Username'] = self.username
+            br['Password'] = self.password
+            br.submit()
+        return br
+
+    def parse_index(self):
+        if self.username is not None and self.password is not None:
+            index_url = 'http://lwn.net/current/bigpage'
+        else:
+            index_url = 'http://lwn.net/free/bigpage'
+        soup = self.index_to_soup(index_url)
+        body = soup.body
+
+        articles = {}
+        ans = []
+        url_re = re.compile('^http://lwn.net/Articles/')
+
+        while True:
+            tag_title = body.findNext(name='p', attrs={'class':'SummaryHL'})
+            if tag_title == None:
+                break
+
+            tag_section = tag_title.findPrevious(name='p', attrs={'class':'Cat1HL'})
+            if tag_section == None:
+                section = 'Front Page'
+            else:
+                section = tag_section.string
+
+            tag_section2 = tag_title.findPrevious(name='p', attrs={'class':'Cat2HL'})
+            if tag_section2 != None:
+                if tag_section2.findPrevious(name='p', attrs={'class':'Cat1HL'}) == tag_section:
+                    section = "%s: %s" %(section, tag_section2.string)
+
+            if section not in articles.keys():
+                articles[section] = []
+            if section not in ans:
+                ans.append(section)
+
+            body = tag_title
+            while True:
+                tag_url = body.findNext(name='a', attrs={'href':url_re})
+                if tag_url == None:
+                    break
+                body = tag_url
+                if tag_url.string == None:
+                    continue
+                elif tag_url.string == 'Full Story':
+                    break
+                elif tag_url.string.startswith('Comments ('):
+                    break
+                else:
+                    continue
+
+            if tag_url == None:
+                break
+
+            article = dict(
+                title=tag_title.string,
+                url=tag_url['href'].split('#')[0],
+                description='', content='', date='')
+            articles[section].append(article)
+
+        ans = [(key, articles[key]) for key in ans if articles.has_key(key)]
+        if not ans:
+            raise Exception('Could not find any articles.')
+
+        return ans
+
+# vim: expandtab:ts=4:sw=4

resources/recipes/mac_world.recipe

@@ -11,7 +11,6 @@ http://www.macworld.co.uk/
 '''
 
 from calibre.web.feeds.news import BasicNewsRecipe
-from calibre.ptempfile import PersistentTemporaryFile
 
 temp_files = []
 articles_are_obfuscated = True
@@ -36,26 +35,17 @@ class macWorld(BasicNewsRecipe):
     remove_javascript     = True
     no_stylesheets        = True
 
-    def get_obfuscated_article(self, url):
-        br = self.get_browser()
-        br.open(url+'&print')
-
-        response = br.follow_link(url, nr = 0)
-        html = response.read()
-
-        self.temp_files.append(PersistentTemporaryFile('_fa.html'))
-        self.temp_files[-1].write(html)
-        self.temp_files[-1].close()
-        return self.temp_files[-1].name
-
     keep_only_tags     = [
-                            dict(name='div', attrs={'id':'article'})
+                            dict(name='div', attrs={'id':'content'})
                         ]
 
     remove_tags        = [
-                            dict(name='div', attrs={'class':['toolBar','mac_tags','toolBar  btmTools','textAds']}),
+                            {'class':['toolBar','mac_tags','toolBar  btmTools','textAds']},
                             dict(name='p', attrs={'class':'breadcrumbs'}),
-                            dict(name='div', attrs={'id':['breadcrumb','sidebar','comments']})
+                            dict(id=['breadcrumb','sidebar','comments','topContentWrapper',
+                                    'rightColumn', 'aboveFootPromo', 'storyCarousel']),
+                            {'class':lambda x: x and ('tools' in x or 'toolBar'
+                                in x)}
 
                         ]
 

resources/recipes/nytimes_sports.recipe

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+from __future__ import with_statement
+__license__ = 'GPL 3'
+__copyright__ = 'zotzo'
+__docformat__ = 'restructuredtext en'
+"""
+http://fifthdown.blogs.nytimes.com/
+http://offthedribble.blogs.nytimes.com/
+http://thequad.blogs.nytimes.com/
+http://slapshot.blogs.nytimes.com/
+http://goal.blogs.nytimes.com/
+http://bats.blogs.nytimes.com/
+http://straightsets.blogs.nytimes.com/
+http://formulaone.blogs.nytimes.com/
+http://onpar.blogs.nytimes.com/
+"""
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+
+class NYTimesSports(BasicNewsRecipe):
+    title = 'New York Times Sports Beat'
+    language = 'en'
+    __author__ = 'rylsfan'
+    description = 'Indepth sports from the New York Times'
+    publisher = 'The New York Times'
+    category = 'Sports'
+    oldest_article = 3
+    max_articles_per_feed = 25
+    no_stylesheets = True
+    language = 'en'
+    #cover_url ='http://bit.ly/h8F4DO'
+    feeds = [
+               (u'The Fifth Down', u'http://fifthdown.blogs.nytimes.com/feed/'),
+               (u'Off The Dribble', u'http://offthedribble.blogs.nytimes.com/feed/'),
+               (u'The Quad', u'http://thequad.blogs.nytimes.com/feed/'),
+               (u'Slap Shot', u'http://slapshot.blogs.nytimes.com/feed/'),
+               (u'Goal', u'http://goal.blogs.nytimes.com/feed/'),
+               (u'Bats', u'http://bats.blogs.nytimes.com/feed/'),
+               (u'Straight Sets', u'http://straightsets.blogs.nytimes.com/feed/'),
+               (u'Formula One', u'http://formulaone.blogs.nytimes.com/feed/'),
+               (u'On Par', u'http://onpar.blogs.nytimes.com/feed/'),
+           ]
+    keep_only_tags = [dict(name='div', attrs={'id':'header'}),
+            dict(name='h1'),
+            dict(name='h2'),
+            dict(name='div', attrs={'class':'entry-content'})]
+    extra_css =    '''
+           h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
+           h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
+           p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
+           body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
+                   '''

resources/recipes/nytimes_tech.recipe

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+from __future__ import with_statement
+__license__ = 'GPL 3'
+__copyright__ = 'zotzo'
+__docformat__ = 'restructuredtext en'
+"""
+http://pogue.blogs.nytimes.com/
+"""
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+
+class NYTimesTechnology(BasicNewsRecipe):
+    title = 'New York Times Technology Beat'
+    language = 'en'
+    __author__ = 'David Pogue'
+    description = 'The latest in technology from David Pogue'
+    publisher = 'The New York Times'
+    category = 'Technology'
+    oldest_article = 14
+    max_articles_per_feed = 25
+    no_stylesheets = True
+    language = 'en'
+    cover_url ='http://bit.ly/g0SKJT'
+    feeds = [
+               (u'Pogues Posts', u'http://pogue.blogs.nytimes.com/feed/'),
+               (u'Bits', u'http://bits.blogs.nytimes.com/feed/'),
+               (u'Gadgetwise', u'http://gadgetwise.blogs.nytimes.com/feed/'),
+               (u'Open', u'http://open.blogs.nytimes.com/feed/')
+                   ]
+    keep_only_tags = [dict(name='div', attrs={'id':'header'}),
+            dict(name='h1'),
+            dict(name='h2'),
+            dict(name='div', attrs={'class':'entry-content'})]
+    extra_css = '''
+           h1{font-family:Arial,Helvetica,sans-serif;
+               font-weight:bold;font-size:large;}
+
+           h2{font-family:Arial,Helvetica,sans-serif;
+               font-weight:normal;font-size:small;}
+
+           p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
+           body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
+               '''

resources/recipes/post_today.recipe

@@ -0,0 +1,21 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1299061355(BasicNewsRecipe):
+    title          = u'Post Today'
+    language       = 'th'
+    __author__     = "Chotechai P."
+    oldest_article = 7
+    max_articles_per_feed = 100
+    cover_url = 'http://upload.wikimedia.org/wikipedia/th/2/2e/Posttoday_Logo.png'
+    feeds          = [(u'Breaking News', u'http://www.posttoday.com/rss/src/breakingnews.xml'), (u'\u0e02\u0e48\u0e32\u0e27', u'http://www.posttoday.com/rss/src/news.xml'), (u'\u0e27\u0e34\u0e40\u0e04\u0e23\u0e32\u0e30\u0e2b\u0e4c', u'http://www.posttoday.com/rss/src/analyse.xml'), (u'\u0e40\u0e21\u0e32\u0e17\u0e4c\u0e01\u0e31\u0e19\u0e43\u0e2b\u0e49 z', u'http://www.posttoday.com/rss/src/mouth.xml'), (u'\u0e44\u0e17\u0e22\u0e42\u0e0b\u0e44\u0e0b\u0e15\u0e35\u0e49', u'http://www.posttoday.com/rss/src/thaisociety.xml'), (u'\u0e44\u0e25\u0e1f\u0e4c\u0e2a\u0e44\u0e15\u0e25\u0e4c', u'http://www.posttoday.com/rss/src/lifestyle.xml'), (u'\u0e0a\u0e35\u0e49\u0e0a\u0e48\u0e2d\u0e07\u0e23\u0e27\u0e22', u'http://www.posttoday.com/rss/src/moneyguide.xml'), (u'\u0e1a\u0e49\u0e32\u0e19-\u0e04\u0e2d\u0e19\u0e42\u0e14', u'http://www.posttoday.com/rss/src/homecondo.xml'), (u'\u0e22\u0e32\u0e19\u0e22\u0e19\u0e15\u0e4c', u'http://www.posttoday.com/rss/src/motor.xml'), (u'\u0e14\u0e34\u0e08\u0e34\u0e15\u0e2d\u0e25\u0e44\u0e25\u0e1f\u0e4c', u'http://www.posttoday.com/rss/src/digitallife.xml'), (u'\u0e01\u0e35\u0e2c\u0e32', u'http://www.posttoday.com/rss/src/sport.xml'), (u'\u0e23\u0e2d\u0e1a\u0e42\u0e25\u0e01', u'http://www.posttoday.com/rss/src/world.xml'), (u'\u0e01\u0e34\u0e19-\u0e40\u0e17\u0e35\u0e48\u0e22\u0e27', u'http://www.posttoday.com/rss/src/eattravel.xml'), (u'Mind & Soul', u'http://www.posttoday.com/rss/src/mindsoul.xml'), (u'\u0e1a\u0e25\u0e47\u0e2d\u0e01 \u0e1a\u0e01.', u'http://www.posttoday.com/rss/src/blogs.xml')]
+    keep_only_tags = []
+    keep_only_tags.append(dict(name = 'div', attrs = {'class' :
+'articleContents'}))
+
+    remove_tags = []
+    remove_tags.append(dict(name = 'label'))
+    remove_tags.append(dict(name = 'span'))
+    remove_tags.append(dict(name = 'div', attrs = {'class' :
+'socialBookmark'}))
+    remove_tags.append(dict(name = 'div', attrs = {'class' :
+'misc'}))

resources/recipes/rbc_ru.recipe

@@ -0,0 +1,49 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1286819935(BasicNewsRecipe):
+    title = u'RBC.ru'
+    __author__ = 'A. Chewi'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets = True
+    use_embedded_content  = False
+    conversion_options = {'linearize_tables' : True}
+    remove_attributes = ['style']
+    language = 'ru'
+    timefmt = ' [%a, %d %b, %Y]'
+
+    keep_only_tags = [dict(name='h2', attrs={}),
+	dict(name='div', attrs={'class': 'box _ga1_on_'}),
+	dict(name='h1', attrs={'class': 'news_section'}),
+	dict(name='div', attrs={'class': 'news_body dotted_border_bottom'}),
+	dict(name='table', attrs={'class': 'newsBody'}),
+	dict(name='h2', attrs={'class': 'black'})]
+
+    feeds = [(u'Главные новости', u'http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/rbc.ru/mainnews.rss'),
+	(u'Политика', u'http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/rbc.ru/politics.rss'),
+	(u'Экономика', u'http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/rbc.ru/economics.rss'),
+	(u'Общество', u'http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/rbc.ru/society.rss'),
+	(u'Происшествия', u'http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/rbc.ru/incidents.rss'),
+	(u'Финансовые новости Quote.rbc.ru', u'http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/quote.ru/mainnews.rss')]
+
+
+    remove_tags = [dict(name='div', attrs={'class': "video-frame"}),
+	dict(name='div', attrs={'class': "photo-container videoContainer videoSWFLinks videoPreviewSlideContainer notes"}),
+	dict(name='div', attrs={'class': "notes"}),
+	dict(name='div', attrs={'class': "publinks"}),
+	dict(name='a', attrs={'class': "print"}),
+	dict(name='div', attrs={'class': "photo-report_new notes newslider"}),
+	dict(name='div', attrs={'class': "videoContainer"}),
+	dict(name='div', attrs={'class': "videoPreviewSlideContainer"}),
+	dict(name='a', attrs={'class': "videoPreviewContainer"}),
+	dict(name='a', attrs={'class': "red"}),]
+
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
+    def print_version(self, url):
+        return url + '?print=true'

resources/recipes/seattle_times.recipe

@@ -69,12 +69,16 @@ class SeattleTimes(BasicNewsRecipe):
                               u'http://seattletimes.nwsource.com/rss/mostreadarticles.xml'),
                          ]
 
+    keep_only_tags = [dict(id='content')]
     remove_tags        = [
-                             dict(name=['object','link','script'])
-                            ,dict(name='p', attrs={'class':'permission'})
+                             dict(name=['object','link','script']),
+                            {'class':['permission', 'note', 'bottomtools',
+                                'homedelivery']},
+                            dict(id=["rightcolumn", 'footer', 'adbottom']),
                          ]
 
     def print_version(self, url):
+        return url
         start_url, sep, rest_url = url.rpartition('_')
         rurl, rsep, article_id = start_url.rpartition('/')
         return u'http://seattletimes.nwsource.com/cgi-bin/PrintStory.pl?document_id=' + article_id

resources/recipes/statesman.recipe

@@ -10,7 +10,9 @@ class AdvancedUserRecipe1278049615(BasicNewsRecipe):
 
     max_articles_per_feed = 100
 
-    feeds          = [(u'News', u'http://www.statesman.com/section-rss.do?source=news&includeSubSections=true'),
+    feeds          = [(u'News',
+        u'http://www.statesman.com/section-rss.do?source=news&includeSubSections=true'),
+    (u'Local', u'http://www.statesman.com/section-rss.do?source=local&includeSubSections=true'),
     (u'Business', u'http://www.statesman.com/section-rss.do?source=business&includeSubSections=true'),
     (u'Life', u'http://www.statesman.com/section-rss.do?source=life&includesubsection=true'),
     (u'Editorial', u'http://www.statesman.com/section-rss.do?source=opinion&includesubsections=true'),
@@ -28,8 +30,11 @@ class AdvancedUserRecipe1278049615(BasicNewsRecipe):
     conversion_options = {'linearize_tables':True}
     remove_tags = [
                     dict(name='div', attrs={'id':'cxArticleOptions'}),
+                    {'class':['perma', 'comments', 'trail', 'share-buttons',
+                        'toggle_show_on']},
                         ]
     keep_only_tags = [
      dict(name='div', attrs={'class':'cxArticleHeader'}),
-                     dict(name='div', attrs={'id':'cxArticleBodyText'}),
+                     dict(name='div', attrs={'id':['cxArticleBodyText',
+                         'content']}),
                                ]

resources/recipes/swiatkindle.recipe

@@ -7,6 +7,7 @@ swiatczytnikow.pl
 '''
 
 import re
+from calibre.web.feeds.news import BasicNewsRecipe
 
 class swiatczytnikow(BasicNewsRecipe):
     title          = u'Swiat Czytnikow'

resources/recipes/thai_post_daily.recipe

@@ -0,0 +1,17 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1299054026(BasicNewsRecipe):
+    title          = u'Thai Post Daily'
+    __author__ = 'Chotechai P.'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    cover_url = 'http://upload.wikimedia.org/wikipedia/th/1/10/ThaiPost_Logo.png'
+    feeds          = [(u'\u0e02\u0e48\u0e32\u0e27\u0e2b\u0e19\u0e49\u0e32\u0e2b\u0e19\u0e36\u0e48\u0e07', u'http://thaipost.net/taxonomy/term/1/all/feed'), (u'\u0e1a\u0e17\u0e1a\u0e23\u0e23\u0e13\u0e32\u0e18\u0e34\u0e01\u0e32\u0e23', u'http://thaipost.net/taxonomy/term/11/all/feed'), (u'\u0e40\u0e1b\u0e25\u0e27 \u0e2a\u0e35\u0e40\u0e07\u0e34\u0e19', u'http://thaipost.net/taxonomy/term/2/all/feed'), (u'\u0e2a\u0e20\u0e32\u0e1b\u0e23\u0e30\u0e0a\u0e32\u0e0a\u0e19', u'http://thaipost.net/taxonomy/term/3/all/feed'), (u'\u0e16\u0e39\u0e01\u0e17\u0e38\u0e01\u0e02\u0e49\u0e2d', u'http://thaipost.net/taxonomy/term/4/all/feed'), (u'\u0e01\u0e32\u0e23\u0e40\u0e21\u0e37\u0e2d\u0e07', u'http://thaipost.net/taxonomy/term/5/all/feed'), (u'\u0e17\u0e48\u0e32\u0e19\u0e02\u0e38\u0e19\u0e19\u0e49\u0e2d\u0e22', u'http://thaipost.net/taxonomy/term/12/all/feed'), (u'\u0e1a\u0e17\u0e04\u0e27\u0e32\u0e21\u0e1e\u0e34\u0e40\u0e28\u0e29', u'http://thaipost.net/taxonomy/term/66/all/feed'), (u'\u0e23\u0e32\u0e22\u0e07\u0e32\u0e19\u0e1e\u0e34\u0e40\u0e28\u0e29', u'http://thaipost.net/taxonomy/term/67/all/feed'), (u'\u0e1a\u0e31\u0e19\u0e17\u0e36\u0e01\u0e2b\u0e19\u0e49\u0e32 4', u'http://thaipost.net/taxonomy/term/13/all/feed'), (u'\u0e40\u0e2a\u0e35\u0e22\u0e1a\u0e0b\u0e36\u0e48\u0e07\u0e2b\u0e19\u0e49\u0e32', u'http://thaipost.net/taxonomy/term/64/all/feed'), (u'\u0e04\u0e31\u0e19\u0e1b\u0e32\u0e01\u0e2d\u0e22\u0e32\u0e01\u0e40\u0e25\u0e48\u0e32', u'http://thaipost.net/taxonomy/term/65/all/feed'), (u'\u0e40\u0e28\u0e23\u0e29\u0e10\u0e01\u0e34\u0e08', u'http://thaipost.net/taxonomy/term/6/all/feed'), (u'\u0e01\u0e23\u0e30\u0e08\u0e01\u0e44\u0e23\u0e49\u0e40\u0e07\u0e32', u'http://thaipost.net/taxonomy/term/14/all/feed'), (u'\u0e01\u0e23\u0e30\u0e08\u0e01\u0e2b\u0e31\u0e01\u0e21\u0e38\u0e21', u'http://thaipost.net/taxonomy/term/71/all/feed'), (u'\u0e04\u0e34\u0e14\u0e40\u0e2b\u0e19\u0e37\u0e2d\u0e01\u0e23\u0e30\u0e41\u0e2a', u'http://thaipost.net/taxonomy/term/69/all/feed'), (u'\u0e23\u0e32\u0e22\u0e07\u0e32\u0e19', u'http://thaipost.net/taxonomy/term/68/all/feed'), (u'\u0e2d\u0e34\u0e42\u0e04\u0e42\u0e1f\u0e01\u0e31\u0e2a', u'http://thaipost.net/taxonomy/term/10/all/feed'), (u'\u0e01\u0e32\u0e23\u0e28\u0e36\u0e01\u0e29\u0e32-\u0e2a\u0e32\u0e18\u0e32\u0e23\u0e13\u0e2a\u0e38\u0e02', u'http://thaipost.net/taxonomy/term/7/all/feed'), (u'\u0e15\u0e48\u0e32\u0e07\u0e1b\u0e23\u0e30\u0e40\u0e17\u0e28', u'http://thaipost.net/taxonomy/term/8/all/feed'), (u'\u0e01\u0e35\u0e2c\u0e32', u'http://thaipost.net/taxonomy/term/9/all/feed')]
+
+    def print_version(self, url):
+      return url.replace(url, 'http://www.thaipost.net/print/' + url [32:])
+
+    remove_tags = []
+    remove_tags.append(dict(name = 'div', attrs = {'class' : 'print-logo'}))
+    remove_tags.append(dict(name = 'div', attrs = {'class' : 'print-site_name'}))
+    remove_tags.append(dict(name = 'div', attrs = {'class' : 'print-breadcrumb'}))

resources/template-functions.json

@@ -5,8 +5,9 @@
     "strcat": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    res = ''\n    for i in range(0, len(args)):\n        res += args[i]\n    return res\n", 
     "substr": "def evaluate(self, formatter, kwargs, mi, locals, str_, start_, end_):\n    return str_[int(start_): len(str_) if int(end_) == 0 else int(end_)]\n", 
     "ifempty": "def evaluate(self, formatter, kwargs, mi, locals, val, value_if_empty):\n    if val:\n        return val\n    else:\n        return value_if_empty\n", 
+    "select": "def evaluate(self, formatter, kwargs, mi, locals, val, key):\n    if not val:\n        return ''\n    vals = [v.strip() for v in val.split(',')]\n    for v in vals:\n        if v.startswith(key+':'):\n            return v[len(key)+1:]\n    return ''\n", 
     "field": "def evaluate(self, formatter, kwargs, mi, locals, name):\n    return formatter.get_value(name, [], kwargs)\n", 
-    "capitalize": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return capitalize(val)\n", 
+    "subtract": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x - y)\n", 
     "list_item": "def evaluate(self, formatter, kwargs, mi, locals, val, index, sep):\n    if not val:\n        return ''\n    index = int(index)\n    val = val.split(sep)\n    try:\n        return val[index]\n    except:\n        return ''\n", 
     "shorten": "def evaluate(self, formatter, kwargs, mi, locals,\n             val, leading, center_string, trailing):\n    l = max(0, int(leading))\n    t = max(0, int(trailing))\n    if len(val) > l + len(center_string) + t:\n        return val[0:l] + center_string + ('' if t == 0 else val[-t:])\n    else:\n        return val\n", 
     "re": "def evaluate(self, formatter, kwargs, mi, locals, val, pattern, replacement):\n    return re.sub(pattern, replacement, val)\n", 
@@ -19,11 +20,13 @@
     "test": "def evaluate(self, formatter, kwargs, mi, locals, val, value_if_set, value_not_set):\n    if val:\n        return value_if_set\n    else:\n        return value_not_set\n", 
     "eval": "def evaluate(self, formatter, kwargs, mi, locals, template):\n    from formatter import eval_formatter\n    template = template.replace('[[', '{').replace(']]', '}')\n    return eval_formatter.safe_format(template, locals, 'EVAL', None)\n", 
     "multiply": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x * y)\n", 
-    "subtract": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x - y)\n", 
+    "format_date": "def evaluate(self, formatter, kwargs, mi, locals, val, format_string):\n    if not val:\n        return ''\n    try:\n        dt = parse_date(val)\n        s = format_date(dt, format_string)\n    except:\n        s = 'BAD DATE'\n    return s\n", 
+    "capitalize": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return capitalize(val)\n", 
     "count": "def evaluate(self, formatter, kwargs, mi, locals, val, sep):\n    return unicode(len(val.split(sep)))\n", 
     "lowercase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return val.lower()\n", 
     "assign": "def evaluate(self, formatter, kwargs, mi, locals, target, value):\n    locals[target] = value\n    return value\n", 
     "switch": "def evaluate(self, formatter, kwargs, mi, locals, val, *args):\n    if (len(args) % 2) != 1:\n        raise ValueError(_('switch requires an odd number of arguments'))\n    i = 0\n    while i < len(args):\n        if i + 1 >= len(args):\n            return args[i]\n        if re.search(args[i], val):\n            return args[i+1]\n        i += 2\n", 
     "strcmp": "def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):\n    v = strcmp(x, y)\n    if v < 0:\n        return lt\n    if v == 0:\n        return eq\n    return gt\n", 
+    "raw_field": "def evaluate(self, formatter, kwargs, mi, locals, name):\n    return unicode(getattr(mi, name, None))\n", 
     "cmp": "def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    if x < y:\n        return lt\n    if x == y:\n        return eq\n    return gt\n"
 }

setup/extensions.py

@@ -68,6 +68,10 @@ if isosx:
 
 extensions = [
 
+    Extension('speedup',
+        ['calibre/utils/speedup.c'],
+        ),
+
     Extension('icu',
         ['calibre/utils/icu.c'],
         libraries=icu_libs,

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.7.47'
+__version__   = '0.7.48'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 
 import re
@@ -69,6 +69,7 @@ if plugins is None:
                 'chmlib',
                 'chm_extra',
                 'icu',
+                'speedup',
             ] + \
                     (['winutil'] if iswindows else []) + \
                     (['usbobserver'] if isosx else []):

src/calibre/devices/apple/driver.py

@@ -701,7 +701,7 @@ class ITUNES(DriverBase):
             self.log.info("ITUNES.get_file(): exporting '%s'" % path)
         outfile.write(open(self.cached_books[path]['lib_book'].location().path).read())
 
-    def open(self):
+    def open(self, library_uuid):
         '''
         Perform any device specific initialization. Called after the device is
         detected but before any other functions that communicate with the device.
@@ -2512,7 +2512,12 @@ class ITUNES(DriverBase):
         # Refresh epub metadata
         with open(fpath,'r+b') as zfo:
             # Touch the OPF timestamp
+            try:
                 zf_opf = ZipFile(fpath,'r')
+            except:
+                raise UserFeedback("'%s' is not a valid EPUB" % metadata.title,
+                                   None,
+                                   level=UserFeedback.WARN)
             fnames = zf_opf.namelist()
             opf = [x for x in fnames if '.opf' in x][0]
             if opf:

src/calibre/devices/bambook/driver.py

@@ -61,7 +61,7 @@ class BAMBOOK(DeviceConfig, DevicePlugin):
             detected_device=None) :
         self.open()
 
-    def open(self):
+    def open(self, library_uuid):
         # Make sure the Bambook library is ready
         if not is_bambook_lib_ready():
             raise OpenFeedback(_("Unable to connect to Bambook, you need to install Bambook library first."))

src/calibre/devices/folder_device/driver.py

@@ -47,6 +47,7 @@ class FOLDER_DEVICE(USBMS):
     #: Icon for this device
     icon = I('devices/folder.png')
     METADATA_CACHE = '.metadata.calibre'
+    DRIVEINFO = '.driveinfo.calibre'
 
     _main_prefix = ''
     _card_a_prefix = None
@@ -77,7 +78,8 @@ class FOLDER_DEVICE(USBMS):
             only_presence=False):
         return self.is_connected, self
 
-    def open(self):
+    def open(self, library_uuid):
+        self.current_library_uuid = library_uuid
         if not self._main_prefix:
             return False
         return True

src/calibre/devices/hanlin/driver.py

@@ -116,6 +116,7 @@ class BOOX(HANLINV3):
     author         = 'Jesus Manuel Marinho Valcarce'
     supported_platforms = ['windows', 'osx', 'linux']
     METADATA_CACHE = '.metadata.calibre'
+    DRIVEINFO = '.driveinfo.calibre'
 
     # Ordered list of supported formats
     FORMATS     = ['epub', 'fb2', 'djvu', 'pdf', 'html', 'txt', 'rtf', 'mobi',

src/calibre/devices/interface.py

@@ -215,7 +215,7 @@ class DevicePlugin(Plugin):
 
         return True
 
-    def open(self):
+    def open(self, library_uuid):
         '''
         Perform any device specific initialization. Called after the device is
         detected but before any other functions that communicate with the device.
@@ -260,6 +260,8 @@ class DevicePlugin(Plugin):
         Ask device for device information. See L{DeviceInfoQuery}.
 
         :return: (device name, device version, software version on device, mime type)
+                 The tuple can optionally have a fifth element, which is a
+                 drive information diction. See usbms.driver for an example.
 
         """
         raise NotImplementedError()
@@ -447,6 +449,15 @@ class DevicePlugin(Plugin):
         '''
         pass
 
+    def set_driveinfo_name(self, location_code, name):
+        '''
+        Set the device name in the driveinfo file to 'name'. This setting will
+        persist until the file is re-created or the name is changed again.
+
+        Non-disk devices will ignore this request.
+        '''
+        pass
+
 class BookList(list):
     '''
     A list of books. Each Book object must have the fields

src/calibre/devices/misc.py

@@ -272,6 +272,7 @@ class NEXTBOOK(USBMS):
     VENDOR_NAME = 'NEXT2'
     WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = '1.0.14'
     SUPPORTS_SUB_DIRS = True
+    THUMBNAIL_HEIGHT = 120
 
     '''
     def upload_cover(self, path, filename, metadata, filepath):

src/calibre/devices/prs500/driver.py

@@ -240,7 +240,7 @@ class PRS500(DeviceConfig, DevicePlugin):
     def set_progress_reporter(self, report_progress):
         self.report_progress = report_progress
 
-    def open(self) :
+    def open(self, library_uuid) :
         """
         Claim an interface on the device for communication.
         Requires write privileges to the device file.

src/calibre/devices/prs505/driver.py

@@ -153,9 +153,6 @@ class PRS505(USBMS):
         # updated on every connect
         self.WANTS_UPDATED_THUMBNAILS = self.settings().extra_customization[2]
 
-    def get_device_information(self, end_session=True):
-        return (self.gui_name, '', '', '')
-
     def filename_callback(self, fname, mi):
         if getattr(mi, 'application_id', None) is not None:
             base = fname.rpartition('.')[0]

src/calibre/devices/usbms/device.py

@@ -700,7 +700,7 @@ class Device(DeviceConfig, DevicePlugin):
 
 
 
-    def open(self):
+    def open(self, library_uuid):
         time.sleep(5)
         self._main_prefix = self._card_a_prefix = self._card_b_prefix = None
         if islinux:
@@ -722,6 +722,7 @@ class Device(DeviceConfig, DevicePlugin):
                 time.sleep(7)
                 self.open_osx()
 
+        self.current_library_uuid = library_uuid
         self.post_open_callback()
 
     def post_open_callback(self):

src/calibre/devices/usbms/driver.py

@@ -10,17 +10,18 @@ driver. It is intended to be subclassed with the relevant parts implemented
 for a particular device.
 '''
 
-import os
-import re
-import time
+import os, re, time, json, uuid
 from itertools import cycle
 
+from calibre.constants import numeric_version
 from calibre import prints, isbytestring
 from calibre.constants import filesystem_encoding, DEBUG
 from calibre.devices.usbms.cli import CLI
 from calibre.devices.usbms.device import Device
 from calibre.devices.usbms.books import BookList, Book
 from calibre.ebooks.metadata.book.json_codec import JsonCodec
+from calibre.utils.config import from_json, to_json
+from calibre.utils.date import now, isoformat
 
 BASE_TIME = None
 def debug_print(*args):
@@ -52,10 +53,59 @@ class USBMS(CLI, Device):
     FORMATS = []
     CAN_SET_METADATA = []
     METADATA_CACHE = 'metadata.calibre'
+    DRIVEINFO = 'driveinfo.calibre'
+
+    def _update_driveinfo_record(self, dinfo, prefix, location_code, name=None):
+        if not isinstance(dinfo, dict):
+            dinfo = {}
+        if dinfo.get('device_store_uuid', None) is None:
+            dinfo['device_store_uuid'] = unicode(uuid.uuid4())
+        if dinfo.get('device_name') is None:
+            dinfo['device_name'] = self.get_gui_name()
+        if name is not None:
+            dinfo['device_name'] = name
+        dinfo['location_code'] = location_code
+        dinfo['last_library_uuid'] = getattr(self, 'current_library_uuid', None)
+        dinfo['calibre_version'] = '.'.join([unicode(i) for i in numeric_version])
+        dinfo['date_last_connected'] = isoformat(now())
+        dinfo['prefix'] = prefix.replace('\\', '/')
+        return dinfo
+
+    def _update_driveinfo_file(self, prefix, location_code, name=None):
+        if os.path.exists(os.path.join(prefix, self.DRIVEINFO)):
+            with open(os.path.join(prefix, self.DRIVEINFO), 'rb') as f:
+                try:
+                    driveinfo = json.loads(f.read(), object_hook=from_json)
+                except:
+                    driveinfo = None
+                driveinfo = self._update_driveinfo_record(driveinfo, prefix,
+                                                          location_code, name)
+            with open(os.path.join(prefix, self.DRIVEINFO), 'wb') as f:
+                f.write(json.dumps(driveinfo, default=to_json))
+        else:
+            driveinfo = self._update_driveinfo_record({}, prefix, location_code, name)
+            with open(os.path.join(prefix, self.DRIVEINFO), 'wb') as f:
+                f.write(json.dumps(driveinfo, default=to_json))
+        return driveinfo
 
     def get_device_information(self, end_session=True):
         self.report_progress(1.0, _('Get device information...'))
-        return (self.get_gui_name(), '', '', '')
+        self.driveinfo = {}
+        if self._main_prefix is not None:
+            self.driveinfo['main'] = self._update_driveinfo_file(self._main_prefix, 'main')
+        if self._card_a_prefix is not None:
+            self.driveinfo['A'] = self._update_driveinfo_file(self._card_a_prefix, 'A')
+        if self._card_b_prefix is not None:
+            self.driveinfo['B'] = self._update_driveinfo_file(self._card_b_prefix, 'B')
+        return (self.get_gui_name(), '', '', '', self.driveinfo)
+
+    def set_driveinfo_name(self, location_code, name):
+        if location_code == 'main':
+            self._update_driveinfo_file(self._main_prefix, location_code, name)
+        elif location_code == 'A':
+            self._update_driveinfo_file(self._card_a_prefix, location_code, name)
+        elif location_code == 'B':
+            self._update_driveinfo_file(self._card_b_prefix, location_code, name)
 
     def books(self, oncard=None, end_session=True):
         from calibre.ebooks.metadata.meta import path_to_ext

src/calibre/ebooks/__init__.py

@@ -28,7 +28,7 @@ class ParserError(ValueError):
 BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'txtz', 'htm', 'xhtm',
                    'html', 'xhtml', 'pdf', 'pdb', 'pdr', 'prc', 'mobi', 'azw', 'doc',
                    'epub', 'fb2', 'djvu', 'lrx', 'cbr', 'cbz', 'cbc', 'oebzip',
-                   'rb', 'imp', 'odt', 'chm', 'tpz', 'azw1', 'pml', 'mbp', 'tan', 'snb']
+                   'rb', 'imp', 'odt', 'chm', 'tpz', 'azw1', 'pml', 'pmlz', 'mbp', 'tan', 'snb']
 
 class HTMLRenderer(object):
 

src/calibre/ebooks/chm/reader.py

@@ -54,6 +54,10 @@ class CHMReader(CHMFile):
         self._extracted = False
 
         # location of '.hhc' file, which is the CHM TOC.
+        if self.topics is None:
+            self.root, ext = os.path.splitext(self.home.lstrip('/'))
+            self.hhc_path = self.root + ".hhc"
+        else:
             self.root, ext = os.path.splitext(self.topics.lstrip('/'))
             self.hhc_path = self.root + ".hhc"
 

src/calibre/ebooks/conversion/preprocess.py

@@ -265,16 +265,28 @@ class CSSPreProcessor(object):
 
     PAGE_PAT   = re.compile(r'@page[^{]*?{[^}]*?}')
     # Remove some of the broken CSS Microsoft products
-    # create, slightly dangerous as it removes to end of line
-    # rather than semi-colon
-    MS_PAT     = re.compile(r'^\s*(mso-|panose-).+?$',
-            re.MULTILINE|re.IGNORECASE)
+    # create
+    MS_PAT     = re.compile(r'''
+        (?P<start>^|;|\{)\s*    # The end of the previous rule or block start
+        (%s).+?                 # The invalid selectors
+        (?P<end>$|;|\})         # The end of the declaration
+        '''%'mso-|panose-|text-underline|tab-interval',
+        re.MULTILINE|re.IGNORECASE|re.VERBOSE)
+
+    def ms_sub(self, match):
+        end = match.group('end')
+        try:
+            start = match.group('start')
+        except:
+            start = ''
+        if end == ';':
+            end = ''
+        return start + end
 
     def __call__(self, data, add_namespace=False):
         from calibre.ebooks.oeb.base import XHTML_CSS_NAMESPACE
         data = self.PAGE_PAT.sub('', data)
-        if '\n' in data:
-            data = self.MS_PAT.sub('', data)
+        data = self.MS_PAT.sub(self.ms_sub, data)
         if not add_namespace:
             return data
         ans, namespaced = [], False

src/calibre/ebooks/metadata/book/__init__.py

@@ -18,14 +18,14 @@ SOCIAL_METADATA_FIELDS = frozenset([
     'series_index',     # A floating point number
     # Of the form { scheme1:value1, scheme2:value2}
     # For example: {'isbn':'123456789', 'doi':'xxxx', ... }
-    'classifiers',
+    'identifiers',
 ])
 
 '''
-The list of names that convert to classifiers when in get and set.
+The list of names that convert to identifiers when in get and set.
 '''
 
-TOP_LEVEL_CLASSIFIERS = frozenset([
+TOP_LEVEL_IDENTIFIERS = frozenset([
     'isbn',
 ])
 
@@ -108,7 +108,7 @@ STANDARD_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union(
 SC_FIELDS_NOT_COPIED =     frozenset(['title', 'title_sort', 'authors',
                                       'author_sort', 'author_sort_map',
                                       'cover_data', 'tags', 'language',
-                                      'classifiers'])
+                                      'identifiers'])
 
 # Metadata fields that smart update should copy only if the source is not None
 SC_FIELDS_COPY_NOT_NULL =  frozenset(['lpath', 'size', 'comments', 'thumbnail'])

src/calibre/ebooks/metadata/book/base.py

@@ -12,7 +12,7 @@ from calibre.constants import DEBUG
 from calibre.ebooks.metadata.book import SC_COPYABLE_FIELDS
 from calibre.ebooks.metadata.book import SC_FIELDS_COPY_NOT_NULL
 from calibre.ebooks.metadata.book import STANDARD_METADATA_FIELDS
-from calibre.ebooks.metadata.book import TOP_LEVEL_CLASSIFIERS
+from calibre.ebooks.metadata.book import TOP_LEVEL_IDENTIFIERS
 from calibre.ebooks.metadata.book import ALL_METADATA_FIELDS
 from calibre.library.field_metadata import FieldMetadata
 from calibre.utils.date import isoformat, format_date
@@ -24,7 +24,7 @@ NULL_VALUES = {
                 'user_metadata': {},
                 'cover_data'   : (None, None),
                 'tags'         : [],
-                'classifiers'  : {},
+                'identifiers'  : {},
                 'languages'    : [],
                 'device_collections': [],
                 'author_sort_map': {},
@@ -41,7 +41,7 @@ class SafeFormat(TemplateFormatter):
     def get_value(self, key, args, kwargs):
         try:
             key = key.lower()
-            if key != 'title_sort':
+            if key != 'title_sort' and key not in TOP_LEVEL_IDENTIFIERS:
                 key = field_metadata.search_term_to_field_key(key)
             b = self.book.get_user_metadata(key, False)
             if b and b['datatype'] == 'int' and self.book.get(key, 0) == 0:
@@ -49,7 +49,7 @@ class SafeFormat(TemplateFormatter):
             elif b and b['datatype'] == 'float' and self.book.get(key, 0.0) == 0.0:
                 v = ''
             else:
-                ign, v = self.book.format_field(key, series_with_index=False)
+                v = self.book.format_field(key, series_with_index=False)[1]
             if v is None:
                 return ''
             if v == '':
@@ -96,8 +96,8 @@ class Metadata(object):
 
     def __getattribute__(self, field):
         _data = object.__getattribute__(self, '_data')
-        if field in TOP_LEVEL_CLASSIFIERS:
-            return _data.get('classifiers').get(field, None)
+        if field in TOP_LEVEL_IDENTIFIERS:
+            return _data.get('identifiers').get(field, None)
         if field in STANDARD_METADATA_FIELDS:
             return _data.get(field, None)
         try:
@@ -123,8 +123,11 @@ class Metadata(object):
 
     def __setattr__(self, field, val, extra=None):
         _data = object.__getattribute__(self, '_data')
-        if field in TOP_LEVEL_CLASSIFIERS:
-            _data['classifiers'].update({field: val})
+        if field in TOP_LEVEL_IDENTIFIERS:
+            field, val = self._clean_identifier(field, val)
+            _data['identifiers'].update({field: val})
+        elif field == 'identifiers':
+            self.set_identifiers(val)
         elif field in STANDARD_METADATA_FIELDS:
             if val is None:
                 val = NULL_VALUES.get(field, None)
@@ -176,17 +179,48 @@ class Metadata(object):
     def set(self, field, val, extra=None):
         self.__setattr__(field, val, extra)
 
-    def get_classifiers(self):
+    def get_identifiers(self):
         '''
-        Return a copy of the classifiers dictionary.
+        Return a copy of the identifiers dictionary.
         The dict is small, and the penalty for using a reference where a copy is
         needed is large. Also, we don't want any manipulations of the returned
         dict to show up in the book.
         '''
-        return copy.deepcopy(object.__getattribute__(self, '_data')['classifiers'])
+        ans = object.__getattribute__(self,
+            '_data')['identifiers']
+        if not ans:
+            ans = {}
+        return copy.deepcopy(ans)
 
-    def set_classifiers(self, classifiers):
-        object.__getattribute__(self, '_data')['classifiers'] = classifiers
+    def _clean_identifier(self, typ, val):
+        typ = icu_lower(typ).strip().replace(':', '').replace(',', '')
+        val = val.strip().replace(',', '|').replace(':', '|')
+        return typ, val
+
+    def set_identifiers(self, identifiers):
+        '''
+        Set all identifiers. Note that if you previously set ISBN, calling
+        this method will delete it.
+        '''
+        cleaned = {}
+        for key, val in identifiers.iteritems():
+            key, val = self._clean_identifier(key, val)
+            if key and val:
+                cleaned[key] = val
+        object.__getattribute__(self, '_data')['identifiers'] = cleaned
+
+    def set_identifier(self, typ, val):
+        'If val is empty, deletes identifier of type typ'
+        typ, val = self._clean_identifier(typ, val)
+        if not typ:
+            return
+        identifiers = object.__getattribute__(self,
+            '_data')['identifiers']
+
+        if not val and typ in identifiers:
+            identifiers.pop(typ)
+        if val:
+            identifiers[typ] = val
 
     # field-oriented interface. Intended to be the same as in LibraryDatabase
 
@@ -229,7 +263,7 @@ class Metadata(object):
             if v is not None:
                 result[attr] = v
         # separate these because it uses the self.get(), not _data.get()
-        for attr in TOP_LEVEL_CLASSIFIERS:
+        for attr in TOP_LEVEL_IDENTIFIERS:
             v = self.get(attr, None)
             if v is not None:
                 result[attr] = v
@@ -400,8 +434,8 @@ class Metadata(object):
             self.set_all_user_metadata(other.get_all_user_metadata(make_copy=True))
             for x in SC_FIELDS_COPY_NOT_NULL:
                 copy_not_none(self, other, x)
-            if callable(getattr(other, 'get_classifiers', None)):
-                self.set_classifiers(other.get_classifiers())
+            if callable(getattr(other, 'get_identifiers', None)):
+                self.set_identifiers(other.get_identifiers())
             # language is handled below
         else:
             for attr in SC_COPYABLE_FIELDS:
@@ -456,15 +490,15 @@ class Metadata(object):
             if len(other_comments.strip()) > len(my_comments.strip()):
                 self.comments = other_comments
 
-            # Copy all the non-none classifiers
-            if callable(getattr(other, 'get_classifiers', None)):
-                d = self.get_classifiers()
-                s = other.get_classifiers()
+            # Copy all the non-none identifiers
+            if callable(getattr(other, 'get_identifiers', None)):
+                d = self.get_identifiers()
+                s = other.get_identifiers()
                 d.update([v for v in s.iteritems() if v[1] is not None])
-                self.set_classifiers(d)
+                self.set_identifiers(d)
             else:
-                # other structure not Metadata. Copy the top-level classifiers
-                for attr in TOP_LEVEL_CLASSIFIERS:
+                # other structure not Metadata. Copy the top-level identifiers
+                for attr in TOP_LEVEL_IDENTIFIERS:
                     copy_not_none(self, other, attr)
 
         other_lang = getattr(other, 'language', None)
@@ -544,9 +578,15 @@ class Metadata(object):
                 res = res/2
             return (name, unicode(res), orig_res, cmeta)
 
+        # convert top-level ids into their value
+        if key in TOP_LEVEL_IDENTIFIERS:
+            fmeta = field_metadata['identifiers']
+            name = key
+            res = self.get(key, None)
+            return (name, res, res, fmeta)
+
         # Translate aliases into the standard field name
         fmkey = field_metadata.search_term_to_field_key(key)
-
         if fmkey in field_metadata and field_metadata[fmkey]['kind'] == 'field':
             res = self.get(key, None)
             fmeta = field_metadata[fmkey]
@@ -561,6 +601,8 @@ class Metadata(object):
             elif key == 'series_index':
                 res = self.format_series_index(res)
             elif datatype == 'text' and fmeta['is_multiple']:
+                if isinstance(res, dict):
+                    res = [k + ':' + v for k,v in res.items()]
                 res = u', '.join(sorted(res, key=sort_key))
             elif datatype == 'series' and series_with_index:
                 res = res + ' [%s]'%self.format_series_index()

src/calibre/ebooks/metadata/book/json_codec.py

@@ -123,6 +123,8 @@ class JsonCodec(object):
                     if key == 'user_metadata':
                         book.set_all_user_metadata(meta)
                     else:
+                        if key == 'classifiers':
+                            key = 'identifiers'
                         setattr(book, key, meta)
                 booklist.append(book)
         except:
@@ -130,6 +132,8 @@ class JsonCodec(object):
             traceback.print_exc()
 
     def decode_metadata(self, key, value):
+        if key == 'classifiers':
+            key = 'identifiers'
         if key == 'user_metadata':
             for k in value:
                 if value[k]['datatype'] == 'datetime':

src/calibre/ebooks/metadata/opf2.py

@@ -596,6 +596,9 @@ class OPF(object): # {{{
         ans = MetaInformation(self)
         for n, v in self._user_metadata_.items():
             ans.set_user_metadata(n, v)
+
+        ans.set_identifiers(self.get_identifiers())
+
         return ans
 
     def write_user_metadata(self):
@@ -855,6 +858,21 @@ class OPF(object): # {{{
 
         return property(fget=fget, fset=fset)
 
+    def get_identifiers(self):
+        identifiers = {}
+        for x in self.XPath(
+            'descendant::*[local-name() = "identifier" and text()]')(
+                    self.metadata):
+            for attr, val in x.attrib.iteritems():
+                if attr.endswith('scheme'):
+                    typ = icu_lower(val)
+                    val = etree.tostring(x, with_tail=False, encoding=unicode,
+                            method='text').strip()
+                    if val and typ not in ('calibre', 'uuid'):
+                        identifiers[typ] = val
+                    break
+        return identifiers
+
     @dynamic_property
     def application_id(self):
 
@@ -1166,8 +1184,8 @@ class OPFCreator(Metadata):
             a(DC_ELEM('description', self.comments))
         if self.publisher:
             a(DC_ELEM('publisher', self.publisher))
-        if self.isbn:
-            a(DC_ELEM('identifier', self.isbn, opf_attrs={'scheme':'ISBN'}))
+        for key, val in self.get_identifiers().iteritems():
+            a(DC_ELEM('identifier', val, opf_attrs={'scheme':icu_upper(key)}))
         if self.rights:
             a(DC_ELEM('rights', self.rights))
         if self.tags:
@@ -1291,8 +1309,8 @@ def metadata_to_opf(mi, as_string=True):
         factory(DC('description'), mi.comments)
     if mi.publisher:
         factory(DC('publisher'), mi.publisher)
-    if mi.isbn:
-        factory(DC('identifier'), mi.isbn, scheme='ISBN')
+    for key, val in mi.get_identifiers().iteritems():
+        factory(DC('identifier'), val, scheme=icu_upper(key))
     if mi.rights:
         factory(DC('rights'), mi.rights)
     factory(DC('language'), mi.language if mi.language and mi.language.lower()
@@ -1342,7 +1360,7 @@ def test_m2o():
     mi.language = 'en'
     mi.comments = 'what a fun book\n\n'
     mi.publisher = 'publisher'
-    mi.isbn = 'boooo'
+    mi.set_identifiers({'isbn':'booo', 'dummy':'dummy'})
     mi.tags = ['a', 'b']
     mi.series = 's"c\'l&<>'
     mi.series_index = 3.34
@@ -1350,7 +1368,7 @@ def test_m2o():
     mi.timestamp = nowf()
     mi.publication_type = 'ooooo'
     mi.rights = 'yes'
-    mi.cover = 'asd.jpg'
+    mi.cover = os.path.abspath('asd.jpg')
     opf = metadata_to_opf(mi)
     print opf
     newmi = MetaInformation(OPF(StringIO(opf)))
@@ -1363,6 +1381,9 @@ def test_m2o():
         o, n = getattr(mi, attr), getattr(newmi, attr)
         if o != n and o.strip() != n.strip():
             print 'FAILED:', attr, getattr(mi, attr), '!=', getattr(newmi, attr)
+    if mi.get_identifiers() != newmi.get_identifiers():
+        print 'FAILED:', 'identifiers', mi.get_identifiers(),
+        print '!=', newmi.get_identifiers()
 
 
 class OPFTest(unittest.TestCase):
@@ -1378,6 +1399,7 @@ class OPFTest(unittest.TestCase):
     <creator opf:role="aut">Next</creator>
     <dc:subject>One</dc:subject><dc:subject>Two</dc:subject>
     <dc:identifier scheme="ISBN">123456789</dc:identifier>
+    <dc:identifier scheme="dummy">dummy</dc:identifier>
     <meta name="calibre:series" content="A one book series" />
     <meta name="calibre:rating" content="4"/>
     <meta name="calibre:publication_type" content="test"/>
@@ -1405,6 +1427,8 @@ class OPFTest(unittest.TestCase):
         self.assertEqual(opf.rating, 4)
         self.assertEqual(opf.publication_type, 'test')
         self.assertEqual(list(opf.itermanifest())[0].get('href'), 'a ~ b')
+        self.assertEqual(opf.get_identifiers(), {'isbn':'123456789',
+            'dummy':'dummy'})
 
     def testWriting(self):
         for test in [('title', 'New & Title'), ('authors', ['One', 'Two']),
@@ -1461,5 +1485,5 @@ def test_user_metadata():
 
 if __name__ == '__main__':
     #test_user_metadata()
-    #test_m2o()
+    test_m2o()
     test()

src/calibre/ebooks/metadata/overdrive.py

@@ -4,7 +4,7 @@ __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 
 '''
-Fetch metadata using Adobe Overdrive
+Fetch metadata using Overdrive Content Reserve
 '''
 import sys, re, random, urllib, mechanize, copy
 from threading import RLock
@@ -24,6 +24,34 @@ cover_url_cache = {}
 cache_lock = RLock()
 base_url = 'http://search.overdrive.com/'
 
+class ContentReserve(Source):
+
+    def create_query(self, title=None, authors=None, identifiers={}):
+        q = ''
+        if title or authors:
+            def build_term(prefix, parts):
+                return ' '.join('in'+prefix + ':' + x for x in parts)
+            title_tokens = list(self.get_title_tokens(title))
+            if title_tokens:
+                q += build_term('title', title_tokens)
+            author_tokens = self.get_author_tokens(authors,
+                    only_first_author=True)
+            if author_tokens:
+                q += ('+' if q else '') + build_term('author',
+                        author_tokens)
+
+        if isinstance(q, unicode):
+            q = q.encode('utf-8')
+        if not q:
+            return None
+        return BASE_URL+urlencode({
+            'q':q,
+            'max-results':20,
+            'start-index':1,
+            'min-viewability':'none',
+            })
+
+
 def get_base_referer():
     choices = [
         'http://overdrive.chipublib.org/82DC601D-7DDE-4212-B43A-09D821935B01/10/375/en/',
@@ -203,9 +231,9 @@ def get_cover_url(isbn, title, author, br):
     print "isbn is "+str(isbn)
     print "title is "+str(title)
     print "author is "+str(author[0])
-    cleanup = Source()
-    author = cleanup.get_author_tokens(author)
-    print "cleansed author is "+str(author)
+    cleanup = ContentReserve()
+    query = cleanup.create_query(author, title)
+    print "cleansed query is "+str(author)
 
     with cache_lock:
         ans = cover_url_cache.get(isbn, None)

src/calibre/ebooks/metadata/sources/base.py

@@ -65,6 +65,7 @@ class Source(Plugin):
                     parts = parts[1:] + parts[:1]
                 for tok in parts:
                     tok = pat.sub('', tok).strip()
+                    if len(tok) > 2 and tok.lower() not in ('von', ):
                         yield tok
 
 

src/calibre/ebooks/mobi/reader.py

@@ -18,6 +18,7 @@ from calibre import xml_entity_to_unicode, CurrentDir, entity_to_unicode, \
     replace_entities
 from calibre.utils.filenames import ascii_filename
 from calibre.utils.date import parse_date
+from calibre.utils.cleantext import clean_ascii_chars
 from calibre.ptempfile import TemporaryDirectory
 from calibre.ebooks import DRMError
 from calibre.ebooks.chardet import ENCODING_PATS
@@ -323,6 +324,7 @@ class MobiReader(object):
         self.cleanup_html()
 
         self.log.debug('Parsing HTML...')
+        self.processed_html = clean_ascii_chars(self.processed_html)
         try:
             root = html.fromstring(self.processed_html)
             if len(root.xpath('//html')) > 5:

src/calibre/ebooks/oeb/base.py

@@ -827,6 +827,24 @@ class Manifest(object):
                 return None
             return etree.fromstring(data, parser=RECOVER_PARSER)
 
+        def clean_word_doc(self, data):
+            prefixes = []
+            for match in re.finditer(r'xmlns:(\S+?)=".*?microsoft.*?"', data):
+                prefixes.append(match.group(1))
+            if prefixes:
+                self.oeb.log.warn('Found microsoft markup, cleaning...')
+                # Remove empty tags as they are not rendered by browsers
+                # but can become renderable HTML tags like <p/> if the
+                # document is parsed by an HTML parser
+                pat = re.compile(
+                        r'<(%s):([a-zA-Z0-9]+)[^>/]*?></\1:\2>'%('|'.join(prefixes)),
+                        re.DOTALL)
+                data = pat.sub('', data)
+                pat = re.compile(
+                        r'<(%s):([a-zA-Z0-9]+)[^>/]*?/>'%('|'.join(prefixes)))
+                data = pat.sub('', data)
+            return data
+
         def _parse_xhtml(self, data):
             self.oeb.log.debug('Parsing', self.href, '...')
             # Convert to Unicode and normalize line endings
@@ -884,6 +902,10 @@ class Manifest(object):
                         except etree.XMLSyntaxError:
                             data = etree.fromstring(data, parser=RECOVER_PARSER)
                 return data
+            try:
+                data = self.clean_word_doc(data)
+            except:
+                pass
             data = first_pass(data)
 
             # Handle weird (non-HTML/fragment) files
@@ -907,6 +929,7 @@ class Manifest(object):
                     parent.append(child)
                 data = nroot
 
+
             # Force into the XHTML namespace
             if not namespace(data.tag):
                 self.oeb.log.warn('Forcing', self.href, 'into XHTML namespace')

src/calibre/ebooks/oeb/output.py

@@ -59,18 +59,32 @@ class OEBOutput(OutputFormatPlugin):
     def workaround_nook_cover_bug(self, root): # {{{
         cov = root.xpath('//*[local-name() = "meta" and @name="cover" and'
                 ' @content != "cover"]')
+
+        def manifest_items_with_id(id_):
+            return root.xpath('//*[local-name() = "manifest"]/*[local-name() = "item" '
+                ' and @id="%s"]'%id_)
+
         if len(cov) == 1:
-            manpath = ('//*[local-name() = "manifest"]/*[local-name() = "item" '
-                ' and @id="%s" and @media-type]')
             cov = cov[0]
-            covid = cov.get('content')
-            manifest_item = root.xpath(manpath%covid)
-            has_cover = root.xpath(manpath%'cover')
-            if len(manifest_item) == 1 and not has_cover and \
+            covid = cov.get('content', '')
+
+            if covid:
+                manifest_item = manifest_items_with_id(covid)
+                if len(manifest_item) == 1 and \
                         manifest_item[0].get('media-type',
                                 '').startswith('image/'):
                     self.log.warn('The cover image has an id != "cover". Renaming'
-                        ' to work around Nook Color bug')
+                            ' to work around bug in Nook Color')
+
+                    import uuid
+                    newid = str(uuid.uuid4())
+
+                    for item in manifest_items_with_id('cover'):
+                        item.set('id', newid)
+
+                    for x in root.xpath('//*[@idref="cover"]'):
+                        x.set('idref', newid)
+
                     manifest_item = manifest_item[0]
                     manifest_item.set('id', 'cover')
                     cov.set('content', 'cover')

src/calibre/ebooks/oeb/stylizer.py

@@ -423,6 +423,7 @@ class Stylizer(object):
 
 class Style(object):
     UNIT_RE = re.compile(r'^(-*[0-9]*[.]?[0-9]*)\s*(%|em|ex|en|px|mm|cm|in|pt|pc)$')
+    MS_PAT = re.compile(r'^\s*(mso-|panose-|text-underline|tab-interval)')
 
     def __init__(self, element, stylizer):
         self._element = element
@@ -447,6 +448,8 @@ class Style(object):
             return
         css = attrib['style'].split(';')
         css = filter(None, (x.strip() for x in css))
+        css = [x.strip() for x in css]
+        css = [x for x in css if self.MS_PAT.match(x) is None]
         try:
             style = CSSStyleDeclaration('; '.join(css))
         except CSSSyntaxError:

src/calibre/ebooks/pdf/reflow.cpp

@@ -887,7 +887,7 @@ vector<char>* Reflow::render_first_page(bool use_crop_box, double x_res,
         }
 
         pg_w *= x_res/72.;
-        pg_h *= x_res/72.;
+        pg_h *= y_res/72.;
 
         int x=0, y=0;
         this->doc->displayPageSlice(out, pg, x_res, y_res, 0,

src/calibre/gui2/actions/add.py

@@ -20,9 +20,26 @@ from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.utils.filenames import ascii_filename
 from calibre.constants import preferred_encoding, filesystem_encoding
 from calibre.gui2.actions import InterfaceAction
-from calibre.gui2 import config
+from calibre.gui2 import config, question_dialog
 from calibre.ebooks.metadata import MetaInformation
 
+def get_filters():
+    return [
+            (_('Books'), BOOK_EXTENSIONS),
+            (_('EPUB Books'), ['epub']),
+            (_('LRF Books'), ['lrf']),
+            (_('HTML Books'), ['htm', 'html', 'xhtm', 'xhtml']),
+            (_('LIT Books'), ['lit']),
+            (_('MOBI Books'), ['mobi', 'prc', 'azw']),
+            (_('Topaz books'), ['tpz','azw1']),
+            (_('Text books'), ['txt', 'rtf']),
+            (_('PDF Books'), ['pdf']),
+            (_('SNB Books'), ['snb']),
+            (_('Comics'), ['cbz', 'cbr', 'cbc']),
+            (_('Archives'), ['zip', 'rar']),
+    ]
+
+
 class AddAction(InterfaceAction):
 
     name = 'Add Books'
@@ -47,6 +64,10 @@ class AddAction(InterfaceAction):
         self.add_menu.addAction(_('Add Empty book. (Book entry with no '
             'formats)'), self.add_empty, _('Shift+Ctrl+E'))
         self.add_menu.addAction(_('Add from ISBN'), self.add_from_isbn)
+        self.add_menu.addSeparator()
+        self.add_menu.addAction(_('Add files to selected book records'),
+                self.add_formats, _('Shift+A'))
+
         self.qaction.setMenu(self.add_menu)
         self.qaction.triggered.connect(self.add_books)
 
@@ -55,6 +76,39 @@ class AddAction(InterfaceAction):
         for action in list(self.add_menu.actions())[1:]:
             action.setEnabled(enabled)
 
+    def add_formats(self, *args):
+        if self.gui.stack.currentIndex() != 0:
+            return
+        view = self.gui.library_view
+        rows = view.selectionModel().selectedRows()
+        if not rows:
+            return
+        ids = [view.model().id(r) for r in rows]
+
+        if len(ids) > 1 and not question_dialog(self.gui,
+                _('Are you sure'),
+            _('Are you sure you want to add the same'
+                ' files to all %d books? If the format'
+                'already exists for a book, it will be replaced.')%len(ids)):
+                return
+
+        books = choose_files(self.gui, 'add formats dialog dir',
+                _('Select book files'), filters=get_filters())
+        if not books:
+            return
+
+        db = view.model().db
+        for id_ in ids:
+            for fpath in books:
+                fmt = os.path.splitext(fpath)[1][1:].upper()
+                if fmt:
+                    db.add_format_with_hooks(id_, fmt, fpath, index_is_id=True,
+                        notify=True)
+        current_idx = self.gui.library_view.currentIndex()
+        if current_idx.isValid():
+            view.model().current_changed(current_idx, current_idx)
+
+
     def add_recursive(self, single):
         root = choose_dir(self.gui, 'recursive book import root dir dialog',
                           'Select root folder')
@@ -207,27 +261,14 @@ class AddAction(InterfaceAction):
         '''
         Add books from the local filesystem to either the library or the device.
         '''
-        filters = [
-                        (_('Books'), BOOK_EXTENSIONS),
-                        (_('EPUB Books'), ['epub']),
-                        (_('LRF Books'), ['lrf']),
-                        (_('HTML Books'), ['htm', 'html', 'xhtm', 'xhtml']),
-                        (_('LIT Books'), ['lit']),
-                        (_('MOBI Books'), ['mobi', 'prc', 'azw']),
-                        (_('Topaz books'), ['tpz','azw1']),
-                        (_('Text books'), ['txt', 'rtf']),
-                        (_('PDF Books'), ['pdf']),
-                        (_('SNB Books'), ['snb']),
-                        (_('Comics'), ['cbz', 'cbr', 'cbc']),
-                        (_('Archives'), ['zip', 'rar']),
-                        ]
+        filters = get_filters()
         to_device = self.gui.stack.currentIndex() != 0
         if to_device:
             fmts = self.gui.device_manager.device.settings().format_map
             filters = [(_('Supported books'), fmts)]
 
-        books = choose_files(self.gui, 'add books dialog dir', 'Select books',
-                             filters=filters)
+        books = choose_files(self.gui, 'add books dialog dir',
+                _('Select books'), filters=filters)
         if not books:
             return
         self._add_books(books, to_device)

src/calibre/gui2/actions/choose_library.py

@@ -355,6 +355,7 @@ class ChooseLibraryAction(InterfaceAction):
                 print
         print 'before:', self.before_mem
         print 'after:', memory()/1024**2
+        print
         self.dbref = self.before_mem = None
 
 

src/calibre/gui2/actions/delete.py

@@ -19,11 +19,11 @@ single_shot = partial(QTimer.singleShot, 10)
 
 class MultiDeleter(QObject):
 
-    def __init__(self, gui, rows, callback):
+    def __init__(self, gui, ids, callback):
         from calibre.gui2.dialogs.progress import ProgressDialog
         QObject.__init__(self, gui)
         self.model = gui.library_view.model()
-        self.ids = list(map(self.model.id, rows))
+        self.ids = ids
         self.gui = gui
         self.failures = []
         self.deleted_ids = []
@@ -231,6 +231,7 @@ class DeleteAction(InterfaceAction):
             return
         # Library view is visible.
         if self.gui.stack.currentIndex() == 0:
+            to_delete_ids = [view.model().id(r) for r in rows]
             # Ask the user if they want to delete the book from the library or device if it is in both.
             if self.gui.device_manager.is_device_connected:
                 on_device = False
@@ -264,10 +265,10 @@ class DeleteAction(InterfaceAction):
             if ci.isValid():
                 row = ci.row()
             if len(rows) < 5:
-                ids_deleted = view.model().delete_books(rows)
-                self.library_ids_deleted(ids_deleted, row)
+                view.model().delete_books_by_id(to_delete_ids)
+                self.library_ids_deleted(to_delete_ids, row)
             else:
-                self.__md = MultiDeleter(self.gui, rows,
+                self.__md = MultiDeleter(self.gui, to_delete_ids,
                         partial(self.library_ids_deleted, current_row=row))
         # Device view is visible.
         else:

src/calibre/gui2/book_details.py

@@ -10,7 +10,7 @@ from Queue import Queue
 
 from PyQt4.Qt import QPixmap, QSize, QWidget, Qt, pyqtSignal, QUrl, \
     QPropertyAnimation, QEasingCurve, QThread, QApplication, QFontInfo, \
-    QSizePolicy, QPainter, QRect, pyqtProperty, QLayout, QPalette
+    QSizePolicy, QPainter, QRect, pyqtProperty, QLayout, QPalette, QMenu
 from PyQt4.QtWebKit import QWebView
 
 from calibre import fit_image, prepare_string_for_xml
@@ -18,7 +18,7 @@ from calibre.gui2.widgets import IMAGE_EXTENSIONS
 from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.constants import preferred_encoding
 from calibre.library.comments import comments_to_html
-from calibre.gui2 import config, open_local_file, open_url
+from calibre.gui2 import config, open_local_file, open_url, pixmap_to_data
 from calibre.utils.icu import sort_key
 
 # render_rows(data) {{{
@@ -70,6 +70,7 @@ def render_rows(data):
 
 class CoverView(QWidget): # {{{
 
+    cover_changed = pyqtSignal(object, object)
 
     def __init__(self, vertical, parent=None):
         QWidget.__init__(self, parent)
@@ -151,6 +152,35 @@ class CoverView(QWidget): # {{{
             fset=setCurrentPixmapSize
             )
 
+    def contextMenuEvent(self, ev):
+        cm = QMenu(self)
+        paste = cm.addAction(_('Paste Cover'))
+        copy = cm.addAction(_('Copy Cover'))
+        if not QApplication.instance().clipboard().mimeData().hasImage():
+            paste.setEnabled(False)
+        copy.triggered.connect(self.copy_to_clipboard)
+        paste.triggered.connect(self.paste_from_clipboard)
+        cm.exec_(ev.globalPos())
+
+    def copy_to_clipboard(self):
+        QApplication.instance().clipboard().setPixmap(self.pixmap)
+
+    def paste_from_clipboard(self):
+        cb = QApplication.instance().clipboard()
+        pmap = cb.pixmap()
+        if pmap.isNull() and cb.supportsSelection():
+            pmap = cb.pixmap(cb.Selection)
+        if not pmap.isNull():
+            self.pixmap = pmap
+            self.do_layout()
+            self.update()
+            if not config['disable_animations']:
+                self.animation.start()
+            id_ = self.data.get('id', None)
+            if id_ is not None:
+                self.cover_changed.emit(id_,
+                    pixmap_to_data(pmap))
+
 
     # }}}
 
@@ -362,7 +392,9 @@ class BookDetails(QWidget): # {{{
     # Drag 'n drop {{{
     DROPABBLE_EXTENSIONS = IMAGE_EXTENSIONS+BOOK_EXTENSIONS
     files_dropped = pyqtSignal(object, object)
+    cover_changed = pyqtSignal(object, object)
 
+    # application/x-moz-file-promise-url
     @classmethod
     def paths_from_event(cls, event):
         '''
@@ -399,6 +431,7 @@ class BookDetails(QWidget): # {{{
         self.setLayout(self._layout)
 
         self.cover_view = CoverView(vertical, self)
+        self.cover_view.cover_changed.connect(self.cover_changed.emit)
         self._layout.addWidget(self.cover_view)
         self.book_info = BookInfo(vertical, self)
         self._layout.addWidget(self.book_info)

src/calibre/gui2/device.py

@@ -140,6 +140,8 @@ class DeviceManager(Thread): # {{{
         self.mount_connection_requests = Queue.Queue(0)
         self.open_feedback_slot = open_feedback_slot
         self.open_feedback_msg = open_feedback_msg
+        self._device_information = None
+        self.current_library_uuid = None
 
     def report_progress(self, *args):
         pass
@@ -159,7 +161,7 @@ class DeviceManager(Thread): # {{{
             try:
                 dev.reset(detected_device=detected_device,
                     report_progress=self.report_progress)
-                dev.open()
+                dev.open(self.current_library_uuid)
             except OpenFeedback, e:
                 if dev not in self.ejected_devices:
                     self.open_feedback_msg(dev.get_gui_name(), e.feedback_msg)
@@ -194,6 +196,7 @@ class DeviceManager(Thread): # {{{
         else:
             self.connected_slot(False, self.connected_device_kind)
         self.connected_device = None
+        self._device_information = None
 
     def detect_device(self):
         self.scanner.scan()
@@ -292,9 +295,13 @@ class DeviceManager(Thread): # {{{
 
     def _get_device_information(self):
         info = self.device.get_device_information(end_session=False)
-        info = [i.replace('\x00', '').replace('\x01', '') for i in info]
+        if len(info) < 5:
+            info = tuple(list(info) + [{}])
+        info = [i.replace('\x00', '').replace('\x01', '') if isinstance(i, basestring) else i
+                 for i in info]
         cp = self.device.card_prefix(end_session=False)
         fs = self.device.free_space()
+        self._device_information = {'info': info, 'prefixes': cp, 'freespace': fs}
         return info, cp, fs
 
     def get_device_information(self, done):
@@ -302,6 +309,9 @@ class DeviceManager(Thread): # {{{
         return self.create_job(self._get_device_information, done,
                     description=_('Get device information'))
 
+    def get_current_device_information(self):
+        return self._device_information
+
     def _books(self):
         '''Get metadata from device'''
         mainlist = self.device.books(oncard=None, end_session=False)
@@ -417,6 +427,13 @@ class DeviceManager(Thread): # {{{
         return self.create_job(self._view_book, done, args=[path, target],
                         description=_('View book on device'))
 
+    def set_current_library_uuid(self, uuid):
+        self.current_library_uuid = uuid
+
+    def set_driveinfo_name(self, location_code, name):
+        if self.connected_device:
+            self.connected_device.set_driveinfo_name(location_code, name)
+
     # }}}
 
 class DeviceAction(QAction): # {{{

src/calibre/gui2/dialogs/book_info.py

@@ -6,7 +6,7 @@ __docformat__ = 'restructuredtext en'
 import textwrap, os, re
 
 from PyQt4.Qt import QCoreApplication, SIGNAL, QModelIndex, QTimer, Qt, \
-    QDialog, QPixmap, QGraphicsScene, QIcon, QSize
+    QDialog, QPixmap, QIcon, QSize
 
 from calibre.gui2.dialogs.book_info_ui import Ui_BookInfo
 from calibre.gui2 import dynamic, open_local_file, open_url
@@ -14,12 +14,14 @@ from calibre import fit_image
 from calibre.library.comments import comments_to_html
 from calibre.utils.icu import sort_key
 
+
 class BookInfo(QDialog, Ui_BookInfo):
 
     def __init__(self, parent, view, row, view_func):
         QDialog.__init__(self, parent)
         Ui_BookInfo.__init__(self)
         self.setupUi(self)
+        self.gui = parent
         self.cover_pixmap = None
         self.comments.sizeHint = self.comments_size_hint
         self.comments.page().setLinkDelegationPolicy(self.comments.page().DelegateAllLinks)
@@ -38,11 +40,26 @@ class BookInfo(QDialog, Ui_BookInfo):
         self.connect(self.text, SIGNAL('linkActivated(QString)'), self.open_book_path)
         self.fit_cover.stateChanged.connect(self.toggle_cover_fit)
         self.cover.resizeEvent = self.cover_view_resized
+        self.cover.cover_changed.connect(self.cover_changed)
 
         desktop = QCoreApplication.instance().desktop()
         screen_height = desktop.availableGeometry().height() - 100
         self.resize(self.size().width(), screen_height)
 
+    def cover_changed(self, data):
+        if self.current_row is not None:
+            id_ = self.view.model().id(self.current_row)
+            self.view.model().db.set_cover(id_, data)
+        if self.gui.cover_flow:
+            self.gui.cover_flow.dataChanged()
+        ci = self.view.currentIndex()
+        if ci.isValid():
+            self.view.model().current_changed(ci, ci)
+        self.cover_pixmap = QPixmap()
+        self.cover_pixmap.loadFromData(data)
+        if self.fit_cover.isChecked():
+            self.resize_cover()
+
     def link_clicked(self, url):
         open_url(url)
 
@@ -83,7 +100,6 @@ class BookInfo(QDialog, Ui_BookInfo):
         if self.cover_pixmap is None:
             return
         self.setWindowIcon(QIcon(self.cover_pixmap))
-        self.scene = QGraphicsScene()
         pixmap = self.cover_pixmap
         if self.fit_cover.isChecked():
             scaled, new_width, new_height = fit_image(pixmap.width(),
@@ -92,8 +108,7 @@ class BookInfo(QDialog, Ui_BookInfo):
             if scaled:
                 pixmap = pixmap.scaled(new_width, new_height,
                         Qt.KeepAspectRatio, Qt.SmoothTransformation)
-        self.scene.addPixmap(pixmap)
-        self.cover.setScene(self.scene)
+        self.cover.set_pixmap(pixmap)
 
     def refresh(self, row):
         if isinstance(row, QModelIndex):

src/calibre/gui2/dialogs/book_info.ui

@@ -25,7 +25,7 @@
     </widget>
    </item>
    <item row="1" column="0">
-    <widget class="QGraphicsView" name="cover"/>
+    <widget class="CoverView" name="cover"/>
    </item>
    <item row="1" column="1">
     <layout class="QVBoxLayout" name="verticalLayout">
@@ -115,6 +115,11 @@
    <extends>QWidget</extends>
    <header>QtWebKit/QWebView</header>
   </customwidget>
+  <customwidget>
+   <class>CoverView</class>
+   <extends>QGraphicsView</extends>
+   <header>calibre/gui2/widgets.h</header>
+  </customwidget>
  </customwidgets>
  <resources>
   <include location="../../../../resources/images.qrc"/>

src/calibre/gui2/dialogs/edit_authors_dialog.py

@@ -66,8 +66,8 @@ class EditAuthorsDialog(QDialog, Ui_EditAuthorsDialog):
         self.sort_by_author_sort.setChecked(True)
         self.author_sort_order = 1
 
-        # set up author sort calc button
         self.recalc_author_sort.clicked.connect(self.do_recalc_author_sort)
+        self.auth_sort_to_author.clicked.connect(self.do_auth_sort_to_author)
 
         if select_item is not None:
             self.table.setCurrentItem(select_item)
@@ -108,6 +108,17 @@ class EditAuthorsDialog(QDialog, Ui_EditAuthorsDialog):
         self.table.setFocus(Qt.OtherFocusReason)
         self.table.cellChanged.connect(self.cell_changed)
 
+    def do_auth_sort_to_author(self):
+        self.table.cellChanged.disconnect()
+        for row in range(0,self.table.rowCount()):
+            item = self.table.item(row, 1)
+            aus  = unicode(item.text()).strip()
+            c = self.table.item(row, 0)
+            # Sometimes trailing commas are left by changing between copy algs
+            c.setText(aus)
+        self.table.setFocus(Qt.OtherFocusReason)
+        self.table.cellChanged.connect(self.cell_changed)
+
     def cell_changed(self, row, col):
         if col == 0:
             item = self.table.item(row, 0)

src/calibre/gui2/dialogs/edit_authors_dialog.ui

@@ -52,13 +52,26 @@
      <item>
       <widget class="QPushButton" name="recalc_author_sort">
        <property name="toolTip">
-        <string>Reset all the author sort values to a value automatically generated from the author. Exactly how this value is automatically generated can be controlled via Preferences-&gt;Advanced-&gt;Tweaks</string>
+        <string>Reset all the author sort values to a value automatically
+generated from the author. Exactly how this value is automatically
+generated can be controlled via Preferences-&gt;Advanced-&gt;Tweaks</string>
        </property>
        <property name="text">
         <string>Recalculate all author sort values</string>
        </property>
       </widget>
      </item>
+     <item>
+      <widget class="QPushButton" name="auth_sort_to_author">
+       <property name="toolTip">
+        <string>Copy author sort to author for every author. You typically use this button
+after changing Preferences-&gt;Advanced-&gt;Tweaks-&gt;Author sort name algorithm</string>
+       </property>
+       <property name="text">
+        <string>Copy all author sort values to author</string>
+       </property>
+      </widget>
+     </item>
      <item>
       <spacer name="horizontalSpacer_3">
        <property name="orientation">

src/calibre/gui2/dialogs/metadata_bulk.ui

@@ -341,7 +341,7 @@ from the value in the box</string>
                 <number>1</number>
                </property>
                <property name="maximum">
-                <number>990000</number>
+                <number>99000000</number>
                </property>
                <property name="value">
                 <number>1</number>

src/calibre/gui2/dialogs/metadata_single.ui

@@ -419,7 +419,7 @@ If the box is colored green, then text matches the individual author's sort stri
                      <string>Book </string>
                     </property>
                     <property name="maximum">
-                     <double>9999.989999999999782</double>
+                     <double>99999999.989999994635582</double>
                     </property>
                    </widget>
                   </item>

src/calibre/gui2/dialogs/scheduler.py

@@ -8,9 +8,11 @@ Scheduler for automated recipe downloads
 '''
 
 from datetime import timedelta
+import calendar, textwrap
 
-from PyQt4.Qt import QDialog, SIGNAL, Qt, QTime, QObject, QMenu, \
-        QAction, QIcon, QMutex, QTimer, pyqtSignal
+from PyQt4.Qt import QDialog, Qt, QTime, QObject, QMenu, QHBoxLayout, \
+        QAction, QIcon, QMutex, QTimer, pyqtSignal, QWidget, QGridLayout, \
+        QCheckBox, QTimeEdit, QLabel, QLineEdit, QDoubleSpinBox
 
 from calibre.gui2.dialogs.scheduler_ui import Ui_Dialog
 from calibre.gui2 import config as gconf, error_dialog
@@ -18,9 +20,173 @@ from calibre.web.feeds.recipes.model import RecipeModel
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.utils.date import utcnow
 from calibre.utils.network import internet_connected
+from calibre.utils.ordered_dict import OrderedDict
+from calibre import force_unicode
+
+def convert_day_time_schedule(val):
+    day_of_week, hour, minute = val
+    if day_of_week == -1:
+        return (tuple(xrange(7)), hour, minute)
+    return ((day_of_week,), hour, minute)
+
+class Base(QWidget):
+
+    def __init__(self, parent=None):
+        QWidget.__init__(self, parent)
+        self.l = QGridLayout()
+        self.setLayout(self.l)
+        self.setToolTip(textwrap.dedent(self.HELP))
+
+class DaysOfWeek(Base):
+
+    HELP = _('''\
+                Download this periodical every week on the specified days after
+                the specified time. For example, if you choose: Monday after
+                9:00 AM, then the periodical will be download every Monday as
+                soon after 9:00 AM as possible.
+            ''')
+
+    def __init__(self, parent=None):
+        Base.__init__(self, parent)
+        self.days = [QCheckBox(force_unicode(calendar.day_abbr[d]),
+            self) for d in xrange(7)]
+        for i, cb in enumerate(self.days):
+            row = i % 2
+            col = i // 2
+            self.l.addWidget(cb, row, col, 1, 1)
+
+        self.time = QTimeEdit(self)
+        self.time.setDisplayFormat('hh:mm AP')
+        self.hl = QHBoxLayout()
+        self.l1 = QLabel(_('&Download after:'))
+        self.l1.setBuddy(self.time)
+        self.hl.addWidget(self.l1)
+        self.hl.addWidget(self.time)
+        self.l.addLayout(self.hl, 1, 3, 1, 1)
+        self.initialize()
+
+    def initialize(self, typ=None, val=None):
+        if typ is None:
+            typ = 'day/time'
+            val = (-1, 9, 0)
+        if typ == 'day/time':
+            val = convert_day_time_schedule(val)
+
+        days_of_week, hour, minute = val
+        for i, d in enumerate(self.days):
+            d.setChecked(i in days_of_week)
+
+        self.time.setTime(QTime(hour, minute))
+
+    @property
+    def schedule(self):
+        days_of_week = tuple([i for i, d in enumerate(self.days) if
+            d.isChecked()])
+        t = self.time.time()
+        hour, minute = t.hour(), t.minute()
+        return 'days_of_week', (days_of_week, int(hour), int(minute))
+
+class DaysOfMonth(Base):
+
+    HELP = _('''\
+                Download this periodical every month, on the specified days.
+                The download will happen as soon after the specified time as
+                possible on the specified days of each month. For example,
+                if you choose the 1st and the 15th after 9:00 AM, the
+                periodical will be downloaded on the 1st and 15th of every
+                month, as soon after 9:00 AM as possible.
+            ''')
+
+    def __init__(self, parent=None):
+        Base.__init__(self, parent)
+
+        self.l1 = QLabel(_('&Days of the month:'))
+        self.days = QLineEdit(self)
+        self.days.setToolTip(_('Comma separated list of days of the month.'
+            ' For example: 1, 15'))
+        self.l1.setBuddy(self.days)
+
+        self.l2 = QLabel(_('Download &after:'))
+        self.time = QTimeEdit(self)
+        self.time.setDisplayFormat('hh:mm AP')
+        self.l2.setBuddy(self.time)
+
+        self.l.addWidget(self.l1, 0, 0, 1, 1)
+        self.l.addWidget(self.days, 0, 1, 1, 1)
+        self.l.addWidget(self.l2, 1, 0, 1, 1)
+        self.l.addWidget(self.time, 1, 1, 1, 1)
+
+    def initialize(self, typ=None, val=None):
+        if val is None:
+            val = ((1,), 9, 0)
+        days_of_month, hour, minute = val
+        self.days.setText(', '.join(map(str, map(int, days_of_month))))
+        self.time.setTime(QTime(hour, minute))
+
+    @property
+    def schedule(self):
+        parts = [x.strip() for x in unicode(self.days.text()).split(',') if
+                x.strip()]
+        try:
+            days_of_month = tuple(map(int, parts))
+        except:
+            days_of_month = (1,)
+        if not days_of_month:
+            days_of_month = (1,)
+        t = self.time.time()
+        hour, minute = t.hour(), t.minute()
+        return 'days_of_month', (days_of_month, int(hour), int(minute))
+
+class EveryXDays(Base):
+
+    HELP = _('''\
+                Download this periodical every x days. For example, if you
+                choose 30 days, the periodical will be downloaded every 30
+                days. Note that you can set periods of less than a day, like
+                0.1 days to download a periodical more than once a day.
+            ''')
+
+    def __init__(self, parent=None):
+        Base.__init__(self, parent)
+        self.l1 = QLabel(_('&Download every:'))
+        self.interval = QDoubleSpinBox(self)
+        self.interval.setMinimum(0.04)
+        self.interval.setSpecialValueText(_('every hour'))
+        self.interval.setMaximum(1000.0)
+        self.interval.setValue(31.0)
+        self.interval.setSuffix(' ' + _('days'))
+        self.interval.setSingleStep(1.0)
+        self.interval.setDecimals(2)
+        self.l1.setBuddy(self.interval)
+        self.l2 = QLabel(_('Note: You can set intervals of less than a day,'
+            ' by typing the value manually.'))
+        self.l2.setWordWrap(True)
+
+        self.l.addWidget(self.l1, 0, 0, 1, 1)
+        self.l.addWidget(self.interval, 0, 1, 1, 1)
+        self.l.addWidget(self.l2, 1, 0, 1, -1)
+
+    def initialize(self, typ=None, val=None):
+        if val is None:
+            val = 31.0
+        self.interval.setValue(val)
+
+    @property
+    def schedule(self):
+        schedule = self.interval.value()
+        return 'interval', schedule
+
 
 class SchedulerDialog(QDialog, Ui_Dialog):
 
+    SCHEDULE_TYPES = OrderedDict([
+            ('days_of_week', DaysOfWeek),
+            ('days_of_month', DaysOfMonth),
+            ('every_x_days', EveryXDays),
+    ])
+
+    download = pyqtSignal(object)
+
     def __init__(self, recipe_model, parent=None):
         QDialog.__init__(self, parent)
         self.setupUi(self)
@@ -30,6 +196,11 @@ class SchedulerDialog(QDialog, Ui_Dialog):
             _('%s news sources') %
                 self.recipe_model.showing_count)
 
+        self.schedule_widgets = []
+        for key in reversed(self.SCHEDULE_TYPES):
+            self.schedule_widgets.insert(0, self.SCHEDULE_TYPES[key](self))
+            self.schedule_stack.insertWidget(0, self.schedule_widgets[0])
+
         self.search.initialize('scheduler_search_history')
         self.search.setMinimumContentsLength(15)
         self.search.search.connect(self.recipe_model.search)
@@ -43,29 +214,43 @@ class SchedulerDialog(QDialog, Ui_Dialog):
         self.detail_box.setVisible(False)
         self.download_button.setVisible(False)
         self.recipes.currentChanged = self.current_changed
-        self.interval_button.setChecked(True)
+        for b, c in self.SCHEDULE_TYPES.iteritems():
+            b = getattr(self, b)
+            b.toggled.connect(self.schedule_type_selected)
+            b.setToolTip(textwrap.dedent(c.HELP))
+        self.days_of_week.setChecked(True)
 
-        self.connect(self.schedule, SIGNAL('stateChanged(int)'),
-                self.toggle_schedule_info)
-        self.connect(self.show_password, SIGNAL('stateChanged(int)'),
-                     lambda state: self.password.setEchoMode(self.password.Normal if state == Qt.Checked else self.password.Password))
-        self.connect(self.download_button, SIGNAL('clicked()'),
-                self.download_clicked)
-        self.connect(self.download_all_button, SIGNAL('clicked()'),
+        self.schedule.stateChanged[int].connect(self.toggle_schedule_info)
+        self.show_password.stateChanged[int].connect(self.set_pw_echo_mode)
+        self.download_button.clicked.connect(self.download_clicked)
+        self.download_all_button.clicked.connect(
                 self.download_all_clicked)
 
         self.old_news.setValue(gconf['oldest_news'])
 
+    def set_pw_echo_mode(self, state):
+        self.password.setEchoMode(self.password.Normal
+                if state == Qt.Checked else self.password.Password)
+
+
+    def schedule_type_selected(self, *args):
+        for i, st in enumerate(self.SCHEDULE_TYPES):
+            if getattr(self, st).isChecked():
+                self.schedule_stack.setCurrentIndex(i)
+                break
+
     def keyPressEvent(self, ev):
         if ev.key() not in (Qt.Key_Enter, Qt.Key_Return):
             return QDialog.keyPressEvent(self, ev)
 
     def break_cycles(self):
-        self.disconnect(self.recipe_model,  SIGNAL('searched(PyQt_PyObject)'),
-                self.search_done)
-        self.disconnect(self.recipe_model,  SIGNAL('searched(PyQt_PyObject)'),
-                self.search.search_done)
+        try:
+            self.recipe_model.searched.disconnect(self.search_done)
+            self.recipe_model.searched.disconnect(self.search.search_done)
             self.search.search.disconnect()
+            self.download.disconnect()
+        except:
+            pass
         self.recipe_model = None
 
     def search_done(self, *args):
@@ -74,8 +259,9 @@ class SchedulerDialog(QDialog, Ui_Dialog):
 
     def toggle_schedule_info(self, *args):
         enabled = self.schedule.isChecked()
-        for x in ('daily_button', 'day', 'time', 'interval_button', 'interval'):
+        for x in self.SCHEDULE_TYPES:
             getattr(self, x).setEnabled(enabled)
+        self.schedule_stack.setEnabled(enabled)
         self.last_downloaded.setVisible(enabled)
 
     def current_changed(self, current, previous):
@@ -97,14 +283,14 @@ class SchedulerDialog(QDialog, Ui_Dialog):
             return False
         return QDialog.accept(self)
 
-    def download_clicked(self):
+    def download_clicked(self, *args):
         self.commit()
         if self.commit() and self.current_urn:
-            self.emit(SIGNAL('download(PyQt_PyObject)'), self.current_urn)
+            self.download.emit(self.current_urn)
 
-    def download_all_clicked(self):
+    def download_all_clicked(self, *args):
         if self.commit() and self.commit():
-            self.emit(SIGNAL('download(PyQt_PyObject)'), None)
+            self.download.emit(None)
 
     @property
     def current_urn(self):
@@ -130,16 +316,8 @@ class SchedulerDialog(QDialog, Ui_Dialog):
                 self.recipe_model.set_account_info(urn, un, pw)
 
         if self.schedule.isChecked():
-            schedule_type = 'interval' if self.interval_button.isChecked() else 'day/time'
-            if schedule_type == 'interval':
-                schedule = self.interval.value()
-                if schedule < 0.1:
-                    schedule = 1./24.
-            else:
-                day_of_week = self.day.currentIndex() - 1
-                t = self.time.time()
-                hour, minute = t.hour(), t.minute()
-                schedule = (day_of_week, hour, minute)
+            schedule_type, schedule = \
+                    self.schedule_stack.currentWidget().schedule
             self.recipe_model.schedule_recipe(urn, schedule_type, schedule)
         else:
             self.recipe_model.un_schedule_recipe(urn)
@@ -192,27 +370,27 @@ class SchedulerDialog(QDialog, Ui_Dialog):
         self.schedule.setChecked(scheduled)
         self.toggle_schedule_info()
         self.last_downloaded.setText(_('Last downloaded: never'))
+        ld_text = _('never')
         if scheduled:
             typ, sch, last_downloaded = schedule_info
-            if typ == 'interval':
-                self.interval_button.setChecked(True)
-                self.interval.setValue(sch)
-            elif typ == 'day/time':
-                self.daily_button.setChecked(True)
-                day, hour, minute = sch
-                self.day.setCurrentIndex(day+1)
-                self.time.setTime(QTime(hour, minute))
-
             d = utcnow() - last_downloaded
             def hm(x): return (x-x%3600)//3600, (x%3600 - (x%3600)%60)//60
             hours, minutes = hm(d.seconds)
             tm = _('%d days, %d hours and %d minutes ago')%(d.days, hours, minutes)
             if d < timedelta(days=366):
-                self.last_downloaded.setText(_('Last downloaded')+': '+tm)
-
+                ld_text = tm
+        else:
+            typ, sch = 'day/time', (-1, 9, 0)
+        sch_widget = {'day/time': 0, 'days_of_week': 0, 'days_of_month':1,
+                'interval':2}[typ]
+        rb = getattr(self, list(self.SCHEDULE_TYPES)[sch_widget])
+        rb.setChecked(True)
+        self.schedule_stack.setCurrentIndex(sch_widget)
+        self.schedule_stack.currentWidget().initialize(typ, sch)
         add_title_tag, custom_tags, keep_issues = customize_info
         self.add_title_tag.setChecked(add_title_tag)
         self.custom_tags.setText(u', '.join(custom_tags))
+        self.last_downloaded.setText(_('Last downloaded:') + ' ' + ld_text)
         try:
             keep_issues = int(keep_issues)
         except:
@@ -233,7 +411,8 @@ class Scheduler(QObject):
         QObject.__init__(self, parent)
         self.internet_connection_failed = False
         self._parent = parent
-        self.recipe_model = RecipeModel(db)
+        self.recipe_model = RecipeModel()
+        self.db = db
         self.lock = QMutex(QMutex.Recursive)
         self.download_queue = set([])
 
@@ -241,9 +420,9 @@ class Scheduler(QObject):
         self.news_icon = QIcon(I('news.png'))
         self.scheduler_action = QAction(QIcon(I('scheduler.png')), _('Schedule news download'), self)
         self.news_menu.addAction(self.scheduler_action)
-        self.connect(self.scheduler_action, SIGNAL('triggered(bool)'), self.show_dialog)
+        self.scheduler_action.triggered[bool].connect(self.show_dialog)
         self.cac = QAction(QIcon(I('user_profile.png')), _('Add a custom news source'), self)
-        self.connect(self.cac, SIGNAL('triggered(bool)'), self.customize_feeds)
+        self.cac.triggered[bool].connect(self.customize_feeds)
         self.news_menu.addAction(self.cac)
         self.news_menu.addSeparator()
         self.all_action = self.news_menu.addAction(
@@ -252,10 +431,12 @@ class Scheduler(QObject):
 
         self.timer = QTimer(self)
         self.timer.start(int(self.INTERVAL * 60 * 1000))
-        self.connect(self.timer, SIGNAL('timeout()'), self.check)
+        self.timer.timeout.connect(self.check)
         self.oldest = gconf['oldest_news']
         QTimer.singleShot(5 * 1000, self.oldest_check)
-        self.database_changed = self.recipe_model.database_changed
+
+    def database_changed(self, db):
+        self.db = db
 
     def oldest_check(self):
         if self.oldest > 0:
@@ -264,10 +445,8 @@ class Scheduler(QObject):
                 ids = list(self.recipe_model.db.tags_older_than(_('News'),
                     delta))
             except:
-                # Should never happen
+                # Happens if library is being switched
                 ids = []
-                import traceback
-                traceback.print_exc()
             if ids:
                 if ids:
                     self.delete_old_news.emit(ids)
@@ -278,8 +457,7 @@ class Scheduler(QObject):
         self.lock.lock()
         try:
             d = SchedulerDialog(self.recipe_model)
-            self.connect(d, SIGNAL('download(PyQt_PyObject)'),
-                self.download_clicked)
+            d.download.connect(self.download_clicked)
             d.exec_()
             gconf['oldest_news'] = self.oldest = d.old_news.value()
             d.break_cycles()
@@ -374,7 +552,6 @@ class Scheduler(QObject):
 if __name__ == '__main__':
     from calibre.gui2 import is_ok_to_use_qt
     is_ok_to_use_qt()
-    from calibre.library.database2 import LibraryDatabase2
-    d = SchedulerDialog(RecipeModel(LibraryDatabase2('/home/kovid/documents/library')))
+    d = SchedulerDialog(RecipeModel())
     d.exec_()
 

src/calibre/gui2/dialogs/scheduler.ui

@@ -61,7 +61,7 @@
           <attribute name="title">
            <string>&amp;Schedule</string>
           </attribute>
-          <layout class="QVBoxLayout" name="verticalLayout_2">
+          <layout class="QVBoxLayout" name="verticalLayout">
            <item>
             <widget class="QLabel" name="blurb">
              <property name="text">
@@ -75,6 +75,28 @@
              </property>
             </widget>
            </item>
+           <item>
+            <spacer name="verticalSpacer_3">
+             <property name="orientation">
+              <enum>Qt::Vertical</enum>
+             </property>
+             <property name="sizeHint" stdset="0">
+              <size>
+               <width>20</width>
+               <height>40</height>
+              </size>
+             </property>
+            </spacer>
+           </item>
+           <item>
+            <widget class="QFrame" name="frame">
+             <property name="frameShape">
+              <enum>QFrame::StyledPanel</enum>
+             </property>
+             <property name="frameShadow">
+              <enum>QFrame::Raised</enum>
+             </property>
+             <layout class="QVBoxLayout" name="verticalLayout_2">
               <item>
                <widget class="QCheckBox" name="schedule">
                 <property name="text">
@@ -82,126 +104,41 @@
                 </property>
                </widget>
               </item>
-           <item>
-            <layout class="QHBoxLayout" name="horizontalLayout_2">
-             <item>
-              <widget class="QRadioButton" name="daily_button">
-               <property name="text">
-                <string>Every </string>
-               </property>
-              </widget>
-             </item>
-             <item>
-              <widget class="QComboBox" name="day">
-               <item>
-                <property name="text">
-                 <string>day</string>
-                </property>
-               </item>
-               <item>
-                <property name="text">
-                 <string>Monday</string>
-                </property>
-               </item>
-               <item>
-                <property name="text">
-                 <string>Tuesday</string>
-                </property>
-               </item>
-               <item>
-                <property name="text">
-                 <string>Wednesday</string>
-                </property>
-               </item>
-               <item>
-                <property name="text">
-                 <string>Thursday</string>
-                </property>
-               </item>
-               <item>
-                <property name="text">
-                 <string>Friday</string>
-                </property>
-               </item>
-               <item>
-                <property name="text">
-                 <string>Saturday</string>
-                </property>
-               </item>
-               <item>
-                <property name="text">
-                 <string>Sunday</string>
-                </property>
-               </item>
-              </widget>
-             </item>
-             <item>
-              <widget class="QLabel" name="label_4">
-               <property name="text">
-                <string>at</string>
-               </property>
-              </widget>
-             </item>
-             <item>
-              <widget class="QTimeEdit" name="time"/>
-             </item>
-             <item>
-              <spacer name="horizontalSpacer">
-               <property name="orientation">
-                <enum>Qt::Horizontal</enum>
-               </property>
-               <property name="sizeHint" stdset="0">
-                <size>
-                 <width>40</width>
-                 <height>20</height>
-                </size>
-               </property>
-              </spacer>
-             </item>
-            </layout>
-           </item>
               <item>
                <layout class="QHBoxLayout" name="horizontalLayout">
                 <item>
-              <widget class="QRadioButton" name="interval_button">
+                 <widget class="QRadioButton" name="days_of_week">
                   <property name="text">
-                <string>Every </string>
+                   <string>Days of  week</string>
                   </property>
                  </widget>
                 </item>
                 <item>
-              <widget class="QDoubleSpinBox" name="interval">
-               <property name="sizePolicy">
-                <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
-                 <horstretch>0</horstretch>
-                 <verstretch>0</verstretch>
-                </sizepolicy>
+                 <widget class="QRadioButton" name="days_of_month">
+                  <property name="text">
+                   <string>Days of month</string>
                   </property>
-               <property name="toolTip">
-                <string>Interval at which to download this recipe. A value of zero means that the recipe will be downloaded every hour.</string>
-               </property>
-               <property name="suffix">
-                <string> days</string>
-               </property>
-               <property name="decimals">
-                <number>1</number>
-               </property>
-               <property name="minimum">
-                <double>0.000000000000000</double>
-               </property>
-               <property name="maximum">
-                <double>365.100000000000023</double>
-               </property>
-               <property name="singleStep">
-                <double>1.000000000000000</double>
-               </property>
-               <property name="value">
-                <double>1.000000000000000</double>
+                 </widget>
+                </item>
+                <item>
+                 <widget class="QRadioButton" name="every_x_days">
+                  <property name="text">
+                   <string>Every x days</string>
                   </property>
                  </widget>
                 </item>
                </layout>
               </item>
+              <item>
+               <widget class="QStackedWidget" name="schedule_stack">
+                <property name="maximumSize">
+                 <size>
+                  <width>16777215</width>
+                  <height>75</height>
+                 </size>
+                </property>
+               </widget>
+              </item>
               <item>
                <widget class="QLabel" name="last_downloaded">
                 <property name="text">
@@ -212,6 +149,22 @@
                 </property>
                </widget>
               </item>
+             </layout>
+            </widget>
+           </item>
+           <item>
+            <spacer name="verticalSpacer_4">
+             <property name="orientation">
+              <enum>Qt::Vertical</enum>
+             </property>
+             <property name="sizeHint" stdset="0">
+              <size>
+               <width>20</width>
+               <height>40</height>
+              </size>
+             </property>
+            </spacer>
+           </item>
            <item>
             <widget class="QGroupBox" name="account">
              <property name="title">
@@ -343,6 +296,19 @@
          </widget>
         </widget>
        </item>
+       <item>
+        <spacer name="verticalSpacer_2">
+         <property name="orientation">
+          <enum>Qt::Vertical</enum>
+         </property>
+         <property name="sizeHint" stdset="0">
+          <size>
+           <width>20</width>
+           <height>40</height>
+          </size>
+         </property>
+        </spacer>
+       </item>
        <item>
         <widget class="QPushButton" name="download_button">
          <property name="text">
@@ -485,54 +451,6 @@
     </hint>
    </hints>
   </connection>
-  <connection>
-   <sender>daily_button</sender>
-   <signal>toggled(bool)</signal>
-   <receiver>day</receiver>
-   <slot>setEnabled(bool)</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>458</x>
-     <y>155</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>573</x>
-     <y>158</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>daily_button</sender>
-   <signal>toggled(bool)</signal>
-   <receiver>time</receiver>
-   <slot>setEnabled(bool)</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>458</x>
-     <y>155</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>684</x>
-     <y>157</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>interval_button</sender>
-   <signal>toggled(bool)</signal>
-   <receiver>interval</receiver>
-   <slot>setEnabled(bool)</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>458</x>
-     <y>212</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>752</x>
-     <y>215</y>
-    </hint>
-   </hints>
-  </connection>
   <connection>
    <sender>add_title_tag</sender>
    <signal>toggled(bool)</signal>

src/calibre/gui2/dialogs/user_profiles.py

@@ -6,11 +6,11 @@ import time, os
 from PyQt4.Qt import SIGNAL, QUrl, QAbstractListModel, Qt, \
         QVariant
 
-from calibre.web.feeds.recipes import compile_recipe
+from calibre.web.feeds.recipes import compile_recipe, custom_recipes
 from calibre.web.feeds.news import AutomaticNewsRecipe
 from calibre.gui2.dialogs.user_profiles_ui import Ui_Dialog
 from calibre.gui2 import error_dialog, question_dialog, open_url, \
-                         choose_files, ResizableDialog, NONE
+                         choose_files, ResizableDialog, NONE, open_local_file
 from calibre.gui2.widgets import PythonHighlighter
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.utils.icu import sort_key
@@ -93,6 +93,7 @@ class UserProfiles(ResizableDialog, Ui_Dialog):
         self.connect(self.load_button, SIGNAL('clicked()'), self.load)
         self.connect(self.builtin_recipe_button, SIGNAL('clicked()'), self.add_builtin_recipe)
         self.connect(self.share_button, SIGNAL('clicked()'), self.share)
+        self.show_recipe_files_button.clicked.connect(self.show_recipe_files)
         self.connect(self.down_button, SIGNAL('clicked()'), self.down)
         self.connect(self.up_button, SIGNAL('clicked()'), self.up)
         self.connect(self.add_profile_button, SIGNAL('clicked(bool)'),
@@ -102,6 +103,10 @@ class UserProfiles(ResizableDialog, Ui_Dialog):
         self.connect(self.toggle_mode_button, SIGNAL('clicked(bool)'), self.toggle_mode)
         self.clear()
 
+    def show_recipe_files(self, *args):
+        bdir = os.path.dirname(custom_recipes.file_path)
+        open_local_file(bdir)
+
     def break_cycles(self):
         self.recipe_model = self._model.recipe_model = None
         self.available_profiles = None
@@ -366,8 +371,7 @@ class %(classname)s(%(base_class)s):
 if __name__ == '__main__':
     from calibre.gui2 import is_ok_to_use_qt
     is_ok_to_use_qt()
-    from calibre.library.database2 import LibraryDatabase2
     from calibre.web.feeds.recipes.model import RecipeModel
-    d=UserProfiles(None, RecipeModel(LibraryDatabase2('/home/kovid/documents/library')))
+    d=UserProfiles(None, RecipeModel())
     d.exec_()
 

src/calibre/gui2/dialogs/user_profiles.ui

@@ -35,7 +35,7 @@
         <x>0</x>
         <y>0</y>
         <width>730</width>
-        <height>600</height>
+        <height>601</height>
        </rect>
       </property>
       <layout class="QVBoxLayout" name="verticalLayout_3">
@@ -102,6 +102,17 @@
                </property>
               </widget>
              </item>
+             <item>
+              <widget class="QPushButton" name="show_recipe_files_button">
+               <property name="text">
+                <string>S&amp;how recipe files</string>
+               </property>
+               <property name="icon">
+                <iconset resource="../../../../resources/images.qrc">
+                 <normaloff>:/images/document_open.png</normaloff>:/images/document_open.png</iconset>
+               </property>
+              </widget>
+             </item>
              <item>
               <widget class="QPushButton" name="builtin_recipe_button">
                <property name="text">

src/calibre/gui2/init.py

@@ -141,6 +141,15 @@ class Stack(QStackedWidget): # {{{
 
 # }}}
 
+class UpdateLabel(QLabel): # {{{
+
+    def __init__(self, *args, **kwargs):
+        QLabel.__init__(self, *args, **kwargs)
+
+    def contextMenuEvent(self, e):
+        pass
+# }}}
+
 class StatusBar(QStatusBar): # {{{
 
     def __init__(self, parent=None):
@@ -148,7 +157,7 @@ class StatusBar(QStatusBar): # {{{
         self.default_message = __appname__ + ' ' + _('version') + ' ' + \
                 self.get_version() + ' ' + _('created by Kovid Goyal')
         self.device_string = ''
-        self.update_label = QLabel('')
+        self.update_label = UpdateLabel('')
         self.addPermanentWidget(self.update_label)
         self.update_label.setVisible(False)
         self._font = QFont()
@@ -253,6 +262,8 @@ class LayoutMixin(object): # {{{
         self.status_bar.initialize(self.system_tray_icon)
         self.book_details.show_book_info.connect(self.iactions['Show Book Details'].show_book_info)
         self.book_details.files_dropped.connect(self.iactions['Add Books'].files_dropped_on_book)
+        self.book_details.cover_changed.connect(self.bd_cover_changed,
+                type=Qt.QueuedConnection)
         self.book_details.open_containing_folder.connect(self.iactions['View'].view_folder_for_id)
         self.book_details.view_specific_format.connect(self.iactions['View'].view_format_by_id)
 
@@ -263,6 +274,10 @@ class LayoutMixin(object): # {{{
                     self.library_view.currentIndex())
         self.library_view.setFocus(Qt.OtherFocusReason)
 
+    def bd_cover_changed(self, id_, cdata):
+        self.library_view.model().db.set_cover(id_, cdata)
+        if self.cover_flow:
+            self.cover_flow.dataChanged()
 
     def save_layout_state(self):
         for x in ('library', 'memory', 'card_a', 'card_b'):

src/calibre/gui2/library/models.py

@@ -616,6 +616,19 @@ class BooksModel(QAbstractTableModel): # {{{
 
         def bool_type_decorator(r, idx=-1, bool_cols_are_tristate=True):
             val = self.db.data[r][idx]
+            if isinstance(val, (str, unicode)):
+                try:
+                    val = icu_lower(val)
+                    if not val:
+                        val = None
+                    elif val in [_('yes'), _('checked'), 'true']:
+                        val = True
+                    elif val in [_('no'), _('unchecked'), 'false']:
+                        val = False
+                    else:
+                        val = bool(int(val))
+                except:
+                    val = None
             if not bool_cols_are_tristate:
                 if val is None or not val:
                     return self.bool_no_icon
@@ -674,8 +687,14 @@ class BooksModel(QAbstractTableModel): # {{{
             idx = self.custom_columns[col]['rec_index']
             datatype = self.custom_columns[col]['datatype']
             if datatype in ('text', 'comments', 'composite', 'enumeration'):
-                self.dc[col] = functools.partial(text_type, idx=idx,
-                                                 mult=self.custom_columns[col]['is_multiple'])
+                mult=self.custom_columns[col]['is_multiple']
+                self.dc[col] = functools.partial(text_type, idx=idx, mult=mult)
+                if datatype in ['text', 'composite', 'enumeration'] and not mult:
+                    if self.custom_columns[col]['display'].get('use_decorations', False):
+                        self.dc_decorator[col] = functools.partial(
+                            bool_type_decorator, idx=idx,
+                            bool_cols_are_tristate=
+                                tweaks['bool_custom_columns_are_tristate'] != 'no')
             elif datatype in ('int', 'float'):
                 self.dc[col] = functools.partial(number_type, idx=idx)
             elif datatype == 'datetime':
@@ -684,7 +703,8 @@ class BooksModel(QAbstractTableModel): # {{{
                 self.dc[col] = functools.partial(bool_type, idx=idx)
                 self.dc_decorator[col] = functools.partial(
                             bool_type_decorator, idx=idx,
-                            bool_cols_are_tristate=tweaks['bool_custom_columns_are_tristate'] != 'no')
+                            bool_cols_are_tristate=
+                                tweaks['bool_custom_columns_are_tristate'] != 'no')
             elif datatype == 'rating':
                 self.dc[col] = functools.partial(rating_type, idx=idx)
             elif datatype == 'series':

src/calibre/gui2/metadata/basic_widgets.py

@@ -351,7 +351,7 @@ class SeriesIndexEdit(QDoubleSpinBox):
         QDoubleSpinBox.__init__(self, parent)
         self.dialog = parent
         self.db = self.original_series_name = None
-        self.setMaximum(1000000)
+        self.setMaximum(10000000)
         self.series_edit = series_edit
         series_edit.currentIndexChanged.connect(self.enable)
         series_edit.editTextChanged.connect(self.enable)

src/calibre/gui2/preferences/__init__.py

@@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
 import textwrap
 
 from PyQt4.Qt import QWidget, pyqtSignal, QCheckBox, QAbstractSpinBox, \
-    QLineEdit, QComboBox, QVariant
+    QLineEdit, QComboBox, QVariant, Qt
 
 from calibre.customize.ui import preferences_plugins
 from calibre.utils.config import ConfigProxy
@@ -82,6 +82,8 @@ class ConfigWidgetInterface(object):
 
 class Setting(object):
 
+    CHOICES_SEARCH_FLAGS = Qt.MatchExactly | Qt.MatchCaseSensitive
+
     def __init__(self, name, config_obj, widget, gui_name=None,
             empty_string_is_None=True, choices=None, restart_required=False):
         self.name, self.gui_name = name, gui_name
@@ -168,7 +170,8 @@ class Setting(object):
         elif self.datatype == 'string':
             self.gui_obj.setText(val if val else '')
         elif self.datatype == 'choice':
-            idx = self.gui_obj.findData(QVariant(val))
+            idx = self.gui_obj.findData(QVariant(val), role=Qt.UserRole,
+                    flags=self.CHOICES_SEARCH_FLAGS)
             if idx == -1:
                 idx = 0
             self.gui_obj.setCurrentIndex(idx)

src/calibre/gui2/preferences/behavior.py

@@ -9,7 +9,7 @@ import re
 
 from PyQt4.Qt import Qt, QVariant, QListWidgetItem
 
-from calibre.gui2.preferences import ConfigWidgetBase, test_widget
+from calibre.gui2.preferences import ConfigWidgetBase, test_widget, Setting
 from calibre.gui2.preferences.behavior_ui import Ui_Form
 from calibre.gui2 import config, info_dialog, dynamic
 from calibre.utils.config import prefs
@@ -20,6 +20,10 @@ from calibre.ebooks.oeb.iterator import is_supported
 from calibre.constants import iswindows
 from calibre.utils.icu import sort_key
 
+class OutputFormatSetting(Setting):
+
+    CHOICES_SEARCH_FLAGS = Qt.MatchFixedString
+
 class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
     def genesis(self, gui):
@@ -43,7 +47,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         output_formats = list(sorted(available_output_formats()))
         output_formats.remove('oeb')
         choices = [(x.upper(), x) for x in output_formats]
-        r('output_format', prefs, choices=choices)
+        r('output_format', prefs, choices=choices, setting=OutputFormatSetting)
 
         restrictions = sorted(saved_searches().names(), key=sort_key)
         choices = [('', '')] + [(x, x) for x in restrictions]

src/calibre/gui2/preferences/columns.ui

@@ -38,6 +38,9 @@
     <layout class="QVBoxLayout" name="verticalLayout_3">
      <item>
       <widget class="QToolButton" name="column_up">
+       <property name="toolTip">
+        <string>Move column up</string>
+       </property>
        <property name="text">
         <string>...</string>
        </property>
@@ -45,6 +48,12 @@
         <iconset resource="../../../../resources/images.qrc">
          <normaloff>:/images/arrow-up.png</normaloff>:/images/arrow-up.png</iconset>
        </property>
+       <property name="iconSize">
+        <size>
+         <width>32</width>
+         <height>32</height>
+        </size>
+       </property>
       </widget>
      </item>
      <item>
@@ -72,6 +81,12 @@
         <iconset resource="../../../../resources/images.qrc">
          <normaloff>:/images/minus.png</normaloff>:/images/minus.png</iconset>
        </property>
+       <property name="iconSize">
+        <size>
+         <width>32</width>
+         <height>32</height>
+        </size>
+       </property>
       </widget>
      </item>
      <item>
@@ -99,6 +114,12 @@
         <iconset resource="../../../../resources/images.qrc">
          <normaloff>:/images/plus.png</normaloff>:/images/plus.png</iconset>
        </property>
+       <property name="iconSize">
+        <size>
+         <width>32</width>
+         <height>32</height>
+        </size>
+       </property>
       </widget>
      </item>
      <item>
@@ -126,6 +147,12 @@
         <iconset resource="../../../../resources/images.qrc">
          <normaloff>:/images/edit_input.png</normaloff>:/images/edit_input.png</iconset>
        </property>
+       <property name="iconSize">
+        <size>
+         <width>32</width>
+         <height>32</height>
+        </size>
+       </property>
       </widget>
      </item>
      <item>
@@ -143,6 +170,9 @@
      </item>
      <item>
       <widget class="QToolButton" name="column_down">
+       <property name="toolTip">
+        <string>Move column down</string>
+       </property>
        <property name="text">
         <string>...</string>
        </property>
@@ -150,6 +180,12 @@
         <iconset resource="../../../../resources/images.qrc">
          <normaloff>:/images/arrow-down.png</normaloff>:/images/arrow-down.png</iconset>
        </property>
+       <property name="iconSize">
+        <size>
+         <width>32</width>
+         <height>32</height>
+        </size>
+       </property>
       </widget>
      </item>
     </layout>

src/calibre/gui2/preferences/create_custom_column.py

@@ -6,7 +6,6 @@ __copyright__ = '2010, Kovid Goyal <kovid at kovidgoyal.net>'
 import re
 from functools import partial
 
-from PyQt4.QtCore import SIGNAL
 from PyQt4.Qt import QDialog, Qt, QListWidgetItem, QVariant
 
 from calibre.gui2.preferences.create_custom_column_ui import Ui_QCreateCustomColumn
@@ -48,6 +47,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
         QDialog.__init__(self, parent)
         Ui_QCreateCustomColumn.__init__(self)
         self.setupUi(self)
+        self.setWindowTitle(_('Create a custom column'))
+        self.heading_label.setText(_('Create a custom column'))
         # Remove help icon on title bar
         icon = self.windowIcon()
         self.setWindowFlags(self.windowFlags()&(~Qt.WindowContextHelpButtonHint))
@@ -55,8 +56,21 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
 
         self.simple_error = partial(error_dialog, self, show=True,
             show_copy_button=False)
-        self.connect(self.button_box, SIGNAL("accepted()"), self.accept)
-        self.connect(self.button_box, SIGNAL("rejected()"), self.reject)
+        self.button_box.accepted.connect(self.accept)
+        self.button_box.rejected.connect(self.reject)
+        self.shortcuts.linkActivated.connect(self.shortcut_activated)
+        text = '<p>'+_('Quick create:')
+        for col, name in [('isbn', _('ISBN')), ('formats', _('Formats')),
+                ('last_modified', _('Modified Date')), ('yesno', _('Yes/No')),
+                ('tags', _('Tags')), ('series', _('Series')), ('rating',
+                    _('Rating'))]:
+            text += ' <a href="col:%s">%s</a>,'%(col, name)
+        text = text[:-1]
+        self.shortcuts.setText(text)
+
+        for sort_by in [_('Text'), _('Number'), _('Date'), _('Yes/No')]:
+            self.composite_sort_by.addItem(sort_by)
+
         self.parent = parent
         self.editing_col = editing
         self.standard_colheads = standard_colheads
@@ -69,6 +83,9 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             self.datatype_changed()
             self.exec_()
             return
+        self.setWindowTitle(_('Edit a custom column'))
+        self.heading_label.setText(_('Edit a custom column'))
+        self.shortcuts.setVisible(False)
         idx = parent.opt_columns.currentRow()
         if idx < 0:
             self.simple_error(_('No column selected'),
@@ -94,11 +111,47 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
                 self.date_format_box.setText(c['display'].get('date_format', ''))
         elif ct == 'composite':
             self.composite_box.setText(c['display'].get('composite_template', ''))
+            sb = c['display'].get('composite_sort', 'text')
+            vals = ['text', 'number', 'date', 'bool']
+            if sb in vals:
+                sb = vals.index(sb)
+            else:
+                sb = 0
+            self.composite_sort_by.setCurrentIndex(sb)
         elif ct == 'enumeration':
             self.enum_box.setText(','.join(c['display'].get('enum_values', [])))
         self.datatype_changed()
+        if ct in ['text', 'composite', 'enumeration']:
+            self.use_decorations.setChecked(c['display'].get('use_decorations', False))
         self.exec_()
 
+    def shortcut_activated(self, url):
+        which = unicode(url).split(':')[-1]
+        self.column_type_box.setCurrentIndex({
+            'yesno': 9,
+            'tags' : 1,
+            'series': 3,
+            'rating': 8,
+            }.get(which, 10))
+        self.column_name_box.setText(which)
+        self.column_heading_box.setText({
+            'isbn':'ISBN',
+            'formats':_('Formats'),
+            'yesno':_('Yes/No'),
+            'tags': _('My Tags'),
+            'series': _('My Series'),
+            'rating': _('My Rating'),
+            'last_modified':_('Modified Date')}[which])
+        if self.composite_box.isVisible():
+            self.composite_box.setText(
+                {
+                    'isbn': '{identifiers:select(isbn)}',
+                    'formats': '{formats}',
+                    'last_modified':'''{last_modified:'format_date($, "dd MMM yy")'}'''
+                    }[which])
+            self.composite_sort_by.setCurrentIndex(2 if which == 'last_modified' else 0)
+
+
     def datatype_changed(self, *args):
         try:
             col_type = self.column_types[self.column_type_box.currentIndex()]['datatype']
@@ -106,10 +159,11 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             col_type = None
         for x in ('box', 'default_label', 'label'):
             getattr(self, 'date_format_'+x).setVisible(col_type == 'datetime')
-        for x in ('box', 'default_label', 'label'):
+        for x in ('box', 'default_label', 'label', 'sort_by', 'sort_by_label'):
             getattr(self, 'composite_'+x).setVisible(col_type == 'composite')
         for x in ('box', 'default_label', 'label'):
             getattr(self, 'enum_'+x).setVisible(col_type == 'enumeration')
+        self.use_decorations.setVisible(col_type in ['text', 'composite', 'enumeration'])
 
     def accept(self):
         col = unicode(self.column_name_box.text()).strip()
@@ -130,10 +184,13 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             is_multiple = False
         if not col_heading:
             return self.simple_error('', _('No column heading was provided'))
+
+        db = self.parent.gui.library_view.model().db
+        key = db.field_metadata.custom_field_prefix+col
         bad_col = False
-        if col in self.parent.custcols:
+        if key in self.parent.custcols:
             if not self.editing_col or \
-                    self.parent.custcols[col]['colnum'] != self.orig_column_number:
+                    self.parent.custcols[key]['colnum'] != self.orig_column_number:
                 bad_col = True
         if bad_col:
             return self.simple_error('', _('The lookup name %s is already used')%col)
@@ -161,7 +218,10 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             if not unicode(self.composite_box.text()).strip():
                 return self.simple_error('', _('You must enter a template for'
                     ' composite columns'))
-            display_dict = {'composite_template':unicode(self.composite_box.text()).strip()}
+            display_dict = {'composite_template':unicode(self.composite_box.text()).strip(),
+                            'composite_sort': ['text', 'number', 'date', 'bool']
+                                    [self.composite_sort_by.currentIndex()]
+                        }
         elif col_type == 'enumeration':
             if not unicode(self.enum_box.text()).strip():
                 return self.simple_error('', _('You must enter at least one'
@@ -176,8 +236,9 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
                     'list more than once').format(l[i]))
             display_dict = {'enum_values': l}
 
-        db = self.parent.gui.library_view.model().db
-        key = db.field_metadata.custom_field_prefix+col
+        if col_type in ['text', 'composite', 'enumeration']:
+            display_dict['use_decorations'] = self.use_decorations.checkState()
+
         if not self.editing_col:
             db.field_metadata
             self.parent.custcols[key] = {

src/calibre/gui2/preferences/create_custom_column.ui

@@ -9,8 +9,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>528</width>
-    <height>212</height>
+    <width>603</width>
+    <height>344</height>
    </rect>
   </property>
   <property name="sizePolicy">
@@ -19,19 +19,20 @@
     <verstretch>0</verstretch>
    </sizepolicy>
   </property>
-  <property name="windowTitle">
-   <string>Create or edit custom columns</string>
+  <property name="windowIcon">
+   <iconset resource="../../../../resources/images.qrc">
+    <normaloff>:/images/column.png</normaloff>:/images/column.png</iconset>
   </property>
   <layout class="QVBoxLayout" name="verticalLayout">
    <item>
-    <layout class="QGridLayout" name="gridLayout_2" rowstretch="0,0,0,0,0,0,0,0,0,0,0,0">
+    <layout class="QGridLayout" name="gridLayout_2" rowstretch="0,0,0,0,0,0,0,0,0,0,0,0,0,0,0">
      <property name="sizeConstraint">
       <enum>QLayout::SetDefaultConstraint</enum>
      </property>
      <property name="margin">
       <number>5</number>
      </property>
-     <item row="2" column="0">
+     <item row="5" column="0">
       <layout class="QGridLayout" name="gridLayout">
        <property name="margin">
         <number>0</number>
@@ -79,7 +80,7 @@
        <item row="2" column="0">
         <widget class="QLabel" name="label_3">
          <property name="text">
-          <string>Column &amp;type</string>
+          <string>&amp;Column type</string>
          </property>
          <property name="buddy">
           <cstring>column_type_box</cstring>
@@ -87,6 +88,8 @@
         </widget>
        </item>
        <item row="2" column="2">
+        <layout class="QHBoxLayout" name="horizontalLayout_3">
+         <item>
           <widget class="QComboBox" name="column_type_box">
            <property name="sizePolicy">
             <sizepolicy hsizetype="Maximum" vsizetype="Fixed">
@@ -105,6 +108,39 @@
            </property>
           </widget>
          </item>
+         <item>
+          <widget class="QCheckBox" name="use_decorations">
+           <property name="text">
+            <string>Show checkmarks</string>
+           </property>
+           <property name="toolTip">
+            <string>Show check marks in the GUI. Values of 'yes', 'checked', and 'true'
+will show a green check. Values of 'no', 'unchecked', and 'false' will show a red X.
+Everything else will show nothing.</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <spacer name="horizontalSpacer_27">
+           <property name="orientation">
+            <enum>Qt::Horizontal</enum>
+           </property>
+           <property name="sizePolicy">
+            <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
+             <horstretch>10</horstretch>
+             <verstretch>0</verstretch>
+            </sizepolicy>
+           </property>
+           <property name="sizeHint" stdset="0">
+            <size>
+             <width>20</width>
+             <height>0</height>
+            </size>
+           </property>
+          </spacer>
+         </item>
+        </layout>
+       </item>
        <item row="4" column="2">
         <layout class="QHBoxLayout" name="horizontalLayout_3">
          <item>
@@ -147,6 +183,16 @@
          </property>
         </widget>
        </item>
+       <item row="5" column="0">
+        <widget class="QLabel" name="composite_label">
+         <property name="text">
+          <string>&amp;Template</string>
+         </property>
+         <property name="buddy">
+          <cstring>composite_box</cstring>
+         </property>
+        </widget>
+       </item>
        <item row="5" column="2">
         <layout class="QHBoxLayout" name="horizontalLayout_4">
          <item>
@@ -174,16 +220,46 @@
          </item>
         </layout>
        </item>
-       <item row="5" column="0">
-        <widget class="QLabel" name="composite_label">
+       <item row="6" column="0">
+        <widget class="QLabel" name="composite_sort_by_label">
          <property name="text">
-          <string>&amp;Template</string>
+          <string>&amp;Sort/search column by</string>
          </property>
          <property name="buddy">
-          <cstring>composite_box</cstring>
+          <cstring>composite_sort_by</cstring>
          </property>
         </widget>
        </item>
+       <item row="6" column="2">
+        <layout class="QHBoxLayout" name="horizontalLayout_4">
+         <item>
+          <widget class="QComboBox" name="composite_sort_by">
+           <property name="toolTip">
+            <string>How this column should handled in the GUI when sorting and searching</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <spacer name="horizontalSpacer_24">
+           <property name="orientation">
+            <enum>Qt::Horizontal</enum>
+           </property>
+           <property name="sizePolicy">
+            <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
+             <horstretch>10</horstretch>
+             <verstretch>0</verstretch>
+            </sizepolicy>
+           </property>
+           <property name="sizeHint" stdset="0">
+            <size>
+             <width>20</width>
+             <height>0</height>
+            </size>
+           </property>
+          </spacer>
+         </item>
+        </layout>
+       </item>
        <item row="11" column="0" colspan="4">
         <spacer name="verticalSpacer_2">
          <property name="orientation">
@@ -238,7 +314,7 @@ four values, the first of them being the empty value.</string>
        </item>
       </layout>
      </item>
-     <item row="11" column="0">
+     <item row="14" column="0">
       <widget class="QDialogButtonBox" name="button_box">
        <property name="orientation">
         <enum>Qt::Horizontal</enum>
@@ -252,7 +328,7 @@ four values, the first of them being the empty value.</string>
       </widget>
      </item>
      <item row="1" column="0">
-      <widget class="QLabel" name="label_6">
+      <widget class="QLabel" name="heading_label">
        <property name="font">
         <font>
          <weight>75</weight>
@@ -260,7 +336,31 @@ four values, the first of them being the empty value.</string>
         </font>
        </property>
        <property name="text">
-        <string>Create or edit custom columns</string>
+        <string/>
+       </property>
+      </widget>
+     </item>
+     <item row="3" column="0">
+      <widget class="QLabel" name="shortcuts">
+       <property name="text">
+        <string/>
+       </property>
+       <property name="wordWrap">
+        <bool>true</bool>
+       </property>
+      </widget>
+     </item>
+     <item row="2" column="0">
+      <widget class="Line" name="line">
+       <property name="orientation">
+        <enum>Qt::Vertical</enum>
+       </property>
+      </widget>
+     </item>
+     <item row="4" column="0">
+      <widget class="Line" name="line_2">
+       <property name="orientation">
+        <enum>Qt::Vertical</enum>
        </property>
       </widget>
      </item>
@@ -276,6 +376,8 @@ four values, the first of them being the empty value.</string>
   <tabstop>composite_box</tabstop>
   <tabstop>button_box</tabstop>
  </tabstops>
- <resources/>
+ <resources>
+  <include location="../../../../resources/images.qrc"/>
+ </resources>
  <connections/>
 </ui>

src/calibre/gui2/preferences/look_feel.py

@@ -66,7 +66,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         choices = set([k for k in db.field_metadata.all_field_keys()
                     if db.field_metadata[k]['is_category'] and
                        db.field_metadata[k]['datatype'] in ['text', 'series', 'enumeration']])
-        choices -= set(['authors', 'publisher', 'formats', 'news'])
+        choices -= set(['authors', 'publisher', 'formats', 'news', 'identifiers'])
         self.opt_categories_using_hierarchy.update_items_cache(choices)
         r('categories_using_hierarchy', db.prefs, setting=CommaSeparatedList,
           choices=sorted(list(choices), key=sort_key))

src/calibre/gui2/tag_view.py

@@ -33,9 +33,6 @@ from calibre.gui2.dialogs.tag_list_editor import TagListEditor
 from calibre.gui2.dialogs.edit_authors_dialog import EditAuthorsDialog
 from calibre.gui2.widgets import HistoryLineEdit
 
-def original_name(t):
-    return getattr(t, 'original_name', t.name)
-
 class TagDelegate(QItemDelegate): # {{{
 
     def paint(self, painter, option, index):
@@ -240,9 +237,9 @@ class TagsView(QTreeView): # {{{
                 tag = index.tag
                 if len(index.children) > 0:
                     for c in index.children:
-                        self.add_item_to_user_cat.emit(category, original_name(c.tag),
+                        self.add_item_to_user_cat.emit(category, c.tag.original_name,
                                                c.tag.category)
-                self.add_item_to_user_cat.emit(category, original_name(tag),
+                self.add_item_to_user_cat.emit(category, tag.original_name,
                                                tag.category)
                 return
             if action == 'add_subcategory':
@@ -258,9 +255,9 @@ class TagsView(QTreeView): # {{{
                 tag = index.tag
                 if len(index.children) > 0:
                     for c in index.children:
-                        self.del_item_from_user_cat.emit(key, original_name(c.tag),
+                        self.del_item_from_user_cat.emit(key, c.tag.original_name,
                                                c.tag.category)
-                self.del_item_from_user_cat.emit(key, original_name(tag), tag.category)
+                self.del_item_from_user_cat.emit(key, tag.original_name, tag.category)
                 return
             if action == 'manage_searches':
                 self.saved_search_edit.emit(category)
@@ -403,7 +400,7 @@ class TagsView(QTreeView): # {{{
                             self.db.field_metadata[key]['is_custom']:
                     self.context_menu.addAction(_('Manage %s')%category,
                             partial(self.context_menu_handler, action='open_editor',
-                                    category=original_name(tag) if tag else None,
+                                    category=tag.original_name if tag else None,
                                     key=key))
                 elif key == 'authors':
                     self.context_menu.addAction(_('Manage %s')%category,
@@ -565,13 +562,22 @@ class TagTreeItem(object): # {{{
             self.tag = Tag(data)
             self.tag.is_hierarchical = category_key.startswith('@')
         elif self.type == self.TAG:
-            icon_map[0] = data.icon
+            self.icon_state_map[0] = QVariant(data.icon)
             self.tag = data
         if tooltip:
             self.tooltip = tooltip + ' '
         else:
             self.tooltip = ''
 
+    def break_cycles(self):
+        for x in self.children:
+            try:
+                x.break_cycles()
+            except:
+                pass
+        self.parent = self.icon_state_map = self.bold_font = self.tag = \
+                self.icon = self.children = None
+
     def __str__(self):
         if self.type == self.ROOT:
             return 'ROOT'
@@ -623,7 +629,7 @@ class TagTreeItem(object): # {{{
             while p.parent.type != self.ROOT:
                 p = p.parent
             if not tag.is_hierarchical:
-                name = original_name(tag)
+                name = tag.original_name
             else:
                 name = tag.name
             tt_author = False
@@ -635,7 +641,7 @@ class TagTreeItem(object): # {{{
             else:
                 return QVariant('[%d] %s'%(count, name))
         if role == Qt.EditRole:
-            return QVariant(original_name(tag))
+            return QVariant(tag.original_name)
         if role == Qt.DecorationRole:
             return self.icon_state_map[tag.state]
         if role == Qt.ToolTipRole:
@@ -654,12 +660,14 @@ class TagTreeItem(object): # {{{
         '''
         set_to: None => advance the state, otherwise a value from TAG_SEARCH_STATES
         '''
+        basic_search_ok = self.tag.is_editable or \
+                self.tag.category == 'formats' or self.tag.category == 'rating'
         if set_to is None:
             while True:
                 self.tag.state = (self.tag.state + 1)%5
                 if self.tag.state == TAG_SEARCH_STATES['mark_plus'] or \
                         self.tag.state == TAG_SEARCH_STATES['mark_minus']:
-                    if self.tag.is_editable:
+                    if basic_search_ok:
                         break
                 elif self.tag.state == TAG_SEARCH_STATES['mark_plusplus'] or\
                         self.tag.state == TAG_SEARCH_STATES['mark_minusminus']:
@@ -778,6 +786,7 @@ class TagsModel(QAbstractItemModel): # {{{
         self.refresh(data=data)
 
     def break_cycles(self):
+        self.root_item.break_cycles()
         self.db = self.root_item = None
 
     def mimeTypes(self):
@@ -798,7 +807,7 @@ class TagsModel(QAbstractItemModel): # {{{
                     p = node
                     while p.type != TagTreeItem.CATEGORY:
                         p = p.parent
-                    d = (node.type, p.category_key, p.is_gst, original_name(t),
+                    d = (node.type, p.category_key, p.is_gst, t.original_name,
                          t.category, path)
                 data.append(d)
             else:
@@ -849,7 +858,7 @@ class TagsModel(QAbstractItemModel): # {{{
             Copy/move an item and all its children to the destination
             '''
             copied = False
-            src_name = original_name(node.tag)
+            src_name = node.tag.original_name
             src_cat = node.tag.category
             # delete the item if the source is a user category and action is move
             if is_uc and not src_parent_is_gst and src_parent in user_cats and \
@@ -1007,7 +1016,7 @@ class TagsModel(QAbstractItemModel): # {{{
 
         fm = self.db.metadata_for_field(key)
         is_multiple = fm['is_multiple']
-        val = original_name(on_node.tag)
+        val = on_node.tag.original_name
         for id in ids:
             mi = self.db.get_metadata(id, index_is_id=True)
 
@@ -1123,7 +1132,7 @@ class TagsModel(QAbstractItemModel): # {{{
                 collapse_model = 'partition'
                 collapse_template = tweaks['categories_collapsed_popularity_template']
 
-        def process_one_node(category, state_map):
+        def process_one_node(category, state_map): # {{{
             collapse_letter = None
             category_index = self.createIndex(category.row(), 0, category)
             category_node = category_index.internalPointer()
@@ -1140,7 +1149,8 @@ class TagsModel(QAbstractItemModel): # {{{
                                 not fm['is_custom'] and \
                                 not fm['kind'] == 'user' \
                             else False
-            tt = key if fm['kind'] == 'user' else None
+            in_uc = fm['kind'] == 'user'
+            tt = key if in_uc else None
 
             if collapse_model == 'first letter':
                 # Build a list of 'equal' first letters by looking for
@@ -1215,11 +1225,10 @@ class TagsModel(QAbstractItemModel): # {{{
                 # category display order is important here. The following works
                 # only of all the non-user categories are displayed before the
                 # user categories
-                components = [t.strip() for t in original_name(tag).split('.')
+                components = [t.strip() for t in tag.original_name.split('.')
                               if t.strip()]
-                if len(components) == 0 or '.'.join(components) != original_name(tag):
-                    components = [original_name(tag)]
-                in_uc = fm['kind'] == 'user'
+                if len(components) == 0 or '.'.join(components) != tag.original_name:
+                    components = [tag.original_name]
                 if (not tag.is_hierarchical) and (in_uc or
                         key in ['authors', 'publisher', 'news', 'formats', 'rating'] or
                         key not in self.db.prefs.get('categories_using_hierarchy', []) or
@@ -1265,6 +1274,7 @@ class TagsModel(QAbstractItemModel): # {{{
                         # This id_set must not be None
                         node_parent.id_set |= tag.id_set
             return
+        # }}}
 
         for category in self.category_nodes:
             if len(category.children) > 0:
@@ -1352,7 +1362,7 @@ class TagsModel(QAbstractItemModel): # {{{
             return True
 
         key = item.tag.category
-        name = original_name(item.tag)
+        name = item.tag.original_name
         # make certain we know about the item's category
         if key not in self.db.field_metadata:
             return False
@@ -1576,10 +1586,14 @@ class TagsModel(QAbstractItemModel): # {{{
                     else:
                         prefix = ''
                     category = tag.category if key != 'news' else 'tag'
+                    add_colon = False
+                    if self.db.field_metadata[tag.category]['is_csp']:
+                        add_colon = True
+
                     if tag.name and tag.name[0] == u'\u2605': # char is a star. Assume rating
                         ans.append('%s%s:%s'%(prefix, category, len(tag.name)))
                     else:
-                        name = original_name(tag)
+                        name = tag.original_name
                         use_prefix = tag.state in [TAG_SEARCH_STATES['mark_plusplus'],
                                                    TAG_SEARCH_STATES['mark_minusminus']]
                         if category == 'tags':
@@ -1589,9 +1603,12 @@ class TagsModel(QAbstractItemModel): # {{{
                         if tag in nodes_seen:
                             continue
                         nodes_seen.add(tag)
-                        ans.append('%s%s:"=%s%s"'%(prefix, category,
-                                                '.' if use_prefix else '',
-                                                 name.replace(r'"', r'\"')))
+                        n = name.replace(r'"', r'\"')
+                        if name.startswith('.'):
+                            n = '.' + n
+                        ans.append('%s%s:"=%s%s%s"'%(prefix, category,
+                                                '.' if use_prefix else '', n,
+                                                ':' if add_colon else ''))
         return ans
 
     def find_item_node(self, key, txt, start_path, equals_match=False):
@@ -1619,7 +1636,7 @@ class TagsModel(QAbstractItemModel): # {{{
             tag = tag_item.tag
             if tag is None:
                 return False
-            name = original_name(tag)
+            name = tag.original_name
             if (equals_match and strcmp(name, txt) == 0) or \
                     (not equals_match and lower(name).find(txt) >= 0):
                 self.path_found = path
@@ -2061,6 +2078,10 @@ class TagBrowserWidget(QWidget): # {{{
                 _('Add your own categories to the Tag Browser'))
         parent.edit_categories.setStatusTip(parent.edit_categories.toolTip())
 
+        # self.leak_test_timer = QTimer(self)
+        # self.leak_test_timer.timeout.connect(self.test_for_leak)
+        # self.leak_test_timer.start(5000)
+
     def set_pane_is_visible(self, to_what):
         self.tags_view.set_pane_is_visible(to_what)
 
@@ -2122,5 +2143,13 @@ class TagBrowserWidget(QWidget): # {{{
     def not_found_label_timer_event(self):
         self.not_found_label.setVisible(False)
 
+    def test_for_leak(self):
+        from calibre.utils.mem import memory
+        import gc
+        before = memory()
+        self.tags_view.recount()
+        for i in xrange(3): gc.collect()
+        print 'Used memory:', memory(before)/(1024.), 'KB'
+
 # }}}
 

src/calibre/gui2/ui.py

@@ -296,6 +296,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
                 traceback.print_exc()
                 if ac.plugin_path is None:
                     raise
+        self.device_manager.set_current_library_uuid(db.library_id)
 
         if show_gui and self.gui_debug is not None:
             info_dialog(self, _('Debug mode'), '<p>' +
@@ -461,6 +462,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
             self.memory_view.reset()
             self.card_a_view.reset()
             self.card_b_view.reset()
+        self.device_manager.set_current_library_uuid(db.library_id)
 
 
     def set_window_title(self):

src/calibre/gui2/widgets.py

@@ -11,9 +11,9 @@ from PyQt4.Qt import QIcon, QFont, QLabel, QListWidget, QAction, \
                         QPixmap, QSplitterHandle, QToolButton, \
                         QAbstractListModel, QVariant, Qt, SIGNAL, pyqtSignal, \
                         QRegExp, QSettings, QSize, QSplitter, \
-                        QPainter, QLineEdit, QComboBox, QPen, \
+                        QPainter, QLineEdit, QComboBox, QPen, QGraphicsScene, \
                         QMenu, QStringListModel, QCompleter, QStringList, \
-                        QTimer, QRect
+                        QTimer, QRect, QFontDatabase, QGraphicsView
 
 from calibre.gui2 import NONE, error_dialog, pixmap_to_data, gprefs
 from calibre.gui2.filename_pattern_ui import Ui_Form
@@ -181,22 +181,16 @@ class FormatList(QListWidget):
         else:
             return QListWidget.keyPressEvent(self, event)
 
-
-class ImageView(QWidget):
-
-    BORDER_WIDTH = 1
-    cover_changed = pyqtSignal(object)
-
-    def __init__(self, parent=None):
-        QWidget.__init__(self, parent)
-        self._pixmap = QPixmap(self)
-        self.setMinimumSize(QSize(150, 200))
-        self.setAcceptDrops(True)
-        self.draw_border = True
-
-    # Drag 'n drop {{{
+class ImageDropMixin(object): # {{{
+    '''
+    Adds support for dropping images onto widgets and a contect menu for
+    copy/pasting images.
+    '''
     DROPABBLE_EXTENSIONS = IMAGE_EXTENSIONS
 
+    def __init__(self):
+        self.setAcceptDrops(True)
+
     @classmethod
     def paths_from_event(cls, event):
         '''
@@ -223,15 +217,59 @@ class ImageView(QWidget):
             pmap = QPixmap()
             pmap.load(path)
             if not pmap.isNull():
-                self.setPixmap(pmap)
+                self.handle_image_drop(path, pmap)
                 event.accept()
-                self.cover_changed.emit(open(path, 'rb').read())
                 break
 
+    def handle_image_drop(self, path, pmap):
+        self.set_pixmap(pmap)
+        self.cover_changed.emit(open(path, 'rb').read())
+
     def dragMoveEvent(self, event):
         event.acceptProposedAction()
+
+    def get_pixmap(self):
+        return self.pixmap()
+
+    def set_pixmap(self, pmap):
+        self.setPixmap(pmap)
+
+    def contextMenuEvent(self, ev):
+        cm = QMenu(self)
+        paste = cm.addAction(_('Paste Cover'))
+        copy = cm.addAction(_('Copy Cover'))
+        if not QApplication.instance().clipboard().mimeData().hasImage():
+            paste.setEnabled(False)
+        copy.triggered.connect(self.copy_to_clipboard)
+        paste.triggered.connect(self.paste_from_clipboard)
+        cm.exec_(ev.globalPos())
+
+    def copy_to_clipboard(self):
+        QApplication.instance().clipboard().setPixmap(self.get_pixmap())
+
+    def paste_from_clipboard(self):
+        cb = QApplication.instance().clipboard()
+        pmap = cb.pixmap()
+        if pmap.isNull() and cb.supportsSelection():
+            pmap = cb.pixmap(cb.Selection)
+        if not pmap.isNull():
+            self.set_pixmap(pmap)
+            self.cover_changed.emit(
+                    pixmap_to_data(pmap))
 # }}}
 
+class ImageView(QWidget, ImageDropMixin):
+
+    BORDER_WIDTH = 1
+    cover_changed = pyqtSignal(object)
+
+    def __init__(self, parent=None):
+        QWidget.__init__(self, parent)
+        self._pixmap = QPixmap(self)
+        self.setMinimumSize(QSize(150, 200))
+        ImageDropMixin.__init__(self)
+        self.draw_border = True
+
     def setPixmap(self, pixmap):
         if not isinstance(pixmap, QPixmap):
             raise TypeError('Must use a QPixmap')
@@ -272,34 +310,23 @@ class ImageView(QWidget):
             p.drawRect(target)
         p.end()
 
+class CoverView(QGraphicsView, ImageDropMixin):
 
-    # Clipboard copy/paste # {{{
-    def contextMenuEvent(self, ev):
-        cm = QMenu(self)
-        copy = cm.addAction(_('Copy Image'))
-        paste = cm.addAction(_('Paste Image'))
-        if not QApplication.instance().clipboard().mimeData().hasImage():
-            paste.setEnabled(False)
-        copy.triggered.connect(self.copy_to_clipboard)
-        paste.triggered.connect(self.paste_from_clipboard)
-        cm.exec_(ev.globalPos())
-
-    def copy_to_clipboard(self):
-        QApplication.instance().clipboard().setPixmap(self.pixmap())
-
-    def paste_from_clipboard(self):
-        cb = QApplication.instance().clipboard()
-        pmap = cb.pixmap()
-        if pmap.isNull() and cb.supportsSelection():
-            pmap = cb.pixmap(cb.Selection)
-        if not pmap.isNull():
-            self.setPixmap(pmap)
-            self.cover_changed.emit(
-                    pixmap_to_data(pmap))
-    # }}}
+    cover_changed = pyqtSignal(object)
 
+    def __init__(self, *args, **kwargs):
+        QGraphicsView.__init__(self, *args, **kwargs)
+        ImageDropMixin.__init__(self)
 
+    def get_pixmap(self):
+        for item in self.scene().items():
+            if hasattr(item, 'pixmap'):
+                return item.pixmap()
 
+    def set_pixmap(self, pmap):
+        self.scene = QGraphicsScene()
+        self.scene.addPixmap(pmap)
+        self.setScene(self.scene)
 
 class FontFamilyModel(QAbstractListModel):
 
@@ -312,6 +339,9 @@ class FontFamilyModel(QAbstractListModel):
             self.families = []
             print 'WARNING: Could not load fonts'
             traceback.print_exc()
+        # Restrict to Qt families as Qt tends to crash
+        qt_families = set([unicode(x) for x in QFontDatabase().families()])
+        self.families = list(qt_families.intersection(set(self.families)))
         self.families.sort()
         self.families[:0] = [_('None')]
 

src/calibre/gui2/wizard/__init__.py

@@ -51,7 +51,7 @@ class Device(object):
     @classmethod
     def set_output_format(cls):
         if cls.output_format:
-            prefs.set('output_format', cls.output_format)
+            prefs.set('output_format', cls.output_format.lower())
 
     @classmethod
     def commit(cls):

src/calibre/library/caches.py

@@ -121,11 +121,16 @@ CONTAINS_MATCH = 0
 EQUALS_MATCH   = 1
 REGEXP_MATCH   = 2
 def _match(query, value, matchkind):
+    if query.startswith('..'):
+        query = query[1:]
+        prefix_match_ok = False
+    else:
+        prefix_match_ok = True
     for t in value:
         t = icu_lower(t)
         try:     ### ignore regexp exceptions, required because search-ahead tries before typing is finished
             if (matchkind == EQUALS_MATCH):
-                if query[0] == '.':
+                if prefix_match_ok and query[0] == '.':
                     if t.startswith(query[1:]):
                         ql = len(query) - 1
                         if (len(t) == ql) or (t[ql:ql+1] == '.'):
@@ -297,14 +302,20 @@ class ResultCache(SearchQueryParser): # {{{
             for id_ in candidates:
                 item = self._data[id_]
                 if item is None: continue
-                if item[loc] is None or item[loc] <= UNDEFINED_DATE:
+                v = item[loc]
+                if isinstance(v, (str, unicode)):
+                    v = parse_date(v)
+                if v is None or v <= UNDEFINED_DATE:
                     matches.add(item[0])
             return matches
         if query == 'true':
             for id_ in candidates:
                 item = self._data[id_]
                 if item is None: continue
-                if item[loc] is not None and item[loc] > UNDEFINED_DATE:
+                v = item[loc]
+                if isinstance(v, (str, unicode)):
+                    v = parse_date(v)
+                if v is not None and v > UNDEFINED_DATE:
                     matches.add(item[0])
             return matches
 
@@ -344,7 +355,10 @@ class ResultCache(SearchQueryParser): # {{{
         for id_ in candidates:
             item = self._data[id_]
             if item is None or item[loc] is None: continue
-            if relop(item[loc], qd, field_count):
+            v = item[loc]
+            if isinstance(v, (str, unicode)):
+                v = parse_date(v)
+            if relop(v, qd, field_count):
                 matches.add(item[0])
         return matches
 
@@ -385,7 +399,7 @@ class ResultCache(SearchQueryParser): # {{{
         elif  dt == 'rating':
             cast = (lambda x: int (x))
             adjust = lambda x: x/2
-        elif dt == 'float':
+        elif dt in ('float', 'composite'):
             cast = lambda x : float (x)
             adjust = lambda x: x
         else: # count operation
@@ -408,19 +422,22 @@ class ResultCache(SearchQueryParser): # {{{
             item = self._data[id_]
             if item is None:
                 continue
-            v = val_func(item)
+            try:
+                v = cast(val_func(item))
+            except:
+                v = 0
             if not v:
-                i = 0
+                v = 0
             else:
-                i = adjust(v)
-            if relop(i, q):
+                v = adjust(v)
+            if relop(v, q):
                 matches.add(item[0])
         return matches
 
     def get_user_category_matches(self, location, query, candidates):
-        res = set([])
+        matches = set([])
         if self.db_prefs is None or len(query) < 2:
-            return  res
+            return  matches
         user_cats = self.db_prefs.get('user_categories', [])
         c = set(candidates)
 
@@ -435,10 +452,118 @@ class ResultCache(SearchQueryParser): # {{{
                 for (item, category, ign) in user_cats[key]:
                     s = self.get_matches(category, '=' + item, candidates=c)
                     c -= s
-                    res |= s
+                    matches |= s
         if query == 'false':
-            return candidates - res
-        return res
+            return candidates - matches
+        return matches
+
+    def get_keypair_matches(self, location, query, candidates):
+        matches = set([])
+        if query.find(':') >= 0:
+            q = [q.strip() for q in query.split(':')]
+            if len(q) != 2:
+                raise ParseException(query, len(query),
+                        'Invalid query format for colon-separated search', self)
+            (keyq, valq) = q
+            keyq_mkind, keyq = self._matchkind(keyq)
+            valq_mkind, valq = self._matchkind(valq)
+        else:
+            keyq = keyq_mkind = ''
+            valq_mkind, valq = self._matchkind(query)
+
+        loc = self.field_metadata[location]['rec_index']
+        split_char = self.field_metadata[location]['is_multiple']
+        for id_ in candidates:
+            item = self._data[id_]
+            if item is None:
+                continue
+
+            if item[loc] is None:
+                if valq == 'false':
+                    matches.add(id_)
+                continue
+
+            pairs = [p.strip() for p in item[loc].split(split_char)]
+            for pair in pairs:
+                parts = pair.split(':')
+                if len(parts) != 2:
+                    continue
+                k = parts[:1]
+                v = parts[1:]
+                if keyq and not _match(keyq, k, keyq_mkind):
+                    continue
+                if valq:
+                    if valq == 'true':
+                        if not v:
+                            continue
+                    elif valq == 'false':
+                        if v:
+                            continue
+                    elif not _match(valq, v, valq_mkind):
+                        continue
+                matches.add(id_)
+        return matches
+
+    def _matchkind(self, query):
+        matchkind = CONTAINS_MATCH
+        if (len(query) > 1):
+            if query.startswith('\\'):
+                query = query[1:]
+            elif query.startswith('='):
+                matchkind = EQUALS_MATCH
+                query = query[1:]
+            elif query.startswith('~'):
+                matchkind = REGEXP_MATCH
+                query = query[1:]
+
+        if matchkind != REGEXP_MATCH:
+            # leave case in regexps because it can be significant e.g. \S \W \D
+            query = icu_lower(query)
+        return matchkind, query
+
+    def get_bool_matches(self, location, query, candidates):
+        bools_are_tristate = tweaks['bool_custom_columns_are_tristate'] != 'no'
+        loc = self.field_metadata[location]['rec_index']
+        matches = set()
+        query = icu_lower(query)
+        for id_ in candidates:
+            item = self._data[id_]
+            if item is None:
+                continue
+
+            val = item[loc]
+            if isinstance(val, (str, unicode)):
+                try:
+                    val = icu_lower(val)
+                    if not val:
+                        val = None
+                    elif val in [_('yes'), _('checked'), 'true']:
+                        val = True
+                    elif val in [_('no'), _('unchecked'), 'false']:
+                        val = False
+                    else:
+                        val = bool(int(val))
+                except:
+                    val = None
+
+            if not bools_are_tristate:
+                if val is None or not val: # item is None or set to false
+                    if query in [_('no'), _('unchecked'), 'false']:
+                        matches.add(item[0])
+                else: # item is explicitly set to true
+                    if query in [_('yes'), _('checked'), 'true']:
+                        matches.add(item[0])
+            else:
+                if val is None:
+                    if query in [_('empty'), _('blank'), 'false']:
+                        matches.add(item[0])
+                elif not val: # is not None and false
+                    if query in [_('no'), _('unchecked'), 'true']:
+                        matches.add(item[0])
+                else: # item is not None and true
+                    if query in [_('yes'), _('checked'), 'true']:
+                        matches.add(item[0])
+        return matches
 
     def get_matches(self, location, query, candidates=None,
             allow_recursion=True):
@@ -455,6 +580,7 @@ class ResultCache(SearchQueryParser): # {{{
         if query and query.strip():
             # get metadata key associated with the search term. Eliminates
             # dealing with plurals and other aliases
+            original_location = location
             location = self.field_metadata.search_term_to_field_key(icu_lower(location.strip()))
             # grouped search terms
             if isinstance(location, list):
@@ -489,13 +615,20 @@ class ResultCache(SearchQueryParser): # {{{
             if location in self.field_metadata:
                 fm = self.field_metadata[location]
                 # take care of dates special case
-                if fm['datatype'] == 'datetime':
+                if fm['datatype'] == 'datetime' or \
+                        (fm['datatype'] == 'composite' and
+                         fm['display'].get('composite_sort', '') == 'date'):
                     return self.get_dates_matches(location, query.lower(), candidates)
 
                 # take care of numbers special case
-                if fm['datatype'] in ('rating', 'int', 'float'):
+                if fm['datatype'] in ('rating', 'int', 'float') or \
+                        (fm['datatype'] == 'composite' and
+                         fm['display'].get('composite_sort', '') == 'number'):
                     return self.get_numeric_matches(location, query.lower(), candidates)
 
+                if fm['datatype'] == 'bool':
+                    return self.get_bool_matches(location, query, candidates)
+
                 # take care of the 'count' operator for is_multiples
                 if fm['is_multiple'] and \
                         len(query) > 1 and query.startswith('#') and \
@@ -505,24 +638,20 @@ class ResultCache(SearchQueryParser): # {{{
                     return self.get_numeric_matches(location, query[1:],
                                                     candidates, val_func=vf)
 
+                # special case: colon-separated fields such as identifiers. isbn
+                # is a special case within the case
+                if fm.get('is_csp', False):
+                    if location == 'identifiers' and original_location == 'isbn':
+                        return self.get_keypair_matches('identifiers',
+                                                   '=isbn:'+query, candidates)
+                    return self.get_keypair_matches(location, query, candidates)
+
             # check for user categories
             if len(location) >= 2 and location.startswith('@'):
                 return self.get_user_category_matches(location[1:], query.lower(),
                                                       candidates)
             # everything else, or 'all' matches
-            matchkind = CONTAINS_MATCH
-            if (len(query) > 1):
-                if query.startswith('\\'):
-                    query = query[1:]
-                elif query.startswith('='):
-                    matchkind = EQUALS_MATCH
-                    query = query[1:]
-                elif query.startswith('~'):
-                    matchkind = REGEXP_MATCH
-                    query = query[1:]
-            if matchkind != REGEXP_MATCH:
-                # leave case in regexps because it can be significant e.g. \S \W \D
-                query = icu_lower(query)
+            matchkind, query = self._matchkind(query)
 
             if not isinstance(query, unicode):
                 query = query.decode('utf-8')
@@ -553,9 +682,6 @@ class ResultCache(SearchQueryParser): # {{{
             for i, loc in enumerate(location):
                 location[i] = db_col[loc]
 
-            # get the tweak here so that the string lookup and compare aren't in the loop
-            bools_are_tristate = tweaks['bool_custom_columns_are_tristate'] != 'no'
-
             for loc in location: # location is now an array of field indices
                 if loc == db_col['authors']:
                     ### DB stores authors with commas changed to bars, so change query
@@ -567,27 +693,6 @@ class ResultCache(SearchQueryParser): # {{{
                     item = self._data[id_]
                     if item is None: continue
 
-                    if col_datatype[loc] == 'bool': # complexity caused by the two-/three-value tweak
-                        v = item[loc]
-                        if not bools_are_tristate:
-                            if v is None or not v: # item is None or set to false
-                                if q in [_('no'), _('unchecked'), 'false']:
-                                    matches.add(item[0])
-                            else: # item is explicitly set to true
-                                if q in [_('yes'), _('checked'), 'true']:
-                                    matches.add(item[0])
-                        else:
-                            if v is None:
-                                if q in [_('empty'), _('blank'), 'false']:
-                                    matches.add(item[0])
-                            elif not v: # is not None and false
-                                if q in [_('no'), _('unchecked'), 'true']:
-                                    matches.add(item[0])
-                            else: # item is not None and true
-                                if q in [_('yes'), _('checked'), 'true']:
-                                    matches.add(item[0])
-                        continue
-
                     if not item[loc]:
                         if q == 'false':
                             matches.add(item[0])
@@ -827,6 +932,34 @@ class SortKeyGenerator(object):
         for name, fm in self.entries:
             dt = fm['datatype']
             val = record[fm['rec_index']]
+            if dt == 'composite':
+                sb = fm['display'].get('composite_sort', 'text')
+                if sb == 'date':
+                    try:
+                        val = parse_date(val)
+                        dt = 'datetime'
+                    except:
+                        pass
+                elif sb == 'number':
+                    try:
+                        val = float(val)
+                    except:
+                        val = 0.0
+                    dt = 'float'
+                elif sb == 'bool':
+                    try:
+                        v = icu_lower(val)
+                        if not val:
+                            val = None
+                        elif v in [_('yes'), _('checked'), 'true']:
+                            val = True
+                        elif v in [_('no'), _('unchecked'), 'false']:
+                            val = False
+                        else:
+                            val = bool(int(val))
+                    except:
+                        val = None
+                    dt = 'bool'
 
             if dt == 'datetime':
                 if val is None:

src/calibre/library/catalog.py

@@ -5103,6 +5103,19 @@ Author '{0}':
                 recommendations.append(('book_producer',opts.output_profile,
                     OptionRecommendation.HIGH))
 
+            # If cover exists, use it
+            try:
+                search_text = 'title:"%s" author:%s' % (
+                        opts.catalog_title.replace('"', '\\"'), 'calibre')
+                matches = db.search(search_text, return_matches=True)
+                if matches:
+                    cpath = db.cover(matches[0], index_is_id=True, as_path=True)
+                    if cpath and os.path.exists(cpath):
+                        recommendations.append(('cover', cpath,
+                            OptionRecommendation.HIGH))
+            except:
+                pass
+
             # Run ebook-convert
             from calibre.ebooks.conversion.plumber import Plumber
             plumber = Plumber(os.path.join(catalog.catalogPath,

src/calibre/library/cli.py

@@ -20,7 +20,8 @@ from calibre.utils.date import isoformat
 
 FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating',
     'timestamp', 'size', 'tags', 'comments', 'series', 'series_index',
-    'formats', 'isbn', 'uuid', 'pubdate', 'cover'])
+    'formats', 'isbn', 'uuid', 'pubdate', 'cover', 'last_modified',
+    'identifiers'])
 
 def send_message(msg=''):
     prints('Notifying calibre of the change')

src/calibre/library/custom_columns.py

@@ -188,7 +188,7 @@ class CustomColumns(object):
                     table=tn, column='value', datatype=v['datatype'],
                     colnum=v['num'], name=v['name'], display=v['display'],
                     is_multiple=is_m, is_category=is_category,
-                    is_editable=v['editable'])
+                    is_editable=v['editable'], is_csp=False)
 
     def get_custom(self, idx, label=None, num=None, index_is_id=False):
         if label is not None:

src/calibre/library/database2.py

@@ -6,7 +6,8 @@ __docformat__ = 'restructuredtext en'
 '''
 The database used to store ebook metadata
 '''
-import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, json
+import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, \
+        json, uuid
 import threading, random
 from itertools import repeat
 from math import ceil
@@ -47,7 +48,7 @@ class Tag(object):
 
     def __init__(self, name, id=None, count=0, state=0, avg=0, sort=None,
                  tooltip=None, icon=None, category=None, id_set=None):
-        self.name = name
+        self.name = self.original_name = name
         self.id = id
         self.count = count
         self.state = state
@@ -94,6 +95,31 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
         return property(doc=doc, fget=fget, fset=fset)
 
+    @dynamic_property
+    def library_id(self):
+        doc = ('The UUID for this library. As long as the user only operates'
+                ' on libraries with calibre, it will be unique')
+
+        def fget(self):
+            if self._library_id_ is None:
+                ans = self.conn.get('SELECT uuid FROM library_id', all=False)
+                if ans is None:
+                    ans = str(uuid.uuid4())
+                    self.library_id = ans
+                else:
+                    self._library_id_ = ans
+            return self._library_id_
+
+        def fset(self, val):
+            self._library_id_ = unicode(val)
+            self.conn.executescript('''
+                    DELETE FROM library_id;
+                    INSERT INTO library_id (uuid) VALUES ("%s");
+                    '''%self._library_id_)
+            self.conn.commit()
+
+        return property(doc=doc, fget=fget, fset=fset)
+
     def connect(self):
         if 'win32' in sys.platform and len(self.library_path) + 4*self.PATH_LIMIT + 10 > 259:
             raise ValueError('Path to library too long. Must be less than %d characters.'%(259-4*self.PATH_LIMIT-10))
@@ -120,6 +146,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
     def __init__(self, library_path, row_factory=False, default_prefs=None,
             read_only=False):
         self.field_metadata = FieldMetadata()
+        self._library_id_ = None
         # Create the lock to be used to guard access to the metadata writer
         # queues. This must be an RLock, not a Lock
         self.dirtied_lock = threading.RLock()
@@ -148,6 +175,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.is_case_sensitive = not iswindows and not isosx and \
             not os.path.exists(self.dbpath.replace('metadata.db', 'MeTAdAtA.dB'))
         SchemaUpgrade.__init__(self)
+        # Guarantee that the library_id is set
+        self.library_id
 
         # if we are to copy the prefs and structure from some other DB, then
         # we need to do it before we call initialize_dynamic
@@ -293,14 +322,14 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
              'sort',
              'author_sort',
              '(SELECT group_concat(format) FROM data WHERE data.book=books.id) formats',
-             'isbn',
              'path',
-             'lccn',
              'pubdate',
-             'flags',
              'uuid',
              'has_cover',
-            ('au_map', 'authors', 'author', 'aum_sortconcat(link.id, authors.name, authors.sort)')
+            ('au_map', 'authors', 'author',
+                'aum_sortconcat(link.id, authors.name, authors.sort)'),
+            'last_modified',
+            '(SELECT identifiers_concat(type, val) FROM identifiers WHERE identifiers.book=books.id) identifiers',
             ]
         lines = []
         for col in columns:
@@ -318,8 +347,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.FIELD_MAP = {'id':0, 'title':1, 'authors':2, 'timestamp':3,
              'size':4, 'rating':5, 'tags':6, 'comments':7, 'series':8,
              'publisher':9, 'series_index':10, 'sort':11, 'author_sort':12,
-             'formats':13, 'isbn':14, 'path':15, 'lccn':16, 'pubdate':17,
-             'flags':18, 'uuid':19, 'cover':20, 'au_map':21}
+             'formats':13, 'path':14, 'pubdate':15, 'uuid':16, 'cover':17,
+             'au_map':18, 'last_modified':19, 'identifiers':20}
 
         for k,v in self.FIELD_MAP.iteritems():
             self.field_metadata.set_field_record_index(k, v, prefer_custom=False)
@@ -391,11 +420,16 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.has_id  = self.data.has_id
         self.count   = self.data.count
 
-        for prop in ('author_sort', 'authors', 'comment', 'comments', 'isbn',
+        for prop in (
+                'author_sort', 'authors', 'comment', 'comments',
                 'publisher', 'rating', 'series', 'series_index', 'tags',
-                     'title', 'timestamp', 'uuid', 'pubdate', 'ondevice'):
+                'title', 'timestamp', 'uuid', 'pubdate', 'ondevice',
+                'metadata_last_modified',
+                ):
+            fm = {'comment':'comments', 'metadata_last_modified':
+                    'last_modified'}.get(prop, prop)
             setattr(self, prop, functools.partial(self.get_property,
-                    loc=self.FIELD_MAP['comments' if prop == 'comment' else prop]))
+                    loc=self.FIELD_MAP[fm]))
         setattr(self, 'title_sort', functools.partial(self.get_property,
                 loc=self.FIELD_MAP['sort']))
 
@@ -681,8 +715,20 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if commit:
             self.conn.commit()
 
+    def update_last_modified(self, book_ids, commit=False, now=None):
+        if now is None:
+            now = nowf()
+        if book_ids:
+            self.conn.executemany(
+                'UPDATE books SET last_modified=? WHERE id=?',
+                [(now, book) for book in book_ids])
+            for book_id in book_ids:
+                self.data.set(book_id, self.FIELD_MAP['last_modified'], now, row_is_id=True)
+            if commit:
+                self.conn.commit()
+
     def dirtied(self, book_ids, commit=True):
-        changed = False
+        self.update_last_modified(book_ids)
         for book in book_ids:
             with self.dirtied_lock:
                 # print 'dirtied: check id', book
@@ -691,21 +737,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                     self.dirtied_sequence += 1
                     continue
                 # print 'book not already dirty'
-                try:
+
                 self.conn.execute(
-                        'INSERT INTO metadata_dirtied (book) VALUES (?)',
+                    'INSERT OR IGNORE INTO metadata_dirtied (book) VALUES (?)',
                     (book,))
-                    changed = True
-                except IntegrityError:
-                    # Already in table
-                    pass
                 self.dirtied_cache[book] = self.dirtied_sequence
                 self.dirtied_sequence += 1
+
         # If the commit doesn't happen, then the DB table will be wrong. This
         # could lead to a problem because on restart, we won't put the book back
         # into the dirtied_cache. We deal with this by writing the dirtied_cache
         # back to the table on GUI exit. Not perfect, but probably OK
-        if commit and changed:
+        if book_ids and commit:
             self.conn.commit()
 
     def get_a_dirtied_book(self):
@@ -790,6 +833,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         mi.pubdate     = row[fm['pubdate']]
         mi.uuid        = row[fm['uuid']]
         mi.title_sort  = row[fm['sort']]
+        mi.last_modified = row[fm['last_modified']]
         formats = row[fm['formats']]
         if not formats:
             formats = None
@@ -803,8 +847,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if mi.series:
             mi.series_index = row[fm['series_index']]
         mi.rating = row[fm['rating']]
-        mi.isbn = row[fm['isbn']]
         id = idx if index_is_id else self.id(idx)
+        mi.set_identifiers(self.get_identifiers(id, index_is_id=True))
         mi.application_id = id
         mi.id = id
         for key, meta in self.field_metadata.custom_iteritems():
@@ -911,10 +955,14 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             except (IOError, OSError):
                 time.sleep(0.2)
                 save_cover_data_to(data, path)
-        self.conn.execute('UPDATE books SET has_cover=1 WHERE id=?', (id,))
+        now = nowf()
+        self.conn.execute(
+            'UPDATE books SET has_cover=1,last_modified=? WHERE id=?',
+              (now, id))
         if commit:
             self.conn.commit()
         self.data.set(id, self.FIELD_MAP['cover'], True, row_is_id=True)
+        self.data.set(id, self.FIELD_MAP['last_modified'], now, row_is_id=True)
         if notify:
             self.notify('cover', [id])
 
@@ -923,8 +971,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
     def set_has_cover(self, id, val):
         dval = 1 if val else 0
-        self.conn.execute('UPDATE books SET has_cover=? WHERE id=?', (dval, id,))
+        now = nowf()
+        self.conn.execute(
+                'UPDATE books SET has_cover=?,last_modified=? WHERE id=?',
+                (dval, now, id))
         self.data.set(id, self.FIELD_MAP['cover'], val, row_is_id=True)
+        self.data.set(id, self.FIELD_MAP['last_modified'], now, row_is_id=True)
 
     def book_on_device(self, id):
         if callable(self.book_on_device_func):
@@ -1135,12 +1187,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.clean_custom()
         self.conn.commit()
 
-    def get_recipes(self):
-        return self.conn.get('SELECT id, script FROM feeds')
-
-    def get_recipe(self, id):
-        return self.conn.get('SELECT script FROM feeds WHERE id=?', (id,), all=False)
-
     def get_books_for_category(self, category, id_):
         ans = set([])
 
@@ -1196,6 +1242,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             else:
                 new_cats['.'.join(comps)] = user_cats[k]
         try:
+            if new_cats != user_cats:
                 self.prefs.set('user_categories', new_cats)
         except:
             pass
@@ -1221,7 +1268,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         for category in tb_cats.keys():
             cat = tb_cats[category]
             if not cat['is_category'] or cat['kind'] in ['user', 'search'] \
-                    or category in ['news', 'formats']:
+                    or category in ['news', 'formats'] or cat.get('is_csp',
+                            False):
                 continue
             # Get the ids for the item values
             if not cat['is_custom']:
@@ -1439,6 +1487,34 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             # No need for ICU here.
             categories['formats'].sort(key = lambda x:x.name)
 
+        # Now do identifiers. This works like formats
+        categories['identifiers'] = []
+        icon = None
+        if icon_map and 'identifiers' in icon_map:
+                icon = icon_map['identifiers']
+        for ident in self.conn.get('SELECT DISTINCT type FROM identifiers'):
+            ident = ident[0]
+            if ids is not None:
+                count = self.conn.get('''SELECT COUNT(book)
+                                       FROM identifiers
+                                       WHERE type="%s" AND
+                                       books_list_filter(book)'''%ident,
+                                       all=False)
+            else:
+                count = self.conn.get('''SELECT COUNT(id)
+                                       FROM identifiers
+                                       WHERE type="%s"'''%ident,
+                                       all=False)
+            if count > 0:
+                categories['identifiers'].append(Tag(ident, count=count, icon=icon,
+                                                 category='identifiers'))
+
+        if sort == 'popularity':
+            categories['identifiers'].sort(key=lambda x: x.count, reverse=True)
+        else: # no ratings exist to sort on
+            # No need for ICU here.
+            categories['identifiers'].sort(key = lambda x:x.name)
+
         #### Now do the user-defined categories. ####
         user_categories = dict.copy(self.clean_user_categories())
 
@@ -1651,8 +1727,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             doit(self.set_tags, id, mi.tags, notify=False, commit=False)
         if mi.comments:
             doit(self.set_comment, id, mi.comments, notify=False, commit=False)
-        if mi.isbn and mi.isbn.strip():
-            doit(self.set_isbn, id, mi.isbn, notify=False, commit=False)
         if mi.series_index:
             doit(self.set_series_index, id, mi.series_index, notify=False,
                     commit=False)
@@ -1662,6 +1736,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             doit(self.set_timestamp, id, mi.timestamp, notify=False,
                     commit=False)
 
+        mi_idents = mi.get_identifiers()
+        if mi_idents:
+            identifiers = self.get_identifiers(id, index_is_id=True)
+            for key, val in mi_idents.iteritems():
+                if val and val.strip(): # Don't delete an existing identifier
+                    identifiers[icu_lower(key)] = val
+            self.set_identifiers(id, identifiers, notify=False, commit=False)
+
+
         user_mi = mi.get_all_user_metadata(make_copy=False)
         for key in user_mi.iterkeys():
             if key in self.field_metadata and \
@@ -2440,14 +2523,84 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if notify:
             self.notify('metadata', [id])
 
-    def set_isbn(self, id, isbn, notify=True, commit=True):
-        self.conn.execute('UPDATE books SET isbn=? WHERE id=?', (isbn, id))
-        self.dirtied([id], commit=False)
+    def isbn(self, idx, index_is_id=False):
+        row = self.data._data[idx] if index_is_id else self.data[idx]
+        if row is not None:
+            raw = row[self.FIELD_MAP['identifiers']]
+            if raw:
+                for x in raw.split(','):
+                    if x.startswith('isbn:'):
+                        return x[5:].strip()
+
+    def get_identifiers(self, idx, index_is_id=False):
+        ans = {}
+        row = self.data._data[idx] if index_is_id else self.data[idx]
+        if row is not None:
+            raw = row[self.FIELD_MAP['identifiers']]
+            if raw:
+                for x in raw.split(','):
+                    key, _, val = x.partition(':')
+                    key, val = key.strip(), val.strip()
+                    if key and val:
+                        ans[key] = val
+
+        return ans
+
+    def _clean_identifier(self, typ, val):
+        typ = icu_lower(typ).strip().replace(':', '').replace(',', '')
+        val = val.strip().replace(',', '|').replace(':', '|')
+        return typ, val
+
+    def set_identifier(self, id_, typ, val, notify=True, commit=True):
+        'If val is empty, deletes identifier of type typ'
+        typ, val = self._clean_identifier(typ, val)
+        identifiers = self.get_identifiers(id_, index_is_id=True)
+        if not typ:
+            return
+        changed = False
+        if not val and typ in identifiers:
+            identifiers.pop(typ)
+            changed = True
+            self.conn.execute(
+                'DELETE from identifiers WHERE book=? AND type=?',
+                    (id_, typ))
+        if val and identifiers.get(typ, None) != val:
+            changed = True
+            identifiers[typ] = val
+            self.conn.execute(
+                'INSERT OR REPLACE INTO identifiers (book, type, val) VALUES (?, ?, ?)',
+                    (id_, typ, val))
+        if changed:
+            raw = ','.join(['%s:%s'%(k, v) for k, v in
+                identifiers.iteritems()])
+            self.data.set(id_, self.FIELD_MAP['identifiers'], raw,
+                    row_is_id=True)
             if commit:
                 self.conn.commit()
-        self.data.set(id, self.FIELD_MAP['isbn'], isbn, row_is_id=True)
             if notify:
-            self.notify('metadata', [id])
+                self.notify('metadata', [id_])
+
+    def set_identifiers(self, id_, identifiers, notify=True, commit=True):
+        cleaned = {}
+        for typ, val in identifiers.iteritems():
+            typ, val = self._clean_identifier(typ, val)
+            if val:
+                cleaned[typ] = val
+        self.conn.execute('DELETE FROM identifiers WHERE book=?', (id_,))
+        self.conn.executemany(
+            'INSERT INTO identifiers (book, type, val) VALUES (?, ?, ?)',
+            [(id_, k, v) for k, v in cleaned.iteritems()])
+        raw = ','.join(['%s:%s'%(k, v) for k, v in
+                cleaned.iteritems()])
+        self.data.set(id_, self.FIELD_MAP['identifiers'], raw,
+                    row_is_id=True)
+        if commit:
+            self.conn.commit()
+        if notify:
+            self.notify('metadata', [id_])
+
+    def set_isbn(self, id_, isbn, notify=True, commit=True):
+        self.set_identifier(id_, 'isbn', isbn, notify=notify, commit=commit)
 
     def add_catalog(self, path, title):
         format = os.path.splitext(path)[1][1:].lower()
@@ -2745,7 +2898,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             prefix = self.library_path
         FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating',
             'timestamp', 'size', 'tags', 'comments', 'series', 'series_index',
-            'isbn', 'uuid', 'pubdate'])
+            'uuid', 'pubdate', 'last_modified', 'identifiers'])
         for x in self.custom_column_num_map:
             FIELDS.add(x)
         data = []
@@ -2760,6 +2913,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             data.append(x)
             x['id'] = db_id
             x['formats'] = []
+            isbn = self.isbn(db_id, index_is_id=True)
+            x['isbn'] = isbn if isbn else ''
             if not x['authors']:
                 x['authors'] = _('Unknown')
             x['authors'] = [i.replace('|', ',') for i in x['authors'].split(',')]
@@ -2952,8 +3107,4 @@ books_series_link      feeds
         s = self.conn.get('''SELECT book FROM books_plugin_data WHERE name=?''', (name,))
         return [x[0] for x in s]
 
-    def get_custom_recipes(self):
-        for id, title, script in self.conn.get('SELECT id,title,script FROM feeds'):
-            yield id, title, script
-
 

src/calibre/library/field_metadata.py

@@ -16,7 +16,8 @@ class TagsIcons(dict):
     '''
 
     category_icons = ['authors', 'series', 'formats', 'publisher', 'rating',
-                      'news',    'tags',   'custom:', 'user:',     'search',]
+                      'news',    'tags',   'custom:', 'user:',     'search',
+                      'identifiers']
     def __init__(self, icon_dict):
         for a in self.category_icons:
             if a not in icon_dict:
@@ -33,7 +34,8 @@ category_icon_map = {
                     'tags'       : 'tags.png',
                     'custom:'    : 'column.png',
                     'user:'      : 'tb_folder.png',
-                    'search'    : 'search.png'
+                    'search'     : 'search.png',
+                    'identifiers': 'id_card.png'
             }
 
 
@@ -80,6 +82,8 @@ class FieldMetadata(dict):
 
     rec_index: the index of the field in the db metadata record.
 
+    is_csp: field contains colon-separated pairs. Must also be text, is_multiple
+
     '''
 
     VALID_DATA_TYPES = frozenset([None, 'rating', 'text', 'comments', 'datetime',
@@ -98,7 +102,8 @@ class FieldMetadata(dict):
                            'name':_('Authors'),
                            'search_terms':['authors', 'author'],
                            'is_custom':False,
-                           'is_category':True}),
+                           'is_category':True,
+                           'is_csp': False}),
             ('series',    {'table':'series',
                            'column':'name',
                            'link_column':'series',
@@ -109,7 +114,8 @@ class FieldMetadata(dict):
                            'name':_('Series'),
                            'search_terms':['series'],
                            'is_custom':False,
-                           'is_category':True}),
+                           'is_category':True,
+                           'is_csp': False}),
             ('formats',   {'table':None,
                            'column':None,
                            'datatype':'text',
@@ -118,7 +124,8 @@ class FieldMetadata(dict):
                            'name':_('Formats'),
                            'search_terms':['formats', 'format'],
                            'is_custom':False,
-                           'is_category':True}),
+                           'is_category':True,
+                           'is_csp': False}),
             ('publisher', {'table':'publishers',
                            'column':'name',
                            'link_column':'publisher',
@@ -129,7 +136,8 @@ class FieldMetadata(dict):
                            'name':_('Publishers'),
                            'search_terms':['publisher'],
                            'is_custom':False,
-                           'is_category':True}),
+                           'is_category':True,
+                           'is_csp': False}),
             ('rating',    {'table':'ratings',
                            'column':'rating',
                            'link_column':'rating',
@@ -140,7 +148,8 @@ class FieldMetadata(dict):
                            'name':_('Ratings'),
                            'search_terms':['rating'],
                            'is_custom':False,
-                           'is_category':True}),
+                           'is_category':True,
+                           'is_csp': False}),
             ('news',      {'table':'news',
                            'column':'name',
                            'category_sort':'name',
@@ -150,7 +159,8 @@ class FieldMetadata(dict):
                            'name':_('News'),
                            'search_terms':[],
                            'is_custom':False,
-                           'is_category':True}),
+                           'is_category':True,
+                           'is_csp': False}),
             ('tags',      {'table':'tags',
                            'column':'name',
                            'link_column': 'tag',
@@ -161,7 +171,18 @@ class FieldMetadata(dict):
                            'name':_('Tags'),
                            'search_terms':['tags', 'tag'],
                            'is_custom':False,
-                           'is_category':True}),
+                           'is_category':True,
+                           'is_csp': False}),
+            ('identifiers',   {'table':None,
+                           'column':None,
+                           'datatype':'text',
+                           'is_multiple':',',
+                           'kind':'field',
+                           'name':_('Identifiers'),
+                           'search_terms':['identifiers', 'identifier', 'isbn'],
+                           'is_custom':False,
+                           'is_category':True,
+                           'is_csp': True}),
             ('author_sort',{'table':None,
                             'column':None,
                             'datatype':'text',
@@ -170,7 +191,8 @@ class FieldMetadata(dict):
                            'name':None,
                            'search_terms':['author_sort'],
                            'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
             ('au_map',    {'table':None,
                            'column':None,
                            'datatype':'text',
@@ -179,7 +201,8 @@ class FieldMetadata(dict):
                            'name':None,
                            'search_terms':[],
                            'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
             ('comments',  {'table':None,
                            'column':None,
                            'datatype':'text',
@@ -187,7 +210,9 @@ class FieldMetadata(dict):
                            'kind':'field',
                            'name':_('Comments'),
                            'search_terms':['comments', 'comment'],
-                           'is_custom':False, 'is_category':False}),
+                           'is_custom':False,
+                           'is_category':False,
+                           'is_csp': False}),
             ('cover',     {'table':None,
                            'column':None,
                            'datatype':'int',
@@ -196,16 +221,8 @@ class FieldMetadata(dict):
                            'name':None,
                            'search_terms':['cover'],
                            'is_custom':False,
-                           'is_category':False}),
-            ('flags',     {'table':None,
-                           'column':None,
-                           'datatype':'text',
-                           'is_multiple':None,
-                           'kind':'field',
-                           'name':None,
-                           'search_terms':[],
-                           'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
             ('id',        {'table':None,
                            'column':None,
                            'datatype':'int',
@@ -214,25 +231,18 @@ class FieldMetadata(dict):
                            'name':None,
                            'search_terms':[],
                            'is_custom':False,
-                           'is_category':False}),
-            ('isbn',      {'table':None,
+                           'is_category':False,
+                           'is_csp': False}),
+            ('last_modified', {'table':None,
                            'column':None,
-                           'datatype':'text',
+                           'datatype':'datetime',
                            'is_multiple':None,
                            'kind':'field',
-                           'name':None,
-                           'search_terms':['isbn'],
+                           'name':_('Date'),
+                           'search_terms':['last_modified'],
                            'is_custom':False,
-                           'is_category':False}),
-            ('lccn',      {'table':None,
-                           'column':None,
-                           'datatype':'text',
-                           'is_multiple':None,
-                           'kind':'field',
-                           'name':None,
-                           'search_terms':[],
-                           'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
             ('ondevice',  {'table':None,
                            'column':None,
                            'datatype':'text',
@@ -241,7 +251,8 @@ class FieldMetadata(dict):
                            'name':_('On Device'),
                            'search_terms':['ondevice'],
                            'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
             ('path',      {'table':None,
                            'column':None,
                            'datatype':'text',
@@ -250,7 +261,8 @@ class FieldMetadata(dict):
                            'name':None,
                            'search_terms':[],
                            'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
             ('pubdate',   {'table':None,
                            'column':None,
                            'datatype':'datetime',
@@ -259,7 +271,8 @@ class FieldMetadata(dict):
                            'name':_('Published'),
                            'search_terms':['pubdate'],
                            'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
             ('series_index',{'table':None,
                              'column':None,
                              'datatype':'float',
@@ -268,7 +281,8 @@ class FieldMetadata(dict):
                              'name':None,
                              'search_terms':['series_index'],
                              'is_custom':False,
-                             'is_category':False}),
+                             'is_category':False,
+                           'is_csp': False}),
             ('sort',      {'table':None,
                            'column':None,
                            'datatype':'text',
@@ -277,7 +291,8 @@ class FieldMetadata(dict):
                            'name':_('Title Sort'),
                            'search_terms':['title_sort'],
                            'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
             ('size',      {'table':None,
                            'column':None,
                            'datatype':'float',
@@ -286,7 +301,8 @@ class FieldMetadata(dict):
                            'name':_('Size (MB)'),
                            'search_terms':['size'],
                            'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
             ('timestamp', {'table':None,
                            'column':None,
                            'datatype':'datetime',
@@ -295,7 +311,8 @@ class FieldMetadata(dict):
                            'name':_('Date'),
                            'search_terms':['date'],
                            'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
             ('title',     {'table':None,
                            'column':None,
                            'datatype':'text',
@@ -304,7 +321,8 @@ class FieldMetadata(dict):
                            'name':_('Title'),
                            'search_terms':['title'],
                            'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
             ('uuid',      {'table':None,
                            'column':None,
                            'datatype':'text',
@@ -313,7 +331,8 @@ class FieldMetadata(dict):
                            'name':None,
                            'search_terms':[],
                            'is_custom':False,
-                           'is_category':False}),
+                           'is_category':False,
+                           'is_csp': False}),
         ]
     # }}}
 
@@ -335,7 +354,8 @@ class FieldMetadata(dict):
             self._tb_cats[k]['display'] = {}
             self._tb_cats[k]['is_editable'] = True
             self._add_search_terms_to_map(k, v['search_terms'])
-        self._tb_cats['timestamp']['display'] = {
+        for x in ('timestamp', 'last_modified'):
+            self._tb_cats[x]['display'] = {
                         'date_format': tweaks['gui_timestamp_display_format']}
         self._tb_cats['pubdate']['display'] = {
                         'date_format': tweaks['gui_pubdate_display_format']}
@@ -441,7 +461,8 @@ class FieldMetadata(dict):
         return l
 
     def add_custom_field(self, label, table, column, datatype, colnum, name,
-                               display, is_editable, is_multiple, is_category):
+                         display, is_editable, is_multiple, is_category,
+                         is_csp=False):
         key = self.custom_field_prefix + label
         if key in self._tb_cats:
             raise ValueError('Duplicate custom field [%s]'%(label))
@@ -454,7 +475,7 @@ class FieldMetadata(dict):
                              'colnum':colnum,      'display':display,
                              'is_custom':True,     'is_category':is_category,
                              'link_column':'value','category_sort':'value',
-                             'is_editable': is_editable,}
+                             'is_csp' : is_csp,     'is_editable': is_editable,}
         self._add_search_terms_to_map(key, [key])
         self.custom_label_to_key_map[label] = key
         if datatype == 'series':
@@ -466,7 +487,7 @@ class FieldMetadata(dict):
                                  'colnum':None,        'display':{},
                                  'is_custom':False,    'is_category':False,
                                  'link_column':None,   'category_sort':None,
-                                 'is_editable': False,}
+                                 'is_editable': False, 'is_csp': False}
             self._add_search_terms_to_map(key, [key])
             self.custom_label_to_key_map[label+'_index'] = key
 
@@ -515,7 +536,7 @@ class FieldMetadata(dict):
                                 'datatype':None,       'is_multiple':None,
                                 'kind':'user',         'name':name,
                                 'search_terms':st,     'is_custom':False,
-                                'is_category':True}
+                                'is_category':True,    'is_csp': False}
         self._add_search_terms_to_map(label, st)
 
     def add_search_category(self, label, name):
@@ -525,7 +546,7 @@ class FieldMetadata(dict):
                                 'datatype':None,     'is_multiple':None,
                                 'kind':'search',     'name':name,
                                 'search_terms':[],   'is_custom':False,
-                                'is_category':True}
+                                'is_category':True,  'is_csp': False}
 
     def set_field_record_index(self, label, index, prefer_custom=False):
         if prefer_custom:

src/calibre/library/prefs.py

@@ -49,8 +49,7 @@ class DBPrefs(dict):
         if self.disable_setting:
             return
         raw = self.to_raw(val)
-        self.db.conn.execute('DELETE FROM preferences WHERE key=?', (key,))
-        self.db.conn.execute('INSERT INTO preferences (key,val) VALUES (?,?)', (key,
+        self.db.conn.execute('INSERT OR REPLACE INTO preferences (key,val) VALUES (?,?)', (key,
             raw))
         self.db.conn.commit()
         dict.__setitem__(self, key, val)

src/calibre/library/restore.py

@@ -13,6 +13,7 @@ from calibre.ptempfile import TemporaryDirectory
 from calibre.ebooks.metadata.opf2 import OPF
 from calibre.library.database2 import LibraryDatabase2
 from calibre.constants import filesystem_encoding
+from calibre.utils.date import utcfromtimestamp
 from calibre import isbytestring
 
 NON_EBOOK_EXTENSIONS = frozenset([
@@ -211,8 +212,8 @@ class Restore(Thread):
                 force_id=book['id'])
         if book['mi'].uuid:
             db.set_uuid(book['id'], book['mi'].uuid, commit=False, notify=False)
-        db.conn.execute('UPDATE books SET path=? WHERE id=?', (book['path'],
-            book['id']))
+        db.conn.execute('UPDATE books SET path=?,last_modified=? WHERE id=?', (book['path'],
+            utcfromtimestamp(book['timestamp']), book['id']))
 
         for fmt, size, name in book['formats']:
             db.conn.execute('''

src/calibre/library/schema_upgrades.py

@@ -8,6 +8,8 @@ __docformat__ = 'restructuredtext en'
 
 import os
 
+from calibre.utils.date import isoformat, DEFAULT_DATE
+
 class SchemaUpgrade(object):
 
     def __init__(self):
@@ -468,4 +470,134 @@ class SchemaUpgrade(object):
         '''
         self.conn.executescript(script)
 
+    def upgrade_version_18(self):
+        '''
+        Add a library UUID.
+        Add an identifiers table.
+        Add a languages table.
+        Add a last_modified column.
+        NOTE: You cannot downgrade after this update, if you do
+        any changes you make to book isbns will be lost.
+        '''
+        script = '''
+        DROP TABLE IF EXISTS library_id;
+        CREATE TABLE library_id ( id   INTEGER PRIMARY KEY,
+                                  uuid TEXT NOT NULL,
+                                  UNIQUE(uuid)
+        );
+
+        DROP TABLE IF EXISTS identifiers;
+        CREATE TABLE identifiers  ( id     INTEGER PRIMARY KEY,
+                                    book   INTEGER NON NULL,
+                                    type   TEXT NON NULL DEFAULT "isbn" COLLATE NOCASE,
+                                    val    TEXT NON NULL COLLATE NOCASE,
+                                    UNIQUE(book, type)
+        );
+
+        DROP TABLE IF EXISTS languages;
+        CREATE TABLE languages    ( id        INTEGER PRIMARY KEY,
+                                    lang_code TEXT NON NULL COLLATE NOCASE,
+                                    UNIQUE(lang_code)
+        );
+
+        DROP TABLE IF EXISTS books_languages_link;
+        CREATE TABLE books_languages_link ( id INTEGER PRIMARY KEY,
+                                            book INTEGER NOT NULL,
+                                            lang_code INTEGER NOT NULL,
+                                            item_order INTEGER NOT NULL DEFAULT 0,
+                                            UNIQUE(book, lang_code)
+        );
+
+        DROP TRIGGER IF EXISTS fkc_delete_on_languages;
+        CREATE TRIGGER fkc_delete_on_languages
+        BEFORE DELETE ON languages
+        BEGIN
+            SELECT CASE
+                WHEN (SELECT COUNT(id) FROM books_languages_link WHERE lang_code=OLD.id) > 0
+                THEN RAISE(ABORT, 'Foreign key violation: language is still referenced')
+            END;
+        END;
+
+        DROP TRIGGER IF EXISTS fkc_delete_on_languages_link;
+        CREATE TRIGGER fkc_delete_on_languages_link
+        BEFORE INSERT ON books_languages_link
+        BEGIN
+          SELECT CASE
+              WHEN (SELECT id from books WHERE id=NEW.book) IS NULL
+              THEN RAISE(ABORT, 'Foreign key violation: book not in books')
+              WHEN (SELECT id from languages WHERE id=NEW.lang_code) IS NULL
+              THEN RAISE(ABORT, 'Foreign key violation: lang_code not in languages')
+          END;
+        END;
+
+        DROP TRIGGER IF EXISTS fkc_update_books_languages_link_a;
+        CREATE TRIGGER fkc_update_books_languages_link_a
+        BEFORE UPDATE OF book ON books_languages_link
+        BEGIN
+            SELECT CASE
+                WHEN (SELECT id from books WHERE id=NEW.book) IS NULL
+                THEN RAISE(ABORT, 'Foreign key violation: book not in books')
+            END;
+        END;
+        DROP TRIGGER IF EXISTS fkc_update_books_languages_link_b;
+        CREATE TRIGGER fkc_update_books_languages_link_b
+        BEFORE UPDATE OF lang_code ON books_languages_link
+        BEGIN
+            SELECT CASE
+                WHEN (SELECT id from languages WHERE id=NEW.lang_code) IS NULL
+                THEN RAISE(ABORT, 'Foreign key violation: lang_code not in languages')
+            END;
+        END;
+
+        DROP INDEX IF EXISTS books_languages_link_aidx;
+        CREATE INDEX books_languages_link_aidx ON books_languages_link (lang_code);
+        DROP INDEX IF EXISTS books_languages_link_bidx;
+        CREATE INDEX books_languages_link_bidx ON books_languages_link (book);
+        DROP INDEX IF EXISTS languages_idx;
+        CREATE INDEX languages_idx ON languages (lang_code COLLATE NOCASE);
+
+        DROP TRIGGER IF EXISTS books_delete_trg;
+        CREATE TRIGGER books_delete_trg
+            AFTER DELETE ON books
+            BEGIN
+                DELETE FROM books_authors_link WHERE book=OLD.id;
+                DELETE FROM books_publishers_link WHERE book=OLD.id;
+                DELETE FROM books_ratings_link WHERE book=OLD.id;
+                DELETE FROM books_series_link WHERE book=OLD.id;
+                DELETE FROM books_tags_link WHERE book=OLD.id;
+                DELETE FROM books_languages_link WHERE book=OLD.id;
+                DELETE FROM data WHERE book=OLD.id;
+                DELETE FROM comments WHERE book=OLD.id;
+                DELETE FROM conversion_options WHERE book=OLD.id;
+                DELETE FROM books_plugin_data WHERE book=OLD.id;
+                DELETE FROM identifiers WHERE book=OLD.id;
+        END;
+
+        INSERT INTO identifiers (book, val) SELECT id,isbn FROM books WHERE isbn;
+
+        ALTER TABLE books ADD COLUMN last_modified TIMESTAMP NOT NULL DEFAULT "%s";
+
+        '''%isoformat(DEFAULT_DATE, sep=' ')
+        # Sqlite does not support non constant default values in alter
+        # statements
+        self.conn.executescript(script)
+
+    def upgrade_version_19(self):
+        recipes = self.conn.get('SELECT id,title,script FROM feeds')
+        if recipes:
+            from calibre.web.feeds.recipes import custom_recipes, \
+                    custom_recipe_filename
+            bdir = os.path.dirname(custom_recipes.file_path)
+            for id_, title, script in recipes:
+                existing = frozenset(map(int, custom_recipes.iterkeys()))
+                if id_ in existing:
+                    id_ = max(existing) + 1000
+                id_ = str(id_)
+                fname = custom_recipe_filename(id_, title)
+                custom_recipes[id_] = (title, fname)
+                if isinstance(script, unicode):
+                    script = script.encode('utf-8')
+                with open(os.path.join(bdir, fname), 'wb') as f:
+                    f.write(script)
+
 

src/calibre/library/server/browse.py

@@ -346,7 +346,7 @@ class BrowseServer(object):
         for category in sorted(categories, key=lambda x: sort_key(getter(x))):
             if len(categories[category]) == 0:
                 continue
-            if category == 'formats':
+            if category in ('formats', 'identifiers'):
                 continue
             meta = category_meta.get(category, None)
             if meta is None:
@@ -666,7 +666,7 @@ class BrowseServer(object):
             if add_category_links:
                 added_key = False
                 fm = mi.metadata_for_field(key)
-                if val and fm and fm['is_category'] and \
+                if val and fm and fm['is_category'] and not fm['is_csp'] and\
                         key != 'formats' and fm['datatype'] not in ['rating']:
                     categories = mi.get(key)
                     if isinstance(categories, basestring):

src/calibre/library/server/opds.py

@@ -580,7 +580,7 @@ class OPDSServer(object):
         for category in sorted(categories, key=lambda x: sort_key(getter(x))):
             if len(categories[category]) == 0:
                 continue
-            if category == 'formats':
+            if category in ('formats', 'identifiers'):
                 continue
             meta = category_meta.get(category, None)
             if meta is None:

src/calibre/library/server/xml.py

@@ -89,13 +89,16 @@ class XMLServer(object):
             for x in ('id', 'title', 'sort', 'author_sort', 'rating', 'size'):
                 kwargs[x] = serialize(record[FM[x]])
 
-            for x in ('isbn', 'formats', 'series', 'tags', 'publisher',
-                    'comments'):
+            for x in ('formats', 'series', 'tags', 'publisher',
+                    'comments', 'identifiers'):
                 y = record[FM[x]]
                 if x == 'tags':
                     y = format_tag_string(y, ',', ignore_max=True)
                 kwargs[x] = serialize(y) if y else ''
 
+            isbn = self.db.isbn(record[FM['id']], index_is_id=True)
+            kwargs['isbn'] = serialize(isbn if isbn else '')
+
             kwargs['safe_title'] = ascii_filename(kwargs['title'])
 
             c = kwargs.pop('comments')

src/calibre/library/sqlite.py

@@ -8,6 +8,7 @@ Wrapper for multi-threaded access to a single sqlite database connection. Serial
 all calls.
 '''
 import sqlite3 as sqlite, traceback, time, uuid, sys, os
+import repr as reprlib
 from sqlite3 import IntegrityError, OperationalError
 from threading import Thread
 from Queue import Queue
@@ -16,18 +17,54 @@ from datetime import datetime
 from functools import partial
 
 from calibre.ebooks.metadata import title_sort, author_to_author_sort
-from calibre.utils.date import parse_date, isoformat
+from calibre.utils.date import parse_date, isoformat, local_tz
 from calibre import isbytestring, force_unicode
-from calibre.constants import iswindows, DEBUG
+from calibre.constants import iswindows, DEBUG, plugins
 from calibre.utils.icu import strcmp
+from calibre import prints
+
+from dateutil.tz import tzoffset
 
 global_lock = RLock()
 
-def convert_timestamp(val):
+_c_speedup = plugins['speedup'][0]
+
+def _c_convert_timestamp(val):
+    if not val:
+        return None
+    try:
+        ret = _c_speedup.parse_date(val.strip())
+    except:
+        ret = None
+    if ret is None:
+        return parse_date(val, as_utc=False)
+    year, month, day, hour, minutes, seconds, tzsecs = ret
+    return datetime(year, month, day, hour, minutes, seconds,
+                tzinfo=tzoffset(None, tzsecs)).astimezone(local_tz)
+
+def _py_convert_timestamp(val):
     if val:
+        tzsecs = 0
+        try:
+            sign = {'+':1, '-':-1}.get(val[-6], None)
+            if sign is not None:
+                tzsecs = 60*((int(val[-5:-3])*60 + int(val[-2:])) * sign)
+            year = int(val[0:4])
+            month = int(val[5:7])
+            day = int(val[8:10])
+            hour = int(val[11:13])
+            min = int(val[14:16])
+            sec = int(val[17:19])
+            return datetime(year, month, day, hour, min, sec,
+                    tzinfo=tzoffset(None, tzsecs))
+        except:
+            pass
         return parse_date(val, as_utc=False)
     return None
 
+convert_timestamp = _py_convert_timestamp if _c_speedup is None else \
+                    _c_convert_timestamp
+
 def adapt_datetime(dt):
     return isoformat(dt, sep=' ')
 
@@ -87,6 +124,18 @@ class SortedConcatenate(object):
 class SafeSortedConcatenate(SortedConcatenate):
     sep = '|'
 
+class IdentifiersConcat(object):
+    '''String concatenation aggregator for the identifiers map'''
+    def __init__(self):
+        self.ans = []
+
+    def step(self, key, val):
+        self.ans.append(u'%s:%s'%(key, val))
+
+    def finalize(self):
+        return ','.join(self.ans)
+
+
 class AumSortedConcatenate(object):
     '''String concatenation aggregator for the author sort map'''
     def __init__(self):
@@ -170,13 +219,13 @@ class DBThread(Thread):
                                    detect_types=sqlite.PARSE_DECLTYPES|sqlite.PARSE_COLNAMES)
         self.conn.execute('pragma cache_size=5000')
         encoding = self.conn.execute('pragma encoding').fetchone()[0]
-        c_ext_loaded = load_c_extensions(self.conn)
+        self.conn.create_aggregate('sortconcat', 2, SortedConcatenate)
+        self.conn.create_aggregate('sort_concat', 2, SafeSortedConcatenate)
+        self.conn.create_aggregate('identifiers_concat', 2, IdentifiersConcat)
+        load_c_extensions(self.conn)
         self.conn.row_factory = sqlite.Row if self.row_factory else  lambda cursor, row : list(row)
         self.conn.create_aggregate('concat', 1, Concatenate)
         self.conn.create_aggregate('aum_sortconcat', 3, AumSortedConcatenate)
-        if not c_ext_loaded:
-            self.conn.create_aggregate('sortconcat', 2, SortedConcatenate)
-            self.conn.create_aggregate('sort_concat', 2, SafeSortedConcatenate)
         self.conn.create_collation('PYNOCASE', partial(pynocase,
             encoding=encoding))
         self.conn.create_function('title_sort', 1, title_sort)
@@ -208,17 +257,21 @@ class DBThread(Thread):
                     except Exception, err:
                         ok, res = False, (err, traceback.format_exc())
                 else:
-                    func = getattr(self.conn, func)
+                    bfunc = getattr(self.conn, func)
                     try:
                         for i in range(3):
                             try:
-                                ok, res = True, func(*args, **kwargs)
+                                ok, res = True, bfunc(*args, **kwargs)
                                 break
                             except OperationalError, err:
                                 # Retry if unable to open db file
-                                if 'unable to open' not in str(err) or i == 2:
+                                e = str(err)
+                                if 'unable to open' not in e or i == 2:
+                                    if 'unable to open' in e:
+                                        prints('Unable to open database for func',
+                                            func, reprlib.repr(args),
+                                            reprlib.repr(kwargs))
                                     raise
-                                traceback.print_exc()
                             time.sleep(0.5)
                     except Exception, err:
                         ok, res = False, (err, traceback.format_exc())

src/calibre/library/sqlite_custom.c

@@ -77,6 +77,7 @@ static void sort_concat_free(SortConcatList *list) {
         free(list->vals[i]->val);
         free(list->vals[i]);
     }
+    free(list->vals);
 }
 
 static int sort_concat_cmp(const void *a_, const void *b_) {
@@ -142,11 +143,102 @@ static void sort_concat_finalize2(sqlite3_context *context) {
 
 // }}}
 
+// identifiers_concat {{{
+
+typedef struct {
+    char *val;
+    size_t length;
+} IdentifiersConcatItem;
+
+typedef struct {
+    IdentifiersConcatItem **vals;
+    size_t count;
+    size_t length;
+} IdentifiersConcatList;
+
+static void identifiers_concat_step(sqlite3_context *context, int argc, sqlite3_value **argv) {
+    const char *key, *val;
+    size_t len = 0;
+    IdentifiersConcatList *list;
+
+    assert(argc == 2);
+
+    list = (IdentifiersConcatList*) sqlite3_aggregate_context(context, sizeof(*list));
+    if (list == NULL) return;
+
+    if (list->vals == NULL) {
+        list->vals = (IdentifiersConcatItem**)calloc(100, sizeof(IdentifiersConcatItem*));
+        if (list->vals == NULL) return;
+        list->length = 100;
+        list->count = 0;
+    }
+
+    if (list->count == list->length) {
+        list->vals = (IdentifiersConcatItem**)realloc(list->vals, list->length + 100);
+        if (list->vals == NULL) return;
+        list->length = list->length + 100;
+    }
+
+    list->vals[list->count] = (IdentifiersConcatItem*)calloc(1, sizeof(IdentifiersConcatItem));
+    if (list->vals[list->count] == NULL) return;
+
+    key = (char*) sqlite3_value_text(argv[0]);
+    val = (char*) sqlite3_value_text(argv[1]);
+    if (key == NULL || val == NULL) {return;}
+    len = strlen(key) + strlen(val) + 1;
+
+    list->vals[list->count]->val = (char*)calloc(len+1, sizeof(char));
+    if (list->vals[list->count]->val == NULL) return;
+    snprintf(list->vals[list->count]->val, len+1, "%s:%s", key, val);
+    list->vals[list->count]->length = len;
+
+    list->count = list->count + 1;
+
+}
+
+
+static void identifiers_concat_finalize(sqlite3_context *context) {
+    IdentifiersConcatList *list;
+    IdentifiersConcatItem *item;
+    char *ans, *pos;
+    size_t sz = 0, i;
+
+    list = (IdentifiersConcatList*) sqlite3_aggregate_context(context, sizeof(*list));
+    if (list == NULL || list->vals == NULL || list->count < 1) return;
+
+    for (i = 0; i < list->count; i++) { 
+        sz += list->vals[i]->length;
+    }
+    sz += list->count; // Space for commas
+    ans = (char*)calloc(sz+2, sizeof(char));
+    if (ans == NULL) return;
+
+    pos = ans;
+
+    for (i = 0; i < list->count; i++) {
+        item = list->vals[i];
+        if (item == NULL || item->val == NULL) continue;
+        memcpy(pos, item->val, item->length);
+        pos += item->length;
+        *pos = ',';
+        pos += 1;
+        free(item->val);
+        free(item);
+    }
+    *(pos-1) = 0; // Remove trailing comma
+    sqlite3_result_text(context, ans, -1, SQLITE_TRANSIENT);
+    free(ans);
+    free(list->vals);
+}
+
+// }}}
+
 MYEXPORT int sqlite3_extension_init(
     sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi){
   SQLITE_EXTENSION_INIT2(pApi);
   sqlite3_create_function(db, "sortconcat", 2, SQLITE_UTF8, NULL, NULL, sort_concat_step, sort_concat_finalize);
   sqlite3_create_function(db, "sort_concat", 2, SQLITE_UTF8, NULL, NULL, sort_concat_step, sort_concat_finalize2);
+  sqlite3_create_function(db, "identifiers_concat", 2, SQLITE_UTF8, NULL, NULL, identifiers_concat_step, identifiers_concat_finalize);
   return 0;
 }
 

src/calibre/manual/faq.rst

@@ -327,10 +327,24 @@ Now coming to author name sorting:
     * When recalculating the author sort values for books, |app| uses the author sort values for each individual author. Therefore, ensure that the individual author sort values are correct before recalculating the books' author sort values.
     * You can control whether the Tag Browser display authors using their names or their sort values by setting the :guilabel:`categories_use_field_for_author_name` tweak in Preferences->Tweaks
 
-With all this flexibility, it is possible to have |app| manage your author names however you like. For example, one common request is to have |app| display author names LN, FN. To do this first set the ``author_sort_copy_method`` to ``copy``. Then change all author names to LN, FN via the Manage authors dialog. Then have |app| recalculate author sort values for both authors and books as described above.
-
 Note that you can set an individual author's sort value to whatever you want using :guilabel:`Manage authors`. This is useful when dealing with names that |app| will not get right, such as complex multi-part names like Miguel de Cervantes Saavedra or when dealing with Asian names like Sun Tzu.
 
+With all this flexibility, it is possible to have |app| manage your author names however you like. For example, one common request is to have |app| display author names LN, FN. To do this, and if the note below does not apply to you, then:
+    * Set the ``author_sort_copy_method`` tweak to ``copy`` as described above.
+    * Restart calibre. Do not change any book metadata before doing the remaining steps.
+    * Change all author names to LN, FN using the Manage authors dialog.
+    * After you have changed all the authors, press the `Recalculate all author sort values` button.
+    * Press OK, at which point |app| will change the authors in all your books. This can take a while.
+    
+.. note::
+
+    When changing from FN LN to LN, FN, it is often the case that the values in author_sort are already in LN, FN format. If this is your case, then do the following:
+        * set the ``author_sort_copy_method`` tweak to ``copy`` as described above.
+        * restart calibre. Do not change any book metadata before doing the remaining steps.
+        * open the Manage authors dialog. Press the ``copy all author sort values to author`` button.
+        * Check through the authors to be sure you are happy. You can still press Cancel to abandon the changes. Once you press OK, there is no undo.
+        * Press OK, at which point |app| will change the authors in all your books. This can take a while.
+
 
 Why doesn't |app| let me store books in my own directory structure?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -350,7 +364,7 @@ Why doesn't |app| have a column for foo?
 |app| is designed to have columns for the most frequently and widely used fields. In addition, you can add any columns you like. Columns can be added via :guilabel:`Preferences->Interface->Add your own columns`.
 Watch the tutorial `UI Power tips <http://calibre-ebook.com/demo#tutorials>`_ to learn how to create your own columns.
 
-You can also create "virtual columns" that contain combinations of the metadata from other columns. In the add column dialog choose the option "Column from other columns" and in the template enter the other column names. For example to create a virtual column containing formats or ISBN, enter ``{formats}`` for formats or ``{isbn}`` for ISBN. For more details, see :ref:`templatelangcalibre`.
+You can also create "virtual columns" that contain combinations of the metadata from other columns. In the add column dialog  use the :guilabel:`Quick create` links to easily create columns to show the book ISBN, formats or the time the book was last modified. For more details, see :ref:`templatelangcalibre`.
 
 
 Can I have a column showing the formats or the ISBN?

src/calibre/manual/gui.rst

@@ -338,7 +338,7 @@ You can build advanced search queries easily using the :guilabel:`Advanced Searc
 clicking the button |sbi|.
 
 Available fields for searching are: ``tag, title, author, publisher, series, series_index, rating, cover,
-comments, format, isbn, date, pubdate, search, size`` and custom columns. If a device is plugged in, the
+comments, format, identifiers, date, pubdate, search, size`` and custom columns. If a device is plugged in, the
 ``ondevice`` field becomes available. To find the search name for a custom column, hover your mouse over the
 column header.
 
@@ -385,6 +385,21 @@ with undefined values in the column. Searching for ``true`` will find all books
 values in the column. Searching for ``yes`` or ``checked`` will find all books with ``Yes`` in the column.
 Searching for ``no`` or ``unchecked`` will find all books with ``No`` in the column.
 
+Hierarchical items (e.g. A.B.C) use an extended syntax to match initial parts of the hierarchy. This is done by adding a period between the exact match indicator (=) and the text. For example, the query ``tags:=.A`` will find the tags `A` and `A.B`, but will not find the tags `AA` or `AA.B`. The query ``tags:=.A.B`` will find the tags `A.B` and `A.C`, but not the tag `A`.
+
+Identifiers (e.g., isbn, doi, lccn etc) also use an extended syntax. First, note that an identifier has the form ``key:value``, as in ``isbn:123456789``. The extended syntax permits you to specify independently which key and value to search for. Both the key and the value parts of the query can use `equality`, `contains`, or `regular expression` matches. Examples:
+
+    * ``identifiers:true`` will find books with any identifier.
+    * ``identifiers:false`` will find books with no identifier.
+    * ``identifiers:123`` will search for books with any key having a value containing `123`.
+    * ``identifiers:=123456789`` will search for books with any key having a value equal to `123456789`.
+    * ``identifiers:=isbn:`` and ``identifiers:isbn:true`` will find books with a key equal to isbn having any value
+    * ``identifiers:=isbn:false`` will find books with no key equal to isbn.
+    * ``identifiers:=isbn:123`` will find books with a key equal to isbn having a value containing `123`.
+    * ``identifiers:=isbn:=123456789`` will find books with a key equal to isbn having a value equal to `123456789`.
+    * ``identifiers:i:1`` will find books with a key containing an `i` having a value containing a `1`.
+    
+
 .. |sbi| image:: images/search_button.png
     :align: middle
 
@@ -482,6 +497,8 @@ Calibre has several keyboard shortcuts to save you time and mouse movement. Thes
       - Edit the metadata of the currently selected field in the book list.
     * - :kbd:`A` 
       - Add Books
+    * - :kbd:`Shift+A` 
+      - Add Formats to the selected books
     * - :kbd:`C` 
       - Convert selected Books
     * - :kbd:`D` 

src/calibre/manual/template_lang.rst

@@ -30,7 +30,7 @@ You can use all the various metadata fields available in calibre in a template,
 In addition to the column based fields, you also can use::
 
     {formats} - A list of formats available in the calibre library for a book
-    {isbn}    - The ISBN number of the book
+    {identifiers:select(isbn)} - The ISBN number of the book
 
 If a particular book does not have a particular piece of metadata, the field in the template is automatically removed for that book. Consider, for example::
 
@@ -95,7 +95,7 @@ Advanced features
 Using templates in custom columns
 ----------------------------------
 
-There are sometimes cases where you want to display metadata that |app| does not normally display, or to display data in a way different from how |app| normally does. For example, you might want to display the ISBN, a field that |app| does not display. You can use custom columns for this by creating a column with the type 'column built from other columns' (hereafter called composite columns), and entering a template. Result: |app| will display a column showing the result of evaluating that template. To display the ISBN, create the column and enter ``{isbn}`` into the template box. To display a column containing the values of two series custom columns separated by a comma, use ``{#series1:||,}{#series2}``.
+There are sometimes cases where you want to display metadata that |app| does not normally display, or to display data in a way different from how |app| normally does. For example, you might want to display the ISBN, a field that |app| does not display. You can use custom columns for this by creating a column with the type 'column built from other columns' (hereafter called composite columns), and entering a template. Result: |app| will display a column showing the result of evaluating that template. To display the ISBN, create the column and enter ``{identifiers:select(isbn)}`` into the template box. To display a column containing the values of two series custom columns separated by a comma, use ``{#series1:||,}{#series2}``.
 
 Composite columns can use any template option, including formatting.
 
@@ -122,10 +122,11 @@ The functions available are:
     * ``count(separator)`` -- interprets the value as a list of items separated by `separator`, returning the number of items in the list. Most lists use a comma as the separator, but authors uses an ampersand. Examples: `{tags:count(,)}`, `{authors:count(&)}`
     * ``ifempty(text)``	-- if the field is not empty, return the value of the field. Otherwise return `text`.
     * ``list_item(index, separator)`` -- interpret the value as a list of items separated by `separator`, returning the `index`th item. The first item is number zero. The last item can be returned using `list_item(-1,separator)`. If the item is not in the list, then the empty value is returned. The separator has the same meaning as in the `count` function.
-    * ``lookup(pattern, field, pattern, field, ..., else_field)`` -- like switch, except the arguments are field (metadata) names, not text. The value of the appropriate field will be fetched and used. Note that because composite columns are fields, you can use this function in one composite field to use the value of some other composite field. This is extremely useful when constructing variable save paths (more later).
     * ``re(pattern, replacement)`` -- return the field after applying the regular expression. All instances of `pattern` are replaced with `replacement`. As in all of |app|, these are python-compatible regular expressions.
     * ``shorten(left chars, middle text, right chars)`` -- Return a shortened version of the field, consisting of `left chars` characters from the beginning of the field, followed by `middle text`, followed by `right chars` characters from the end of the string. `Left chars` and `right chars` must be integers. For example, assume the title of the book is `Ancient English Laws in the Times of Ivanhoe`, and you want it to fit in a space of at most 15 characters. If you use ``{title:shorten(9,-,5)}``, the result will be `Ancient E-nhoe`. If the field's length is less than ``left chars`` + ``right chars`` + the length of ``middle text``, then the field will be used intact. For example, the title `The Dome` would not be changed.
     * ``switch(pattern, value, pattern, value, ..., else_value)`` -- for each ``pattern, value`` pair, checks if the field matches the regular expression ``pattern`` and if so, returns that ``value``. If no ``pattern`` matches, then ``else_value`` is returned. You can have as many ``pattern, value`` pairs as you want.
+    * ``lookup(pattern, field, pattern, field, ..., else_field)`` -- like switch, except the arguments are field (metadata) names, not text. The value of the appropriate field will be fetched and used. Note that because composite columns are fields, you can use this function in one composite field to use the value of some other composite field. This is extremely useful when constructing variable save paths (more later).
+    * ``select(key)`` -- interpret the field as a comma-separated list of items, with the items being of the form "id:value". Find the pair with the id equal to key, and return the corresponding value. This function is particularly useful for extracting a value such as an isbn from the set of identifiers for a book.
     * ``test(text if not empty, text if empty)`` -- return `text if not empty` if the field is not empty, otherwise return `text if empty`.
 
 
@@ -143,6 +144,8 @@ Using functions in templates - template program mode
 
 The template language program mode differs from single-function mode in that it permits you to write template expressions that refer to other metadata fields, modify values, and do arithmetic. It is a reasonably complete programming language. 
 
+You can use the functions documented above in template program mode. See below for details.
+
 Beginning with an example, assume that you want your template to show the series for a book if it has one, otherwise show the value of a custom field #genre. You cannot do this in the basic language because you cannot make reference to another metadata field within a template expression. In program mode, you can. The following expression works::
 
     {#series:'ifempty($, field('#genre'))'}
@@ -203,7 +206,7 @@ For various values of series_index, the program returns:
     * series_index == 2, result = ``prefix 2->eq suffix``
     * series_index == 3, result = ``prefix 3->gt suffix``
 
-All the functions listed under single-function mode can be used in program mode, noting that unlike the functions described below you must supply a first parameter providing the value the function is to act upon. 
+**All the functions listed under single-function mode can be used in program mode**. To do so, you must supply the value that the function is to act upon as the first parameter, in addition to the parameters documented above. For example, in program mode the parameters of the `test` function are ``test(x, text_if_not_empty, text_if_empty)``. The `x` parameter, which is the value to be tested, will almost always be a variable or a function call, often `field()`.
 
 The following functions are available in addition to those described in single-function mode. Remember from the example above that the single-function mode functions require an additional first parameter specifying the field to operate on. With the exception of the ``id`` parameter of assign, all parameters can be statements (sequences of expressions):
 
@@ -212,9 +215,23 @@ The following functions are available in addition to those described in single-f
     * ``cmp(x, y, lt, eq, gt)`` -- compares x and y after converting both to numbers. Returns ``lt`` if x < y. Returns ``eq`` if x == y. Otherwise returns ``gt``.
     * ``divide(x, y)`` -- returns x / y. Throws an exception if either x or y are not numbers.
     * ``field(name)`` -- returns the metadata field named by ``name``.
+    * ``format_date(x, date_format)`` -- format_date(val, format_string) -- format the value, which must be a date field, using the format_string, returning a string. The formatting codes are::
+    
+        d    : the day as number without a leading zero (1 to 31)
+        dd   : the day as number with a leading zero (01 to 31) '
+        ddd  : the abbreviated localized day name (e.g. "Mon" to "Sun"). '
+        dddd : the long localized day name (e.g. "Monday" to "Sunday"). '
+        M    : the month as number without a leading zero (1 to 12). '
+        MM   : the month as number with a leading zero (01 to 12) '
+        MMM  : the abbreviated localized month name (e.g. "Jan" to "Dec"). '
+        MMMM : the long localized month name (e.g. "January" to "December"). '
+        yy   : the year as two digit number (00 to 99). '
+        yyyy : the year as four digit number.'
+    
     * ``eval(string)`` -- evaluates the string as a program, passing the local variables (those ``assign`` ed to). This permits using the template processor to construct complex results from local variables.
     * ``multiply(x, y)`` -- returns x * y. Throws an exception if either x or y are not numbers.
     * ``print(a, b, ...)`` -- prints the arguments to standard output. Unless you start calibre from the command line (``calibre-debug -g``), the output will go to a black hole.
+    * ``raw_field(name)`` -- returns the metadata field named by name without applying any formatting.
     * ``strcat(a, b, ...)`` -- can take any number of arguments. Returns a string formed by concatenating all the arguments.
     * ``strcmp(x, y, lt, eq, gt)`` -- does a case-insensitive comparison x and y as strings. Returns ``lt`` if x < y. Returns ``eq`` if x == y. Otherwise returns ``gt``.
     * ``substr(str, start, end)`` -- returns the ``start``'th through the ``end``'th characters of ``str``. The first character in ``str`` is the zero'th character. If end is negative, then it indicates that many characters counting from the right. If end is zero, then it indicates the last character. For example, ``substr('12345', 1, 0)`` returns ``'2345'``, and ``substr('12345', 1, -1)`` returns ``'234'``.

src/calibre/manual/templates/layout.html

@@ -1,23 +1,6 @@
 {% extends "!layout.html" %}
 
 {% block extrahead %}
-    {% if not embedded %}
-    <script type="text/javascript">
-
-        var _gaq = _gaq || [];
-        _gaq.push(['_setAccount', 'UA-20736318-1']);
-        _gaq.push(['_setDomainName', '.calibre-ebook.com']);
-        _gaq.push(['_trackPageview']);
-
-        (function() {
-            var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
-            ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
-            var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
-        })();
-
-    </script>
-    {% endif %}
-
     <style type="text/css">
         .float-left-img { float: left; margin-right: 1em; margin-bottom: 1em }
         .float-right-img { float: right; margin-left: 1em; margin-bottom: 1em }
@@ -52,6 +35,23 @@
         </div>
       {%- endif %}
       </div>
+        {% if not embedded %}
+            <script type="text/javascript">
+
+                var _gaq = _gaq || [];
+                _gaq.push(['_setAccount', 'UA-20736318-1']);
+                _gaq.push(['_setDomainName', '.calibre-ebook.com']);
+                _gaq.push(['_trackPageview']);
+
+                (function() {
+                    var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+                    ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+                    var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+                })();
+
+            </script>
+        {%- endif %}
+
 {% endblock %}
 
 {% block sidebarlogo %}