Sync to trunk.

Changelog.yaml

@@ -19,9 +19,93 @@
 #  new recipes:
 #    - title: 
 
-#    - title: "Launch of a new website that catalogues DRM free books. http://drmfree.calibre-ebook.com"
+- version: 0.7.48
-#      description: "A growing catalogue of DRM free books. Books that you actually own after buying instead of renting."
+  date: 2011-03-04
-#      type: major
+
+  new features:
+    - title: "Changes to the internal database structure used by calibre"
+      description: >
+        "These changes will allow calibre, in the future, to support book language, arbitrary book identifiers and keep track of when the metadata for a book was last modified. WARNING: Because of these changes, if you downgrade calibre versions after upgrading to 0.7.48, you will lose any changes you make to the ISBN of book entries in your calibre database, so do not downgrade unless you really have to. Also note that the first time you start calibre after this update, the startup will be slow as the database structure is being changed."
+
+    - title: "Launch of a new website that catalogues DRM free ebooks. http://drmfree.calibre-ebook.com"
+      description: "A growing catalogue of DRM free ebooks. Ebooks that you actually own after paying, instead of just renting."
+      type: major
+
+    - title: "News download: Add an option to keep at most x issues of a particular periodical in the calibre library. Use the Advanced tab in the Fetch news dialog for your news source to set this option."
+      tickets: [9168]
+
+    - title: "You can now right click on the cover in the book details panel to copy/paste a new cover."
+      tickets: [9255]
+
+    - title: "Add an entry to the add books drop down menu to easily add formats to an existing book record"
+
+    - title: "Tag browser: Clicking on a nested category now searches for the category alone. Clicking twice searches for the category and all its descendants and so on."
+      tickets: [9166, 9169]
+
+    - title: "Add a button to the Manage authors dialog to copy author sort values to author"
+
+    - title: "Decrease startup times on large libraries by using a faster algorithm to parse stored dates"
+
+    - title: "Add quick create links to easily create custom columns of commonly used types to the add custom column dialog"
+
+    - title: "Allow drag drop of images to change cover in book details window."
+      tickets: [9226]
+
+    - title: "Device susbsytem: Create a drive info file named driveinfo.calibre in the root of each device drive for USB connected devices. This file contains various useful data. API Change: The open method of the device plugins now accepts an extra parameter library_uuid which is the id of the calibre library connected tot eh device"
+
+  bug fixes:
+    - title: "Conversion pipeline: Fix regression in 0.7.46 that caused loss of some CSS information when converting HTML produced by Microsoft Word. Also remove empty tags from microsoft namespaces when parsing HTML"
+
+    - title: "Try harder to ensure that the worker log temporary files are deleted in windows"
+
+    - title: "CHM Input: Handle CHM files that dont specify a topics file."
+      tickets: [9253]
+
+    - title: "Fix regression that caused memory leak in Tag Browser. This would show up as the memory usage of calibre increasing when switching libraries."
+      tickets: [9246]
+
+    - title: "Fix bug that caused preferences->behavior to not show the output format set by the welcome wizard, and instead default to showing EPUB"
+
+    - title: "Fix bug that caused wrong books to be deleted from library if you choose 'delete from library and device' while the library is sorted by the On device column"
+
+    - title: "MOBI Input: Ignore all ASCII control codes except CR, NL and Tab."
+      tickets: [9219]
+
+  improved recipes:
+    - Credit Slips
+    - Seattle Times
+    - MacWorld
+    - Austin Statesman
+    - EPL Talk
+    - Gawker
+    - Deadspin
+
+  new recipes:
+    - title: "Thai Post Today and Daily Post"
+      author: "Chotechai P."
+
+    - title: "RBC.ru"
+      author: Chewi
+
+    - title: Helsingin Sanomat
+      author: oneillpt
+
+    - title: "LWN Weekly"
+      author: David Cavalca
+
+    - title: "New York Times Sports and Technology Blogs"
+      author: rylsfan
+
+    - title: "Historia and Buctaras"
+      author: Silviu Coatara
+
+    - title: "Buffalo News"
+      author: ChappyOnIce
+
+    - title: "Dotpod"
+      author: Federico Escalada
+
+
 
 - version: 0.7.47
   date: 2011-02-25

resources/recipes/leduc.recipe

@@ -1,40 +1,52 @@
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class AdvancedUserRecipe1292550626(BasicNewsRecipe):
-   title = 'Leduc - Wetaskiwin Pipestone Flyer'
+     title = 'Leduc - Wetaskiwin Pipestone Flyer'
-   __author__ = 'Brian Hahn'
+     __author__ = 'Brian Hahn'
-   description = 'News from Alberta, Canada'
+     description = '''Provides news from central Alberta, Canada.  This is a
-   oldest_article = 56
+ weekly publication that provides coverage from the Cities of Leduc and
-   max_articles_per_feed = 100
+ Wetaskiwin, including news from two complete counties, plus the towns and
-   no_stylesheets = True
+ villages within.  The counties of Leduc and Wetaskiwin provide news
-   #delay = 1
+ coverage of agriculture, sports, government, family, events and opinion.
-   use_embedded_content = False
+ This publication updated weekly every Thursday.'''
-   publisher = 'Pipestone Publishing'
+     oldest_article = 13
-   category = 'News, Alberta, Canada'
+     max_articles_per_feed = 100
-   language = 'en_CA'
+     no_stylesheets = True
-   encoding = 'iso-8859-1'
+     #delay = 1
-   cover_url = 'http://www.pipestoneflyer.ca/images/calibre-cover.jpg'
+     use_embedded_content = False
-   remove_tags_before = dict(id='ContentPanel')
+     publisher = 'Pipestone Publishing'
-   remove_tags_after  = dict(id='ContentPanel')
+     category = 'News, Alberta, Canada'
-   remove_tags = [dict(name='div', attrs={'id':'StoryNav'}),dict(name='div', attrs={'id':'BottomAds'}),dict(name='div', attrs={'id':'MoreStoryLinks'})]
+     language = 'en_CA'
-   extra_css = 'img { margin:5px }'
+     encoding = 'iso-8859-1'
-   feeds = [
+     cover_url = 'http://www.pipestoneflyer.ca/images/calibre-cover.jpg'
-('Feature', 'http://www.pipestoneflyer.ca/Feature.rss'),
+     remove_tags_before = dict(id='ContentPanel')
-('Editors Desk', 'http://www.pipestoneflyer.ca/Editor%27s%20Desk.rss'),
+     remove_tags_after  = dict(id='ContentPanel')
-('Letters', 'http://www.pipestoneflyer.ca/Letters.rss'),
+     remove_tags = [dict(name='div',
-('A Loco Viewpoint', 'http://www.pipestoneflyer.ca/A%20Loco%20Viewpoint.rss'),
+ attrs={'id':'StoryNav'}),dict(name='div',
-('Lifes Doorway', 'http://www.pipestoneflyer.ca/Life%27s%20Doorway.rss'),
+ attrs={'id':'BottomAds'}),dict(name='div', attrs={'id':'MoreStoryLinks'})]
-('From the Otherside', 'http://www.pipestoneflyer.ca/From%20the%20Otherside.rss'),
+     extra_css = 'img { margin:5px }'
-('Opinion', 'http://www.pipestoneflyer.ca/Opinion.rss'),
+     feeds = [
-('Community', 'http://www.pipestoneflyer.ca/Community.rss'),
+ ('Feature', 'http://www.pipestoneflyer.ca/Feature.rss'),
-('Sports', 'http://www.pipestoneflyer.ca/Sports.rss'),
+ ('Editors Desk', 'http://www.pipestoneflyer.ca/Editor%27s%20Desk.rss'),
-('Chambers', 'http://www.pipestoneflyer.ca/Chambers.rss'),
+ ('Letters', 'http://www.pipestoneflyer.ca/Letters.rss'),
-('Government', 'http://www.pipestoneflyer.ca/Government.rss'),
+ ('A Loco Viewpoint',
-('Environment', 'http://www.pipestoneflyer.ca/Environment.rss'),
+ 'http://www.pipestoneflyer.ca/A%20Loco%20Viewpoint.rss'),
-('Health', 'http://www.pipestoneflyer.ca/Health.rss'),
+ ('Lifes Doorway', 'http://www.pipestoneflyer.ca/Life%27s%20Doorway.rss'),
-('Funnies', 'http://www.pipestoneflyer.ca/Funnies.rss'),
+ ('From the Otherside',
-('Faith', 'http://www.pipestoneflyer.ca/Faith.rss'),
+ 'http://www.pipestoneflyer.ca/From%20the%20Otherside.rss'),
-('News and Views', 'http://www.pipestoneflyer.ca/News%20and%20Views.rss'),
+ ('Opinion', 'http://www.pipestoneflyer.ca/Opinion.rss'),
-('Obituaries', 'http://www.pipestoneflyer.ca/Obituaries.rss'),
+ ('Community', 'http://www.pipestoneflyer.ca/Community.rss'),
-('Police Blotter', 'http://www.pipestoneflyer.ca/Police%20Blotter.rss'),
+ ('Sports', 'http://www.pipestoneflyer.ca/Sports.rss'),
-]
+ ('Chambers', 'http://www.pipestoneflyer.ca/Chambers.rss'),
+ ('Government', 'http://www.pipestoneflyer.ca/Government.rss'),
+ ('Travel ', 'http://www.pipestoneflyer.ca/Travel%20.rss'),
+ ('Environment', 'http://www.pipestoneflyer.ca/Environment.rss'),
+ ('Health', 'http://www.pipestoneflyer.ca/Health.rss'),
+ ('Funnies', 'http://www.pipestoneflyer.ca/Funnies.rss'),
+ ('Events', 'http://www.pipestoneflyer.ca/Events.rss'),
+ ('Faith', 'http://www.pipestoneflyer.ca/Faith.rss'),
+ ('News and Views', 'http://www.pipestoneflyer.ca/News%20and%20Views.rss'),
+ ('Obituaries', 'http://www.pipestoneflyer.ca/Obituaries.rss'),
+ ('Police Blotter', 'http://www.pipestoneflyer.ca/Police%20Blotter.rss'),
+ ('Careers', 'http://www.pipestoneflyer.ca/Careers.rss'),
+ ]

resources/recipes/post_today.recipe

@@ -3,7 +3,7 @@ from calibre.web.feeds.news import BasicNewsRecipe
 class AdvancedUserRecipe1299061355(BasicNewsRecipe):
     title          = u'Post Today'
     language       = 'th'
-    __author__     = "Chotechai"
+    __author__     = "Chotechai P."
     oldest_article = 7
     max_articles_per_feed = 100
     cover_url = 'http://upload.wikimedia.org/wikipedia/th/2/2e/Posttoday_Logo.png'

resources/recipes/thai_post_daily.recipe

@@ -1,18 +1,17 @@
-from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.web.feeds.news import BasicNewsRecipe
-
+
-class AdvancedUserRecipe1299054026(BasicNewsRecipe):
+class AdvancedUserRecipe1299054026(BasicNewsRecipe):
-    title          = u'Thai Post Daily'
+    title          = u'Thai Post Daily'
-    __author__ = 'Chotechai P.'
+    __author__ = 'Chotechai P.'
-    language = 'th'
+    oldest_article = 7
-    oldest_article = 7
+    max_articles_per_feed = 100
-    max_articles_per_feed = 100
+    cover_url = 'http://upload.wikimedia.org/wikipedia/th/1/10/ThaiPost_Logo.png'
-
+    feeds          = [(u'\u0e02\u0e48\u0e32\u0e27\u0e2b\u0e19\u0e49\u0e32\u0e2b\u0e19\u0e36\u0e48\u0e07', u'http://thaipost.net/taxonomy/term/1/all/feed'), (u'\u0e1a\u0e17\u0e1a\u0e23\u0e23\u0e13\u0e32\u0e18\u0e34\u0e01\u0e32\u0e23', u'http://thaipost.net/taxonomy/term/11/all/feed'), (u'\u0e40\u0e1b\u0e25\u0e27 \u0e2a\u0e35\u0e40\u0e07\u0e34\u0e19', u'http://thaipost.net/taxonomy/term/2/all/feed'), (u'\u0e2a\u0e20\u0e32\u0e1b\u0e23\u0e30\u0e0a\u0e32\u0e0a\u0e19', u'http://thaipost.net/taxonomy/term/3/all/feed'), (u'\u0e16\u0e39\u0e01\u0e17\u0e38\u0e01\u0e02\u0e49\u0e2d', u'http://thaipost.net/taxonomy/term/4/all/feed'), (u'\u0e01\u0e32\u0e23\u0e40\u0e21\u0e37\u0e2d\u0e07', u'http://thaipost.net/taxonomy/term/5/all/feed'), (u'\u0e17\u0e48\u0e32\u0e19\u0e02\u0e38\u0e19\u0e19\u0e49\u0e2d\u0e22', u'http://thaipost.net/taxonomy/term/12/all/feed'), (u'\u0e1a\u0e17\u0e04\u0e27\u0e32\u0e21\u0e1e\u0e34\u0e40\u0e28\u0e29', u'http://thaipost.net/taxonomy/term/66/all/feed'), (u'\u0e23\u0e32\u0e22\u0e07\u0e32\u0e19\u0e1e\u0e34\u0e40\u0e28\u0e29', u'http://thaipost.net/taxonomy/term/67/all/feed'), (u'\u0e1a\u0e31\u0e19\u0e17\u0e36\u0e01\u0e2b\u0e19\u0e49\u0e32 4', u'http://thaipost.net/taxonomy/term/13/all/feed'), (u'\u0e40\u0e2a\u0e35\u0e22\u0e1a\u0e0b\u0e36\u0e48\u0e07\u0e2b\u0e19\u0e49\u0e32', u'http://thaipost.net/taxonomy/term/64/all/feed'), (u'\u0e04\u0e31\u0e19\u0e1b\u0e32\u0e01\u0e2d\u0e22\u0e32\u0e01\u0e40\u0e25\u0e48\u0e32', u'http://thaipost.net/taxonomy/term/65/all/feed'), (u'\u0e40\u0e28\u0e23\u0e29\u0e10\u0e01\u0e34\u0e08', u'http://thaipost.net/taxonomy/term/6/all/feed'), (u'\u0e01\u0e23\u0e30\u0e08\u0e01\u0e44\u0e23\u0e49\u0e40\u0e07\u0e32', u'http://thaipost.net/taxonomy/term/14/all/feed'), (u'\u0e01\u0e23\u0e30\u0e08\u0e01\u0e2b\u0e31\u0e01\u0e21\u0e38\u0e21', u'http://thaipost.net/taxonomy/term/71/all/feed'), (u'\u0e04\u0e34\u0e14\u0e40\u0e2b\u0e19\u0e37\u0e2d\u0e01\u0e23\u0e30\u0e41\u0e2a', u'http://thaipost.net/taxonomy/term/69/all/feed'), (u'\u0e23\u0e32\u0e22\u0e07\u0e32\u0e19', u'http://thaipost.net/taxonomy/term/68/all/feed'), (u'\u0e2d\u0e34\u0e42\u0e04\u0e42\u0e1f\u0e01\u0e31\u0e2a', u'http://thaipost.net/taxonomy/term/10/all/feed'), (u'\u0e01\u0e32\u0e23\u0e28\u0e36\u0e01\u0e29\u0e32-\u0e2a\u0e32\u0e18\u0e32\u0e23\u0e13\u0e2a\u0e38\u0e02', u'http://thaipost.net/taxonomy/term/7/all/feed'), (u'\u0e15\u0e48\u0e32\u0e07\u0e1b\u0e23\u0e30\u0e40\u0e17\u0e28', u'http://thaipost.net/taxonomy/term/8/all/feed'), (u'\u0e01\u0e35\u0e2c\u0e32', u'http://thaipost.net/taxonomy/term/9/all/feed')]
-    feeds          = [(u'\u0e02\u0e48\u0e32\u0e27\u0e2b\u0e19\u0e49\u0e32\u0e2b\u0e19\u0e36\u0e48\u0e07', u'http://thaipost.net/taxonomy/term/1/all/feed'), (u'\u0e1a\u0e17\u0e1a\u0e23\u0e23\u0e13\u0e32\u0e18\u0e34\u0e01\u0e32\u0e23', u'http://thaipost.net/taxonomy/term/11/all/feed'), (u'\u0e40\u0e1b\u0e25\u0e27 \u0e2a\u0e35\u0e40\u0e07\u0e34\u0e19', u'http://thaipost.net/taxonomy/term/2/all/feed'), (u'\u0e2a\u0e20\u0e32\u0e1b\u0e23\u0e30\u0e0a\u0e32\u0e0a\u0e19', u'http://thaipost.net/taxonomy/term/3/all/feed'), (u'\u0e16\u0e39\u0e01\u0e17\u0e38\u0e01\u0e02\u0e49\u0e2d', u'http://thaipost.net/taxonomy/term/4/all/feed'), (u'\u0e01\u0e32\u0e23\u0e40\u0e21\u0e37\u0e2d\u0e07', u'http://thaipost.net/taxonomy/term/5/all/feed'), (u'\u0e17\u0e48\u0e32\u0e19\u0e02\u0e38\u0e19\u0e19\u0e49\u0e2d\u0e22', u'http://thaipost.net/taxonomy/term/12/all/feed'), (u'\u0e1a\u0e17\u0e04\u0e27\u0e32\u0e21\u0e1e\u0e34\u0e40\u0e28\u0e29', u'http://thaipost.net/taxonomy/term/66/all/feed'), (u'\u0e23\u0e32\u0e22\u0e07\u0e32\u0e19\u0e1e\u0e34\u0e40\u0e28\u0e29', u'http://thaipost.net/taxonomy/term/67/all/feed'), (u'\u0e1a\u0e31\u0e19\u0e17\u0e36\u0e01\u0e2b\u0e19\u0e49\u0e32 4', u'http://thaipost.net/taxonomy/term/13/all/feed'), (u'\u0e40\u0e2a\u0e35\u0e22\u0e1a\u0e0b\u0e36\u0e48\u0e07\u0e2b\u0e19\u0e49\u0e32', u'http://thaipost.net/taxonomy/term/64/all/feed'), (u'\u0e04\u0e31\u0e19\u0e1b\u0e32\u0e01\u0e2d\u0e22\u0e32\u0e01\u0e40\u0e25\u0e48\u0e32', u'http://thaipost.net/taxonomy/term/65/all/feed'), (u'\u0e40\u0e28\u0e23\u0e29\u0e10\u0e01\u0e34\u0e08', u'http://thaipost.net/taxonomy/term/6/all/feed'), (u'\u0e01\u0e23\u0e30\u0e08\u0e01\u0e44\u0e23\u0e49\u0e40\u0e07\u0e32', u'http://thaipost.net/taxonomy/term/14/all/feed'), (u'\u0e01\u0e23\u0e30\u0e08\u0e01\u0e2b\u0e31\u0e01\u0e21\u0e38\u0e21', u'http://thaipost.net/taxonomy/term/71/all/feed'), (u'\u0e04\u0e34\u0e14\u0e40\u0e2b\u0e19\u0e37\u0e2d\u0e01\u0e23\u0e30\u0e41\u0e2a', u'http://thaipost.net/taxonomy/term/69/all/feed'), (u'\u0e23\u0e32\u0e22\u0e07\u0e32\u0e19', u'http://thaipost.net/taxonomy/term/68/all/feed'), (u'\u0e2d\u0e34\u0e42\u0e04\u0e42\u0e1f\u0e01\u0e31\u0e2a', u'http://thaipost.net/taxonomy/term/10/all/feed'), (u'\u0e01\u0e32\u0e23\u0e28\u0e36\u0e01\u0e29\u0e32-\u0e2a\u0e32\u0e18\u0e32\u0e23\u0e13\u0e2a\u0e38\u0e02', u'http://thaipost.net/taxonomy/term/7/all/feed'), (u'\u0e15\u0e48\u0e32\u0e07\u0e1b\u0e23\u0e30\u0e40\u0e17\u0e28', u'http://thaipost.net/taxonomy/term/8/all/feed'), (u'\u0e01\u0e35\u0e2c\u0e32', u'http://thaipost.net/taxonomy/term/9/all/feed')]
+
-
+    def print_version(self, url):
-    def print_version(self, url):
+      return url.replace(url, 'http://www.thaipost.net/print/' + url [32:])
-      return url.replace(url, 'http://www.thaipost.net/print/' + url [32:])
+
-
+    remove_tags = []
-    remove_tags = []
+    remove_tags.append(dict(name = 'div', attrs = {'class' : 'print-logo'}))
-    remove_tags.append(dict(name = 'div', attrs = {'class' : 'print-logo'}))
+    remove_tags.append(dict(name = 'div', attrs = {'class' : 'print-site_name'}))
-    remove_tags.append(dict(name = 'div', attrs = {'class' : 'print-site_name'}))
+    remove_tags.append(dict(name = 'div', attrs = {'class' : 'print-breadcrumb'}))
-    remove_tags.append(dict(name = 'div', attrs = {'class' : 'print-breadcrumb'}))

resources/template-functions.json

@@ -20,7 +20,7 @@
     "test": "def evaluate(self, formatter, kwargs, mi, locals, val, value_if_set, value_not_set):\n    if val:\n        return value_if_set\n    else:\n        return value_not_set\n", 
     "eval": "def evaluate(self, formatter, kwargs, mi, locals, template):\n    from formatter import eval_formatter\n    template = template.replace('[[', '{').replace(']]', '}')\n    return eval_formatter.safe_format(template, locals, 'EVAL', None)\n", 
     "multiply": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x * y)\n", 
-    "format_date": "def evaluate(self, formatter, kwargs, mi, locals, val, format_string):\n    print val\n    if not val:\n        return ''\n    try:\n        dt = parse_date(val)\n        s = format_date(dt, format_string)\n    except:\n        s = 'BAD DATE'\n    return s\n", 
+    "format_date": "def evaluate(self, formatter, kwargs, mi, locals, val, format_string):\n    if not val:\n        return ''\n    try:\n        dt = parse_date(val)\n        s = format_date(dt, format_string)\n    except:\n        s = 'BAD DATE'\n    return s\n", 
     "capitalize": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return capitalize(val)\n", 
     "count": "def evaluate(self, formatter, kwargs, mi, locals, val, sep):\n    return unicode(len(val.split(sep)))\n", 
     "lowercase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return val.lower()\n", 

setup/extensions.py

@@ -68,6 +68,10 @@ if isosx:
 
 extensions = [
 
+    Extension('speedup',
+        ['calibre/utils/speedup.c'],
+        ),
+
     Extension('icu',
         ['calibre/utils/icu.c'],
         libraries=icu_libs,

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.7.47'
+__version__   = '0.7.48'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 
 import re
@@ -69,6 +69,7 @@ if plugins is None:
                 'chmlib',
                 'chm_extra',
                 'icu',
+                'speedup',
             ] + \
                     (['winutil'] if iswindows else []) + \
                     (['usbobserver'] if isosx else []):

src/calibre/ebooks/chm/reader.py

@@ -54,8 +54,12 @@ class CHMReader(CHMFile):
         self._extracted = False
 
         # location of '.hhc' file, which is the CHM TOC.
-        self.root, ext = os.path.splitext(self.topics.lstrip('/'))
+        if self.topics is None:
-        self.hhc_path = self.root + ".hhc"
+            self.root, ext = os.path.splitext(self.home.lstrip('/'))
+            self.hhc_path = self.root + ".hhc"
+        else:
+            self.root, ext = os.path.splitext(self.topics.lstrip('/'))
+            self.hhc_path = self.root + ".hhc"
 
     def _parse_toc(self, ul, basedir=os.getcwdu()):
         toc = TOC(play_order=self._playorder, base_path=basedir, text='')

src/calibre/ebooks/metadata/book/base.py

@@ -41,7 +41,7 @@ class SafeFormat(TemplateFormatter):
     def get_value(self, key, args, kwargs):
         try:
             key = key.lower()
-            if key != 'title_sort':
+            if key != 'title_sort' and key not in TOP_LEVEL_IDENTIFIERS:
                 key = field_metadata.search_term_to_field_key(key)
             b = self.book.get_user_metadata(key, False)
             if b and b['datatype'] == 'int' and self.book.get(key, 0) == 0:
@@ -49,7 +49,7 @@ class SafeFormat(TemplateFormatter):
             elif b and b['datatype'] == 'float' and self.book.get(key, 0.0) == 0.0:
                 v = ''
             else:
-                ign, v = self.book.format_field(key, series_with_index=False)
+                v = self.book.format_field(key, series_with_index=False)[1]
             if v is None:
                 return ''
             if v == '':
@@ -578,9 +578,15 @@ class Metadata(object):
                 res = res/2
             return (name, unicode(res), orig_res, cmeta)
 
+        # convert top-level ids into their value
+        if key in TOP_LEVEL_IDENTIFIERS:
+            fmeta = field_metadata['identifiers']
+            name = key
+            res = self.get(key, None)
+            return (name, res, res, fmeta)
+
         # Translate aliases into the standard field name
         fmkey = field_metadata.search_term_to_field_key(key)
-
         if fmkey in field_metadata and field_metadata[fmkey]['kind'] == 'field':
             res = self.get(key, None)
             fmeta = field_metadata[fmkey]

src/calibre/ebooks/metadata/book/json_codec.py

@@ -119,12 +119,12 @@ class JsonCodec(object):
             for item in js:
                 book = book_class(prefix, item.get('lpath', None))
                 for key in item.keys():
-                    if key == 'classifiers':
-                        key = 'identifiers'
                     meta = self.decode_metadata(key, item[key])
                     if key == 'user_metadata':
                         book.set_all_user_metadata(meta)
                     else:
+                        if key == 'classifiers':
+                            key = 'identifiers'
                         setattr(book, key, meta)
                 booklist.append(book)
         except:
@@ -132,6 +132,8 @@ class JsonCodec(object):
             traceback.print_exc()
 
     def decode_metadata(self, key, value):
+        if key == 'classifiers':
+            key = 'identifiers'
         if key == 'user_metadata':
             for k in value:
                 if value[k]['datatype'] == 'datetime':

src/calibre/ebooks/metadata/sources/base.py

@@ -65,7 +65,8 @@ class Source(Plugin):
                     parts = parts[1:] + parts[:1]
                 for tok in parts:
                     tok = pat.sub('', tok).strip()
-                    yield tok
+                    if len(tok) > 2 and tok.lower() not in ('von', ):
+                        yield tok
 
 
     def get_title_tokens(self, title):

src/calibre/ebooks/oeb/output.py

@@ -59,20 +59,34 @@ class OEBOutput(OutputFormatPlugin):
     def workaround_nook_cover_bug(self, root): # {{{
         cov = root.xpath('//*[local-name() = "meta" and @name="cover" and'
                 ' @content != "cover"]')
+
+        def manifest_items_with_id(id_):
+            return root.xpath('//*[local-name() = "manifest"]/*[local-name() = "item" '
+                ' and @id="%s"]'%id_)
+
         if len(cov) == 1:
-            manpath = ('//*[local-name() = "manifest"]/*[local-name() = "item" '
-                ' and @id="%s" and @media-type]')
             cov = cov[0]
-            covid = cov.get('content')
+            covid = cov.get('content', '')
-            manifest_item = root.xpath(manpath%covid)
+
-            has_cover = root.xpath(manpath%'cover')
+            if covid:
-            if len(manifest_item) == 1 and not has_cover and \
+                manifest_item = manifest_items_with_id(covid)
-                    manifest_item[0].get('media-type',
+                if len(manifest_item) == 1 and \
-                            '').startswith('image/'):
+                        manifest_item[0].get('media-type',
-                self.log.warn('The cover image has an id != "cover". Renaming'
+                                '').startswith('image/'):
-                        ' to work around Nook Color bug')
+                    self.log.warn('The cover image has an id != "cover". Renaming'
-                manifest_item = manifest_item[0]
+                            ' to work around bug in Nook Color')
-                manifest_item.set('id', 'cover')
+
-                cov.set('content', 'cover')
+                    import uuid
+                    newid = str(uuid.uuid4())
+
+                    for item in manifest_items_with_id('cover'):
+                        item.set('id', newid)
+
+                    for x in root.xpath('//*[@idref="cover"]'):
+                        x.set('idref', newid)
+
+                    manifest_item = manifest_item[0]
+                    manifest_item.set('id', 'cover')
+                    cov.set('content', 'cover')
     # }}}
 

src/calibre/gui2/book_details.py

@@ -10,7 +10,7 @@ from Queue import Queue
 
 from PyQt4.Qt import QPixmap, QSize, QWidget, Qt, pyqtSignal, QUrl, \
     QPropertyAnimation, QEasingCurve, QThread, QApplication, QFontInfo, \
-    QSizePolicy, QPainter, QRect, pyqtProperty, QLayout, QPalette
+    QSizePolicy, QPainter, QRect, pyqtProperty, QLayout, QPalette, QMenu
 from PyQt4.QtWebKit import QWebView
 
 from calibre import fit_image, prepare_string_for_xml
@@ -18,7 +18,7 @@ from calibre.gui2.widgets import IMAGE_EXTENSIONS
 from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.constants import preferred_encoding
 from calibre.library.comments import comments_to_html
-from calibre.gui2 import config, open_local_file, open_url
+from calibre.gui2 import config, open_local_file, open_url, pixmap_to_data
 from calibre.utils.icu import sort_key
 
 # render_rows(data) {{{
@@ -70,6 +70,7 @@ def render_rows(data):
 
 class CoverView(QWidget): # {{{
 
+    cover_changed = pyqtSignal(object, object)
 
     def __init__(self, vertical, parent=None):
         QWidget.__init__(self, parent)
@@ -151,6 +152,35 @@ class CoverView(QWidget): # {{{
             fset=setCurrentPixmapSize
             )
 
+    def contextMenuEvent(self, ev):
+        cm = QMenu(self)
+        paste = cm.addAction(_('Paste Cover'))
+        copy = cm.addAction(_('Copy Cover'))
+        if not QApplication.instance().clipboard().mimeData().hasImage():
+            paste.setEnabled(False)
+        copy.triggered.connect(self.copy_to_clipboard)
+        paste.triggered.connect(self.paste_from_clipboard)
+        cm.exec_(ev.globalPos())
+
+    def copy_to_clipboard(self):
+        QApplication.instance().clipboard().setPixmap(self.pixmap)
+
+    def paste_from_clipboard(self):
+        cb = QApplication.instance().clipboard()
+        pmap = cb.pixmap()
+        if pmap.isNull() and cb.supportsSelection():
+            pmap = cb.pixmap(cb.Selection)
+        if not pmap.isNull():
+            self.pixmap = pmap
+            self.do_layout()
+            self.update()
+            if not config['disable_animations']:
+                self.animation.start()
+            id_ = self.data.get('id', None)
+            if id_ is not None:
+                self.cover_changed.emit(id_,
+                    pixmap_to_data(pmap))
+
 
     # }}}
 
@@ -362,7 +392,9 @@ class BookDetails(QWidget): # {{{
     # Drag 'n drop {{{
     DROPABBLE_EXTENSIONS = IMAGE_EXTENSIONS+BOOK_EXTENSIONS
     files_dropped = pyqtSignal(object, object)
+    cover_changed = pyqtSignal(object, object)
 
+    # application/x-moz-file-promise-url
     @classmethod
     def paths_from_event(cls, event):
         '''
@@ -399,6 +431,7 @@ class BookDetails(QWidget): # {{{
         self.setLayout(self._layout)
 
         self.cover_view = CoverView(vertical, self)
+        self.cover_view.cover_changed.connect(self.cover_changed.emit)
         self._layout.addWidget(self.cover_view)
         self.book_info = BookInfo(vertical, self)
         self._layout.addWidget(self.book_info)

src/calibre/gui2/dialogs/edit_authors_dialog.py

@@ -66,8 +66,8 @@ class EditAuthorsDialog(QDialog, Ui_EditAuthorsDialog):
         self.sort_by_author_sort.setChecked(True)
         self.author_sort_order = 1
 
-        # set up author sort calc button
         self.recalc_author_sort.clicked.connect(self.do_recalc_author_sort)
+        self.auth_sort_to_author.clicked.connect(self.do_auth_sort_to_author)
 
         if select_item is not None:
             self.table.setCurrentItem(select_item)
@@ -108,6 +108,17 @@ class EditAuthorsDialog(QDialog, Ui_EditAuthorsDialog):
         self.table.setFocus(Qt.OtherFocusReason)
         self.table.cellChanged.connect(self.cell_changed)
 
+    def do_auth_sort_to_author(self):
+        self.table.cellChanged.disconnect()
+        for row in range(0,self.table.rowCount()):
+            item = self.table.item(row, 1)
+            aus  = unicode(item.text()).strip()
+            c = self.table.item(row, 0)
+            # Sometimes trailing commas are left by changing between copy algs
+            c.setText(aus)
+        self.table.setFocus(Qt.OtherFocusReason)
+        self.table.cellChanged.connect(self.cell_changed)
+
     def cell_changed(self, row, col):
         if col == 0:
             item = self.table.item(row, 0)

src/calibre/gui2/dialogs/edit_authors_dialog.ui

@@ -52,13 +52,26 @@
      <item>
       <widget class="QPushButton" name="recalc_author_sort">
        <property name="toolTip">
-        <string>Reset all the author sort values to a value automatically generated from the author. Exactly how this value is automatically generated can be controlled via Preferences-&gt;Advanced-&gt;Tweaks</string>
+        <string>Reset all the author sort values to a value automatically
+generated from the author. Exactly how this value is automatically
+generated can be controlled via Preferences-&gt;Advanced-&gt;Tweaks</string>
        </property>
        <property name="text">
         <string>Recalculate all author sort values</string>
        </property>
       </widget>
      </item>
+     <item>
+      <widget class="QPushButton" name="auth_sort_to_author">
+       <property name="toolTip">
+        <string>Copy author sort to author for every author. You typically use this button
+after changing Preferences-&gt;Advanced-&gt;Tweaks-&gt;Author sort name algorithm</string>
+       </property>
+       <property name="text">
+        <string>Copy all author sort values to author</string>
+       </property>
+      </widget>
+     </item>
      <item>
       <spacer name="horizontalSpacer_3">
        <property name="orientation">

src/calibre/gui2/dialogs/scheduler.py

@@ -264,10 +264,8 @@ class Scheduler(QObject):
                 ids = list(self.recipe_model.db.tags_older_than(_('News'),
                     delta))
             except:
-                # Should never happen
+                # Happens if library is being switched
                 ids = []
-                import traceback
-                traceback.print_exc()
             if ids:
                 if ids:
                     self.delete_old_news.emit(ids)

src/calibre/gui2/init.py

@@ -262,6 +262,8 @@ class LayoutMixin(object): # {{{
         self.status_bar.initialize(self.system_tray_icon)
         self.book_details.show_book_info.connect(self.iactions['Show Book Details'].show_book_info)
         self.book_details.files_dropped.connect(self.iactions['Add Books'].files_dropped_on_book)
+        self.book_details.cover_changed.connect(self.bd_cover_changed,
+                type=Qt.QueuedConnection)
         self.book_details.open_containing_folder.connect(self.iactions['View'].view_folder_for_id)
         self.book_details.view_specific_format.connect(self.iactions['View'].view_format_by_id)
 
@@ -272,6 +274,10 @@ class LayoutMixin(object): # {{{
                     self.library_view.currentIndex())
         self.library_view.setFocus(Qt.OtherFocusReason)
 
+    def bd_cover_changed(self, id_, cdata):
+        self.library_view.model().db.set_cover(id_, cdata)
+        if self.cover_flow:
+            self.cover_flow.dataChanged()
 
     def save_layout_state(self):
         for x in ('library', 'memory', 'card_a', 'card_b'):

src/calibre/gui2/library/models.py

@@ -616,6 +616,19 @@ class BooksModel(QAbstractTableModel): # {{{
 
         def bool_type_decorator(r, idx=-1, bool_cols_are_tristate=True):
             val = self.db.data[r][idx]
+            if isinstance(val, (str, unicode)):
+                try:
+                    val = icu_lower(val)
+                    if not val:
+                        val = None
+                    elif val in [_('yes'), _('checked'), 'true']:
+                        val = True
+                    elif val in [_('no'), _('unchecked'), 'false']:
+                        val = False
+                    else:
+                        val = bool(int(val))
+                except:
+                    val = None
             if not bool_cols_are_tristate:
                 if val is None or not val:
                     return self.bool_no_icon
@@ -676,6 +689,12 @@ class BooksModel(QAbstractTableModel): # {{{
             if datatype in ('text', 'comments', 'composite', 'enumeration'):
                 self.dc[col] = functools.partial(text_type, idx=idx,
                                                  mult=self.custom_columns[col]['is_multiple'])
+                if datatype == 'composite':
+                    csort = self.custom_columns[col]['display'].get('composite_sort', 'text')
+                    if csort == 'bool':
+                        self.dc_decorator[col] = functools.partial(
+                                    bool_type_decorator, idx=idx,
+                                    bool_cols_are_tristate=tweaks['bool_custom_columns_are_tristate'] != 'no')
             elif datatype in ('int', 'float'):
                 self.dc[col] = functools.partial(number_type, idx=idx)
             elif datatype == 'datetime':

src/calibre/gui2/preferences/create_custom_column.py

@@ -68,6 +68,9 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
         text = text[:-1]
         self.shortcuts.setText(text)
 
+        for sort_by in [_('Text'), _('Number'), _('Date'), _('Yes/No')]:
+            self.composite_sort_by.addItem(sort_by)
+
         self.parent = parent
         self.editing_col = editing
         self.standard_colheads = standard_colheads
@@ -108,6 +111,13 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
                 self.date_format_box.setText(c['display'].get('date_format', ''))
         elif ct == 'composite':
             self.composite_box.setText(c['display'].get('composite_template', ''))
+            sb = c['display'].get('composite_sort', 'text')
+            vals = ['text', 'number', 'date', 'bool']
+            if sb in vals:
+                sb = vals.index(sb)
+            else:
+                sb = 0
+            self.composite_sort_by.setCurrentIndex(sb)
         elif ct == 'enumeration':
             self.enum_box.setText(','.join(c['display'].get('enum_values', [])))
         self.datatype_changed()
@@ -135,8 +145,9 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
                 {
                     'isbn': '{identifiers:select(isbn)}',
                     'formats': '{formats}',
-                    'last_modified':'''{last_modified:'format_date($, "%d %m, %Y")'}'''
+                    'last_modified':'''{last_modified:'format_date($, "dd MMM yy")'}'''
                     }[which])
+            self.composite_sort_by.setCurrentIndex(2 if which == 'last_modified' else 0)
 
 
     def datatype_changed(self, *args):
@@ -146,7 +157,7 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             col_type = None
         for x in ('box', 'default_label', 'label'):
             getattr(self, 'date_format_'+x).setVisible(col_type == 'datetime')
-        for x in ('box', 'default_label', 'label'):
+        for x in ('box', 'default_label', 'label', 'sort_by', 'sort_by_label'):
             getattr(self, 'composite_'+x).setVisible(col_type == 'composite')
         for x in ('box', 'default_label', 'label'):
             getattr(self, 'enum_'+x).setVisible(col_type == 'enumeration')
@@ -170,10 +181,13 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             is_multiple = False
         if not col_heading:
             return self.simple_error('', _('No column heading was provided'))
+
+        db = self.parent.gui.library_view.model().db
+        key = db.field_metadata.custom_field_prefix+col
         bad_col = False
-        if col in self.parent.custcols:
+        if key in self.parent.custcols:
             if not self.editing_col or \
-                    self.parent.custcols[col]['colnum'] != self.orig_column_number:
+                    self.parent.custcols[key]['colnum'] != self.orig_column_number:
                 bad_col = True
         if bad_col:
             return self.simple_error('', _('The lookup name %s is already used')%col)
@@ -201,7 +215,10 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             if not unicode(self.composite_box.text()).strip():
                 return self.simple_error('', _('You must enter a template for'
                     ' composite columns'))
-            display_dict = {'composite_template':unicode(self.composite_box.text()).strip()}
+            display_dict = {'composite_template':unicode(self.composite_box.text()).strip(),
+                            'composite_sort': ['text', 'number', 'date', 'bool']
+                                    [self.composite_sort_by.currentIndex()]
+                        }
         elif col_type == 'enumeration':
             if not unicode(self.enum_box.text()).strip():
                 return self.simple_error('', _('You must enter at least one'
@@ -216,8 +233,6 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
                     'list more than once').format(l[i]))
             display_dict = {'enum_values': l}
 
-        db = self.parent.gui.library_view.model().db
-        key = db.field_metadata.custom_field_prefix+col
         if not self.editing_col:
             db.field_metadata
             self.parent.custcols[key] = {

src/calibre/gui2/preferences/create_custom_column.ui

@@ -80,7 +80,7 @@
        <item row="2" column="0">
         <widget class="QLabel" name="label_3">
          <property name="text">
-          <string>Column &amp;type</string>
+          <string>&amp;Column type</string>
          </property>
          <property name="buddy">
           <cstring>column_type_box</cstring>
@@ -148,6 +148,16 @@
          </property>
         </widget>
        </item>
+       <item row="5" column="0">
+        <widget class="QLabel" name="composite_label">
+         <property name="text">
+          <string>&amp;Template</string>
+         </property>
+         <property name="buddy">
+          <cstring>composite_box</cstring>
+         </property>
+        </widget>
+       </item>
        <item row="5" column="2">
         <layout class="QHBoxLayout" name="horizontalLayout_4">
          <item>
@@ -175,16 +185,46 @@
          </item>
         </layout>
        </item>
-       <item row="5" column="0">
+       <item row="6" column="0">
-        <widget class="QLabel" name="composite_label">
+        <widget class="QLabel" name="composite_sort_by_label">
          <property name="text">
-          <string>&amp;Template</string>
+          <string>&amp;Sort/search column by</string>
          </property>
          <property name="buddy">
-          <cstring>composite_box</cstring>
+          <cstring>composite_sort_by</cstring>
          </property>
         </widget>
        </item>
+       <item row="6" column="2">
+        <layout class="QHBoxLayout" name="horizontalLayout_4">
+         <item>
+          <widget class="QComboBox" name="composite_sort_by">
+           <property name="toolTip">
+            <string>How this column should handled in the GUI when sorting and searching</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <spacer name="horizontalSpacer_24">
+           <property name="orientation">
+            <enum>Qt::Horizontal</enum>
+           </property>
+           <property name="sizePolicy">
+            <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
+             <horstretch>10</horstretch>
+             <verstretch>0</verstretch>
+            </sizepolicy>
+           </property>
+           <property name="sizeHint" stdset="0">
+            <size>
+             <width>20</width>
+             <height>0</height>
+            </size>
+           </property>
+          </spacer>
+         </item>
+        </layout>
+       </item>
        <item row="11" column="0" colspan="4">
         <spacer name="verticalSpacer_2">
          <property name="orientation">

src/calibre/gui2/tag_view.py

@@ -33,9 +33,6 @@ from calibre.gui2.dialogs.tag_list_editor import TagListEditor
 from calibre.gui2.dialogs.edit_authors_dialog import EditAuthorsDialog
 from calibre.gui2.widgets import HistoryLineEdit
 
-def original_name(t):
-    return getattr(t, 'original_name', t.name)
-
 class TagDelegate(QItemDelegate): # {{{
 
     def paint(self, painter, option, index):
@@ -240,9 +237,9 @@ class TagsView(QTreeView): # {{{
                 tag = index.tag
                 if len(index.children) > 0:
                     for c in index.children:
-                        self.add_item_to_user_cat.emit(category, original_name(c.tag),
+                        self.add_item_to_user_cat.emit(category, c.tag.original_name,
                                                c.tag.category)
-                self.add_item_to_user_cat.emit(category, original_name(tag),
+                self.add_item_to_user_cat.emit(category, tag.original_name,
                                                tag.category)
                 return
             if action == 'add_subcategory':
@@ -258,9 +255,9 @@ class TagsView(QTreeView): # {{{
                 tag = index.tag
                 if len(index.children) > 0:
                     for c in index.children:
-                        self.del_item_from_user_cat.emit(key, original_name(c.tag),
+                        self.del_item_from_user_cat.emit(key, c.tag.original_name,
                                                c.tag.category)
-                self.del_item_from_user_cat.emit(key, original_name(tag), tag.category)
+                self.del_item_from_user_cat.emit(key, tag.original_name, tag.category)
                 return
             if action == 'manage_searches':
                 self.saved_search_edit.emit(category)
@@ -403,7 +400,7 @@ class TagsView(QTreeView): # {{{
                             self.db.field_metadata[key]['is_custom']:
                     self.context_menu.addAction(_('Manage %s')%category,
                             partial(self.context_menu_handler, action='open_editor',
-                                    category=original_name(tag) if tag else None,
+                                    category=tag.original_name if tag else None,
                                     key=key))
                 elif key == 'authors':
                     self.context_menu.addAction(_('Manage %s')%category,
@@ -632,7 +629,7 @@ class TagTreeItem(object): # {{{
             while p.parent.type != self.ROOT:
                 p = p.parent
             if not tag.is_hierarchical:
-                name = original_name(tag)
+                name = tag.original_name
             else:
                 name = tag.name
             tt_author = False
@@ -644,7 +641,7 @@ class TagTreeItem(object): # {{{
             else:
                 return QVariant('[%d] %s'%(count, name))
         if role == Qt.EditRole:
-            return QVariant(original_name(tag))
+            return QVariant(tag.original_name)
         if role == Qt.DecorationRole:
             return self.icon_state_map[tag.state]
         if role == Qt.ToolTipRole:
@@ -810,7 +807,7 @@ class TagsModel(QAbstractItemModel): # {{{
                     p = node
                     while p.type != TagTreeItem.CATEGORY:
                         p = p.parent
-                    d = (node.type, p.category_key, p.is_gst, original_name(t),
+                    d = (node.type, p.category_key, p.is_gst, t.original_name,
                          t.category, path)
                 data.append(d)
             else:
@@ -861,7 +858,7 @@ class TagsModel(QAbstractItemModel): # {{{
             Copy/move an item and all its children to the destination
             '''
             copied = False
-            src_name = original_name(node.tag)
+            src_name = node.tag.original_name
             src_cat = node.tag.category
             # delete the item if the source is a user category and action is move
             if is_uc and not src_parent_is_gst and src_parent in user_cats and \
@@ -1019,7 +1016,7 @@ class TagsModel(QAbstractItemModel): # {{{
 
         fm = self.db.metadata_for_field(key)
         is_multiple = fm['is_multiple']
-        val = original_name(on_node.tag)
+        val = on_node.tag.original_name
         for id in ids:
             mi = self.db.get_metadata(id, index_is_id=True)
 
@@ -1135,7 +1132,7 @@ class TagsModel(QAbstractItemModel): # {{{
                 collapse_model = 'partition'
                 collapse_template = tweaks['categories_collapsed_popularity_template']
 
-        def process_one_node(category, state_map):
+        def process_one_node(category, state_map): # {{{
             collapse_letter = None
             category_index = self.createIndex(category.row(), 0, category)
             category_node = category_index.internalPointer()
@@ -1152,7 +1149,8 @@ class TagsModel(QAbstractItemModel): # {{{
                                 not fm['is_custom'] and \
                                 not fm['kind'] == 'user' \
                             else False
-            tt = key if fm['kind'] == 'user' else None
+            in_uc = fm['kind'] == 'user'
+            tt = key if in_uc else None
 
             if collapse_model == 'first letter':
                 # Build a list of 'equal' first letters by looking for
@@ -1227,11 +1225,10 @@ class TagsModel(QAbstractItemModel): # {{{
                 # category display order is important here. The following works
                 # only of all the non-user categories are displayed before the
                 # user categories
-                components = [t.strip() for t in original_name(tag).split('.')
+                components = [t.strip() for t in tag.original_name.split('.')
                               if t.strip()]
-                if len(components) == 0 or '.'.join(components) != original_name(tag):
+                if len(components) == 0 or '.'.join(components) != tag.original_name:
-                    components = [original_name(tag)]
+                    components = [tag.original_name]
-                in_uc = fm['kind'] == 'user'
                 if (not tag.is_hierarchical) and (in_uc or
                         key in ['authors', 'publisher', 'news', 'formats', 'rating'] or
                         key not in self.db.prefs.get('categories_using_hierarchy', []) or
@@ -1277,6 +1274,7 @@ class TagsModel(QAbstractItemModel): # {{{
                         # This id_set must not be None
                         node_parent.id_set |= tag.id_set
             return
+        # }}}
 
         for category in self.category_nodes:
             if len(category.children) > 0:
@@ -1364,7 +1362,7 @@ class TagsModel(QAbstractItemModel): # {{{
             return True
 
         key = item.tag.category
-        name = original_name(item.tag)
+        name = item.tag.original_name
         # make certain we know about the item's category
         if key not in self.db.field_metadata:
             return False
@@ -1588,10 +1586,14 @@ class TagsModel(QAbstractItemModel): # {{{
                     else:
                         prefix = ''
                     category = tag.category if key != 'news' else 'tag'
+                    add_colon = False
+                    if self.db.field_metadata[tag.category]['is_csp']:
+                        add_colon = True
+
                     if tag.name and tag.name[0] == u'\u2605': # char is a star. Assume rating
                         ans.append('%s%s:%s'%(prefix, category, len(tag.name)))
                     else:
-                        name = original_name(tag)
+                        name = tag.original_name
                         use_prefix = tag.state in [TAG_SEARCH_STATES['mark_plusplus'],
                                                    TAG_SEARCH_STATES['mark_minusminus']]
                         if category == 'tags':
@@ -1604,8 +1606,9 @@ class TagsModel(QAbstractItemModel): # {{{
                         n = name.replace(r'"', r'\"')
                         if name.startswith('.'):
                             n = '.' + n
-                        ans.append('%s%s:"=%s%s"'%(prefix, category,
+                        ans.append('%s%s:"=%s%s%s"'%(prefix, category,
-                                                '.' if use_prefix else '', n))
+                                                '.' if use_prefix else '', n,
+                                                ':' if add_colon else ''))
         return ans
 
     def find_item_node(self, key, txt, start_path, equals_match=False):
@@ -1633,7 +1636,7 @@ class TagsModel(QAbstractItemModel): # {{{
             tag = tag_item.tag
             if tag is None:
                 return False
-            name = original_name(tag)
+            name = tag.original_name
             if (equals_match and strcmp(name, txt) == 0) or \
                     (not equals_match and lower(name).find(txt) >= 0):
                 self.path_found = path
@@ -2075,6 +2078,10 @@ class TagBrowserWidget(QWidget): # {{{
                 _('Add your own categories to the Tag Browser'))
         parent.edit_categories.setStatusTip(parent.edit_categories.toolTip())
 
+        # self.leak_test_timer = QTimer(self)
+        # self.leak_test_timer.timeout.connect(self.test_for_leak)
+        # self.leak_test_timer.start(5000)
+
     def set_pane_is_visible(self, to_what):
         self.tags_view.set_pane_is_visible(to_what)
 
@@ -2136,5 +2143,13 @@ class TagBrowserWidget(QWidget): # {{{
     def not_found_label_timer_event(self):
         self.not_found_label.setVisible(False)
 
+    def test_for_leak(self):
+        from calibre.utils.mem import memory
+        import gc
+        before = memory()
+        self.tags_view.recount()
+        for i in xrange(3): gc.collect()
+        print 'Used memory:', memory(before)/(1024.), 'KB'
+
 # }}}
 

src/calibre/gui2/widgets.py

@@ -236,8 +236,8 @@ class ImageDropMixin(object): # {{{
 
     def contextMenuEvent(self, ev):
         cm = QMenu(self)
-        copy = cm.addAction(_('Copy Image'))
+        paste = cm.addAction(_('Paste Cover'))
-        paste = cm.addAction(_('Paste Image'))
+        copy = cm.addAction(_('Copy Cover'))
         if not QApplication.instance().clipboard().mimeData().hasImage():
             paste.setEnabled(False)
         copy.triggered.connect(self.copy_to_clipboard)

src/calibre/library/caches.py

@@ -302,14 +302,20 @@ class ResultCache(SearchQueryParser): # {{{
             for id_ in candidates:
                 item = self._data[id_]
                 if item is None: continue
-                if item[loc] is None or item[loc] <= UNDEFINED_DATE:
+                v = item[loc]
+                if isinstance(v, (str, unicode)):
+                    v = parse_date(v)
+                if v is None or v <= UNDEFINED_DATE:
                     matches.add(item[0])
             return matches
         if query == 'true':
             for id_ in candidates:
                 item = self._data[id_]
                 if item is None: continue
-                if item[loc] is not None and item[loc] > UNDEFINED_DATE:
+                v = item[loc]
+                if isinstance(v, (str, unicode)):
+                    v = parse_date(v)
+                if v is not None and v > UNDEFINED_DATE:
                     matches.add(item[0])
             return matches
 
@@ -349,7 +355,10 @@ class ResultCache(SearchQueryParser): # {{{
         for id_ in candidates:
             item = self._data[id_]
             if item is None or item[loc] is None: continue
-            if relop(item[loc], qd, field_count):
+            v = item[loc]
+            if isinstance(v, (str, unicode)):
+                v = parse_date(v)
+            if relop(v, qd, field_count):
                 matches.add(item[0])
         return matches
 
@@ -390,7 +399,7 @@ class ResultCache(SearchQueryParser): # {{{
         elif  dt == 'rating':
             cast = (lambda x: int (x))
             adjust = lambda x: x/2
-        elif dt == 'float':
+        elif dt in ('float', 'composite'):
             cast = lambda x : float (x)
             adjust = lambda x: x
         else: # count operation
@@ -413,12 +422,15 @@ class ResultCache(SearchQueryParser): # {{{
             item = self._data[id_]
             if item is None:
                 continue
-            v = val_func(item)
+            try:
+                v = cast(val_func(item))
+            except:
+                v = 0
             if not v:
-                i = 0
+                v = 0
             else:
-                i = adjust(v)
+                v = adjust(v)
-            if relop(i, q):
+            if relop(v, q):
                 matches.add(item[0])
         return matches
 
@@ -509,6 +521,50 @@ class ResultCache(SearchQueryParser): # {{{
             query = icu_lower(query)
         return matchkind, query
 
+    def get_bool_matches(self, location, query, candidates):
+        bools_are_tristate = tweaks['bool_custom_columns_are_tristate'] != 'no'
+        loc = self.field_metadata[location]['rec_index']
+        matches = set()
+        query = icu_lower(query)
+        for id_ in candidates:
+            item = self._data[id_]
+            if item is None:
+                continue
+
+            val = item[loc]
+            if isinstance(val, (str, unicode)):
+                try:
+                    val = icu_lower(val)
+                    if not val:
+                        val = None
+                    elif val in [_('yes'), _('checked'), 'true']:
+                        val = True
+                    elif val in [_('no'), _('unchecked'), 'false']:
+                        val = False
+                    else:
+                        val = bool(int(val))
+                except:
+                    val = None
+
+            if not bools_are_tristate:
+                if val is None or not val: # item is None or set to false
+                    if query in [_('no'), _('unchecked'), 'false']:
+                        matches.add(item[0])
+                else: # item is explicitly set to true
+                    if query in [_('yes'), _('checked'), 'true']:
+                        matches.add(item[0])
+            else:
+                if val is None:
+                    if query in [_('empty'), _('blank'), 'false']:
+                        matches.add(item[0])
+                elif not val: # is not None and false
+                    if query in [_('no'), _('unchecked'), 'true']:
+                        matches.add(item[0])
+                else: # item is not None and true
+                    if query in [_('yes'), _('checked'), 'true']:
+                        matches.add(item[0])
+        return matches
+
     def get_matches(self, location, query, candidates=None,
             allow_recursion=True):
         matches = set([])
@@ -559,13 +615,20 @@ class ResultCache(SearchQueryParser): # {{{
             if location in self.field_metadata:
                 fm = self.field_metadata[location]
                 # take care of dates special case
-                if fm['datatype'] == 'datetime':
+                if fm['datatype'] == 'datetime' or \
+                        (fm['datatype'] == 'composite' and
+                         fm['display'].get('composite_sort', '') == 'date'):
                     return self.get_dates_matches(location, query.lower(), candidates)
 
                 # take care of numbers special case
-                if fm['datatype'] in ('rating', 'int', 'float'):
+                if fm['datatype'] in ('rating', 'int', 'float') or \
+                        (fm['datatype'] == 'composite' and
+                         fm['display'].get('composite_sort', '') == 'number'):
                     return self.get_numeric_matches(location, query.lower(), candidates)
 
+                if fm['datatype'] == 'bool':
+                    return self.get_bool_matches(location, query, candidates)
+
                 # take care of the 'count' operator for is_multiples
                 if fm['is_multiple'] and \
                         len(query) > 1 and query.startswith('#') and \
@@ -619,9 +682,6 @@ class ResultCache(SearchQueryParser): # {{{
             for i, loc in enumerate(location):
                 location[i] = db_col[loc]
 
-            # get the tweak here so that the string lookup and compare aren't in the loop
-            bools_are_tristate = tweaks['bool_custom_columns_are_tristate'] != 'no'
-
             for loc in location: # location is now an array of field indices
                 if loc == db_col['authors']:
                     ### DB stores authors with commas changed to bars, so change query
@@ -633,27 +693,6 @@ class ResultCache(SearchQueryParser): # {{{
                     item = self._data[id_]
                     if item is None: continue
 
-                    if col_datatype[loc] == 'bool': # complexity caused by the two-/three-value tweak
-                        v = item[loc]
-                        if not bools_are_tristate:
-                            if v is None or not v: # item is None or set to false
-                                if q in [_('no'), _('unchecked'), 'false']:
-                                    matches.add(item[0])
-                            else: # item is explicitly set to true
-                                if q in [_('yes'), _('checked'), 'true']:
-                                    matches.add(item[0])
-                        else:
-                            if v is None:
-                                if q in [_('empty'), _('blank'), 'false']:
-                                    matches.add(item[0])
-                            elif not v: # is not None and false
-                                if q in [_('no'), _('unchecked'), 'true']:
-                                    matches.add(item[0])
-                            else: # item is not None and true
-                                if q in [_('yes'), _('checked'), 'true']:
-                                    matches.add(item[0])
-                        continue
-
                     if not item[loc]:
                         if q == 'false':
                             matches.add(item[0])
@@ -893,6 +932,34 @@ class SortKeyGenerator(object):
         for name, fm in self.entries:
             dt = fm['datatype']
             val = record[fm['rec_index']]
+            if dt == 'composite':
+                sb = fm['display'].get('composite_sort', 'text')
+                if sb == 'date':
+                    try:
+                        val = parse_date(val)
+                        dt = 'datetime'
+                    except:
+                        pass
+                elif sb == 'number':
+                    try:
+                        val = float(val)
+                    except:
+                        val = 0.0
+                    dt = 'float'
+                elif sb == 'bool':
+                    try:
+                        v = icu_lower(val)
+                        if not val:
+                            val = None
+                        elif v in [_('yes'), _('checked'), 'true']:
+                            val = True
+                        elif v in [_('no'), _('unchecked'), 'false']:
+                            val = False
+                        else:
+                            val = bool(int(val))
+                    except:
+                        val = None
+                    dt = 'bool'
 
             if dt == 'datetime':
                 if val is None:

src/calibre/library/database2.py

@@ -48,7 +48,7 @@ class Tag(object):
 
     def __init__(self, name, id=None, count=0, state=0, avg=0, sort=None,
                  tooltip=None, icon=None, category=None, id_set=None):
-        self.name = name
+        self.name = self.original_name = name
         self.id = id
         self.count = count
         self.state = state
@@ -833,7 +833,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         mi.pubdate     = row[fm['pubdate']]
         mi.uuid        = row[fm['uuid']]
         mi.title_sort  = row[fm['sort']]
-        mi.metadata_last_modified = row[fm['last_modified']]
+        mi.last_modified = row[fm['last_modified']]
         formats = row[fm['formats']]
         if not formats:
             formats = None
@@ -1493,6 +1493,34 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             # No need for ICU here.
             categories['formats'].sort(key = lambda x:x.name)
 
+        # Now do identifiers. This works like formats
+        categories['identifiers'] = []
+        icon = None
+        if icon_map and 'identifiers' in icon_map:
+                icon = icon_map['identifiers']
+        for ident in self.conn.get('SELECT DISTINCT type FROM identifiers'):
+            ident = ident[0]
+            if ids is not None:
+                count = self.conn.get('''SELECT COUNT(book)
+                                       FROM identifiers
+                                       WHERE type="%s" AND
+                                       books_list_filter(book)'''%ident,
+                                       all=False)
+            else:
+                count = self.conn.get('''SELECT COUNT(id)
+                                       FROM identifiers
+                                       WHERE type="%s"'''%ident,
+                                       all=False)
+            if count > 0:
+                categories['identifiers'].append(Tag(ident, count=count, icon=icon,
+                                                 category='identifiers'))
+
+        if sort == 'popularity':
+            categories['identifiers'].sort(key=lambda x: x.count, reverse=True)
+        else: # no ratings exist to sort on
+            # No need for ICU here.
+            categories['identifiers'].sort(key = lambda x:x.name)
+
         #### Now do the user-defined categories. ####
         user_categories = dict.copy(self.clean_user_categories())
 

src/calibre/library/field_metadata.py

@@ -16,7 +16,8 @@ class TagsIcons(dict):
     '''
 
     category_icons = ['authors', 'series', 'formats', 'publisher', 'rating',
-                      'news',    'tags',   'custom:', 'user:',     'search',]
+                      'news',    'tags',   'custom:', 'user:',     'search',
+                      'identifiers']
     def __init__(self, icon_dict):
         for a in self.category_icons:
             if a not in icon_dict:
@@ -24,16 +25,17 @@ class TagsIcons(dict):
             self[a] = icon_dict[a]
 
 category_icon_map = {
-                    'authors'   : 'user_profile.png',
+                    'authors'    : 'user_profile.png',
-                    'series'    : 'series.png',
+                    'series'     : 'series.png',
-                    'formats'   : 'book.png',
+                    'formats'    : 'book.png',
-                    'publisher' : 'publisher.png',
+                    'publisher'  : 'publisher.png',
-                    'rating'    : 'rating.png',
+                    'rating'     : 'rating.png',
-                    'news'      : 'news.png',
+                    'news'       : 'news.png',
-                    'tags'      : 'tags.png',
+                    'tags'       : 'tags.png',
-                    'custom:'   : 'column.png',
+                    'custom:'    : 'column.png',
-                    'user:'     : 'tb_folder.png',
+                    'user:'      : 'tb_folder.png',
-                    'search'    : 'search.png'
+                    'search'     : 'search.png',
+                    'identifiers': 'id_card.png'
             }
 
 

src/calibre/library/server/browse.py

@@ -346,7 +346,7 @@ class BrowseServer(object):
         for category in sorted(categories, key=lambda x: sort_key(getter(x))):
             if len(categories[category]) == 0:
                 continue
-            if category == 'formats':
+            if category in ('formats', 'identifiers'):
                 continue
             meta = category_meta.get(category, None)
             if meta is None:

src/calibre/library/server/opds.py

@@ -580,7 +580,7 @@ class OPDSServer(object):
         for category in sorted(categories, key=lambda x: sort_key(getter(x))):
             if len(categories[category]) == 0:
                 continue
-            if category == 'formats':
+            if category in ('formats', 'identifiers'):
                 continue
             meta = category_meta.get(category, None)
             if meta is None:

src/calibre/library/sqlite.py

@@ -17,19 +17,54 @@ from datetime import datetime
 from functools import partial
 
 from calibre.ebooks.metadata import title_sort, author_to_author_sort
-from calibre.utils.date import parse_date, isoformat
+from calibre.utils.date import parse_date, isoformat, local_tz
 from calibre import isbytestring, force_unicode
-from calibre.constants import iswindows, DEBUG
+from calibre.constants import iswindows, DEBUG, plugins
 from calibre.utils.icu import strcmp
 from calibre import prints
 
+from dateutil.tz import tzoffset
+
 global_lock = RLock()
 
-def convert_timestamp(val):
+_c_speedup = plugins['speedup'][0]
+
+def _c_convert_timestamp(val):
+    if not val:
+        return None
+    try:
+        ret = _c_speedup.parse_date(val.strip())
+    except:
+        ret = None
+    if ret is None:
+        return parse_date(val, as_utc=False)
+    year, month, day, hour, minutes, seconds, tzsecs = ret
+    return datetime(year, month, day, hour, minutes, seconds,
+                tzinfo=tzoffset(None, tzsecs)).astimezone(local_tz)
+
+def _py_convert_timestamp(val):
     if val:
+        tzsecs = 0
+        try:
+            sign = {'+':1, '-':-1}.get(val[-6], None)
+            if sign is not None:
+                tzsecs = 60*((int(val[-5:-3])*60 + int(val[-2:])) * sign)
+            year = int(val[0:4])
+            month = int(val[5:7])
+            day = int(val[8:10])
+            hour = int(val[11:13])
+            min = int(val[14:16])
+            sec = int(val[17:19])
+            return datetime(year, month, day, hour, min, sec,
+                    tzinfo=tzoffset(None, tzsecs))
+        except:
+            pass
         return parse_date(val, as_utc=False)
     return None
 
+convert_timestamp = _py_convert_timestamp if _c_speedup is None else \
+                    _c_convert_timestamp
+
 def adapt_datetime(dt):
     return isoformat(dt, sep=' ')
 

src/calibre/manual/faq.rst

@@ -327,10 +327,24 @@ Now coming to author name sorting:
     * When recalculating the author sort values for books, |app| uses the author sort values for each individual author. Therefore, ensure that the individual author sort values are correct before recalculating the books' author sort values.
     * You can control whether the Tag Browser display authors using their names or their sort values by setting the :guilabel:`categories_use_field_for_author_name` tweak in Preferences->Tweaks
 
-With all this flexibility, it is possible to have |app| manage your author names however you like. For example, one common request is to have |app| display author names LN, FN. To do this first set the ``author_sort_copy_method`` to ``copy``. Then change all author names to LN, FN via the Manage authors dialog. Then have |app| recalculate author sort values for both authors and books as described above.
-
 Note that you can set an individual author's sort value to whatever you want using :guilabel:`Manage authors`. This is useful when dealing with names that |app| will not get right, such as complex multi-part names like Miguel de Cervantes Saavedra or when dealing with Asian names like Sun Tzu.
 
+With all this flexibility, it is possible to have |app| manage your author names however you like. For example, one common request is to have |app| display author names LN, FN. To do this, and if the note below does not apply to you, then:
+    * Set the ``author_sort_copy_method`` tweak to ``copy`` as described above.
+    * Restart calibre. Do not change any book metadata before doing the remaining steps.
+    * Change all author names to LN, FN using the Manage authors dialog.
+    * After you have changed all the authors, press the `Recalculate all author sort values` button.
+    * Press OK, at which point |app| will change the authors in all your books. This can take a while.
+    
+.. note::
+
+    When changing from FN LN to LN, FN, it is often the case that the values in author_sort are already in LN, FN format. If this is your case, then do the following:
+        * set the ``author_sort_copy_method`` tweak to ``copy`` as described above.
+        * restart calibre. Do not change any book metadata before doing the remaining steps.
+        * open the Manage authors dialog. Press the ``copy all author sort values to author`` button.
+        * Check through the authors to be sure you are happy. You can still press Cancel to abandon the changes. Once you press OK, there is no undo.
+        * Press OK, at which point |app| will change the authors in all your books. This can take a while.
+
 
 Why doesn't |app| let me store books in my own directory structure?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

src/calibre/utils/date.py

@@ -71,6 +71,8 @@ def parse_date(date_string, assume_utc=False, as_utc=True, default=None):
     :param default: Missing fields are filled in from default. If None, the
     current date is used.
     '''
+    if not date_string:
+        return UNDEFINED_DATE
     if default is None:
         func = datetime.utcnow if assume_utc else datetime.now
         default = func().replace(hour=0, minute=0, second=0, microsecond=0,

src/calibre/utils/formatter_functions.py

@@ -469,7 +469,6 @@ class BuiltinFormat_date(BuiltinFormatterFunction):
             'yyyy : the year as four digit number.')
 
     def evaluate(self, formatter, kwargs, mi, locals, val, format_string):
-        print val
         if not val:
             return ''
         try:

src/calibre/utils/ipc/launch.py

@@ -94,7 +94,14 @@ class Worker(object):
         if not hasattr(self, 'child'): return None
         return getattr(self.child, 'pid', None)
 
+    def close_log_file(self):
+        try:
+            self._file.close()
+        except:
+            pass
+
     def kill(self):
+        self.close_log_file()
         try:
             if self.is_alive:
                 if iswindows:

src/calibre/utils/ipc/server.py

@@ -34,6 +34,7 @@ class ConnectedWorker(Thread):
         self.killed = False
         self.log_path = worker.log_path
         self.rfile = rfile
+        self.close_log_file = getattr(worker, 'close_log_file', None)
 
     def start_job(self, job):
         notification = PARALLEL_FUNCS[job.name][-1] is not None
@@ -185,6 +186,10 @@ class Server(Thread):
 
             # Remove finished jobs
             for worker in [w for w in self.workers if not w.is_alive]:
+                try:
+                    worker.close_log_file()
+                except:
+                    pass
                 self.workers.remove(worker)
                 job = worker.job
                 if worker.returncode != 0:

src/calibre/utils/speedup.c

@@ -0,0 +1,80 @@
+#define UNICODE
+#include <Python.h>
+
+#include <stdlib.h>
+
+static PyObject *
+speedup_parse_date(PyObject *self, PyObject *args) {
+    const char *raw, *orig, *tz;
+    char *end;
+    long year, month, day, hour, minute, second, tzh = 0, tzm = 0, sign = 0;
+    size_t len;
+    if(!PyArg_ParseTuple(args, "s", &raw)) return NULL;
+    len = strlen(raw);
+    if (len < 19) Py_RETURN_NONE;
+
+    orig = raw;
+
+
+    year = strtol(raw, &end, 10);
+    if ((end - raw) != 4) Py_RETURN_NONE;
+    raw += 5;
+    
+
+    month = strtol(raw, &end, 10);
+    if ((end - raw) != 2) Py_RETURN_NONE;
+    raw += 3;
+    
+
+    day = strtol(raw, &end, 10);
+    if ((end - raw) != 2) Py_RETURN_NONE;
+    raw += 3;
+    
+    hour = strtol(raw, &end, 10);
+    if ((end - raw) != 2) Py_RETURN_NONE;
+    raw += 3;
+
+    minute = strtol(raw, &end, 10);
+    if ((end - raw) != 2) Py_RETURN_NONE;
+    raw += 3;
+
+    second = strtol(raw, &end, 10);
+    if ((end - raw) != 2) Py_RETURN_NONE;
+
+    tz = orig + len - 6;
+
+    if (*tz == '+') sign = +1;
+    if (*tz == '-') sign = -1;
+    if (sign != 0) {
+        // We have TZ info
+        tz += 1;
+
+        tzh = strtol(tz, &end, 10);
+        if ((end - tz) != 2) Py_RETURN_NONE;
+        tz += 3;
+
+        tzm = strtol(tz, &end, 10);
+        if ((end - tz) != 2) Py_RETURN_NONE;
+    }
+
+    return Py_BuildValue("lllllll", year, month, day, hour, minute, second,
+            (tzh*60 + tzm)*sign*60);
+}
+
+static PyMethodDef speedup_methods[] = {
+    {"parse_date", speedup_parse_date, METH_VARARGS,
+        "parse_date()\n\nParse ISO dates faster."
+    },
+
+    {NULL, NULL, 0, NULL}
+};
+
+
+PyMODINIT_FUNC
+initspeedup(void) {
+    PyObject *m;
+    m = Py_InitModule3("speedup", speedup_methods,
+    "Implementation of methods in C for speed."
+    );
+    if (m == NULL) return;
+}