Add functions to the template language that allow getting the last modified time and size of the individual format files for a book. Fix sorting of composite custom columns that display numbers.

src/calibre/db/__init__.py

@@ -63,5 +63,5 @@ Various things that require other things before they can be migrated:
                     columns/categories/searches info into
                     self.field_metadata. Finally, implement metadata dirtied
                     functionality.
-
+    2. Test Schema upgrades
 '''

src/calibre/library/caches.py

@@ -1024,7 +1024,15 @@ class SortKeyGenerator(object):
                     dt = 'datetime'
                 elif sb == 'number':
                     try:
-                        val = float(val)
+                        val = val.replace(',', '').strip()
+                        p = 1
+                        for i, candidate in enumerate(
+                                    (' B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB')):
+                            if val.endswith(candidate):
+                                p = 1024**(i)
+                                val = val[:-len(candidate)].strip()
+                                break
+                        val = float(val) * p
                     except:
                         val = 0.0
                     dt = 'float'

src/calibre/library/database2.py

@@ -8,6 +8,7 @@ The database used to store ebook metadata
 '''
 import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, \
         json, uuid, tempfile, hashlib
+from collections import defaultdict
 import threading, random
 from itertools import repeat
 from math import ceil
@@ -487,6 +488,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self)
         self.refresh()
         self.last_update_check = self.last_modified()
+        self.format_metadata_cache = defaultdict(dict)
 
     def break_cycles(self):
         self.data.break_cycles()
@@ -914,11 +916,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         mi.book_size   = row[fm['size']]
         mi.ondevice_col= row[fm['ondevice']]
         mi.last_modified = row[fm['last_modified']]
+        id = idx if index_is_id else self.id(idx)
         formats = row[fm['formats']]
+        mi.format_metadata = {}
         if not formats:
             formats = None
         else:
             formats = formats.split(',')
+            for f in formats:
+                mi.format_metadata[f] = self.format_metadata(id, f)
         mi.formats = formats
         tags = row[fm['tags']]
         if tags:
@@ -927,7 +933,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if mi.series:
             mi.series_index = row[fm['series_index']]
         mi.rating = row[fm['rating']]
-        id = idx if index_is_id else self.id(idx)
         mi.set_identifiers(self.get_identifiers(id, index_is_id=True))
         mi.application_id = id
         mi.id = id
@@ -1127,13 +1132,21 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if m:
             return m['mtime']
 
-    def format_metadata(self, id_, fmt):
+    def format_metadata(self, id_, fmt, allow_cache=True):
+        if not fmt:
+            return {}
+        fmt = fmt.upper()
+        if allow_cache:
+            x = self.format_metadata_cache[id_].get(fmt, None)
+            if x is not None:
+                return x
         path = self.format_abspath(id_, fmt, index_is_id=True)
         ans = {}
         if path is not None:
             stat = os.stat(path)
             ans['size'] = stat.st_size
             ans['mtime'] = utcfromtimestamp(stat.st_mtime)
+            self.format_metadata_cache[id_][fmt] = ans
         return ans
 
     def format_hash(self, id_, fmt):
@@ -1269,6 +1282,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
     def add_format(self, index, format, stream, index_is_id=False, path=None,
             notify=True, replace=True):
         id = index if index_is_id else self.id(index)
+        if format:
+            self.format_metadata_cache[id].pop(format.upper(), None)
         if path is None:
             path = os.path.join(self.library_path, self.path(id, index_is_id=True))
         name = self.conn.get('SELECT name FROM data WHERE book=? AND format=?', (id, format), all=False)
@@ -1321,6 +1336,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
     def remove_format(self, index, format, index_is_id=False, notify=True,
                       commit=True, db_only=False):
         id = index if index_is_id else self.id(index)
+        if format:
+            self.format_metadata_cache[id].pop(format.upper(), None)
         name = self.conn.get('SELECT name FROM data WHERE book=? AND format=?', (id, format), all=False)
         if name:
             if not db_only:

src/calibre/manual/template_lang.rst

@@ -124,6 +124,8 @@ The functions available are listed below. Note that the definitive documentation
     * ``capitalize()``	-- return the value with the first letter upper case and the rest lower case.
     * ``contains(pattern, text if match, text if not match)`` -- checks if field contains matches for the regular expression `pattern`. Returns `text if match` if matches are found, otherwise it returns `text if no match`.
     * ``count(separator)`` -- interprets the value as a list of items separated by `separator`, returning the number of items in the list. Most lists use a comma as the separator, but authors uses an ampersand. Examples: `{tags:count(,)}`, `{authors:count(&)}`
+    * ``format_number(template)`` -- interprets the value as a number and format that number using a python formatting template such as "{0:5.2f}" or "{0:,d}" or "${0:5,.2f}". The field_name part of the template must be a 0 (zero) (the "{0:" in the above examples). See the template language and python documentation for more examples. Returns the empty string if formatting fails.
+    * ``human_readable()`` -- expects the value to be a number and returns a string representing that number in KB, MB, GB, etc.
     * ``ifempty(text)``	-- if the field is not empty, return the value of the field. Otherwise return `text`.
     * ``in_list(separator, pattern, found_val, not_found_val)`` -- interpret the field as a list of items separated by `separator`, comparing the `pattern` against each value in the list. If the pattern matches a value, return `found_val`, otherwise return `not_found_val`.
     * ``list_item(index, separator)`` -- interpret the field as a list of items separated by `separator`, returning the `index`th item. The first item is number zero. The last item can be returned using `list_item(-1,separator)`. If the item is not in the list, then the empty value is returned. The separator has the same meaning as in the `count` function.
@@ -257,6 +259,8 @@ The following functions are available in addition to those described in single-f
         iso  : the date with time and timezone. Must be the only format present.
     
     * ``eval(string)`` -- evaluates the string as a program, passing the local variables (those ``assign`` ed to). This permits using the template processor to construct complex results from local variables.
+    * ``formats_modtimes(date_format)`` -- return a comma-separated list of colon_separated items representing modification times for the formats of a book. The date_format parameter specifies how the date is to be formatted. See the date_format function for details. You can use the select function to get the mod time for a specific format. Note that format names are always uppercase, as in EPUB.
+    * ``formats_sizes()`` -- return a comma-separated list of colon_separated items representing sizes in bytes of the formats of a book. You can use the select function to get the size for a specific format. Note that format names are always uppercase, as in EPUB.
     * ``has_cover()`` -- return ``Yes`` if the book has a cover, otherwise return the empty string
     * ``not(value)`` -- returns the string "1" if the value is empty, otherwise returns the empty string. This function works well with test or first_non_empty. You can have as many values as you want.
     * ``merge_lists(list1, list2, separator)`` -- return a list made by merging the items in list1 and list2, removing duplicate items using a case-insensitive compare. If items differ in case, the one in list1 is used. The items in list1 and list2 are separated by separator, as are the items in the returned list.

src/calibre/utils/formatter_functions.py

@@ -10,6 +10,7 @@ __docformat__ = 'restructuredtext en'
 
 import inspect, re, traceback
 
+from calibre import human_readable
 from calibre.utils.titlecase import titlecase
 from calibre.utils.icu import capitalize, strcmp, sort_key
 from calibre.utils.date import parse_date, format_date, now, UNDEFINED_DATE
@@ -519,6 +520,80 @@ class BuiltinSelect(BuiltinFormatterFunction):
                 return v[len(key)+1:]
         return ''
 
+class BuiltinFormatsModtimes(BuiltinFormatterFunction):
+    name = 'formats_modtimes'
+    arg_count = 1
+    category = 'Get values from metadata'
+    __doc__ = doc = _('formats_modtimes(date_format) -- return a comma-separated '
+                  'list of colon_separated items representing modification times '
+                  'for the formats of a book. The date_format parameter '
+                  'specifies how the date is to be formatted. See the '
+                  'date_format function for details. You can use the select '
+                  'function to get the mod time for a specific '
+                  'format. Note that format names are always uppercase, '
+                  'as in EPUB.'
+            )
+
+    def evaluate(self, formatter, kwargs, mi, locals, fmt):
+        fmt_data = mi.get('format_metadata', {})
+        return ','.join(k.upper()+':'+format_date(v['mtime'], fmt)
+                        for k,v in fmt_data.iteritems())
+
+class BuiltinFormatsSizes(BuiltinFormatterFunction):
+    name = 'formats_sizes'
+    arg_count = 0
+    category = 'Get values from metadata'
+    __doc__ = doc = _('formats_sizes() -- return a comma-separated list of '
+                      'colon_separated items representing sizes in bytes'
+                      'of the formats of a book. You can use the select '
+                      'function to get the size for a specific '
+                      'format. Note that format names are always uppercase, '
+                      'as in EPUB.'
+            )
+
+    def evaluate(self, formatter, kwargs, mi, locals):
+        fmt_data = mi.get('format_metadata', {})
+        return ','.join(k.upper()+':'+str(v['size']) for k,v in fmt_data.iteritems())
+
+class BuiltinHumanReadable(BuiltinFormatterFunction):
+    name = 'human_readable'
+    arg_count = 1
+    category = 'Formatting values'
+    __doc__ = doc = _('human_readable(v) -- return a string '
+                      'representing the number v in KB, MB, GB, etc.'
+            )
+
+    def evaluate(self, formatter, kwargs, mi, locals, val):
+        try:
+            return human_readable(long(val))
+        except:
+            return ''
+
+class BuiltinFormatNumber(BuiltinFormatterFunction):
+    name = 'format_number'
+    arg_count = 2
+    category = 'Formatting values'
+    __doc__ = doc = _('format_number(v, template) -- format the number v using '
+                  'a python formatting template such as "{0:5.2f}" or '
+                  '"{0:,d}" or "${0:5,.2f}". The field_name part of the '
+                  'template must be a 0 (zero) (the "{0:" in the above examples). '
+                  'See the template language and python documentation for more '
+                  'examples. Returns the empty string if formatting fails.'
+            )
+
+    def evaluate(self, formatter, kwargs, mi, locals, val, template):
+        if val == '' or val == 'None':
+            return ''
+        try:
+            return template.format(float(val))
+        except:
+            pass
+        try:
+            return template.format(int(val))
+        except:
+            pass
+        return ''
+
 class BuiltinSublist(BuiltinFormatterFunction):
     name = 'sublist'
     arg_count = 4
@@ -591,7 +666,7 @@ class BuiltinSubitems(BuiltinFormatterFunction):
 class BuiltinFormatDate(BuiltinFormatterFunction):
     name = 'format_date'
     arg_count = 2
-    category = 'Date functions'
+    category = 'Formatting values'
     __doc__ = doc = _('format_date(val, format_string) -- format the value, '
             'which must be a date, using the format_string, returning a string. '
             'The formatting codes are: '
@@ -811,52 +886,22 @@ class BuiltinDaysBetween(BuiltinFormatterFunction):
         i = d1 - d2
         return str('%d.%d'%(i.days, i.seconds/8640))
 
-
+formatter_builtins = [
-builtin_add         = BuiltinAdd()
+    BuiltinAdd(), BuiltinAnd(), BuiltinAssign(), BuiltinBooksize(),
-builtin_and         = BuiltinAnd()
+    BuiltinCapitalize(), BuiltinCmp(), BuiltinContains(), BuiltinCount(),
-builtin_assign      = BuiltinAssign()
+    BuiltinDaysBetween(), BuiltinDivide(), BuiltinEval(),
-builtin_booksize    = BuiltinBooksize()
+    BuiltinFirstNonEmpty(), BuiltinField(), BuiltinFormatDate(),
-builtin_capitalize  = BuiltinCapitalize()
+    BuiltinFormatNumber(), BuiltinFormatsModtimes(), BuiltinFormatsSizes(),
-builtin_cmp         = BuiltinCmp()
+    BuiltinHasCover(), BuiltinHumanReadable(), BuiltinIdentifierInList(),
-builtin_contains    = BuiltinContains()
+    BuiltinIfempty(), BuiltinInList(), BuiltinListitem(), BuiltinLookup(),
-builtin_count       = BuiltinCount()
+    BuiltinLowercase(), BuiltinMergeLists(), BuiltinMultiply(), BuiltinNot(),
-builtin_days_between= BuiltinDaysBetween()
+    BuiltinOndevice(), BuiltinOr(), BuiltinPrint(), BuiltinRawField(),
-builtin_divide      = BuiltinDivide()
+    BuiltinRe(), BuiltinSelect(), BuiltinShorten(), BuiltinStrcat(),
-builtin_eval        = BuiltinEval()
+    BuiltinStrcmp(), BuiltinStrInList(), BuiltinSubitems(), BuiltinSublist(),
-builtin_first_non_empty = BuiltinFirstNonEmpty()
+    BuiltinSubstr(), BuiltinSubtract(), BuiltinSwapAroundComma(),
-builtin_field       = BuiltinField()
+    BuiltinSwitch(), BuiltinTemplate(), BuiltinTest(), BuiltinTitlecase(),
-builtin_format_date = BuiltinFormatDate()
+    BuiltinToday(), BuiltinUppercase(),
-builtin_has_cover   = BuiltinHasCover()
+]
-builtin_identifier_in_list = BuiltinIdentifierInList()
-builtin_ifempty     = BuiltinIfempty()
-builtin_in_list     = BuiltinInList()
-builtin_list_item   = BuiltinListitem()
-builtin_lookup      = BuiltinLookup()
-builtin_lowercase   = BuiltinLowercase()
-builtin_merge_lists = BuiltinMergeLists()
-builtin_multiply    = BuiltinMultiply()
-builtin_not         = BuiltinNot()
-builtin_ondevice    = BuiltinOndevice()
-builtin_or          = BuiltinOr()
-builtin_print       = BuiltinPrint()
-builtin_raw_field   = BuiltinRawField()
-builtin_re          = BuiltinRe()
-builtin_select      = BuiltinSelect()
-builtin_shorten     = BuiltinShorten()
-builtin_strcat      = BuiltinStrcat()
-builtin_strcmp      = BuiltinStrcmp()
-builtin_str_in_list = BuiltinStrInList()
-builtin_subitems    = BuiltinSubitems()
-builtin_sublist     = BuiltinSublist()
-builtin_substr      = BuiltinSubstr()
-builtin_subtract    = BuiltinSubtract()
-builtin_swaparound  = BuiltinSwapAroundComma()
-builtin_switch      = BuiltinSwitch()
-builtin_template    = BuiltinTemplate()
-builtin_test        = BuiltinTest()
-builtin_titlecase   = BuiltinTitlecase()
-builtin_today       = BuiltinToday()
-builtin_uppercase   = BuiltinUppercase()
 
 class FormatterUserFunction(FormatterFunction):
     def __init__(self, name, doc, arg_count, program_text):