Add tags like composite custom column. Fixes #759663 (Add option to have Composite columns treated like tags). Add Add the ability to sort the library view by a specific named column (field). Useful for sorting by fields that are not visible. Add a new date format code 'iso'. Permits formatting dates to see the complete time.

src/calibre/devices/usbms/books.py

@@ -203,6 +203,8 @@ class CollectionsBookList(BookList):
                     val = [orig_val]
                 elif fm['datatype'] == 'text' and fm['is_multiple']:
                     val = orig_val
+                elif fm['datatype'] == 'composite' and fm['is_multiple']:
+                    val = [v.strip() for v in val.split(fm['is_multiple'])]
                 else:
                     val = [val]
 

src/calibre/ebooks/metadata/book/base.py

@@ -483,7 +483,7 @@ class Metadata(object):
                         self_tags = self.get(x, [])
                         self.set_user_metadata(x, meta) # get... did the deepcopy
                         other_tags = other.get(x, [])
-                        if meta['is_multiple']:
+                        if meta['datatype'] == 'text' and meta['is_multiple']:
                             # Case-insensitive but case preserving merging
                             lotags = [t.lower() for t in other_tags]
                             lstags = [t.lower() for t in self_tags]

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -519,6 +519,8 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
                 val = [] if fm['is_multiple'] else ['']
             elif not fm['is_multiple']:
                 val = [val]
+            elif fm['datatype'] == 'composite':
+                val = [v.strip() for v in val.split(fm['is_multiple'])]
             elif field == 'authors':
                 val = [v.replace('|', ',') for v in val]
         else:

src/calibre/gui2/library/models.py

@@ -314,6 +314,13 @@ class BooksModel(QAbstractTableModel): # {{{
         if not isinstance(order, bool):
             order = order == Qt.AscendingOrder
         label = self.column_map[col]
+        self._sort(label, order, reset)
+
+    def sort_by_named_field(self, field, order, reset=True):
+        if field in self.db.field_metadata.keys():
+            self._sort(field, order, reset)
+
+    def _sort(self, label, order, reset):
         self.db.sort(label, order)
         if reset:
             self.reset()

src/calibre/gui2/library/views.py

@@ -236,6 +236,16 @@ class BooksView(QTableView): # {{{
                 sm.select(idx, sm.Select|sm.Rows)
             self.scroll_to_row(indices[0].row())
         self.selected_ids = []
+
+    def sort_by_named_field(self, field, order, reset=True):
+        if field in self.column_map:
+            idx = self.column_map.index(field)
+            if order:
+                self.sortByColumn(idx, Qt.AscendingOrder)
+            else:
+                self.sortByColumn(idx, Qt.DescendingOrder)
+        else:
+            self._model.sort_by_named_field(field, order, reset)
     # }}}
 
     # Ondevice column {{{

src/calibre/gui2/preferences/columns.py

@@ -163,8 +163,9 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
             elif '*edited' in self.custcols[c]:
                 cc = self.custcols[c]
                 db.set_custom_column_metadata(cc['colnum'], name=cc['name'],
-                                                   label=cc['label'],
+                                              label=cc['label'],
-                                                   display = self.custcols[c]['display'])
+                                              display = self.custcols[c]['display'],
+                                              notify=False)
                 if '*must_restart' in self.custcols[c]:
                     must_restart = True
         return must_restart

src/calibre/gui2/preferences/create_custom_column.py

@@ -41,6 +41,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
                         'text':_('Yes/No'), 'is_multiple':False},
                     10:{'datatype':'composite',
                         'text':_('Column built from other columns'), 'is_multiple':False},
+                    11:{'datatype':'*composite',
+                        'text':_('Column built from other columns, behaves like tags'), 'is_multiple':True},
                 }
 
     def __init__(self, parent, editing, standard_colheads, standard_colnames):
@@ -99,7 +101,9 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
         c = parent.custcols[col]
         self.column_name_box.setText(c['label'])
         self.column_heading_box.setText(c['name'])
-        ct = c['datatype'] if not c['is_multiple'] else '*text'
+        ct = c['datatype']
+        if c['is_multiple']:
+            ct = '*' + ct
         self.orig_column_number = c['colnum']
         self.orig_column_name = col
         column_numbers = dict(map(lambda x:(self.column_types[x]['datatype'], x),
@@ -109,7 +113,7 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
         if ct == 'datetime':
             if c['display'].get('date_format', None):
                 self.date_format_box.setText(c['display'].get('date_format', ''))
-        elif ct == 'composite':
+        elif ct in ['composite', '*composite']:
             self.composite_box.setText(c['display'].get('composite_template', ''))
             sb = c['display'].get('composite_sort', 'text')
             vals = ['text', 'number', 'date', 'bool']
@@ -167,7 +171,7 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             getattr(self, 'date_format_'+x).setVisible(col_type == 'datetime')
         for x in ('box', 'default_label', 'label', 'sort_by', 'sort_by_label',
                   'make_category'):
-            getattr(self, 'composite_'+x).setVisible(col_type == 'composite')
+            getattr(self, 'composite_'+x).setVisible(col_type in ['composite', '*composite'])
         for x in ('box', 'default_label', 'label'):
             getattr(self, 'enum_'+x).setVisible(col_type == 'enumeration')
         self.use_decorations.setVisible(col_type in ['text', 'composite', 'enumeration'])
@@ -187,8 +191,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
                     'because these names are reserved for the index of a series column.'))
         col_heading = unicode(self.column_heading_box.text()).strip()
         col_type = self.column_types[self.column_type_box.currentIndex()]['datatype']
-        if col_type == '*text':
+        if col_type[0] == '*':
-            col_type='text'
+            col_type = col_type[1:]
             is_multiple = True
         else:
             is_multiple = False
@@ -249,11 +253,10 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
         elif col_type == 'text' and is_multiple:
             display_dict = {'is_names': self.is_names.isChecked()}
 
-        if col_type in ['text', 'composite', 'enumeration']:
+        if col_type in ['text', 'composite', 'enumeration'] and not is_multiple:
             display_dict['use_decorations'] = self.use_decorations.checkState()
 
         if not self.editing_col:
-            db.field_metadata
             self.parent.custcols[key] = {
                     'label':col,
                     'name':col_heading,

src/calibre/library/caches.py

@@ -751,7 +751,7 @@ class ResultCache(SearchQueryParser): # {{{
 
                     if loc not in exclude_fields: # time for text matching
                         if is_multiple_cols[loc] is not None:
-                            vals = item[loc].split(is_multiple_cols[loc])
+                            vals = [v.strip() for v in item[loc].split(is_multiple_cols[loc])]
                         else:
                             vals = [item[loc]] ### make into list to make _match happy
                         if _match(q, vals, matchkind):

src/calibre/library/custom_columns.py

@@ -182,7 +182,7 @@ class CustomColumns(object):
             else:
                 is_category = False
             if v['is_multiple']:
-                is_m = '|'
+                is_m = ',' if v['datatype'] == 'composite' else '|'
             else:
                 is_m = None
             tn = 'custom_column_{0}'.format(v['num'])
@@ -318,7 +318,7 @@ class CustomColumns(object):
         self.conn.commit()
 
     def set_custom_column_metadata(self, num, name=None, label=None,
-            is_editable=None, display=None):
+            is_editable=None, display=None, notify=True):
         changed = False
         if name is not None:
             self.conn.execute('UPDATE custom_columns SET name=? WHERE id=?',
@@ -340,6 +340,9 @@ class CustomColumns(object):
 
         if changed:
             self.conn.commit()
+        if notify:
+            self.notify('metadata', [])
+
         return changed
 
     def set_custom_bulk_multiple(self, ids, add=[], remove=[],
@@ -595,7 +598,7 @@ class CustomColumns(object):
             raise ValueError('%r is not a supported data type'%datatype)
         normalized  = datatype not in ('datetime', 'comments', 'int', 'bool',
                 'float', 'composite')
-        is_multiple = is_multiple and datatype in ('text',)
+        is_multiple = is_multiple and datatype in ('text', 'composite')
         num = self.conn.execute(
                 ('INSERT INTO '
                 'custom_columns(label,name,datatype,is_multiple,editable,display,normalized)'

src/calibre/library/database2.py

@@ -1224,7 +1224,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if field['datatype'] == 'composite':
             dex = field['rec_index']
             for book in self.data.iterall():
-                if book[dex] == id_:
+                if field['is_multiple']:
+                    vals = [v.strip() for v in book[dex].split(field['is_multiple'])
+                            if v.strip()]
+                    if id_ in vals:
+                        ans.add(book[0])
+                elif book[dex] == id_:
                     ans.add(book[0])
             return ans
 
@@ -1354,6 +1359,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             cat = tb_cats[category]
             if cat['datatype'] == 'composite' and \
                                 cat['display'].get('make_category', False):
+                tids[category] = {}
                 tcategories[category] = {}
                 md.append((category, cat['rec_index'], cat['is_multiple'],
                            cat['datatype'] == 'composite'))
@@ -1402,8 +1408,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                         prints('get_categories: item', val, 'is not in', cat, 'list!')
                 else:
                     vals = book[dex].split(mult)
+                    if is_comp:
+                        vals = [v.strip() for v in vals if v.strip()]
+                        for val in vals:
+                            if val not in tids:
+                                tids[cat][val] = (val, val)
+                            item = tcategories[cat].get(val, None)
+                            if not item:
+                                item = tag_class(val, val)
+                                tcategories[cat][val] = item
+                            item.c += 1
+                            item.id = val
                     for val in vals:
-                        if not val: continue
                         try:
                             (item_id, sort_val) = tids[cat][val] # let exceptions fly
                             item = tcategories[cat].get(val, None)

src/calibre/library/field_metadata.py

@@ -364,11 +364,11 @@ class FieldMetadata(dict):
             self._tb_cats[k]['display'] = {}
             self._tb_cats[k]['is_editable'] = True
             self._add_search_terms_to_map(k, v['search_terms'])
-        for x in ('timestamp', 'last_modified'):
+        self._tb_cats['timestamp']['display'] = {
-            self._tb_cats[x]['display'] = {
                         'date_format': tweaks['gui_timestamp_display_format']}
         self._tb_cats['pubdate']['display'] = {
                         'date_format': tweaks['gui_pubdate_display_format']}
+        self._tb_cats['last_modified']['display'] = {'date_format': 'iso'}
         self.custom_field_prefix = '#'
         self.get = self._tb_cats.get
 

src/calibre/manual/template_lang.rst

@@ -236,15 +236,16 @@ The following functions are available in addition to those described in single-f
     * ``format_date(x, date_format)`` -- format_date(val, format_string) -- format the value, which must be a date field, using the format_string, returning a string. The formatting codes are::
     
         d    : the day as number without a leading zero (1 to 31)
-        dd   : the day as number with a leading zero (01 to 31) '
+        dd   : the day as number with a leading zero (01 to 31) 
-        ddd  : the abbreviated localized day name (e.g. "Mon" to "Sun"). '
+        ddd  : the abbreviated localized day name (e.g. "Mon" to "Sun"). 
-        dddd : the long localized day name (e.g. "Monday" to "Sunday"). '
+        dddd : the long localized day name (e.g. "Monday" to "Sunday"). 
-        M    : the month as number without a leading zero (1 to 12). '
+        M    : the month as number without a leading zero (1 to 12). 
-        MM   : the month as number with a leading zero (01 to 12) '
+        MM   : the month as number with a leading zero (01 to 12) 
-        MMM  : the abbreviated localized month name (e.g. "Jan" to "Dec"). '
+        MMM  : the abbreviated localized month name (e.g. "Jan" to "Dec"). 
-        MMMM : the long localized month name (e.g. "January" to "December"). '
+        MMMM : the long localized month name (e.g. "January" to "December"). 
-        yy   : the year as two digit number (00 to 99). '
+        yy   : the year as two digit number (00 to 99). 
-        yyyy : the year as four digit number.'
+        yyyy : the year as four digit number.
+        iso  : the date with time and timezone. Must be the only format present.
     
     * ``eval(string)`` -- evaluates the string as a program, passing the local variables (those ``assign`` ed to). This permits using the template processor to construct complex results from local variables.
     * ``multiply(x, y)`` -- returns x * y. Throws an exception if either x or y are not numbers.

src/calibre/utils/date.py

@@ -142,6 +142,10 @@ def format_date(dt, format, assume_utc=False, as_utc=False):
             dt = dt.replace(tzinfo=_utc_tz if assume_utc else
                     _local_tz)
         dt = dt.astimezone(_utc_tz if as_utc else _local_tz)
+
+    if format == 'iso':
+        return isoformat(dt, assume_utc=assume_utc, as_utc=as_utc)
+
     strf = partial(strftime, t=dt.timetuple())
 
     def format_day(mo):

src/calibre/utils/formatter_functions.py

@@ -504,7 +504,8 @@ class BuiltinFormat_date(BuiltinFormatterFunction):
             'MMM  : the abbreviated localized month name (e.g. "Jan" to "Dec"). '
             'MMMM : the long localized month name (e.g. "January" to "December"). '
             'yy   : the year as two digit number (00 to 99). '
-            'yyyy : the year as four digit number.')
+            'yyyy : the year as four digit number. '
+            'iso  : the date with time and timezone. Must be the only format present')
 
     def evaluate(self, formatter, kwargs, mi, locals, val, format_string):
         if not val: