Tag browser: Add subcategories, when a category has many items, it will be automaically split up. Also add a search to quickly find an item in the Tag Browser. The sub categories can be controlled via preferences->Tweaks. Add an API to store custom data for each book in the calibre db. Fix #8080 (Sort in library doesn't work after bulk metadata edit). Add support for the iRiver Story Wifi. New template language features (program mode, eval function, etc.), see User Manual for details.

resources/default_tweaks.py

@@ -55,6 +55,32 @@ author_sort_copy_method = 'invert'
 #   categories_use_field_for_author_name = 'author_sort'
 categories_use_field_for_author_name = 'author'
 
+# Control how the tags pane displays categories containing many items. If the
+# number of items is larger than categories_collapse_more_than, a sub-category
+# will be added. If sorting by name, then the subcategories can be organized by
+# first letter (categories_collapse_model = 'first letter') or into equal-sized
+# groups (categories_collapse_model = 'partition'). If sorting by average rating
+# or by popularity, then 'partition' is always used. The addition of
+# subcategories can be disabled by setting categories_collapse_more_than = 0.
+# When using partition, the format of the subcategory label is controlled by a
+# template: categories_collapsed_name_template if sorting by name,
+# categories_collapsed_rating_template if sorting by average rating, and
+# categories_collapsed_popularity_template if sorting by popularity. There are
+# two variables available to the template: first and last. The variable 'first'
+# is the initial item in the subcategory, and the variable 'last' is the final
+# item in the subcategory. Both variables are 'objects'; they each have multiple
+# values that are obtained by using a suffix. For example, first.name for an
+# author category will be the name of the author. The sub-values available are:
+#  name: the printable name of the item
+#  count: the number of books that references this item
+#  avg_rating: the averate rating of all the books referencing this item
+#  sort: the sort value. For authors, this is the author_sort for that author
+#  category: the category (e.g., authors, series) that the item is in.
+categories_collapse_more_than = 50
+categories_collapsed_name_template = '{first.name:shorten(4,'',0)}{last.name::shorten(4,'',0)| - |}'
+categories_collapsed_rating_template = '{first.avg_rating:4.2f}{last.avg_rating:4.2f| - |}'
+categories_collapsed_popularity_template = '{first.count:d}{last.count:d| - |}'
+categories_collapse_model = 'first letter'
 
 # Set whether boolean custom columns are two- or three-valued.
 #  Two-values for true booleans

src/calibre/devices/folder_device/driver.py

@@ -18,9 +18,9 @@ class FOLDER_DEVICE_FOR_CONFIG(USBMS):
     supported_platforms = ['windows', 'osx', 'linux']
     FORMATS     = ['epub', 'fb2', 'mobi', 'azw', 'lrf', 'tcr', 'pmlz', 'lit',
                    'rtf', 'rb', 'pdf', 'oeb', 'txt', 'pdb', 'prc']
-    VENDOR_ID   = 0xffff
+    VENDOR_ID   = [0xffff]
-    PRODUCT_ID  = 0xffff
+    PRODUCT_ID  = [0xffff]
-    BCD         = 0xffff
+    BCD         = [0xffff]
     DEVICE_PLUGBOARD_NAME = 'FOLDER_DEVICE'
 
 
@@ -34,9 +34,9 @@ class FOLDER_DEVICE(USBMS):
     supported_platforms = ['windows', 'osx', 'linux']
     FORMATS     = FOLDER_DEVICE_FOR_CONFIG.FORMATS
 
-    VENDOR_ID   = 0xffff
+    VENDOR_ID   = [0xffff]
-    PRODUCT_ID  = 0xffff
+    PRODUCT_ID  = [0xffff]
-    BCD         = 0xffff
+    BCD         = [0xffff]
     DEVICE_PLUGBOARD_NAME = 'FOLDER_DEVICE'
 
     THUMBNAIL_HEIGHT = 68 # Height for thumbnails on device

src/calibre/devices/iriver/driver.py

@@ -20,11 +20,11 @@ class IRIVER_STORY(USBMS):
     FORMATS     = ['epub', 'fb2', 'pdf', 'djvu', 'txt']
 
     VENDOR_ID   = [0x1006]
-    PRODUCT_ID  = [0x4023, 0x4025]
+    PRODUCT_ID  = [0x4023, 0x4024, 0x4025]
     BCD         = [0x0323]
 
     VENDOR_NAME = 'IRIVER'
-    WINDOWS_MAIN_MEM = ['STORY', 'STORY_EB05']
+    WINDOWS_MAIN_MEM = ['STORY', 'STORY_EB05', 'STORY_WI-FI']
     WINDOWS_CARD_A_MEM = ['STORY', 'STORY_SD']
 
     #OSX_MAIN_MEM = 'Kindle Internal Storage Media'

src/calibre/ebooks/metadata/book/base.py

@@ -159,6 +159,11 @@ class Metadata(object):
         try:
             return self.__getattribute__(field)
         except AttributeError:
+            if field.startswith('#') and field.endswith('_index'):
+                try:
+                    return self.get_extra(field[:-6])
+                except:
+                    pass
             return default
 
     def get_extra(self, field):

src/calibre/gui2/dialogs/template_dialog.py

@@ -19,7 +19,7 @@ class TemplateDialog(QDialog, Ui_TemplateDialog):
 
         if text is not None:
             self.textbox.setPlainText(text)
-        self.textbox.setTabChangesFocus(True)
+        self.textbox.setTabStopWidth(50)
         self.buttonBox.button(QDialogButtonBox.Ok).setText(_('&OK'))
         self.buttonBox.button(QDialogButtonBox.Cancel).setText(_('&Cancel'))
 

src/calibre/gui2/dialogs/template_dialog.ui

@@ -6,7 +6,7 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>336</width>
+    <width>500</width>
     <height>235</height>
    </rect>
   </property>

src/calibre/gui2/library/__init__.py

@@ -5,6 +5,4 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-from PyQt4.Qt import Qt
+DEFAULT_SORT = ('timestamp', False)
-
-DEFAULT_SORT = ('timestamp', Qt.DescendingOrder)

src/calibre/gui2/library/models.py

@@ -247,9 +247,10 @@ class BooksModel(QAbstractTableModel): # {{{
         if not self.db:
             return
         self.about_to_be_sorted.emit(self.db.id)
-        ascending = order == Qt.AscendingOrder
+        if not isinstance(order, bool):
+            order = order == Qt.AscendingOrder
         label = self.column_map[col]
-        self.db.sort(label, ascending)
+        self.db.sort(label, order)
         if reset:
             self.reset()
         self.sorted_on = (label, order)

src/calibre/gui2/library/views.py

@@ -165,7 +165,7 @@ class BooksView(QTableView): # {{{
                     partial(self.column_header_context_handler,
                         action='descending', column=col))
             if self._model.sorted_on[0] == col:
-                ac = a if self._model.sorted_on[1] == Qt.AscendingOrder else d
+                ac = a if self._model.sorted_on[1] else d
                 ac.setCheckable(True)
                 ac.setChecked(True)
             if col not in ('ondevice', 'rating', 'inlibrary') and \
@@ -282,17 +282,21 @@ class BooksView(QTableView): # {{{
     def cleanup_sort_history(self, sort_history):
         history = []
         for col, order in sort_history:
+            if not isinstance(order, bool):
+                continue
             if col == 'date':
                 col = 'timestamp'
-            if col in self.column_map and (not history or history[0][0] != col):
+            if col in self.column_map:
-                history.append([col, order])
+                if (not history or history[-1][0] != col):
+                    history.append([col, order])
         return history
 
     def apply_sort_history(self, saved_history):
         if not saved_history:
             return
         for col, order in reversed(self.cleanup_sort_history(saved_history)[:3]):
-            self.sortByColumn(self.column_map.index(col), order)
+            self.sortByColumn(self.column_map.index(col),
+                              Qt.AscendingOrder if order else Qt.DescendingOrder)
 
     def apply_state(self, state):
         h = self.column_header

src/calibre/gui2/tag_view.py

@@ -10,22 +10,25 @@ Browsing book collection by tags.
 from itertools import izip
 from functools import partial
 
-from PyQt4.Qt import Qt, QTreeView, QApplication, pyqtSignal, \
+from PyQt4.Qt import Qt, QTreeView, QApplication, pyqtSignal, QFont, QSize, \
-                     QFont, QSize, QIcon, QPoint, QVBoxLayout, QComboBox, \
+                     QIcon, QPoint, QVBoxLayout, QHBoxLayout, QComboBox,\
                      QAbstractItemModel, QVariant, QModelIndex, QMenu, \
-                     QPushButton, QWidget, QItemDelegate
+                     QPushButton, QWidget, QItemDelegate, QString
 
 from calibre.ebooks.metadata import title_sort
 from calibre.gui2 import config, NONE
 from calibre.library.field_metadata import TagsIcons, category_icon_map
+from calibre.library.database2 import Tag
 from calibre.utils.config import tweaks
-from calibre.utils.icu import sort_key
+from calibre.utils.icu import sort_key, upper, lower
 from calibre.utils.search_query_parser import saved_searches
-from calibre.gui2 import error_dialog
+from calibre.utils.formatter import eval_formatter
+from calibre.gui2 import error_dialog, warning_dialog
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.dialogs.tag_categories import TagCategories
 from calibre.gui2.dialogs.tag_list_editor import TagListEditor
 from calibre.gui2.dialogs.edit_authors_dialog import EditAuthorsDialog
+from calibre.gui2.widgets import HistoryLineEdit
 
 class TagDelegate(QItemDelegate): # {{{
 
@@ -52,6 +55,8 @@ class TagDelegate(QItemDelegate): # {{{
             painter.setClipRect(r)
 
         # Paint the text
+        if item.boxed:
+            painter.drawRoundedRect(r.adjusted(1,1,-1,-1), 5, 5)
         r.setLeft(r.left()+r.height()+3)
         painter.drawText(r, Qt.AlignLeft|Qt.AlignVCenter,
                         model.data(index, Qt.DisplayRole).toString())
@@ -331,12 +336,13 @@ class TagsView(QTreeView): # {{{
     # If the number of user categories changed,  if custom columns have come or
     # gone, or if columns have been hidden or restored, we must rebuild the
     # model. Reason: it is much easier than reconstructing the browser tree.
-    def set_new_model(self):
+    def set_new_model(self, filter_categories_by=None):
         try:
             self._model = TagsModel(self.db, parent=self,
                                     hidden_categories=self.hidden_categories,
                                     search_restriction=self.search_restriction,
-                                    drag_drop_finished=self.drag_drop_finished)
+                                    drag_drop_finished=self.drag_drop_finished,
+                                    filter_categories_by=filter_categories_by)
             self.setModel(self._model)
         except:
             # The DB must be gone. Set the model to None and hope that someone
@@ -355,6 +361,7 @@ class TagTreeItem(object): # {{{
                  parent=None, tooltip=None, category_key=None):
         self.parent = parent
         self.children = []
+        self.boxed = False
         if self.parent is not None:
             self.parent.append(self)
         if data is None:
@@ -400,7 +407,7 @@ class TagTreeItem(object): # {{{
 
     def category_data(self, role):
         if role == Qt.DisplayRole:
-            return QVariant(self.py_name + ' [%d]'%len(self.children))
+            return QVariant(self.py_name + ' [%d]'%len(self.child_tags()))
         if role == Qt.DecorationRole:
             return self.icon
         if role == Qt.FontRole:
@@ -441,12 +448,22 @@ class TagTreeItem(object): # {{{
         if self.type == self.TAG:
             self.tag.state = (self.tag.state + 1)%3
 
+    def child_tags(self):
+        res = []
+        for t in self.children:
+            if t.type == TagTreeItem.CATEGORY:
+                for c in t.children:
+                    res.append(c)
+            else:
+                res.append(t)
+        return res
     # }}}
 
 class TagsModel(QAbstractItemModel): # {{{
 
     def __init__(self, db, parent, hidden_categories=None,
-            search_restriction=None, drag_drop_finished=None):
+            search_restriction=None, drag_drop_finished=None,
+            filter_categories_by=None):
         QAbstractItemModel.__init__(self, parent)
 
         # must do this here because 'QPixmap: Must construct a QApplication
@@ -466,6 +483,7 @@ class TagsModel(QAbstractItemModel): # {{{
         self.hidden_categories = hidden_categories
         self.search_restriction = search_restriction
         self.row_map = []
+        self.filter_categories_by = filter_categories_by
 
         # get_node_tree cannot return None here, because row_map is empty
         data = self.get_node_tree(config['sort_tags_by'])
@@ -477,19 +495,11 @@ class TagsModel(QAbstractItemModel): # {{{
                 tt = _('The lookup/search name is "{0}"').format(r)
             else:
                 tt = ''
-            c = TagTreeItem(parent=self.root_item,
+            TagTreeItem(parent=self.root_item,
                     data=self.categories[i],
                     category_icon=self.category_icon_map[r],
                     tooltip=tt, category_key=r)
-            # This duplicates code in refresh(). Having it here as well
+        self.refresh(data=data)
-            # can save seconds during startup, because we avoid a second
-            # call to get_node_tree.
-            for tag in data[r]:
-                if r not in self.categories_with_ratings and \
-                            not self.db.field_metadata[r]['is_custom'] and \
-                            not self.db.field_metadata[r]['kind'] == 'user':
-                    tag.avg_rating = None
-                TagTreeItem(parent=c, data=tag, icon_map=self.icon_state_map)
 
     def mimeTypes(self):
         return ["application/calibre+from_library"]
@@ -641,6 +651,11 @@ class TagsModel(QAbstractItemModel): # {{{
         else:
             data = self.db.get_categories(sort=sort, icon_map=self.category_icon_map)
 
+        if self.filter_categories_by:
+            for category in data.keys():
+                data[category] = [t for t in data[category]
+                        if lower(t.name).find(self.filter_categories_by) >= 0]
+
         tb_categories = self.db.field_metadata
         for category in tb_categories:
             if category in data: # The search category can come and go
@@ -652,35 +667,85 @@ class TagsModel(QAbstractItemModel): # {{{
             return None
         return data
 
-    def refresh(self):
+    def refresh(self, data=None):
-        data = self.get_node_tree(config['sort_tags_by']) # get category data
+        sort_by = config['sort_tags_by']
+        if data is None:
+            data = self.get_node_tree(sort_by) # get category data
         if data is None:
             return False
         row_index = -1
+        empty_tag = Tag('')
+        collapse = tweaks['categories_collapse_more_than']
+        collapse_model = tweaks['categories_collapse_model']
+        if sort_by == 'name':
+            collapse_template = tweaks['categories_collapsed_name_template']
+        elif sort_by == 'rating':
+            collapse_model = 'partition'
+            collapse_template = tweaks['categories_collapsed_rating_template']
+        else:
+            collapse_model = 'partition'
+            collapse_template = tweaks['categories_collapsed_popularity_template']
+        collapse_letter = None
+
         for i, r in enumerate(self.row_map):
             if self.hidden_categories and self.categories[i] in self.hidden_categories:
                 continue
             row_index += 1
             category = self.root_item.children[row_index]
-            names = [t.tag.name for t in category.children]
+            names = []
-            states = [t.tag.state for t in category.children]
+            states = []
+            children = category.child_tags()
+            states = [t.tag.state for t in children]
+            names = [t.tag.name for names in children]
             state_map = dict(izip(names, states))
             category_index = self.index(row_index, 0, QModelIndex())
+            category_node = category_index.internalPointer()
             if len(category.children) > 0:
                 self.beginRemoveRows(category_index, 0,
                         len(category.children)-1)
                 category.children = []
                 self.endRemoveRows()
-            if len(data[r]) > 0:
+            cat_len = len(data[r])
-                self.beginInsertRows(category_index, 0, len(data[r])-1)
+            if cat_len <= 0:
-                for tag in data[r]:
+                continue
-                    if r not in self.categories_with_ratings and \
+
+            self.beginInsertRows(category_index, 0, len(data[r])-1)
+            clear_rating = True if r not in self.categories_with_ratings and \
                                 not self.db.field_metadata[r]['is_custom'] and \
-                                not self.db.field_metadata[r]['kind'] == 'user':
+                                not self.db.field_metadata[r]['kind'] == 'user' \
-                        tag.avg_rating = None
+                            else False
-                    tag.state = state_map.get(tag.name, 0)
+            for idx,tag in enumerate(data[r]):
+                if clear_rating:
+                    tag.avg_rating = None
+                tag.state = state_map.get(tag.name, 0)
+
+                if collapse > 0 and cat_len > collapse:
+                    if collapse_model == 'partition':
+                        if (idx % collapse) == 0:
+                            d = {'first': tag}
+                            if cat_len > idx + collapse:
+                                d['last'] = data[r][idx+collapse-1]
+                            else:
+                                d['last'] = empty_tag
+                            name = eval_formatter.safe_format(collapse_template,
+                                                              d, 'TAG_VIEW', None)
+                            sub_cat = TagTreeItem(parent=category,
+                                     data = name, tooltip = None,
+                                     category_icon = category_node.icon,
+                                     category_key=category_node.category_key)
+                    else:
+                        if upper(tag.sort[0]) != collapse_letter:
+                            collapse_letter = upper(tag.name[0])
+                            sub_cat = TagTreeItem(parent=category,
+                                     data = collapse_letter,
+                                     category_icon = category_node.icon,
+                                     tooltip = None,
+                                     category_key=category_node.category_key)
+                    t = TagTreeItem(parent=sub_cat, data=tag,
+                                        icon_map=self.icon_state_map)
+                else:
                     t = TagTreeItem(parent=category, data=tag, icon_map=self.icon_state_map)
-                self.endInsertRows()
+            self.endInsertRows()
         return True
 
     def columnCount(self, parent):
@@ -824,19 +889,27 @@ class TagsModel(QAbstractItemModel): # {{{
 
     def reset_all_states(self, except_=None):
         update_list = []
-        for i in xrange(self.rowCount(QModelIndex())):
+        def process_tag(tag_index, tag_item):
-            category_index = self.index(i, 0, QModelIndex())
+            tag = tag_item.tag
+            if tag is except_:
+                self.dataChanged.emit(tag_index, tag_index)
+                return
+            if tag.state != 0 or tag in update_list:
+                tag.state = 0
+                update_list.append(tag)
+                self.dataChanged.emit(tag_index, tag_index)
+
+        def process_level(category_index):
             for j in xrange(self.rowCount(category_index)):
                 tag_index = self.index(j, 0, category_index)
                 tag_item = tag_index.internalPointer()
-                tag = tag_item.tag
+                if tag_item.type == TagTreeItem.CATEGORY:
-                if tag is except_:
+                    process_level(tag_index)
-                    self.dataChanged.emit(tag_index, tag_index)
+                else:
-                    continue
+                    process_tag(tag_index, tag_item)
-                if tag.state != 0 or tag in update_list:
+
-                    tag.state = 0
+        for i in xrange(self.rowCount(QModelIndex())):
-                    update_list.append(tag)
+            process_level(self.index(i, 0, QModelIndex()))
-                    self.dataChanged.emit(tag_index, tag_index)
 
     def clear_state(self):
         self.reset_all_states()
@@ -856,6 +929,7 @@ class TagsModel(QAbstractItemModel): # {{{
         ans = []
         tags_seen = set()
         row_index = -1
+
         for i, key in enumerate(self.row_map):
             if self.hidden_categories and self.categories[i] in self.hidden_categories:
                 continue
@@ -863,7 +937,7 @@ class TagsModel(QAbstractItemModel): # {{{
             if key.endswith(':'): # User category, so skip it. The tag will be marked in its real category
                 continue
             category_item = self.root_item.children[row_index]
-            for tag_item in category_item.children:
+            for tag_item in category_item.child_tags():
                 tag = tag_item.tag
                 if tag.state > 0:
                     prefix = ' not ' if tag.state == 2 else ''
@@ -878,6 +952,82 @@ class TagsModel(QAbstractItemModel): # {{{
                         ans.append('%s%s:"=%s"'%(prefix, category, tag.name))
         return ans
 
+    def find_node(self, key, txt, start_index):
+        if not txt:
+            return None
+        txt = lower(txt)
+        if start_index is None or not start_index.isValid():
+            start_index = QModelIndex()
+        self.node_found = None
+
+        def process_tag(depth, tag_index, tag_item, start_path):
+            path = self.path_for_index(tag_index)
+            if depth < len(start_path) and path[depth] <= start_path[depth]:
+                return False
+            tag = tag_item.tag
+            if tag is None:
+                return False
+            if lower(tag.name).find(txt) >= 0:
+                self.node_found = tag_index
+                return True
+            return False
+
+        def process_level(depth, category_index, start_path):
+            path = self.path_for_index(category_index)
+            if depth < len(start_path):
+                if path[depth] < start_path[depth]:
+                    return False
+                if path[depth] > start_path[depth]:
+                    start_path = path
+            if key and category_index.internalPointer().category_key != key:
+                return False
+            for j in xrange(self.rowCount(category_index)):
+                tag_index = self.index(j, 0, category_index)
+                tag_item = tag_index.internalPointer()
+                if tag_item.type == TagTreeItem.CATEGORY:
+                    if process_level(depth+1, tag_index, start_path):
+                        return True
+                else:
+                    if process_tag(depth+1, tag_index, tag_item, start_path):
+                        return True
+            return False
+
+        for i in xrange(self.rowCount(QModelIndex())):
+            if process_level(0, self.index(i, 0, QModelIndex()),
+                                  self.path_for_index(start_index)):
+                break
+        return self.node_found
+
+    def show_item_at_index(self, idx, box=False):
+        if idx.isValid():
+            tag_item = idx.internalPointer()
+            self.tags_view.setCurrentIndex(idx)
+            self.tags_view.scrollTo(idx, QTreeView.PositionAtCenter)
+            if box:
+                tag_item.boxed = True
+                self.dataChanged.emit(idx, idx)
+
+    def clear_boxed(self):
+        def process_tag(tag_index, tag_item):
+            if tag_item.boxed:
+                tag_item.boxed = False
+                self.dataChanged.emit(tag_index, tag_index)
+
+        def process_level(category_index):
+            for j in xrange(self.rowCount(category_index)):
+                tag_index = self.index(j, 0, category_index)
+                tag_item = tag_index.internalPointer()
+                if tag_item.type == TagTreeItem.CATEGORY:
+                    process_level(tag_index)
+                else:
+                    process_tag(tag_index, tag_item)
+
+        for i in xrange(self.rowCount(QModelIndex())):
+            process_level(self.index(i, 0, QModelIndex()))
+
+    def get_filter_categories_by(self):
+        return self.filter_categories_by
+
     # }}}
 
 class TagBrowserMixin(object): # {{{
@@ -993,10 +1143,40 @@ class TagBrowserWidget(QWidget): # {{{
 
     def __init__(self, parent):
         QWidget.__init__(self, parent)
+        self.parent = parent
         self._layout = QVBoxLayout()
         self.setLayout(self._layout)
         self._layout.setContentsMargins(0,0,0,0)
 
+        search_layout = QHBoxLayout()
+        self._layout.addLayout(search_layout)
+        self.item_search = HistoryLineEdit(parent)
+        try:
+            self.item_search.lineEdit().setPlaceholderText(_('Find item in tag browser'))
+        except:
+            # Using Qt < 4.7
+            pass
+        self.item_search.setToolTip(_(
+        'Search for items. This is a "contains" search; items containing the\n'
+        'text anywhere in the name will be found. You can limit the search\n'
+        'to particular categories using syntax similar to search. For example,\n'
+        'tags:foo will find foo in any tag, but not in authors etc. Entering\n'
+        '*foo will filter all categories at once, showing only those items\n'
+        'containing the text "foo"'))
+        search_layout.addWidget(self.item_search)
+        self.search_button = QPushButton()
+        self.search_button.setText(_('&Find'))
+        self.search_button.setToolTip(_('Find the first/next matching item'))
+        self.search_button.setFixedWidth(40)
+        search_layout.addWidget(self.search_button)
+        self.current_position = None
+        self.search_button.clicked.connect(self.find)
+        self.item_search.initialize('tag_browser_search')
+        self.item_search.lineEdit().returnPressed.connect(self.do_find)
+        self.item_search.lineEdit().textEdited.connect(self.find_text_changed)
+        self.item_search.activated[QString].connect(self.do_find)
+        self.item_search.completer().setCaseSensitivity(Qt.CaseSensitive)
+
         parent.tags_view = TagsView(parent)
         self.tags_view = parent.tags_view
         self._layout.addWidget(parent.tags_view)
@@ -1031,6 +1211,59 @@ class TagBrowserWidget(QWidget): # {{{
     def set_pane_is_visible(self, to_what):
         self.tags_view.set_pane_is_visible(to_what)
 
+    def find_text_changed(self, str):
+        self.current_position = None
+
+    def do_find(self, str=None):
+        self.current_position = None
+        self.find()
+
+    def find(self):
+        model = self.tags_view.model()
+        model.clear_boxed()
+        txt = unicode(self.item_search.currentText()).strip()
+
+        if txt.startswith('*'):
+            self.tags_view.set_new_model(filter_categories_by=txt[1:])
+            self.current_position = None
+            return
+        if model.get_filter_categories_by():
+            self.tags_view.set_new_model(filter_categories_by=None)
+            self.current_position = None
+            model = self.tags_view.model()
+
+        if not txt:
+            return
+
+        self.item_search.blockSignals(True)
+        self.item_search.lineEdit().blockSignals(True)
+        self.search_button.setFocus(True)
+        idx = self.item_search.findText(txt, Qt.MatchFixedString)
+        if idx < 0:
+            self.item_search.insertItem(0, txt)
+        else:
+            t = self.item_search.itemText(idx)
+            self.item_search.removeItem(idx)
+            self.item_search.insertItem(0, t)
+        self.item_search.setCurrentIndex(0)
+        self.item_search.blockSignals(False)
+        self.item_search.lineEdit().blockSignals(False)
+
+        colon = txt.find(':')
+        key = None
+        if colon > 0:
+            key = self.parent.library_view.model().db.\
+                        field_metadata.search_term_to_field_key(txt[:colon])
+            txt = txt[colon+1:]
+
+        self.current_position = model.find_node(key, txt, self.current_position)
+        if self.current_position:
+            model.show_item_at_index(self.current_position, box=True)
+        elif self.item_search.text():
+            warning_dialog(self.tags_view, _('No item found'),
+                        _('No (more) matches for that search')).exec_()
+
+
 
 # }}}
 

src/calibre/library/caches.py

@@ -669,6 +669,9 @@ class ResultCache(SearchQueryParser): # {{{
             fields = [('timestamp', False)]
 
         keyg = SortKeyGenerator(fields, self.field_metadata, self._data)
+        # For efficiency, the key generator returns a plain value if only one
+        # field is in the sort field list. Because the normal cmp function will
+        # always assume asc, we must deal with asc/desc here.
         if len(fields) == 1:
             self._map.sort(key=keyg, reverse=not fields[0][1])
         else:
@@ -697,7 +700,7 @@ class SortKeyGenerator(object):
     def __init__(self, fields, field_metadata, data):
         from calibre.utils.icu import sort_key
         self.field_metadata = field_metadata
-        self.orders = [-1 if x[1] else 1 for x in fields]
+        self.orders = [1 if x[1] else -1 for x in fields]
         self.entries = [(x[0], field_metadata[x[0]]) for x in fields]
         self.library_order = tweaks['title_series_sorting'] == 'library_order'
         self.data = data

src/calibre/library/database2.py

@@ -6,7 +6,7 @@ __docformat__ = 'restructuredtext en'
 '''
 The database used to store ebook metadata
 '''
-import os, sys, shutil, cStringIO, glob, time, functools, traceback, re
+import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, json
 from itertools import repeat
 from math import ceil
 from Queue import Queue
@@ -32,7 +32,7 @@ from calibre.customize.ui import run_plugins_on_import
 from calibre import isbytestring
 from calibre.utils.filenames import ascii_filename
 from calibre.utils.date import utcnow, now as nowf, utcfromtimestamp
-from calibre.utils.config import prefs, tweaks
+from calibre.utils.config import prefs, tweaks, from_json, to_json
 from calibre.utils.icu import sort_key
 from calibre.utils.search_query_parser import saved_searches, set_saved_searches
 from calibre.ebooks import BOOK_EXTENSIONS, check_ebook_format
@@ -2700,6 +2700,34 @@ books_series_link      feeds
 
         return duplicates
 
+    def add_custom_book_data(self, book_id, name, val):
+        x = self.conn.get('SELECT id FROM books WHERE ID=?', (book_id,), all=False)
+        if x is None:
+            raise ValueError('add_custom_book_data: no such book_id %d'%book_id)
+        # Do the json encode first, in case it throws an exception
+        s = json.dumps(val, default=to_json)
+        self.conn.execute('DELETE FROM books_plugin_data WHERE book=? AND name=?',
+                          (book_id, name))
+        self.conn.execute('''INSERT INTO books_plugin_data(book, name, val)
+                             VALUES(?, ?, ?)''', (book_id, name, s))
+        self.commit()
+
+    def get_custom_book_data(self, book_id, name, default=None):
+        try:
+            s = self.conn.get('''select val FROM books_plugin_data
+                    WHERE book=? AND name=?''', (book_id, name), all=False)
+            if s is None:
+                return default
+            return json.loads(s, object_hook=from_json)
+        except:
+            pass
+        return default
+
+    def delete_custom_book_data(self, book_id, name):
+        self.conn.execute('DELETE FROM books_plugin_data WHERE book=? AND name=?',
+                          (book_id, name))
+        self.commit()
+
     def get_custom_recipes(self):
         for id, title, script in self.conn.get('SELECT id,title,script FROM feeds'):
             yield id, title, script

src/calibre/library/schema_upgrades.py

@@ -441,3 +441,31 @@ class SchemaUpgrade(object):
                          WHERE id=NEW.id AND OLD.title <> NEW.title;
             END;
         ''')
+
+    def upgrade_version_17(self):
+        'custom book data table (for plugins)'
+        script = '''
+        DROP TABLE IF EXISTS books_plugin_data;
+        CREATE TABLE books_plugin_data(id INTEGER PRIMARY KEY,
+                                     book INTEGER NON NULL,
+                                     name TEXT NON NULL,
+                                     val TEXT NON NULL,
+                                     UNIQUE(book,name));
+        DROP TRIGGER IF EXISTS books_delete_trg;
+        CREATE TRIGGER books_delete_trg
+            AFTER DELETE ON books
+            BEGIN
+                DELETE FROM books_authors_link WHERE book=OLD.id;
+                DELETE FROM books_publishers_link WHERE book=OLD.id;
+                DELETE FROM books_ratings_link WHERE book=OLD.id;
+                DELETE FROM books_series_link WHERE book=OLD.id;
+                DELETE FROM books_tags_link WHERE book=OLD.id;
+                DELETE FROM data WHERE book=OLD.id;
+                DELETE FROM comments WHERE book=OLD.id;
+                DELETE FROM conversion_options WHERE book=OLD.id;
+                DELETE FROM books_plugin_data WHERE book=OLD.id;
+        END;
+        '''
+        self.conn.executescript(script)
+
+

src/calibre/manual/template_lang.rst

@@ -137,8 +137,8 @@ Note that you can use the prefix and suffix as well. If you want the number to a
     {#myint:0>3s:ifempty(0)|[|]}
     
 
-Using functions in templates - program mode
+Using functions in templates - template program mode
--------------------------------------------
+----------------------------------------------------
 
 The template language program mode differs from single-function mode in that it permits you to write template expressions that refer to other metadata fields, modify values, and do arithmetic. It is a reasonably complete programming language.
 
@@ -161,10 +161,13 @@ The syntax of the language is shown by the following grammar::
     constant   ::= " string " | ' string ' | number
     identifier ::= sequence of letters or ``_`` characters
     function   ::= identifier ( statement [ , statement ]* )
-    expression ::= identifier | constant | function
+    expression ::= identifier | constant | function | assignment
+    assignment ::= identifier '=' expression
     statement  ::= expression [ ; expression ]*
     program    ::= statement
 
+Comments are lines with a '#' character at the beginning of the line.
+
 An ``expression`` always has a value, either the value of the constant, the value contained in the identifier, or the value returned by a function. The value of a ``statement`` is the value of the last expression in the sequence of statements. As such, the value of the program (statement)::
 
     1; 2; 'foobar'; 3
@@ -208,13 +211,102 @@ The following functions are available in addition to those described in single-f
     * ``cmp(x, y, lt, eq, gt)`` -- compares x and y after converting both to numbers. Returns ``lt`` if x < y. Returns ``eq`` if x == y. Otherwise returns ``gt``.
     * ``divide(x, y)`` -- returns x / y. Throws an exception if either x or y are not numbers.
     * ``field(name)`` -- returns the metadata field named by ``name``.
+    * ``eval(string)`` -- evaluates the string as a program, passing the local variables (those ``assign`` ed to). This permits using the template processor to construct complex results from local variables.
     * ``multiply(x, y)`` -- returns x * y. Throws an exception if either x or y are not numbers.
+    * ``print(a, b, ...)`` -- prints the arguments to standard output. Unless you start calibre from the command line (``calibre-debug -g``), the output will go to a black hole.
     * ``strcat(a, b, ...)`` -- can take any number of arguments. Returns a string formed by concatenating all the arguments.
     * ``strcmp(x, y, lt, eq, gt)`` -- does a case-insensitive comparison x and y as strings. Returns ``lt`` if x < y. Returns ``eq`` if x == y. Otherwise returns ``gt``.
     * ``substr(str, start, end)`` -- returns the ``start``'th through the ``end``'th characters of ``str``. The first character in ``str`` is the zero'th character. If end is negative, then it indicates that many characters counting from the right. If end is zero, then it indicates the last character. For example, ``substr('12345', 1, 0)`` returns ``'2345'``, and ``substr('12345', 1, -1)`` returns ``'234'``.
     * ``subtract(x, y)`` -- returns x - y. Throws an exception if either x or y are not numbers.
     * ``template(x)`` -- evaluates x as a template. The evaluation is done in its own context, meaning that variables are not shared between the caller and the template evaluation. Because the `{` and `}` characters are special, you must use `[[` for the `{` character and `]]` for the '}' character; they are converted automatically. For example, ``template('[[title_sort]]') will evaluate the template ``{title_sort}`` and return its value.
-    
+
+Using general program mode
+-----------------------------------
+
+For more complicated template programs, it is sometimes easier to avoid template syntax (all the `{` and `}` characters), instead writing a more classical-looking program. You can do this in |app| by beginning the template with `program:`. In this case, no template processing is done. The special variable `$` is not set. It is up to your program to produce the correct results.
+
+One advantage of `program:` mode is that the brackets are no longer special. For example, it is not necessary to use `[[` and `]]` when using the `template()` function.
+
+The following example is a `program:` mode implementation of a recipe on the MobileRead forum: "Put series into the title, using either initials or a shortened form. Strip leading articles from the series name (any)." For example, for the book The Two Towers in the Lord of the Rings series, the recipe gives `LotR [02] The Two Towers`. Using standard templates, the recipe requires three custom columns and a plugboard, as explained in the following:
+
+The solution requires creating three composite columns. The first column is used to remove the leading articles. The second is used to compute the 'shorten' form. The third is to compute the 'initials' form. Once you have these columns, the plugboard selects between them. You can hide any or all of the three columns on the library view.
+
+    First column:
+    Name: #stripped_series. 
+    Template: {series:re(^(A|The|An)\s+,)||}
+
+    Second column (the shortened form):
+    Name: #shortened. 
+    Template: {#stripped_series:shorten(4,-,4)}
+
+    Third column (the initials form):
+    Name: #initials. 
+    Template: {#stripped_series:re(([^\s])[^\s]+(\s|$),\1)}
+
+    Plugboard expression:
+    Template:{#stripped_series:lookup(.\s,#initials,.,#shortened,series)}{series_index:0>2.0f| [|] }{title}
+    Destination field: title
+
+    This set of fields and plugboard produces:
+    Series: The Lord of the Rings
+    Series index: 2
+    Title: The Two Towers
+    Output: LotR [02] The Two Towers
+
+    Series: Dahak
+    Series index: 1
+    Title: Mutineers Moon
+    Output: Dahak [01] Mutineers Moon
+
+    Series: Berserkers
+    Series Index: 4
+    Title: Berserker Throne
+    Output: Bers-kers [04] Berserker Throne
+
+    Series: Meg Langslow Mysteries
+    Series Index: 3
+    Title: Revenge of the Wrought-Iron Flamingos
+    Output: MLM [03] Revenge of the Wrought-Iron Flamingos
+
+The following program produces the same results as the original recipe, using only one custom column to hold the results of a program that computes the special title value::
+
+    Custom column: 
+    Name: #special_title
+    Template: (the following with all leading spaces removed)
+        program:
+        #	compute the equivalent of the composite fields and store them in local variables
+            stripped = re(field('series'), '^(A|The|An)\s+', '');
+            shortened = shorten(stripped, 4, '-' ,4);
+            initials = re(stripped, '[^\w]*(\w?)[^\s]+(\s|$)', '\1');
+
+        #	Format the series index. Ends up as empty if there is no series index.
+        #	Note that leading and trailing spaces will be removed by the formatter,
+        # 	so we cannot add them here. We will do that in the strcat below.
+        #	Also note that because we are in 'program' mode, we can freely use
+        #	curly brackets in strings, something we cannot do in template mode.
+            s_index = template('{series_index:0>2.0f}');
+
+        #	print(stripped, shortened, initials, s_index);
+
+        #	Now concatenate all the bits together. The switch picks between 
+        # 	initials and shortened, depending on whether there is a space
+        #	in stripped. We then add the brackets around s_index if it is
+        #	not empty. Finally, add the title. As this is the last function in
+        # 	the program, its value will be returned.
+            strcat(
+                switch(	stripped, 
+                        '.\s', initials, 
+                        '.', shortened,
+                        field('series')),
+                test(s_index, strcat(' [', s_index, '] '), ''),
+                field('title'));
+
+    Plugboard expression:
+    Template:{#special_title}
+    Destination field: title
+
+It would be possible to do the above with no custom columns by putting the program into the template box of the plugboard. However, to do so, all comments must be removed because the plugboard text box does not support multi-line editing. It is debatable whether the gain of not having the custom column is worth the vast increase in difficulty caused by the program being one giant line.
+
 Special notes for save/send templates
 -------------------------------------
 
@@ -257,4 +349,4 @@ You might find the following tips useful.
     * Templates can use other templates by referencing a composite custom column.
     * In a plugboard, you can set a field to empty (or whatever is equivalent to empty) by using the special template ``{null}``. This template will always evaluate to an empty string.
     * The technique described above to show numbers even if they have a zero value works with the standard field series_index.
-    
+    

src/calibre/utils/formatter.py

@@ -66,6 +66,10 @@ class _Parser(object):
         template = template.replace('[[', '{').replace(']]', '}')
         return eval_formatter.safe_format(template, self.variables, 'EVAL', None)
 
+    def _print(self, *args):
+        print args
+        return None
+
     local_functions = {
             'add'      : (2, partial(_math, op='+')),
             'assign'   : (2, _assign),
@@ -74,6 +78,7 @@ class _Parser(object):
             'eval'     : (1, _eval),
             'field'    : (1, lambda s, x: s.parent.get_value(x, [], s.parent.kwargs)),
             'multiply' : (2, partial(_math, op='*')),
+            'print'    : (-1, _print),
             'strcat'   : (-1, _concat),
             'strcmp'   : (5, _strcmp),
             'substr'   : (3, lambda s, x, y, z: x[int(y): len(x) if int(z) == 0 else int(z)]),
@@ -143,12 +148,18 @@ class _Parser(object):
             if not self.token_op_is_a(';'):
                 return val
             self.consume()
+            if self.token_is_eof():
+                return val
 
     def expr(self):
         if self.token_is_id():
             # We have an identifier. Determine if it is a function
             id = self.token()
             if not self.token_op_is_a('('):
+                if self.token_op_is_a('='):
+                    # classic assignment statement
+                    self.consume()
+                    return self._assign(id, self.expr())
                 return self.variables.get(id, _('unknown id ') + id)
             # We have a function.
             # Check if it is a known one. We do this here so error reporting is
@@ -339,6 +350,7 @@ class TemplateFormatter(string.Formatter):
                 (r'\w+',                lambda x,t: (2, t)),
                 (r'".*?((?<!\\)")',     lambda x,t: (3, t[1:-1])),
                 (r'\'.*?((?<!\\)\')',   lambda x,t: (3, t[1:-1])),
+                (r'\n#.*?(?=\n)',       None),
                 (r'\s',                 None)
         ])
 
@@ -422,15 +434,15 @@ class TemplateFormatter(string.Formatter):
         self.kwargs = kwargs
         self.book = book
         self.composite_values = {}
-        if fmt.startswith('program:'):
+        try:
-            ans = self._eval_program(None, fmt[8:])
+            if fmt.startswith('program:'):
-        else:
+                ans = self._eval_program(None, fmt[8:])
-            try:
+            else:
                 ans = self.vformat(fmt, [], kwargs).strip()
-            except Exception, e:
+        except Exception, e:
-                if DEBUG:
+            if DEBUG:
-                    traceback.print_exc()
+                traceback.print_exc()
-                ans = error_value + ' ' + e.message
+            ans = error_value + ' ' + e.message
         return ans
 
 class ValidateFormatter(TemplateFormatter):