Search and replace: When using regular expression mode, add a special input field '{template}' that allows use the templating language to create complex input fields. Also allow setting of series_index by search and replace using the same syntax as in the book list, namely, Series Name [series number]

src/calibre/ebooks/metadata/__init__.py

@@ -55,9 +55,11 @@ except:
 
 _ignore_starts = u'\'"'+u''.join(unichr(x) for x in range(0x2018, 0x201e)+[0x2032, 0x2033])
 
-def title_sort(title):
+def title_sort(title, order=None):
+    if order is None:
+        order = tweaks['title_series_sorting']
     title = title.strip()
-    if tweaks['title_series_sorting'] == 'strictly_alphabetic':
+    if order == 'strictly_alphabetic':
         return title
     if title and title[0] in _ignore_starts:
         title = title[1:]

src/calibre/gui2/custom_column_widgets.py

@@ -587,8 +587,6 @@ class BulkSeries(BulkBase):
                 else:
                     s_index = self.db.get_custom_extra(book_id, num=self.col_id,
                                                        index_is_id=True)
-                    if s_index is None:
-                        s_index = 1.0
                 extras.append(s_index)
             self.db.set_custom_bulk(book_ids, val, extras=extras,
                                    num=self.col_id, notify=notify)

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -12,6 +12,7 @@ from PyQt4 import QtGui
 from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
 from calibre.gui2.dialogs.tag_editor import TagEditor
 from calibre.ebooks.metadata import string_to_authors, authors_to_string
+from calibre.ebooks.metadata.book.base import composite_formatter
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.gui2.custom_column_widgets import populate_metadata_page
 from calibre.gui2 import error_dialog
@@ -311,6 +312,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
     def prepare_search_and_replace(self):
         self.search_for.initialize('bulk_edit_search_for')
         self.replace_with.initialize('bulk_edit_replace_with')
+        self.s_r_template.initialize('bulk_edit_template')
         self.test_text.initialize('bulk_edit_test_test')
         self.all_fields = ['']
         self.writable_fields = ['']
@@ -325,9 +327,10 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             if f in ['sort'] or fm[f]['datatype'] == 'composite':
                 self.all_fields.append(f)
         self.all_fields.sort()
+        self.all_fields.insert(1, '{template}')
         self.writable_fields.sort()
-        self.search_field.setMaxVisibleItems(20)
+        self.search_field.setMaxVisibleItems(25)
-        self.destination_field.setMaxVisibleItems(20)
+        self.destination_field.setMaxVisibleItems(25)
         offset = 10
         self.s_r_number_of_books = min(10, len(self.ids))
         for i in range(1,self.s_r_number_of_books+1):
@@ -403,22 +406,28 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
         self.test_text.editTextChanged[str].connect(self.s_r_paint_results)
         self.comma_separated.stateChanged.connect(self.s_r_paint_results)
         self.case_sensitive.stateChanged.connect(self.s_r_paint_results)
+        self.s_r_template.lost_focus.connect(self.s_r_template_changed)
         self.central_widget.setCurrentIndex(0)
 
         self.search_for.completer().setCaseSensitivity(Qt.CaseSensitive)
         self.replace_with.completer().setCaseSensitivity(Qt.CaseSensitive)
+        self.s_r_template.completer().setCaseSensitivity(Qt.CaseSensitive)
 
         self.s_r_search_mode_changed(self.search_mode.currentIndex())
 
     def s_r_get_field(self, mi, field):
         if field:
+            if field == '{template}':
+                v = composite_formatter.safe_format\
+                    (unicode(self.s_r_template.text()), mi, _('S/R TEMPLATE ERROR'), mi)
+                return [v]
             fm = self.db.metadata_for_field(field)
             if field == 'sort':
                 val = mi.get('title_sort', None)
             else:
                 val = mi.get(field, None)
             if val is None:
-                val = []
+                val = [] if fm['is_multiple'] else ['']
             elif not fm['is_multiple']:
                 val = [val]
             elif field == 'authors':
@@ -427,7 +436,16 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             val = []
         return val
 
+    def s_r_template_changed(self):
+        self.s_r_search_field_changed(self.search_field.currentIndex())
+
     def s_r_search_field_changed(self, idx):
+        if self.search_mode.currentIndex() != 0 and idx == 1: # Template
+            self.s_r_template.setVisible(True)
+            self.template_label.setVisible(True)
+        else:
+            self.s_r_template.setVisible(False)
+            self.template_label.setVisible(False)
         for i in range(0, self.s_r_number_of_books):
             w = getattr(self, 'book_%d_text'%(i+1))
             mi = self.db.get_metadata(self.ids[i], index_is_id=True)
@@ -590,11 +608,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
         if not dest:
             dest = source
         dfm = self.db.field_metadata[dest]
-
         mi = self.db.get_metadata(id, index_is_id=True,)
-        val = mi.get(source)
-        if val is None:
-            return
         val = self.s_r_do_regexp(mi)
         val = self.s_r_do_destination(mi, val)
         if dfm['is_multiple']:

src/calibre/gui2/dialogs/metadata_bulk.ui

@@ -508,6 +508,29 @@ Future conversion of these books will use the default settings.</string>
           </layout>
          </item>
          <item row="4" column="0">
+          <widget class="QLabel" name="template_label">
+           <property name="text">
+            <string>Te&amp;mplate:</string>
+           </property>
+           <property name="buddy">
+            <cstring>s_r_template</cstring>
+           </property>
+          </widget>
+         </item>
+         <item row="4" column="1">
+          <widget class="HistoryLineEdit" name="s_r_template">
+           <property name="sizePolicy">
+            <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
+             <horstretch>100</horstretch>
+             <verstretch>0</verstretch>
+            </sizepolicy>
+           </property>
+           <property name="toolTip">
+            <string>Enter a template to be used as the source for the search/replace</string>
+           </property>
+          </widget>
+         </item>
+         <item row="5" column="0">
           <widget class="QLabel" name="xlabel_2">
            <property name="text">
             <string>&amp;Search for:</string>
@@ -517,7 +540,7 @@ Future conversion of these books will use the default settings.</string>
            </property>
           </widget>
          </item>
-         <item row="4" column="1">
+         <item row="5" column="1">
           <widget class="HistoryLineEdit" name="search_for">
            <property name="sizePolicy">
             <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
@@ -530,7 +553,7 @@ Future conversion of these books will use the default settings.</string>
            </property>
           </widget>
          </item>
-         <item row="4" column="2">
+         <item row="5" column="2">
           <widget class="QCheckBox" name="case_sensitive">
            <property name="toolTip">
             <string>Check this box if the search string must match exactly upper and lower case. Uncheck it if case is to be ignored</string>
@@ -543,7 +566,7 @@ Future conversion of these books will use the default settings.</string>
            </property>
           </widget>
          </item>
-         <item row="5" column="0">
+         <item row="6" column="0">
           <widget class="QLabel" name="xlabel_4">
            <property name="text">
             <string>&amp;Replace with:</string>
@@ -553,14 +576,14 @@ Future conversion of these books will use the default settings.</string>
            </property>
           </widget>
          </item>
-         <item row="5" column="1">
+         <item row="6" column="1">
           <widget class="HistoryLineEdit" name="replace_with">
            <property name="toolTip">
             <string>The replacement text. The matched search text will be replaced with this string</string>
            </property>
           </widget>
          </item>
-         <item row="5" column="2">
+         <item row="6" column="2">
           <layout class="QHBoxLayout" name="verticalLayout">
            <item>
             <widget class="QLabel" name="label_41">
@@ -595,7 +618,7 @@ field is processed. In regular expression mode, only the matched text is process
            </item>
           </layout>
          </item>
-         <item row="6" column="0">
+         <item row="7" column="0">
           <widget class="QLabel" name="destination_field_label">
            <property name="text">
             <string>&amp;Destination field:</string>
@@ -605,14 +628,15 @@ field is processed. In regular expression mode, only the matched text is process
            </property>
           </widget>
          </item>
-         <item row="6" column="1">
+         <item row="7" column="1">
           <widget class="QComboBox" name="destination_field">
            <property name="toolTip">
-            <string>The field that the text will be put into after all replacements. If blank, the source field is used.</string>
+            <string>The field that the text will be put into after all replacements.
+If blank, the source field is used if the field is modifiable</string>
            </property>
           </widget>
          </item>
-         <item row="6" column="2">
+         <item row="7" column="2">
           <layout class="QHBoxLayout" name="verticalLayout">
            <item>
             <widget class="QLabel" name="replace_mode_label">
@@ -660,7 +684,7 @@ nothing should be put between the original text and the inserted text</string>
            </item>
           </layout>
          </item>
-         <item row="7" column="1">
+         <item row="8" column="1">
           <widget class="QLabel" name="xlabel_3">
            <property name="text">
             <string>Test &amp;text</string>
@@ -670,7 +694,7 @@ nothing should be put between the original text and the inserted text</string>
            </property>
           </widget>
          </item>
-         <item row="7" column="2">
+         <item row="8" column="2">
           <widget class="QLabel" name="label_51">
            <property name="text">
             <string>Test re&amp;sult</string>
@@ -791,6 +815,7 @@ nothing should be put between the original text and the inserted text</string>
   <tabstop>central_widget</tabstop>
   <tabstop>search_field</tabstop>
   <tabstop>search_mode</tabstop>
+  <tabstop>s_r_template</tabstop>
   <tabstop>search_for</tabstop>
   <tabstop>case_sensitive</tabstop>
   <tabstop>replace_with</tabstop>

src/calibre/gui2/library/models.py

@@ -725,9 +725,7 @@ class BooksModel(QAbstractTableModel): # {{{
                     return False
                 val = qt_to_dt(val, as_utc=False)
         elif typ == 'series':
-            val, s_index = parse_series_string(self.db, label, value.toString())
+            val = unicode(value.toString()).strip()
-            if not val:
-                val = s_index = None
         elif typ == 'composite':
             tmpl = unicode(value.toString()).strip()
             disp = cc['display']

src/calibre/gui2/widgets.py

@@ -524,6 +524,8 @@ class EnComboBox(QComboBox):
 
 class HistoryLineEdit(QComboBox):
 
+    lost_focus = pyqtSignal()
+
     def __init__(self, *args):
         QComboBox.__init__(self, *args)
         self.setEditable(True)
@@ -559,6 +561,10 @@ class HistoryLineEdit(QComboBox):
     def text(self):
         return self.currentText()
 
+    def focusOutEvent(self, e):
+        QComboBox.focusOutEvent(self, e)
+        self.lost_focus.emit()
+
 class ComboBoxWithHelp(QComboBox):
     '''
     A combobox where item 0 is help text. CurrentText will return '' for item 0.

src/calibre/library/custom_columns.py

@@ -445,6 +445,9 @@ class CustomColumns(object):
                 index_is_id=True)
         val = self.custom_data_adapters[data['datatype']](val, data)
 
+        if data['datatype'] == 'series' and extra is None:
+            (val, extra) = self._get_series_values(val)
+
         if data['normalized']:
             if data['datatype'] == 'enumeration' and (
                     val and val not in data['display']['enum_values']):

src/calibre/library/database2.py

@@ -2133,9 +2133,27 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 self.conn.execute('DELETE FROM tags WHERE id=?', (id,))
                 self.conn.commit()
 
+    series_index_pat = re.compile(r'(.*)\s+\[([.0-9]+)\]$')
+
+    def _get_series_values(self, val):
+        if not val:
+            return (val, None)
+        match = self.series_index_pat.match(val.strip())
+        if match is not None:
+            idx = match.group(2)
+            try:
+                idx = float(idx)
+                return (match.group(1).strip(), idx)
+            except:
+                pass
+        return (val, None)
+
     def set_series(self, id, series, notify=True, commit=True):
         self.conn.execute('DELETE FROM books_series_link WHERE book=?',(id,))
-        self.conn.execute('DELETE FROM series WHERE (SELECT COUNT(id) FROM books_series_link WHERE series=series.id) < 1')
+        self.conn.execute('''DELETE FROM series
+                             WHERE (SELECT COUNT(id) FROM books_series_link
+                                    WHERE series=series.id) < 1''')
+        (series, idx) = self._get_series_values(series)
         if series:
             if not isinstance(series, unicode):
                 series = series.decode(preferred_encoding, 'replace')
@@ -2147,6 +2165,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             else:
                 aid = self.conn.execute('INSERT INTO series(name) VALUES (?)', (series,)).lastrowid
             self.conn.execute('INSERT INTO books_series_link(book, series) VALUES (?,?)', (id, aid))
+            if idx:
+                self.set_series_index(id, idx, notify=notify, commit=commit)
         self.dirtied([id], commit=False)
         if commit:
             self.conn.commit()

src/calibre/library/save_to_disk.py

@@ -7,6 +7,7 @@ __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import os, traceback, cStringIO, re, shutil
+from functools import partial
 
 from calibre.constants import DEBUG
 from calibre.utils.config import Config, StringConfig, tweaks
@@ -139,8 +140,7 @@ class SafeFormat(TemplateFormatter):
 def get_components(template, mi, id, timefmt='%b %Y', length=250,
         sanitize_func=ascii_filename, replace_whitespace=False,
         to_lowercase=False):
-    library_order = tweaks['save_template_title_series_sorting'] == 'library_order'
+    tsfmt = partial(title_sort, order=tweaks['save_template_title_series_sorting'])
-    tsfmt = title_sort if library_order else lambda x: x
     format_args = FORMAT_ARGS.copy()
     format_args.update(mi.all_non_none_fields())
     if mi.title:

src/calibre/manual/template_lang.rst

@@ -203,16 +203,17 @@ All the functions listed under single-function mode can be used in program mode,
 
 The following functions are available in addition to those described in single-function mode. With the exception of the ``id`` parameter of assign, all parameters can be statements (sequences of expressions):
 
-    * ``add(x, y)``	-- returns x + y. Throws an exception if either x or y are not numbers.
+    * ``add(x, y)`` -- returns x + y. Throws an exception if either x or y are not numbers.
     * ``assign(id, val)`` -- assigns val to id, then returns val. id must be an identifier, not an expression
     * ``cmp(x, y, lt, eq, gt)`` -- compares x and y after converting both to numbers. Returns ``lt`` if x < y. Returns ``eq`` if x == y. Otherwise returns ``gt``.
-    * ``divide(x, y)``	-- returns x / y. Throws an exception if either x or y are not numbers.
+    * ``divide(x, y)`` -- returns x / y. Throws an exception if either x or y are not numbers.
     * ``field(name)`` -- returns the metadata field named by ``name``.
-    * ``multiply``	-- returns x * y. Throws an exception if either x or y are not numbers.
+    * ``multiply(x, y)`` -- returns x * y. Throws an exception if either x or y are not numbers.
     * ``strcat(a, b, ...)`` -- can take any number of arguments. Returns a string formed by concatenating all the arguments.
     * ``strcmp(x, y, lt, eq, gt)`` -- does a case-insensitive comparison x and y as strings. Returns ``lt`` if x < y. Returns ``eq`` if x == y. Otherwise returns ``gt``.
     * ``substr(str, start, end)`` -- returns the ``start``'th through the ``end``'th characters of ``str``. The first character in ``str`` is the zero'th character. If end is negative, then it indicates that many characters counting from the right. If end is zero, then it indicates the last character. For example, ``substr('12345', 1, 0)`` returns ``'2345'``, and ``substr('12345', 1, -1)`` returns ``'234'``.
-    * ``subtract``	-- returns x - y. Throws an exception if either x or y are not numbers.
+    * ``subtract(x, y)`` -- returns x - y. Throws an exception if either x or y are not numbers.
+    * ``template(x)`` -- evaluates x as a template. The evaluation is done in its own context, meaning that variables are not shared between the caller and the template evaluation. Because the `{` and `}` characters are special, you must use `[[` for the `{` character and `]]` for the '}' character; they are converted automatically. For example, ``template('[[title_sort]]') will evaluate the template ``{title_sort}`` and return its value.
     
 Special notes for save/send templates
 -------------------------------------

src/calibre/utils/formatter.py

@@ -57,6 +57,11 @@ class _Parser(object):
         y = float(y if y else 0)
         return ops[op](x, y)
 
+    def _template(self, template):
+        template = template.replace('[[', '{').replace(']]', '}')
+        return self.parent.safe_format(template, self.parent.kwargs, 'TEMPLATE',
+                                       self.parent.book)
+
     local_functions = {
             'add'      : (2, partial(_math, op='+')),
             'assign'   : (2, _assign),
@@ -68,6 +73,7 @@ class _Parser(object):
             'strcmp'   : (5, _strcmp),
             'substr'   : (3, lambda s, x, y, z: x[int(y): len(x) if int(z) == 0 else int(z)]),
             'subtract' : (2, partial(_math, op='-')),
+            'template' : (1, _template)
     }
 
     def __init__(self, val, prog, parent):