Add search and replace feature to bulk metadata editor

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -4,8 +4,10 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 '''Dialog to edit metadata in bulk'''
 
 from threading import Thread
+import os, re, shutil
 
 from PyQt4.Qt import QDialog, QGridLayout
+from PyQt4 import QtGui
 
 from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
 from calibre.gui2.dialogs.tag_editor import TagEditor
@@ -83,7 +85,6 @@ class Worker(Thread):
             w.commit(self.ids)
         self.db.bulk_modify_tags(self.ids, add=add, remove=remove,
                 notify=False)
-        self.db.clean()
 
     def run(self):
         try:
@@ -101,6 +102,13 @@ class Worker(Thread):
 
 class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
 
+    s_r_functions = {
+                    ''          : lambda x: x,
+                    'lower'     : lambda x: x.lower(),
+                    'upper'     : lambda x: x.upper(),
+                    'title'     : lambda x: x.title(),
+            }
+
     def __init__(self, window, rows, db):
         QDialog.__init__(self, window)
         Ui_MetadataBulkDialog.__init__(self)
@@ -127,12 +135,196 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
         self.series.currentIndexChanged[int].connect(self.series_changed)
         self.series.editTextChanged.connect(self.series_changed)
         self.tag_editor_button.clicked.connect(self.tag_editor)
+
         if len(db.custom_column_label_map) == 0:
-            self.central_widget.tabBar().setVisible(False)
+            self.central_widget.removeTab(1)
         else:
             self.create_custom_column_editors()
+
+        self.prepare_search_and_replace()
         self.exec_()
 
+    def prepare_search_and_replace(self):
+        self.search_for.initialize('bulk_edit_search_for')
+        self.replace_with.initialize('bulk_edit_replace_with')
+        self.test_text.initialize('bulk_edit_test_test')
+        fields = ['']
+        fm = self.db.field_metadata
+        for f in fm:
+            if (f in ['author_sort'] or (
+                fm[f]['datatype'] == 'text' or fm[f]['datatype'] == 'series')
+                    and fm[f].get('search_terms', None)
+                    and f not in ['formats', 'ondevice']):
+                fields.append(f)
+        fields.sort()
+        self.search_field.addItems(fields)
+        self.search_field.setMaxVisibleItems(min(len(fields), 20))
+        offset = 10
+        self.s_r_number_of_books = min(7, len(self.ids))
+        for i in range(1,self.s_r_number_of_books+1):
+            w = QtGui.QLabel(self.tabWidgetPage3)
+            w.setText(_('Book %d:'%i))
+            self.gridLayout1.addWidget(w, i+offset, 0, 1, 1)
+            w = QtGui.QLineEdit(self.tabWidgetPage3)
+            w.setReadOnly(True)
+            name = 'book_%d_text'%i
+            setattr(self, name, w)
+            self.book_1_text.setObjectName(name)
+            self.gridLayout1.addWidget(w, i+offset, 1, 1, 1)
+            w = QtGui.QLineEdit(self.tabWidgetPage3)
+            w.setReadOnly(True)
+            name = 'book_%d_result'%i
+            setattr(self, name, w)
+            self.book_1_text.setObjectName(name)
+            self.gridLayout1.addWidget(w, i+offset, 2, 1, 1)
+
+        self.s_r_heading.setText(
+                           _('Search and replace in text fields using '
+                             'regular expressions. The search text is an '
+                             'arbitrary python-compatible regular expression. '
+                             'The replacement text can contain backreferences '
+                             'to parenthesized expressions in the pattern. '
+                             'The search is not anchored, and can match and '
+                             'replace times on the same string. See '
+                             '<a href="http://docs.python.org/library/re.html"> '
+                             'http://docs.python.org/library/re.html</a> '
+                             'for more information, and in particular the \'sub\' '
+                             'function. <br>'
+                             'Note: <b>you can destroy your library</b> '
+                             'using this feature. Changes are permanent. There '
+                             'is no undo function. You are strongly encouraged '
+                             'to backup the metadata.db file in your library '
+                             'before proceeding.'))
+        self.s_r_error = None
+        self.s_r_obj = None
+
+        self.replace_func.addItems(sorted(self.s_r_functions.keys()))
+        self.search_field.currentIndexChanged[str].connect(self.s_r_field_changed)
+        self.replace_func.currentIndexChanged[str].connect(self.s_r_paint_results)
+        self.search_for.editTextChanged[str].connect(self.s_r_paint_results)
+        self.replace_with.editTextChanged[str].connect(self.s_r_paint_results)
+        self.test_text.editTextChanged[str].connect(self.s_r_paint_results)
+
+    def s_r_field_changed(self, txt):
+        txt = unicode(txt)
+        for i in range(0,self.s_r_number_of_books):
+            if txt:
+                fm = self.db.field_metadata[txt]
+                id = self.ids[i]
+                val = self.db.get_property(id, index_is_id=True,
+                                           loc=fm['rec_index'])
+                if val is None:
+                    val = ''
+                if fm['is_multiple']:
+                    val = [t.strip() for t in val.split(fm['is_multiple']) if t.strip()]
+                    if val:
+                        val.sort(cmp=lambda x,y: cmp(x.lower(), y.lower()))
+                        val = val[0]
+                    else:
+                        val = ''
+            else:
+                val = ''
+            w = getattr(self, 'book_%d_text'%(i+1))
+            w.setText(val)
+        self.s_r_paint_results(None)
+
+    def s_r_set_colors(self):
+        if self.s_r_error is not None:
+            col = 'rgb(255, 0, 0, 20%)'
+            self.test_result.setText(self.s_r_error.message)
+        else:
+            col = 'rgb(0, 255, 0, 20%)'
+        self.test_result.setStyleSheet('QLineEdit { color: black; '
+                                       'background-color: %s; }'%col)
+        for i in range(0,self.s_r_number_of_books):
+            getattr(self, 'book_%d_result'%(i+1)).setText('')
+
+    def s_r_func(self, match):
+        rf = self.s_r_functions[unicode(self.replace_func.currentText())]
+        rv = unicode(self.replace_with.text())
+        val = match.expand(rv)
+        return rf(val)
+
+    def s_r_paint_results(self, txt):
+        self.s_r_error = None
+        self.s_r_set_colors()
+        try:
+            self.s_r_obj = re.compile(unicode(self.search_for.text()))
+        except re.error as e:
+            self.s_r_obj = None
+            self.s_r_error = e
+            self.s_r_set_colors()
+            return
+
+        try:
+            self.test_result.setText(self.s_r_obj.sub(self.s_r_func,
+                                     unicode(self.test_text.text())))
+        except re.error as e:
+            self.s_r_error = e
+            self.s_r_set_colors()
+            return
+
+        for i in range(0,self.s_r_number_of_books):
+            wt = getattr(self, 'book_%d_text'%(i+1))
+            wr = getattr(self, 'book_%d_result'%(i+1))
+            try:
+                wr.setText(self.s_r_obj.sub(self.s_r_func, unicode(wt.text())))
+            except re.error as e:
+                self.s_r_error = e
+                self.s_r_set_colors()
+                break
+
+    def do_search_replace(self):
+        field = unicode(self.search_field.currentText())
+        if not field or not self.s_r_obj:
+            return
+        if self.s_r_backup_db.isChecked():
+            self.db.commit()
+            src = self.db.dbpath
+            dest = self.db.dbpath+'.backup'
+            if os.path.exists(dest):
+                os.remove(dest)
+            shutil.copyfile(src, dest)
+
+        fm = self.db.field_metadata[field]
+
+        def apply_pattern(val):
+            try:
+                return self.s_r_obj.sub(self.s_r_func, val)
+            except:
+                return val
+
+        for id in self.ids:
+            val = self.db.get_property(id, index_is_id=True,
+                                       loc=fm['rec_index'])
+            if val is None:
+                continue
+            if fm['is_multiple']:
+                res = []
+                for val in [t.strip() for t in val.split(fm['is_multiple'])]:
+                    v = apply_pattern(val).strip()
+                    if v:
+                        res.append(v)
+                val = res
+                if fm['is_custom']:
+                    # The standard tags and authors values want to be lists.
+                    # All custom columns are to be strings
+                    val = fm['is_multiple'].join(val)
+            else:
+                val = apply_pattern(val)
+
+            if fm['is_custom']:
+                extra = self.db.get_custom_extra(id, label=fm['label'], index_is_id=True)
+                self.db.set_custom(id, val, label=fm['label'], extra=extra,
+                                   commit=False)
+            else:
+                if field == 'comments':
+                    setter = self.db.set_comment
+                else:
+                    setter = getattr(self.db, 'set_'+field)
+                setter(id, val, notify=False, commit=False)
+        self.db.commit()
+
     def create_custom_column_editors(self):
         w = self.central_widget.widget(1)
         layout = QGridLayout()
@@ -193,6 +385,11 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
         if len(self.ids) < 1:
             return QDialog.accept(self)
 
+        if self.s_r_error is not None:
+            error_dialog(self, _('Search/replace invalid'),
+                    _('Search pattern is invalid: %s')%self.s_r_error.message,
+                    show=True)
+            return False
         self.changed = bool(self.ids)
         # Cache values from GUI so that Qt widgets are not used in
         # non GUI thread
@@ -234,6 +431,10 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             return error_dialog(self, _('Failed'),
                     self.worker.error[0], det_msg=self.worker.error[1],
                     show=True)
+
+        self.do_search_replace()
+
+        self.db.clean()
         return QDialog.accept(self)
 
 

src/calibre/gui2/dialogs/metadata_bulk.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>526</width>
+    <width>572</width>
-    <height>499</height>
+    <height>554</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -200,14 +200,15 @@
          </item>
          <item row="6" column="2">
           <widget class="QCheckBox" name="remove_all_tags">
-           <property name="text">
-            <string>Remove all</string>
-           </property>
            <property name="toolTip">
             <string>Check this box to remove all tags from the books.</string>
            </property>
+           <property name="text">
+            <string>Remove all</string>
+           </property>
           </widget>
-         </item><item row="7" column="0">
+         </item>
+         <item row="7" column="0">
           <widget class="QLabel" name="label_7">
            <property name="text">
             <string>&amp;Series:</string>
@@ -301,6 +302,113 @@ Future conversion of these books will use the default settings.</string>
          <string>&amp;Custom metadata</string>
         </attribute>
        </widget>
+       <widget class="QWidget" name="tabWidgetPage3">
+        <attribute name="title">
+         <string>&amp;Search and replace (experimental)</string>
+        </attribute>
+        <layout class="QGridLayout" name="gridLayout">
+         <property name="sizeConstraint">
+          <enum>QLayout::SetMinimumSize</enum>
+         </property>
+         <item row="1" column="0" colspan="3">
+          <widget class="QLabel" name="s_r_heading">
+           <property name="wordWrap">
+            <bool>true</bool>
+           </property>
+          </widget>
+         </item>
+         <item row="2" column="0" colspan="3">
+          <widget class="QCheckBox" name="s_r_backup_db">
+           <property name="text">
+            <string>Backup 'metadata.db' to 'metadata.db.backup' before applying changes</string>
+           </property>
+           <property name="checked">
+            <bool>true</bool>
+           </property>
+          </widget>
+         </item>
+         <item row="3" column="0">
+          <widget class="QLabel" name="label_2">
+           <property name="text">
+            <string>Search field:</string>
+           </property>
+          </widget>
+         </item>
+         <item row="3" column="1">
+          <widget class="QLabel" name="label_2">
+           <property name="text">
+            <string>Search for:</string>
+           </property>
+          </widget>
+         </item>
+         <item row="3" column="2">
+          <widget class="QLabel" name="label_4">
+           <property name="text">
+            <string>Replace with:</string>
+           </property>
+          </widget>
+         </item>
+         <item row="4" column="0">
+          <widget class="QComboBox" name="search_field"/>
+         </item>
+         <item row="4" column="1">
+          <widget class="HistoryLineEdit" name="search_for"/>
+         </item>
+         <item row="4" column="2">
+          <widget class="HistoryLineEdit" name="replace_with"/>
+         </item>
+         <item row="5" column="1">
+          <widget class="QLabel" name="label_41">
+           <property name="text">
+            <string>Apply function after replace:</string>
+           </property>
+          </widget>
+         </item>
+         <item row="5" column="2">
+          <widget class="QComboBox" name="replace_func"/>
+         </item>
+         <item row="6" column="1">
+          <widget class="QLabel" name="label_3">
+           <property name="text">
+            <string>Test text</string>
+           </property>
+          </widget>
+         </item>
+         <item row="6" column="2">
+          <widget class="QLabel" name="label_5">
+           <property name="text">
+            <string>Test result</string>
+           </property>
+          </widget>
+         </item>
+         <item row="7" column="0">
+          <widget class="QLabel" name="label_3">
+           <property name="text">
+            <string>Your test:</string>
+           </property>
+          </widget>
+         </item>
+         <item row="7" column="1">
+          <widget class="HistoryLineEdit" name="test_text"/>
+         </item>
+         <item row="7" column="2">
+          <widget class="QLineEdit" name="test_result"/>
+         </item>
+         <item row="20" column="1">
+          <spacer name="verticalSpacer">
+           <property name="orientation">
+            <enum>Qt::Vertical</enum>
+           </property>
+           <property name="sizeHint" stdset="0">
+            <size>
+             <width>20</width>
+             <height>40</height>
+            </size>
+           </property>
+          </spacer>
+         </item>
+        </layout>
+       </widget>
       </widget>
      </item>
     </layout>
@@ -333,6 +441,11 @@ Future conversion of these books will use the default settings.</string>
    <extends>QLineEdit</extends>
    <header>widgets.h</header>
   </customwidget>
+  <customwidget>
+   <class>HistoryLineEdit</class>
+   <extends>QLineEdit</extends>
+   <header>widgets.h</header>
+  </customwidget>
  </customwidgets>
  <tabstops>
   <tabstop>authors</tabstop>

src/calibre/library/database2.py

@@ -143,6 +143,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         SchemaUpgrade.__init__(self)
         self.initialize_dynamic()
 
+    def get_property(self, idx, index_is_id=False, loc=-1):
+        row = self.data._data[idx] if index_is_id else self.data[idx]
+        if row is not None:
+            return row[loc]
+
     def initialize_dynamic(self):
         self.field_metadata = FieldMetadata() #Ensure we start with a clean copy
         self.prefs = DBPrefs(self)
@@ -324,17 +329,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.last_update_check = self.last_modified()
 
 
-        def get_property(idx, index_is_id=False, loc=-1):
-            row = self.data._data[idx] if index_is_id else self.data[idx]
-            if row is not None:
-                return row[loc]
-
         for prop in ('author_sort', 'authors', 'comment', 'comments', 'isbn',
                      'publisher', 'rating', 'series', 'series_index', 'tags',
                      'title', 'timestamp', 'uuid', 'pubdate', 'ondevice'):
-            setattr(self, prop, functools.partial(get_property,
+            setattr(self, prop, functools.partial(self.get_property,
                     loc=self.FIELD_MAP['comments' if prop == 'comment' else prop]))
-        setattr(self, 'title_sort', functools.partial(get_property,
+        setattr(self, 'title_sort', functools.partial(self.get_property,
                 loc=self.FIELD_MAP['sort']))
 
     def initialize_database(self):
@@ -1129,7 +1129,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 result.append(r)
         return ' & '.join(result).replace('|', ',')
 
-    def set_authors(self, id, authors, notify=True):
+    def set_authors(self, id, authors, notify=True, commit=True):
         '''
         `authors`: A list of authors.
         '''
@@ -1157,16 +1157,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         ss = self.author_sort_from_book(id, index_is_id=True)
         self.conn.execute('UPDATE books SET author_sort=? WHERE id=?',
                           (ss, id))
-        self.conn.commit()
+        if commit:
+            self.conn.commit()
         self.data.set(id, self.FIELD_MAP['authors'],
                       ','.join([a.replace(',', '|') for a in authors]),
                       row_is_id=True)
         self.data.set(id, self.FIELD_MAP['author_sort'], ss, row_is_id=True)
-        self.set_path(id, True)
+        self.set_path(id, index_is_id=True, commit=commit)
         if notify:
             self.notify('metadata', [id])
 
-    def set_title(self, id, title, notify=True):
+    def set_title(self, id, title, notify=True, commit=True):
         if not title:
             return
         if not isinstance(title, unicode):
@@ -1177,8 +1178,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             self.data.set(id, self.FIELD_MAP['sort'], title_sort(title), row_is_id=True)
         else:
             self.data.set(id, self.FIELD_MAP['sort'], title, row_is_id=True)
-        self.set_path(id, True)
+        self.set_path(id, index_is_id=True, commit=commit)
-        self.conn.commit()
+        if commit:
+            self.conn.commit()
         if notify:
             self.notify('metadata', [id])