Bulk metadata edit dialog: Add an action to set metadata from ebook files. Fixes #1717755 [Reading metadata fails when adding many books](https://bugs.launchpad.net/calibre/+bug/1717755)

src/calibre/db/cache.py

@@ -746,8 +746,8 @@ class Cache(object):
         Instead use, :meth:`copy_format_to`.
 
         Currently used only in calibredb list, the viewer, edit book,
-        compare_format to original format, open with and the catalogs (via
+        compare_format to original format, open with, bulk metadata edit and
-        get_data_as_dict()).
+        the catalogs (via get_data_as_dict()).
 
         Apart from the viewer, open with and edit book, I don't believe any of
         the others do any file write I/O with the results of this call.

src/calibre/ebooks/metadata/worker.py

@@ -33,6 +33,23 @@ def serialize_metadata_for(paths, tdir, group_id):
     return mi, opf, has_cover
 
 
+def read_metadata_bulk(get_opf, get_cover, paths):
+    mi = metadata_from_formats(paths)
+    mi.cover = None
+    cdata = None
+    if mi.cover_data:
+        cdata = mi.cover_data[-1]
+    mi.cover_data = (None, None)
+    if not mi.application_id:
+        mi.application_id = '__calibre_dummy__'
+    ans = {'opf': None, 'cdata': None}
+    if get_opf:
+        ans['opf'] = metadata_to_opf(mi, default_lang='und')
+    if get_cover:
+        ans['cdata'] = cdata
+    return ans
+
+
 def run_import_plugins(paths, group_id, tdir):
     final_paths = []
     for path in paths:

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -3,7 +3,8 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
 '''Dialog to edit metadata in bulk'''
 
-import re, os
+import re
+from io import BytesIO
 from collections import namedtuple, defaultdict
 from threading import Thread
 
@@ -11,6 +12,8 @@ from PyQt5.Qt import Qt, QDialog, QGridLayout, QVBoxLayout, QFont, QLabel, \
                      pyqtSignal, QDialogButtonBox, QInputDialog, QLineEdit, \
                      QDateTime, QCompleter, QCoreApplication, QSize
 
+from calibre import prints
+from calibre.ebooks.metadata.opf2 import OPF
 from calibre.constants import DEBUG
 from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
 from calibre.gui2.dialogs.tag_editor import TagEditor
@@ -25,43 +28,15 @@ from calibre.utils.config import dynamic, JSONConfig
 from calibre.utils.titlecase import titlecase
 from calibre.utils.icu import sort_key, capitalize
 from calibre.utils.config import prefs, tweaks
-from calibre.utils.imghdr import identify
 from calibre.utils.date import qt_to_dt
 from calibre.db import _get_next_series_num_for_list
 
 
-def get_cover_data(stream, ext):  # {{{
-    from calibre.ebooks.metadata.meta import get_metadata
-    old = prefs['read_file_metadata']
-    if not old:
-        prefs['read_file_metadata'] = True
-    cdata = area = None
-
-    try:
-        with stream:
-            mi = get_metadata(stream, ext)
-        if mi.cover and os.access(mi.cover, os.R_OK):
-            cdata = open(mi.cover).read()
-        elif mi.cover_data[1] is not None:
-            cdata = mi.cover_data[1]
-        if cdata:
-            fmt, width, height = identify(cdata)
-            area = width*height
-    except:
-        cdata = area = None
-
-    if old != prefs['read_file_metadata']:
-        prefs['read_file_metadata'] = old
-
-    return cdata, area
-# }}}
-
-
 Settings = namedtuple('Settings',
     'remove_all remove add au aus do_aus rating pub do_series do_autonumber '
     'do_swap_ta do_remove_conv do_auto_author series do_series_restart series_start_value series_increment '
     'do_title_case cover_action clear_series clear_pub pubdate adddate do_title_sort languages clear_languages '
-    'restore_original comments generate_cover_settings')
+    'restore_original comments generate_cover_settings read_file_metadata')
 
 null = object()
 
@@ -134,9 +109,49 @@ class MyBlockingBusy(QDialog):  # {{{
 
         self.all_done.emit()
 
+    def read_file_metadata(self, args):
+        from calibre.utils.ipc.simple_worker import offload_worker
+        db = self.db.new_api
+        worker = offload_worker()
+        try:
+            for book_id in self.ids:
+                fmts = db.formats(book_id, verify_formats=False)
+                paths = filter(None, [db.format_abspath(book_id, fmt) for fmt in fmts])
+                if paths:
+                    ret = worker(
+                        'calibre.ebooks.metadata.worker', 'read_metadata_bulk',
+                        args.read_file_metadata, args.cover_action == 'fromfmt', paths)
+                    if ret['tb'] is not None:
+                        prints(ret['tb'])
+                    else:
+                        ans = ret['result']
+                        opf, cdata = ans['opf'], ans['cdata']
+                        if opf is not None:
+                            try:
+                                mi = OPF(BytesIO(opf), populate_spine=False, try_to_guess_cover=False).to_book_metadata()
+                            except Exception:
+                                import traceback
+                                traceback.print_exc()
+                            else:
+                                db.set_metadata(book_id, mi, allow_case_change=True)
+                        if cdata is not None:
+                            db.set_cover({book_id: cdata})
+        finally:
+            worker.shutdown()
+
     def do_all(self):
         cache = self.db.new_api
         args = self.args
+        from_file = args.cover_action == 'fromfmt' or args.read_file_metadata
+        if from_file:
+            old = prefs['read_file_metadata']
+            if not old:
+                prefs['read_file_metadata'] = True
+            try:
+                self.read_file_metadata(args)
+            finally:
+                if old != prefs['read_file_metadata']:
+                    prefs['read_file_metadata'] = old
 
         # Title and authors
         if args.do_swap_ta:
@@ -190,21 +205,6 @@ class MyBlockingBusy(QDialog):  # {{{
                 mi = self.db.get_metadata(book_id, index_is_id=True)
                 cdata = generate_cover(mi, prefs=args.generate_cover_settings)
                 cache.set_cover({book_id:cdata})
-        elif args.cover_action == 'fromfmt':
-            for book_id in self.ids:
-                fmts = cache.formats(book_id, verify_formats=False)
-                if fmts:
-                    covers = []
-                    for fmt in fmts:
-                        fmtf = cache.format(book_id, fmt, as_file=True)
-                        if fmtf is None:
-                            continue
-                        cdata, area = get_cover_data(fmtf, fmt)
-                        if cdata:
-                            covers.append((cdata, area))
-                    covers.sort(key=lambda x: x[1])
-                    if covers:
-                        cache.set_cover({book_id:covers[-1][0]})
         elif args.cover_action == 'trim':
             from calibre.utils.img import remove_borders_from_image, image_to_data, image_from_data
             for book_id in self.ids:
@@ -1016,6 +1016,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
         do_auto_author = self.auto_author_sort.isChecked()
         do_title_case = self.change_title_to_title_case.isChecked()
         do_title_sort = self.update_title_sort.isChecked()
+        read_file_metadata = self.read_file_metadata.isChecked()
         clear_languages = self.clear_languages.isChecked()
         restore_original = self.restore_original.isChecked()
         languages = self.languages.lang_codes
@@ -1037,12 +1038,14 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
         elif self.cover_clone.isChecked():
             cover_action = 'clone'
 
-        args = Settings(remove_all, remove, add, au, aus, do_aus, rating, pub, do_series,
+        args = Settings(
-                do_autonumber, do_swap_ta,
+            remove_all, remove, add, au, aus, do_aus, rating, pub, do_series,
-                do_remove_conv, do_auto_author, series, do_series_restart,
+            do_autonumber, do_swap_ta, do_remove_conv, do_auto_author, series,
-                series_start_value, series_increment, do_title_case, cover_action, clear_series, clear_pub,
+            do_series_restart, series_start_value, series_increment,
-                pubdate, adddate, do_title_sort, languages, clear_languages,
+            do_title_case, cover_action, clear_series, clear_pub, pubdate,
-                restore_original, self.comments, self.generate_cover_settings)
+            adddate, do_title_sort, languages, clear_languages,
+            restore_original, self.comments, self.generate_cover_settings,
+            read_file_metadata)
         if DEBUG:
             print('Running bulk metadata operation with settings:')
             print(args)

src/calibre/gui2/dialogs/metadata_bulk.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>962</width>
-    <height>669</height>
+    <height>709</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -70,8 +70,8 @@ is completed. This can be slow on large libraries.</string>
        <rect>
         <x>0</x>
         <y>0</y>
-        <width>930</width>
+        <width>944</width>
-        <height>603</height>
+        <height>638</height>
        </rect>
       </property>
       <layout class="QVBoxLayout" name="verticalLayout_2">
@@ -695,6 +695,16 @@ as that of the first selected book.</string>
              </item>
             </layout>
            </item>
+           <item row="14" column="0" colspan="4">
+            <widget class="QCheckBox" name="read_file_metadata">
+             <property name="toolTip">
+              <string>Set the metadata in calibre from the metadata in the e-book files associated with each book. Note that this does not change the cover, for that, use the separate option below.</string>
+             </property>
+             <property name="text">
+              <string>Set &amp;metadata (except cover) from the e-book files</string>
+             </property>
+            </widget>
+           </item>
           </layout>
          </widget>
          <widget class="QWidget" name="tab">
@@ -1181,8 +1191,8 @@ not multiple and the destination field is multiple</string>
                <rect>
                 <x>0</x>
                 <y>0</y>
-                <width>908</width>
+                <width>203</width>
-                <height>253</height>
+                <height>70</height>
                </rect>
               </property>
               <layout class="QGridLayout" name="testgrid">