Fix a regression in 2.10 that causes Adding books from sub-folders, one book per folder to incorrectly add OPF files present in the sub-folders as an extra format. See #1392864 (Errors in book import in 2.10)

src/calibre/db/adding.py

@@ -7,44 +7,58 @@ __license__ = 'GPL v3'
 __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import os
+from collections import defaultdict
+from future_builtins import map
+
 from calibre.ebooks import BOOK_EXTENSIONS
 
+def splitext(path):
+    key, ext = os.path.splitext(path)
+    return key, ext[1:].lower()
+
+def formats_ok(formats):
+    if formats and (len(formats) > 1 or tuple(formats.iterkeys()) != ('opf',)):
+        return True
+    return False
+
+def path_ok(path):
+    return not os.path.isdir(path) and os.access(path, os.R_OK)
+
+_metadata_extensions = None
+
+def metadata_extensions():
+    # Set of all known book extensions + OPF (the OPF is used to read metadata,
+    # but not actually added)
+    global _metadata_extensions
+    if _metadata_extensions is None:
+        _metadata_extensions =  frozenset(map(unicode, BOOK_EXTENSIONS)) | {'opf'}
+    return _metadata_extensions
+
+def listdir(root):
+    for path in os.listdir(root):
+        yield os.path.abspath(os.path.join(root, path))
+
 def find_books_in_directory(dirpath, single_book_per_directory):
     dirpath = os.path.abspath(dirpath)
+    book_extentions = metadata_extensions()
     if single_book_per_directory:
-        formats = []
+        formats = {}
-        for path in os.listdir(dirpath):
+        for path in listdir(dirpath):
-            path = os.path.abspath(os.path.join(dirpath, path))
+            key, ext = splitext(path)
-            if os.path.isdir(path) or not os.access(path, os.R_OK):
+            if ext in book_extentions and path_ok(path):
-                continue
+                formats[ext] = path
-            ext = os.path.splitext(path)[1]
+        if formats_ok(formats):
-            if not ext:
+            yield list(formats.itervalues())
-                continue
-            ext = ext[1:].lower()
-            if ext not in BOOK_EXTENSIONS and ext != 'opf':
-                continue
-            formats.append(path)
-        yield formats
     else:
-        books = {}
+        books = defaultdict(dict)
-        for path in os.listdir(dirpath):
+        for path in listdir(dirpath):
-            path = os.path.abspath(os.path.join(dirpath, path))
+            key, ext = splitext(path)
-            if os.path.isdir(path) or not os.access(path, os.R_OK):
+            if ext in book_extentions and path_ok(path):
-                continue
+                books[icu_lower(key) if isinstance(key, unicode) else key.lower()][ext] = path
-            ext = os.path.splitext(path)[1]
-            if not ext:
-                continue
-            ext = ext[1:].lower()
-            if ext not in BOOK_EXTENSIONS:
-                continue
 
-            key = os.path.splitext(path)[0]
+        for formats in books.itervalues():
-            if key not in books:
+            if formats_ok(formats):
-                books[key] = []
+                yield list(formats.itervalues())
-            books[key].append(path)
-
-        for formats in books.values():
-            yield formats
 
 def import_book_directory_multiple(db, dirpath, callback=None,
         added_ids=None):

src/calibre/gui2/add.py

@@ -62,6 +62,7 @@ class Adder(QObject):
             return
         QObject.__init__(self, parent)
         self.single_book_per_directory = single_book_per_directory
+        self.ignore_opf = False
         self.list_of_archives = list_of_archives
         self.callback = callback
         self.add_formats_to_existing = prefs['add_formats_to_existing']
@@ -123,8 +124,7 @@ class Adder(QObject):
                 for files in find_books_in_directory(dirpath, self.single_book_per_directory):
                     if self.abort_scan:
                         return
-                    if files:
+                    self.file_groups[len(self.file_groups)] = files
-                        self.file_groups[len(self.file_groups)] = files
 
         def extract(source):
             tdir = tempfile.mkdtemp(suffix='_archive', dir=self.tdir)
@@ -145,6 +145,7 @@ class Adder(QObject):
         try:
             if isinstance(self.source, basestring):
                 find_files(self.source)
+                self.ignore_opf = True
             else:
                 unreadable_files = []
                 for path in self.source:
@@ -153,6 +154,7 @@ class Adder(QObject):
                     if os.access(path, os.R_OK):
                         if self.list_of_archives:
                             find_files(extract(path))
+                            self.ignore_opf = True
                         else:
                             self.file_groups[len(self.file_groups)] = [path]
                     else:
@@ -377,6 +379,8 @@ class Adder(QObject):
         for fmt, path in fmap.iteritems():
             # The onimport plugins have already been run by the read metadata
             # worker
+            if self.ignore_opf and fmt.lower() == 'opf':
+                continue
             try:
                 if self.db.add_format(book_id, fmt, path, run_hooks=False, replace=replace):
                     run_plugins_on_postimport(self.dbref(), book_id, fmt)