Linux: When adding books from directories ignore files that have undecodable names. Fixes #1685819 [adding books from directory crashes with unicode error](https://bugs.launchpad.net/calibre/+bug/1685819)

src/calibre/db/adding.py

@@ -10,6 +10,8 @@ import os, time, re
 from collections import defaultdict
 from future_builtins import map
 
+from calibre import prints
+from calibre.constants import iswindows, isosx, filesystem_encoding
 from calibre.ebooks import BOOK_EXTENSIONS
 
 
@@ -56,6 +58,7 @@ def filter_filename(compiled_rules, filename):
         if q(filename):
             return action
 
+
 _metadata_extensions = None
 
 
@@ -68,8 +71,21 @@ def metadata_extensions():
     return _metadata_extensions
 
 
+if iswindows or isosx:
+    unicode_listdir = os.listdir
+else:
+    def unicode_listdir(root):
+        root = root.encode(filesystem_encoding)
+        for x in os.listdir(root):
+            try:
+                yield x.decode(filesystem_encoding)
+            except UnicodeDecodeError:
+                prints('Ignoring un-decodable file:', x)
+
+
 def listdir(root, sort_by_mtime=False):
-    items = (os.path.join(root, x) for x in os.listdir(root))
+
+    items = (os.path.join(root, x) for x in unicode_listdir(root))
     if sort_by_mtime:
         def safe_mtime(x):
             try:
@@ -236,5 +252,3 @@ def add_news(cache, path, arg, dbapi=None):
     if not hasattr(path, 'read'):
         stream.close()
     return db_id
-
-

src/calibre/gui2/add.py

@@ -16,7 +16,7 @@ from future_builtins import map
 from PyQt5.Qt import QObject, Qt, pyqtSignal
 
 from calibre import prints, as_unicode
-from calibre.constants import DEBUG
+from calibre.constants import DEBUG, iswindows, isosx, filesystem_encoding
 from calibre.customize.ui import run_plugins_on_postimport, run_plugins_on_postadd
 from calibre.db.adding import find_books_in_directory, compile_rule
 from calibre.db.utils import find_identical_books
@@ -129,12 +129,27 @@ class Adder(QObject):
             import traceback
             traceback.print_exc()
 
-        def find_files(root):
-            for dirpath, dirnames, filenames in os.walk(root):
-                for files in find_books_in_directory(dirpath, self.single_book_per_directory, compiled_rules=compiled_rules):
-                    if self.abort_scan:
-                        return
-                    self.file_groups[len(self.file_groups)] = files
+        if iswindows or isosx:
+            def find_files(root):
+                for dirpath, dirnames, filenames in os.walk(root):
+                    for files in find_books_in_directory(dirpath, self.single_book_per_directory, compiled_rules=compiled_rules):
+                        if self.abort_scan:
+                            return
+                        self.file_groups[len(self.file_groups)] = files
+        else:
+            def find_files(root):
+                if isinstance(root, type(u'')):
+                    root = root.encode(filesystem_encoding)
+                for dirpath, dirnames, filenames in os.walk(root):
+                    try:
+                        dirpath = dirpath.decode(filesystem_encoding)
+                    except UnicodeDecodeError:
+                        prints('Ignoring non-decodable directory:', dirpath)
+                        continue
+                    for files in find_books_in_directory(dirpath, self.single_book_per_directory, compiled_rules=compiled_rules):
+                        if self.abort_scan:
+                            return
+                        self.file_groups[len(self.file_groups)] = files
 
         def extract(source):
             tdir = tempfile.mkdtemp(suffix='_archive', dir=self.tdir)