Fix #4721 (Bulk import stops because of utf-8 errors)

src/calibre/gui2/add.py

@@ -12,7 +12,7 @@ from calibre.gui2.dialogs.progress import ProgressDialog
 from calibre.gui2 import question_dialog, error_dialog, info_dialog
 from calibre.ebooks.metadata.opf2 import OPF
 from calibre.ebooks.metadata import MetaInformation
-from calibre.constants import preferred_encoding
+from calibre.constants import preferred_encoding, filesystem_encoding
 
 class DuplicatesAdder(QThread):
 
@@ -46,6 +46,8 @@ class RecursiveFind(QThread):
     def run(self):
         root = os.path.abspath(self.path)
         self.books = []
+        if isinstance(root, unicode):
+            root = root.encode(filesystem_encoding)
         try:
             for dirpath in os.walk(root):
                 if self.canceled:
@@ -55,6 +57,8 @@ class RecursiveFind(QThread):
                 self.books += list(self.db.find_books_in_directory(dirpath[0],
                                                 self.single_book_per_directory))
         except Exception, err:
+            import traceback
+            traceback.print_exc()
             try:
                 msg = unicode(err)
             except:

src/calibre/gui2/ui.py

@@ -1048,6 +1048,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         if self._adder.critical:
             det_msg = []
             for name, log in self._adder.critical.items():
+                if isinstance(name, str):
+                    name = name.decode(filesystem_encoding, 'replace')
                 det_msg.append(name+'\n'+log)
             warning_dialog(self, _('Failed to read metadata'),
                     _('Failed to read metadata from the following')+':',

src/calibre/startup.py

@@ -84,6 +84,7 @@ if not _run_once:
         return res
 
     os.path.abspath = my_abspath
+
     _join = os.path.join
     def my_join(a, *p):
         encoding=sys.getfilesystemencoding()

src/calibre/web/feeds/news.py

@@ -558,8 +558,6 @@ class BasicNewsRecipe(Recipe):
             '--max-recursions', str(self.recursions),
             '--delay', str(self.delay),
             ]
-        if self.encoding is not None:
-            web2disk_cmdline.extend(['--encoding', self.encoding])
 
         if self.verbose:
             web2disk_cmdline.append('--verbose')
@@ -578,6 +576,7 @@ class BasicNewsRecipe(Recipe):
                       'preprocess_html', 'remove_tags_after', 'remove_tags_before'):
             setattr(self.web2disk_options, extra, getattr(self, extra))
         self.web2disk_options.postprocess_html = self._postprocess_html
+        self.web2disk_options.encoding = self.encoding
 
         if self.delay > 0:
             self.simultaneous_downloads = 1