More py3 unicode fixes

the csv module needs to write out to a stream with newline='' on
python3. And use PolyglotStringIO instead of PolyglotBytesIO so we remain unicode as long as possible.

src/calibre/db/cli/cmd_check_library.py

@@ -4,6 +4,7 @@
 
 
 import csv
+import io
 import sys
 
 from calibre import prints
@@ -71,10 +72,11 @@ def _print_check_library_results(checker, check, as_csv=False, out=sys.stdout):
 
     if as_csv:
         to_output = [(check[1], i[0], i[1]) for i in list]
-        csv_print = csv.writer(out)
+        buf = io.StringIO(newline='')
+        csv_print = csv.writer(buf)
         for line in to_output:
             csv_print.writerow(line)
-
+        out.write(buf.getvalue())
     else:
         print(check[1], file=out)
         for i in list:

src/calibre/devices/__init__.py

@@ -74,11 +74,11 @@ def debug(ioreg_to_tmp=False, buf=None, plugins=None,
     from calibre.devices.scanner import DeviceScanner
     from calibre.constants import iswindows, isosx
     from calibre import prints
-    from polyglot.io import PolyglotBytesIO
+    from polyglot.io import PolyglotStringIO
     oldo, olde = sys.stdout, sys.stderr
 
     if buf is None:
-        buf = PolyglotBytesIO()
+        buf = PolyglotStringIO()
     sys.stdout = sys.stderr = buf
     out = partial(prints, file=buf)
 
@@ -187,7 +187,7 @@ def debug(ioreg_to_tmp=False, buf=None, plugins=None,
                     out(ioreg)
 
         if hasattr(buf, 'getvalue'):
-            return buf.getvalue().decode('utf-8', 'replace')
+            return buf.getvalue()
     finally:
         sys.stdout = oldo
         sys.stderr = olde

src/calibre/devices/cli.py

@@ -18,7 +18,7 @@ from calibre.customize.ui import device_plugins
 from calibre.devices.scanner import DeviceScanner
 from calibre.utils.config import device_prefs
 from polyglot.builtins import unicode_type
-from polyglot.io import PolyglotBytesIO
+from polyglot.io import PolyglotStringIO
 
 MINIMUM_COL_WIDTH = 12  # : Minimum width of columns in ls output
 
@@ -113,7 +113,7 @@ def ls(dev, path, recurse=False, human_readable_size=False, ll=False, cols=0):
                 c += 1
         return rowwidths
 
-    output = PolyglotBytesIO()
+    output = PolyglotStringIO()
     if path.endswith("/") and len(path) > 1:
         path = path[:-1]
     dirs = dev.list(path, recurse)
@@ -165,7 +165,7 @@ def ls(dev, path, recurse=False, human_readable_size=False, ll=False, cols=0):
                     prints(trytablecol[r][c], "".ljust(padding), end=' ', file=output)
                 prints(file=output)
         prints(file=output)
-    listing = output.getvalue().rstrip().decode('utf-8') + "\n"
+    listing = output.getvalue().rstrip() + "\n"
     output.close()
     return listing
 

src/calibre/gui2/tweak_book/function_replace.py

@@ -22,7 +22,7 @@ from calibre.utils.icu import capitalize, upper, lower, swapcase
 from calibre.utils.titlecase import titlecase
 from calibre.utils.localization import localize_user_manual_link
 from polyglot.builtins import iteritems, unicode_type
-from polyglot.io import PolyglotBytesIO
+from polyglot.io import PolyglotStringIO
 
 user_functions = JSONConfig('editor-search-replace-functions')
 
@@ -67,7 +67,7 @@ class Function(object):
         self.match_index = 0
         self.boss = get_boss()
         self.data = {}
-        self.debug_buf = PolyglotBytesIO()
+        self.debug_buf = PolyglotStringIO()
         self.functions = {name:func.mod for name, func in iteritems(functions()) if func.mod is not None}
 
     def __hash__(self):

src/calibre/gui2/tweak_book/reports.py

@@ -11,7 +11,7 @@ from operator import itemgetter
 from functools import partial
 from collections import defaultdict
 from csv import writer as csv_writer
-from io import BytesIO
+from io import StringIO
 
 import regex
 from PyQt5.Qt import (
@@ -35,7 +35,7 @@ from calibre.gui2.progress_indicator import ProgressIndicator
 from calibre.utils.icu import primary_contains, numeric_sort_key
 from calibre.utils.unicode_names import character_name_from_code
 from calibre.utils.localization import calibre_langcode_to_name, canonicalize_lang
-from polyglot.builtins import filter, iteritems, map, range, unicode_type
+from polyglot.builtins import filter, iteritems, map, range, unicode_type, as_bytes
 
 # Utils {{{
 
@@ -194,7 +194,7 @@ class FilesView(QTableView):
             m.exec_(pos)
 
     def to_csv(self):
-        buf = BytesIO()
+        buf = StringIO(newline='')
         w = csv_writer(buf)
         w.writerow(self.proxy.sourceModel().COLUMN_HEADERS)
         cols = self.proxy.columnCount()
@@ -1093,7 +1093,7 @@ class CSSWidget(QWidget):
         self.proxy.sort(0, self.sort_order)
 
     def to_csv(self):
-        buf = BytesIO()
+        buf = StringIO(newline='')
         w = csv_writer(buf)
         w.writerow([_('Style Rule'), _('Number of matches')])
         for r in range(self.proxy.rowCount()):
@@ -1239,7 +1239,7 @@ class ClassesWidget(CSSWidget):
         self.summary.setText(_('{0} classes, {1} unused').format(self.model.rowCount(), self.model.num_unused))
 
     def to_csv(self):
-        buf = BytesIO()
+        buf = StringIO(newline='')
         w = csv_writer(buf)
         w.writerow([_('Class'), _('Number of matches')])
         for r in range(self.proxy.rowCount()):
@@ -1361,7 +1361,7 @@ class ReportsWidget(QWidget):
             (_('CSV files'), ['csv'])], all_files=False, initial_filename='%s.csv' % category)
         if fname:
             with open(fname, 'wb') as f:
-                f.write(data)
+                f.write(as_bytes(data))
 
 
 class Reports(Dialog):

src/calibre/gui2/wizard/send_email.py

@@ -21,7 +21,7 @@ from calibre.utils.smtp import config as smtp_prefs
 from calibre.gui2 import error_dialog, question_dialog
 from polyglot.builtins import unicode_type
 from polyglot.binary import as_hex_unicode, from_hex_unicode
-from polyglot.io import PolyglotBytesIO
+from polyglot.io import PolyglotStringIO
 
 
 class TestEmail(QDialog):
@@ -194,7 +194,7 @@ class SendEmail(QWidget, Ui_Form):
     def test_email_settings(self, to):
         opts = smtp_prefs().parse()
         from calibre.utils.smtp import sendmail, create_mail
-        buf = PolyglotBytesIO()
+        buf = PolyglotStringIO()
         debug_out = partial(prints, file=buf)
         oout, oerr = sys.stdout, sys.stderr
         sys.stdout = sys.stderr = buf
@@ -210,7 +210,7 @@ class SendEmail(QWidget, Ui_Form):
         except:
             import traceback
             tb = traceback.format_exc()
-            tb += '\n\nLog:\n' + buf.getvalue().decode('utf-8', 'replace')
+            tb += '\n\nLog:\n' + buf.getvalue()
         finally:
             sys.stdout, sys.stderr = oout, oerr
         return tb