From 20b065fb49c2732a181384ae56ab45f15b378e24 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 1 Jun 2019 12:37:04 +0530
Subject: [PATCH] py3: Various fixes for the last py3 merge

---
 .../ebooks/conversion/plugins/epub_output.py       | 14 +++++++-------
 .../ebooks/conversion/plugins/html_input.py        |  4 ++--
 src/calibre/ebooks/conversion/plugins/snb_input.py |  3 +--
 src/calibre/ebooks/docx/writer/container.py        |  4 ++--
 src/calibre/ebooks/docx/writer/links.py            |  2 +-
 src/calibre/ebooks/epub/__init__.py                |  4 ++--
 src/calibre/ebooks/epub/periodical.py              |  2 +-
 src/calibre/ebooks/html/input.py                   |  6 ++++--
 src/calibre/ebooks/lrf/meta.py                     |  8 +++++---
 src/calibre/ebooks/lrf/objects.py                  |  6 +++---
 src/calibre/utils/date.py                          |  6 +++---
 11 files changed, 31 insertions(+), 28 deletions(-)

diff --git a/src/calibre/ebooks/conversion/plugins/epub_output.py b/src/calibre/ebooks/conversion/plugins/epub_output.py
index aaa8739225..2265508c6e 100644
--- a/src/calibre/ebooks/conversion/plugins/epub_output.py
+++ b/src/calibre/ebooks/conversion/plugins/epub_output.py
@@ -12,7 +12,7 @@ from calibre.customize.conversion import (OutputFormatPlugin,
         OptionRecommendation)
 from calibre.ptempfile import TemporaryDirectory
 from calibre import CurrentDir
-from polyglot.builtins import unicode_type, filter, map, zip
+from polyglot.builtins import unicode_type, filter, map, zip, range, as_bytes
 
 block_level_tags = (
       'address',
@@ -269,7 +269,7 @@ class EPUBOutput(OutputFormatPlugin):
                     extra_entries=extra_entries) as epub:
                 epub.add_dir(tdir)
                 if encryption is not None:
-                    epub.writestr('META-INF/encryption.xml', encryption)
+                    epub.writestr('META-INF/encryption.xml', as_bytes(encryption))
                 if metadata_xml is not None:
                     epub.writestr('META-INF/metadata.xml',
                             metadata_xml.encode('utf-8'))
@@ -331,9 +331,9 @@ class EPUBOutput(OutputFormatPlugin):
                 with lopen(path, 'r+b') as f:
                     data = f.read(1024)
                     if len(data) >= 1024:
+                        data = bytearray(data)
                         f.seek(0)
-                        for i in range(1024):
-                            f.write(chr(ord(data[i]) ^ key[i%16]))
+                        f.write(bytes(bytearray(data[i] ^ key[i%16] for i in range(1024))))
                     else:
                         self.log.warn('Font', path, 'is invalid, ignoring')
                 if not isinstance(uri, unicode_type):
@@ -347,13 +347,13 @@ class EPUBOutput(OutputFormatPlugin):
                 </enc:EncryptedData>
                 '''%(uri.replace('"', '\\"')))
             if fonts:
-                ans = b'''<encryption
+                ans = '''<encryption
                     xmlns="urn:oasis:names:tc:opendocument:xmlns:container"
                     xmlns:enc="http://www.w3.org/2001/04/xmlenc#"
                     xmlns:deenc="http://ns.adobe.com/digitaleditions/enc">
                     '''
-                ans += '\n'.join(fonts).encode('utf-8')
-                ans += b'\n</encryption>'
+                ans += '\n'.join(fonts)
+                ans += '\n</encryption>'
                 return ans
     # }}}
 
diff --git a/src/calibre/ebooks/conversion/plugins/html_input.py b/src/calibre/ebooks/conversion/plugins/html_input.py
index a714036ef0..89e4353a13 100644
--- a/src/calibre/ebooks/conversion/plugins/html_input.py
+++ b/src/calibre/ebooks/conversion/plugins/html_input.py
@@ -15,7 +15,7 @@ from calibre.customize.conversion import (InputFormatPlugin,
 from calibre.utils.localization import get_lang
 from calibre.utils.filenames import ascii_filename
 from calibre.utils.imghdr import what
-from polyglot.builtins import unicode_type, zip, getcwd
+from polyglot.builtins import unicode_type, zip, getcwd, as_unicode
 
 
 def sanitize_file_name(x):
@@ -290,7 +290,7 @@ class HTMLInput(InputFormatPlugin):
             # file, therefore we quote it here.
             if isinstance(bhref, unicode_type):
                 bhref = bhref.encode('utf-8')
-            item.html_input_href = unicode_type(quote(bhref))
+            item.html_input_href = as_unicode(quote(bhref))
             if guessed in self.OEB_STYLES:
                 item.override_css_fetch = partial(
                         self.css_import_handler, os.path.dirname(link))
diff --git a/src/calibre/ebooks/conversion/plugins/snb_input.py b/src/calibre/ebooks/conversion/plugins/snb_input.py
index 007adea1b9..03d213b6e1 100644
--- a/src/calibre/ebooks/conversion/plugins/snb_input.py
+++ b/src/calibre/ebooks/conversion/plugins/snb_input.py
@@ -27,8 +27,7 @@ class SNBInput(InputFormatPlugin):
     file_types  = {'snb'}
     commit_name = 'snb_input'
 
-    options = set([
-    ])
+    options = set()
 
     def convert(self, stream, options, file_ext, log,
                 accelerators):
diff --git a/src/calibre/ebooks/docx/writer/container.py b/src/calibre/ebooks/docx/writer/container.py
index e28e5c41cb..ae6108dbfb 100644
--- a/src/calibre/ebooks/docx/writer/container.py
+++ b/src/calibre/ebooks/docx/writer/container.py
@@ -18,7 +18,7 @@ from calibre.utils.date import utcnow
 from calibre.utils.localization import canonicalize_lang, lang_as_iso639_1
 from calibre.utils.zipfile import ZipFile
 from calibre.ebooks.pdf.render.common import PAPER_SIZES
-from polyglot.builtins import iteritems, map, unicode_type
+from polyglot.builtins import iteritems, map, unicode_type, native_string_type
 
 
 def xml2str(root, pretty_print=False, with_tail=False):
@@ -243,7 +243,7 @@ class DOCX(object):
         namespaces = self.namespace.namespaces
         E = ElementMaker(namespace=namespaces['cp'], nsmap={x:namespaces[x] for x in 'cp dc dcterms xsi'.split()})
         cp = E.coreProperties(E.revision("1"), E.lastModifiedBy('calibre'))
-        ts = utcnow().isoformat(unicode_type('T')).rpartition('.')[0] + 'Z'
+        ts = utcnow().isoformat(native_string_type('T')).rpartition('.')[0] + 'Z'
         for x in 'created modified'.split():
             x = cp.makeelement('{%s}%s' % (namespaces['dcterms'], x), **{'{%s}type' % namespaces['xsi']:'dcterms:W3CDTF'})
             x.text = ts
diff --git a/src/calibre/ebooks/docx/writer/links.py b/src/calibre/ebooks/docx/writer/links.py
index ba8bb8aeb4..6eb80b8fd8 100644
--- a/src/calibre/ebooks/docx/writer/links.py
+++ b/src/calibre/ebooks/docx/writer/links.py
@@ -70,7 +70,7 @@ class LinksManager(object):
         self.namespace = namespace
         self.log = log
         self.document_relationships = document_relationships
-        self.top_anchor = uuid4().hex
+        self.top_anchor = unicode_type(uuid4().hex)
         self.anchor_map = {}
         self.used_bookmark_names = set()
         self.bmark_id = 0
diff --git a/src/calibre/ebooks/epub/__init__.py b/src/calibre/ebooks/epub/__init__.py
index a750203b81..4629218f4a 100644
--- a/src/calibre/ebooks/epub/__init__.py
+++ b/src/calibre/ebooks/epub/__init__.py
@@ -41,8 +41,8 @@ def initialize_container(path_to_container, opf_name='metadata.opf',
                 path, mimetype)
     CONTAINER = simple_container_xml(opf_name, rootfiles).encode('utf-8')
     zf = ZipFile(path_to_container, 'w')
-    zf.writestr('mimetype', 'application/epub+zip', compression=ZIP_STORED)
-    zf.writestr('META-INF/', '', 0o755)
+    zf.writestr('mimetype', b'application/epub+zip', compression=ZIP_STORED)
+    zf.writestr('META-INF/', b'', 0o755)
     zf.writestr('META-INF/container.xml', CONTAINER)
     for path, _, data in extra_entries:
         zf.writestr(path, data)
diff --git a/src/calibre/ebooks/epub/periodical.py b/src/calibre/ebooks/epub/periodical.py
index ebadc990dc..e2c5760303 100644
--- a/src/calibre/ebooks/epub/periodical.py
+++ b/src/calibre/ebooks/epub/periodical.py
@@ -134,7 +134,7 @@ def sony_metadata(oeb):
         toc.nodes.append(section)
 
     entries = []
-    seen_titles = set([])
+    seen_titles = set()
     for i, section in enumerate(toc):
         if not section.href:
             continue
diff --git a/src/calibre/ebooks/html/input.py b/src/calibre/ebooks/html/input.py
index 91d02d7297..c87af5dc7a 100644
--- a/src/calibre/ebooks/html/input.py
+++ b/src/calibre/ebooks/html/input.py
@@ -65,7 +65,7 @@ class Link(object):
         return self.path == getattr(other, 'path', other)
 
     def __str__(self):
-        return u'Link: %s --> %s'%(self.url, self.path)
+        return 'Link: %s --> %s'%(self.url, self.path)
 
     if not is_py3:
         __unicode__ = __str__
@@ -176,8 +176,10 @@ class HTMLFile(object):
         return Link(url, self.base)
 
 
-def depth_first(root, flat, visited=set([])):
+def depth_first(root, flat, visited=None):
     yield root
+    if visited is None:
+        visited = set()
     visited.add(root)
     for link in root.links:
         if link.path is not None and link not in visited:
diff --git a/src/calibre/ebooks/lrf/meta.py b/src/calibre/ebooks/lrf/meta.py
index 9a78ea2e4a..d48f05a1f9 100644
--- a/src/calibre/ebooks/lrf/meta.py
+++ b/src/calibre/ebooks/lrf/meta.py
@@ -21,7 +21,7 @@ from functools import wraps
 
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.ebooks.metadata import MetaInformation, string_to_authors
-from polyglot.builtins import unicode_type
+from polyglot.builtins import unicode_type, string_or_bytes
 
 BYTE      = "<B"  #: Unsigned char little endian encoded in 1 byte
 WORD      = "<H"  #: Unsigned short little endian encoded in 2 bytes
@@ -97,8 +97,10 @@ class fixed_stringfield(object):
         return obj.unpack(start=self._start, fmt="<"+length+"s")[0]
 
     def __set__(self, obj, val):
-        if not isinstance(val, unicode_type):
+        if not isinstance(val, string_or_bytes):
             val = unicode_type(val)
+        if isinstance(val, unicode_type):
+            val = val.encode('utf-8')
         if len(val) != self._length:
             raise LRFException("Trying to set fixed_stringfield with a " +
                                "string of  incorrect length")
@@ -710,7 +712,7 @@ def main(args=sys.argv):
         lrf.book_id = options.book_id
     if options.comment:
         path = os.path.expanduser(os.path.expandvars(options.comment))
-        lrf.free_text = open(path).read()
+        lrf.free_text = open(path, 'rb').read().decode('utf-8', 'replace')
     if options.get_thumbnail:
         t = lrf.thumbnail
         td = "None"
diff --git a/src/calibre/ebooks/lrf/objects.py b/src/calibre/ebooks/lrf/objects.py
index 91868da236..1695bc2697 100644
--- a/src/calibre/ebooks/lrf/objects.py
+++ b/src/calibre/ebooks/lrf/objects.py
@@ -123,7 +123,7 @@ class LRFContentObject(LRFObject):
         if tag.id in self.tag_map:
             action = self.tag_map[tag.id]
             if isinstance(action, string_or_bytes):
-                func, args = action, tuple([])
+                func, args = action, ()
             else:
                 func, args = action[0], (action[1],)
             getattr(self, func)(tag, *args)
@@ -495,7 +495,7 @@ class BlockAttr(StyleObject, LRFObject):
             return ans
 
         if hasattr(obj, 'sidemargin'):
-            margin = str(obj.sidemargin) + 'px'
+            margin = unicode_type(obj.sidemargin) + 'px'
             ans += item('margin-left: %(m)s; margin-right: %(m)s;'%dict(m=margin))
         if hasattr(obj, 'topskip'):
             ans += item('margin-top: %dpx;'%obj.topskip)
@@ -1109,7 +1109,7 @@ class Button(LRFObject):
     tag_map.update(LRFObject.tag_map)
 
     def __init__(self, document, stream, id, scramble_key, boundary):
-        self.xml = u''
+        self.xml = ''
         self.refimage = {}
         self.actions = {}
         self.to_dump = True
diff --git a/src/calibre/utils/date.py b/src/calibre/utils/date.py
index 0035c88dfb..1e07598dd1 100644
--- a/src/calibre/utils/date.py
+++ b/src/calibre/utils/date.py
@@ -14,7 +14,7 @@ from calibre import strftime
 from calibre.constants import iswindows, isosx, plugins, preferred_encoding
 from calibre.utils.iso8601 import utc_tz, local_tz, UNDEFINED_DATE
 from calibre.utils.localization import lcdata
-from polyglot.builtins import unicode_type
+from polyglot.builtins import unicode_type, native_string_type
 
 _utc_tz = utc_tz
 _local_tz = local_tz
@@ -194,8 +194,8 @@ def isoformat(date_time, assume_utc=False, as_utc=True, sep='T'):
         date_time = date_time.replace(tzinfo=_utc_tz if assume_utc else
                 _local_tz)
     date_time = date_time.astimezone(_utc_tz if as_utc else _local_tz)
-    # str(sep) because isoformat barfs with unicode sep on python 2.x
-    return unicode_type(date_time.isoformat(str(sep)))
+    # native_string_type(sep) because isoformat barfs with unicode sep on python 2.x
+    return unicode_type(date_time.isoformat(native_string_type(sep)))
 
 
 def internal_iso_format_string():