py3: Port bytes()

src/calibre/devices/mtp/windows/remote.py

@@ -9,6 +9,7 @@ __docformat__ = 'restructuredtext en'
 
 import subprocess, sys, os, pprint, signal, time, glob, io
 pprint, io
+from polyglot.builtins import environ_item
 
 
 def build(mod='wpd'):
@@ -42,9 +43,9 @@ def build(mod='wpd'):
 
 def main():
     fp, d = os.path.abspath(__file__), os.path.dirname
-    if b'CALIBRE_DEVELOP_FROM' not in os.environ:
+    if 'CALIBRE_DEVELOP_FROM' not in os.environ:
         env = os.environ.copy()
-        env[b'CALIBRE_DEVELOP_FROM'] = bytes(d(d(d(d(d(fp))))))
+        env['CALIBRE_DEVELOP_FROM'] = environ_item(d(d(d(d(d(fp))))))
         subprocess.call(['calibre-debug', '-e', fp], env=env)
         return
 

src/calibre/devices/smart_device_app/driver.py

@@ -569,7 +569,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             raise
 
     def _read_string_from_net(self):
-        data = bytes(0)
+        data = b'\0'
         while True:
             dex = data.find(b'[')
             if dex >= 0:

src/calibre/ebooks/conversion/plugins/epub_input.py

@@ -7,7 +7,7 @@ import os, re, posixpath
 from itertools import cycle
 
 from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation
-from polyglot.builtins import unicode_type
+from polyglot.builtins import unicode_type, as_bytes
 
 ADOBE_OBFUSCATION =  'http://ns.adobe.com/pdf/enc#RC'
 IDPF_OBFUSCATION = 'http://www.idpf.org/2008/embedding'
@@ -55,8 +55,8 @@ class EPUBInput(InputFormatPlugin):
             if (scheme and scheme.lower() == 'uuid') or \
                     (item.text and item.text.startswith('urn:uuid:')):
                 try:
-                    key = bytes(item.text).rpartition(':')[-1]
-                    key = uuid.UUID(key).bytes
+                    key = item.text.rpartition(':')[-1]
+                    key = uuid.UUID(as_bytes(key)).bytes
                 except:
                     import traceback
                     traceback.print_exc()

src/calibre/ebooks/conversion/plugins/pdf_input.py

@@ -7,6 +7,7 @@ __docformat__ = 'restructuredtext en'
 import os
 
 from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation
+from polyglot.builtins import as_bytes
 
 
 class PDFInput(InputFormatPlugin):
@@ -73,7 +74,7 @@ class PDFInput(InputFormatPlugin):
             ncxid = opf.manifest.id_for_path('toc.ncx')
             if ncxid:
                 with open(u'metadata.opf', 'r+b') as f:
-                    raw = f.read().replace(b'<spine', b'<spine toc="%s"' % bytes(ncxid))
+                    raw = f.read().replace(b'<spine', b'<spine toc="%s"' % as_bytes(ncxid))
                     f.seek(0)
                     f.write(raw)
 

src/calibre/ebooks/metadata/book/json_codec.py

@@ -12,7 +12,7 @@ from calibre.ebooks.metadata.book import SERIALIZABLE_FIELDS
 from calibre.constants import filesystem_encoding, preferred_encoding
 from calibre.library.field_metadata import FieldMetadata
 from calibre import isbytestring
-from polyglot.builtins import iteritems, itervalues
+from polyglot.builtins import iteritems, itervalues, as_bytes
 from polyglot.binary import as_base64_unicode, from_base64_bytes
 
 # Translate datetimes to and from strings. The string form is the datetime in
@@ -51,7 +51,7 @@ def encode_thumbnail(thumbnail):
         return None
     if not isinstance(thumbnail, (tuple, list)):
         try:
-            width, height = identify(bytes(thumbnail))[1:]
+            width, height = identify(as_bytes(thumbnail))[1:]
             if width < 0 or height < 0:
                 return None
             thumbnail = (width, height, thumbnail)

src/calibre/ebooks/metadata/docx.py

@@ -27,7 +27,7 @@ def get_cover(docx):
         if rid in rid_map:
             try:
                 raw = docx.read(rid_map[rid])
-                fmt, width, height = identify(bytes(raw))
+                fmt, width, height = identify(raw)
             except Exception:
                 continue
             if width < 0 or height < 0:

src/calibre/ebooks/metadata/fb2.py

@@ -237,7 +237,7 @@ def _parse_cover_data(root, imgid, mi, ctx):
             pic_data = elm_binary[0].text
             if pic_data:
                 cdata = base64_decode(pic_data.strip())
-                fmt = identify(bytes(cdata))[0]
+                fmt = identify(cdata)[0]
                 mi.cover_data = (fmt, cdata)
         else:
             prints("WARNING: Unsupported coverpage mime-type '%s' (id=#%s)" % (mimetype, imgid))

src/calibre/ebooks/metadata/odt.py

@@ -241,7 +241,7 @@ def read_cover(stream, zin, mi, opfmeta, extract_cover):
         except KeyError:
             continue
         try:
-            fmt, width, height = identify(bytes(raw))
+            fmt, width, height = identify(raw)
         except Exception:
             continue
         imgnum += 1
@@ -264,7 +264,7 @@ def read_cover(stream, zin, mi, opfmeta, extract_cover):
             if not cover_data:
                 raw = zin.read(cover_href)
                 try:
-                    fmt = identify(bytes(raw))[0]
+                    fmt = identify(raw)[0]
                 except Exception:
                     pass
                 else:

src/calibre/ebooks/mobi/debug/headers.py

@@ -14,7 +14,7 @@ from calibre.ebooks.mobi.reader.headers import NULL_INDEX
 from calibre.ebooks.mobi.langcodes import main_language, sub_language
 from calibre.ebooks.mobi.debug import format_bytes
 from calibre.ebooks.mobi.utils import get_trailing_data
-from polyglot.builtins import iteritems, range
+from polyglot.builtins import iteritems, range, as_bytes
 
 # PalmDB {{{
 
@@ -208,7 +208,7 @@ class EXTHRecord(object):
             else:
                 self.data, = struct.unpack(b'>L', self.data)
         elif self.type in {209, 300}:
-            self.data = bytes(self.data.encode('hex'))
+            self.data = as_bytes(self.data.encode('hex'))
 
     def __str__(self):
         return '%s (%d): %r'%(self.name, self.type, self.data)

src/calibre/ebooks/mobi/debug/mobi6.py

@@ -20,7 +20,7 @@ from calibre.ebooks.mobi.utils import (decode_hex_number, decint,
 from calibre.utils.imghdr import what
 from calibre.ebooks.mobi.debug import format_bytes
 from calibre.ebooks.mobi.debug.headers import TextRecord
-from polyglot.builtins import unicode_type, range, iteritems
+from polyglot.builtins import unicode_type, range, iteritems, as_bytes
 
 
 class TagX(object):  # {{{
@@ -606,7 +606,7 @@ class TBSIndexing(object):  # {{{
 
         def bin4(num):
             ans = bin(num)[2:]
-            return bytes('0'*(4-len(ans)) + ans)
+            return as_bytes('0'*(4-len(ans)) + ans)
 
         def repr_extra(x):
             return str({bin4(k):v for k, v in iteritems(extra)})

src/calibre/ebooks/mobi/utils.py

@@ -14,7 +14,7 @@ from io import BytesIO
 from calibre.utils.img import save_cover_data_to, scale_image, image_to_data, image_from_data, resize_image
 from calibre.utils.imghdr import what
 from calibre.ebooks import normalize
-from polyglot.builtins import unicode_type, range
+from polyglot.builtins import unicode_type, range, as_bytes
 from tinycss.color3 import parse_color_string
 
 IMAGE_MAX_SIZE = 10 * 1024 * 1024
@@ -65,7 +65,7 @@ def decode_hex_number(raw, codec='utf-8'):
 
 
 def encode_string(raw):
-    ans = bytearray(bytes(raw))
+    ans = bytearray(as_bytes(raw))
     ans.insert(0, len(ans))
     return bytes(ans)
 

src/calibre/ebooks/mobi/writer8/header.py

@@ -13,7 +13,7 @@ from collections import OrderedDict
 from struct import pack
 
 from calibre.ebooks.mobi.utils import align_block
-from polyglot.builtins import iteritems
+from polyglot.builtins import iteritems, as_bytes
 
 NULL = 0xffffffff
 zeroes = lambda x: b'\0'*x
@@ -62,7 +62,7 @@ class Header(OrderedDict):
             self[name] = val
 
         buf = BytesIO()
-        buf.write(bytes(self.HEADER_NAME))
+        buf.write(as_bytes(self.HEADER_NAME))
         for name, val in iteritems(self):
             val = self.format_value(name, val)
             positions[name] = buf.tell()

src/calibre/ebooks/oeb/polish/container.py

@@ -14,7 +14,7 @@ import time
 import unicodedata
 import uuid
 from collections import defaultdict
-from polyglot.builtins import iteritems, unicode_type, zip
+from polyglot.builtins import iteritems, unicode_type, zip, as_bytes
 from io import BytesIO
 from itertools import count
 
@@ -1289,9 +1289,9 @@ class EpubContainer(Container):
             if (scheme and scheme.lower() == 'uuid') or \
                     (item.text and item.text.startswith('urn:uuid:')):
                 try:
-                    key = bytes(item.text).rpartition(':')[-1]
-                    key = uuid.UUID(key).bytes
-                except:
+                    key = item.text.rpartition(':')[-1]
+                    key = uuid.UUID(as_bytes(key)).bytes
+                except Exception:
                     self.log.exception('Failed to parse obfuscation key')
                     key = None
 

src/calibre/ebooks/pdf/render/fonts.py

@@ -11,7 +11,7 @@ import re
 from itertools import groupby
 from operator import itemgetter
 from collections import Counter, OrderedDict
-from polyglot.builtins import iteritems, map, zip
+from polyglot.builtins import iteritems, map, zip, unicode_type
 
 from calibre import as_unicode
 from calibre.ebooks.pdf.render.common import (Array, String, Stream,
@@ -62,7 +62,10 @@ class FontStream(Stream):
 
 
 def to_hex_string(c):
-    return bytes(hex(int(c))[2:]).rjust(4, b'0').decode('ascii')
+    ans = hex(int(c))[2:].rjust(4, '0')
+    if isinstance(ans, bytes):
+        ans = ans.decode('ascii')
+    return ans
 
 
 class CMap(Stream):
@@ -120,8 +123,9 @@ class Font(object):
     def __init__(self, metrics, num, objects, compress):
         self.metrics, self.compress = metrics, compress
         self.is_otf = self.metrics.is_otf
-        self.subset_tag = bytes(re.sub('.', lambda m: chr(int(m.group())+ord('A')),
-                                  oct(num))).rjust(6, b'A').decode('ascii')
+        self.subset_tag = unicode_type(
+            re.sub('.', lambda m: chr(int(m.group())+ord('A')), oct(num).replace('o', '')
+        )).rjust(6, 'A')
         self.font_stream = FontStream(metrics.is_otf, compress=compress)
         try:
             psname = metrics.postscript_name

src/calibre/gui2/dnd.py

@@ -18,14 +18,14 @@ from calibre.ptempfile import PersistentTemporaryFile
 from calibre import browser, as_unicode, prints
 from calibre.gui2 import error_dialog
 from calibre.utils.imghdr import what
-from polyglot.builtins import unicode_type
+from polyglot.builtins import unicode_type, as_unicode as as_unicode_polyglot
 from polyglot.urllib import unquote, urlparse
 from polyglot.queue import Queue, Empty
 
 
 def image_extensions():
     if not hasattr(image_extensions, 'ans'):
-        image_extensions.ans = [bytes(x).decode('utf-8') for x in QImageReader.supportedImageFormats()]
+        image_extensions.ans = [as_unicode_polyglot(x) for x in QImageReader.supportedImageFormats()]
     return image_extensions.ans
 
 
@@ -162,7 +162,7 @@ def path_from_qurl(qurl):
     raw = bytes(qurl.toEncoded(
         QUrl.PreferLocalFile | QUrl.RemoveScheme | QUrl.RemovePassword | QUrl.RemoveUserInfo |
         QUrl.RemovePort | QUrl.RemoveAuthority | QUrl.RemoveQuery | QUrl.RemoveFragment))
-    ans = unquote(raw).decode('utf-8', 'replace')
+    ans = as_unicode_polyglot(unquote(raw), errors='replace')
     if iswindows and ans.startswith('/'):
         ans = ans[1:]
     return ans
@@ -172,7 +172,7 @@ def remote_urls_from_qurl(qurls, allowed_exts):
     for qurl in qurls:
         if qurl.scheme() in {'http', 'https', 'ftp'} and posixpath.splitext(
                 qurl.path())[1][1:].lower() in allowed_exts:
-            yield bytes(qurl.toEncoded()), posixpath.basename(qurl.path())
+            yield bytes(qurl.toEncoded()).decode('utf-8'), posixpath.basename(qurl.path())
 
 
 def dnd_has_extension(md, extensions, allow_all_extensions=False):
@@ -189,7 +189,7 @@ def dnd_has_extension(md, extensions, allow_all_extensions=False):
     paths = [path_from_qurl(u) for u in urls]
     exts = frozenset([posixpath.splitext(u)[1][1:].lower() for u in paths if u])
     if DEBUG:
-        repr_urls = [bytes(u.toEncoded()) for u in urls]
+        repr_urls = [bytes(u.toEncoded()).decode('utf-8') for u in urls]
         prints('URLS:', repr(repr_urls))
         prints('Paths:', paths)
         prints('Extensions:', exts)

src/calibre/gui2/tweak_book/char_select.py

@@ -529,7 +529,7 @@ class CharModel(QAbstractListModel):
     def dropMimeData(self, md, action, row, column, parent):
         if action != Qt.MoveAction or not md.hasFormat('application/calibre_charcode_indices') or row < 0 or column != 0:
             return False
-        indices = map(int, bytes(md.data('application/calibre_charcode_indices')).split(','))
+        indices = map(int, bytes(md.data('application/calibre_charcode_indices')).decode('ascii').split(','))
         codes = [self.chars[x] for x in indices]
         for x in indices:
             self.chars[x] = None

src/calibre/gui2/tweak_book/diff/view.py

@@ -12,7 +12,7 @@ from math import ceil
 from functools import partial
 from collections import namedtuple, OrderedDict
 from difflib import SequenceMatcher
-from polyglot.builtins import iteritems, unicode_type, zip, range
+from polyglot.builtins import iteritems, unicode_type, zip, range, as_bytes
 
 import regex
 from PyQt5.Qt import (
@@ -589,7 +589,7 @@ class DiffSplit(QSplitter):  # {{{
     def add_image_diff(self, left_data, right_data):
         def load(data):
             p = QPixmap()
-            p.loadFromData(bytes(data))
+            p.loadFromData(as_bytes(data))
             try:
                 dpr = self.devicePixelRatioF()
             except AttributeError:

src/calibre/gui2/tweak_book/editor/canvas.py

@@ -25,6 +25,7 @@ from calibre.utils.img import (
     remove_borders_from_image, gaussian_sharpen_image, gaussian_blur_image, image_to_data, despeckle_image,
     normalize_image, oil_paint_image
 )
+from polyglot.builtins import as_unicode
 
 
 def painter(func):
@@ -325,7 +326,7 @@ class Canvas(QWidget):
         if not self.is_modified:
             return self.original_image_data
         fmt = self.original_image_format or 'JPEG'
-        if fmt.lower() not in set(map(lambda x:bytes(x).decode('ascii'), QImageWriter.supportedImageFormats())):
+        if fmt.lower() not in set(map(as_unicode, QImageWriter.supportedImageFormats())):
             if fmt.lower() == 'gif':
                 data = image_to_data(self.current_image, fmt='PNG', png_compression_level=0)
                 from PIL import Image

src/calibre/gui2/tweak_book/editor/text.py

@@ -8,7 +8,7 @@ import os
 import re
 import textwrap
 import unicodedata
-from polyglot.builtins import unicode_type, map, range
+from polyglot.builtins import unicode_type, map, range, as_unicode
 
 from PyQt5.Qt import (
     QColor, QColorDialog, QFont, QFontDatabase, QKeySequence, QPainter, QPalette,
@@ -131,7 +131,7 @@ class TextEdit(PlainTextEdit):
             )
 
         if md.hasFormat(CONTAINER_DND_MIMETYPE):
-            for line in bytes(md.data(CONTAINER_DND_MIMETYPE)).decode('utf-8').splitlines():
+            for line in as_unicode(bytes(md.data(CONTAINER_DND_MIMETYPE))).splitlines():
                 mt = current_container().mime_map.get(line, 'application/octet-stream')
                 if is_mt_ok(mt):
                     yield line, mt, True

src/calibre/srv/legacy.py

@@ -19,7 +19,7 @@ from calibre.srv.routes import endpoint
 from calibre.srv.utils import get_library_data, http_date
 from calibre.utils.cleantext import clean_xml_chars
 from calibre.utils.date import dt_as_local, is_date_undefined, timestampfromdt
-from polyglot.builtins import iteritems, string_or_bytes, filter
+from polyglot.builtins import iteritems, string_or_bytes, filter, as_bytes
 from polyglot.urllib import urlencode
 
 # /mobile {{{
@@ -245,7 +245,7 @@ def mobile(ctx, rd):
         books = [db.get_metadata(book_id) for book_id in book_ids[(start-1):(start-1)+num]]
     rd.outheaders['Last-Modified'] = http_date(timestampfromdt(db.last_modified()))
     order = 'ascending' if ascending else 'descending'
-    q = {b'search':search.encode('utf-8'), b'order':bytes(order), b'sort':sort_by.encode('utf-8'), b'num':bytes(num), 'library_id':library_id}
+    q = {b'search':search.encode('utf-8'), b'order':order.encode('ascii'), b'sort':sort_by.encode('utf-8'), b'num':as_bytes(num), 'library_id':library_id}
     url_base = ctx.url_for('/mobile') + '?' + urlencode(q)
     lm = {k:v for k, v in iteritems(library_map) if k != library_id}
     return build_index(rd, books, num, search, sort_by, order, start, total, url_base, db.field_metadata, ctx, lm, library_id)

src/calibre/utils/__init__.py

@@ -8,6 +8,7 @@ Miscelleaneous utilities.
 '''
 
 from time import time
+from polyglot.builtins import as_bytes
 
 
 def join_with_timeout(q, timeout=2):
@@ -48,5 +49,5 @@ def pickle_binary_string(data):
     # Maintains compatibility with python's pickle module protocol version 2
     import struct
     PROTO, STOP, BINSTRING = b'\x80', b'.', b'T'
-    data = bytes(data)
+    data = as_bytes(data)
     return PROTO + b'\x02' + BINSTRING + struct.pack(b'<i', len(data)) + data + STOP

src/calibre/utils/fonts/utils.py

@@ -11,7 +11,7 @@ import struct
 from io import BytesIO
 from collections import defaultdict
 
-from polyglot.builtins import iteritems, itervalues, unicode_type, range
+from polyglot.builtins import iteritems, itervalues, unicode_type, range, as_bytes
 
 
 class UnsupportedFont(ValueError):
@@ -42,7 +42,7 @@ def get_tables(raw):
 
 def get_table(raw, name):
     ''' Get the raw table bytes for the specified table in the font '''
-    name = bytes(name.lower())
+    name = as_bytes(name.lower())
     for table_tag, table, table_index, table_offset, table_checksum in get_tables(raw):
         if table_tag.lower() == name:
             return table, table_index, table_offset, table_checksum

src/polyglot/builtins.py

@@ -14,6 +14,27 @@ def iterkeys(d):
     return iter(d)
 
 
+def as_bytes(x, encoding='utf-8'):
+    if isinstance(x, unicode_type):
+        return x.encode(encoding)
+    if isinstance(x, bytes):
+        return x
+    if isinstance(x, bytearray):
+        return bytes(x)
+    if isinstance(x, memoryview):
+        return x.tobytes()
+    ans = unicode_type(x)
+    if isinstance(ans, unicode_type):
+        ans = ans.encode(encoding)
+    return ans
+
+
+def as_unicode(x, encoding='utf-8', errors='strict'):
+    if isinstance(x, bytes):
+        return x.decode(encoding, errors)
+    return unicode_type(x)
+
+
 if is_py3:
     def reraise(tp, value, tb=None):
         try: