py3: Port use of (un)hexlify()

src/calibre/db/cache.py

@@ -2206,8 +2206,8 @@ class Cache(object):
 
     @read_api
     def export_library(self, library_key, exporter, progress=None, abort=None):
-        from binascii import hexlify
+        from polyglot.binary import as_hex_unicode
-        key_prefix = hexlify(library_key)
+        key_prefix = as_hex_unicode(library_key)
         book_ids = self._all_book_ids()
         total = len(book_ids) + 1
         format_metadata = {}

src/calibre/ebooks/conversion/plugins/epub_output.py

@@ -311,13 +311,12 @@ class EPUBOutput(OutputFormatPlugin):
             pass
 
     def encrypt_fonts(self, uris, tdir, uuid):  # {{{
-        from binascii import unhexlify
+        from polyglot.binary import from_hex_bytes
 
         key = re.sub(r'[^a-fA-F0-9]', '', uuid)
         if len(key) < 16:
             raise ValueError('UUID identifier %r is invalid'%uuid)
-        key = unhexlify((key + key)[:32])
+        key = bytearray(from_hex_bytes((key + key)[:32]))
-        key = tuple(map(ord, key))
         paths = []
         with CurrentDir(tdir):
             paths = [os.path.join(*x.split('/')) for x in uris]

src/calibre/ebooks/metadata/book/render.py

@@ -8,7 +8,6 @@ __copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import os
 from functools import partial
-from binascii import hexlify
 
 from calibre import prepare_string_for_xml, force_unicode
 from calibre.ebooks.metadata import fmt_sidx, rating_to_stars
@@ -22,6 +21,7 @@ from calibre.utils.date import is_date_undefined
 from calibre.utils.localization import calibre_langcode_to_name
 from calibre.utils.serialize import json_dumps
 from polyglot.builtins import unicode_type
+from polyglot.binary import as_hex_unicode
 
 default_sort = ('title', 'title_sort', 'authors', 'author_sort', 'series', 'rating', 'pubdate', 'tags', 'publisher', 'identifiers')
 
@@ -55,7 +55,7 @@ def get_field_list(mi):
 
 def search_href(search_term, value):
     search = '%s:"=%s"' % (search_term, value.replace('"', '\\"'))
-    return prepare_string_for_xml('search:' + hexlify(search.encode('utf-8')), True)
+    return prepare_string_for_xml('search:' + as_hex_unicode(search.encode('utf-8')), True)
 
 
 DEFAULT_AUTHOR_LINK = 'search-{}'.format(DEFAULT_AUTHOR_SOURCE)
@@ -80,7 +80,7 @@ def author_search_href(which, title=None, author=None):
 
 
 def item_data(field_name, value, book_id):
-    return hexlify(json_dumps((field_name, value, book_id)))
+    return as_hex_unicode(json_dumps((field_name, value, book_id)))
 
 
 def mi_to_html(mi, field_list=None, default_author_link=None, use_roman_numbers=True, rating_font='Liberation Serif', rtl=False):

src/calibre/ebooks/pdf/render/common.py

@@ -10,11 +10,11 @@ __docformat__ = 'restructuredtext en'
 import codecs, zlib, numbers
 from io import BytesIO
 from datetime import datetime
-from binascii import hexlify
 
 from calibre.constants import plugins, ispy3
 from calibre.utils.logging import default_log
 from polyglot.builtins import iteritems, unicode_type
+from polyglot.binary import as_hex_bytes
 
 pdf_float = plugins['speedup'][0].pdf_float
 
@@ -137,7 +137,7 @@ class UTF16String(unicode_type):
         if False:
             # Disabled as the parentheses based strings give easier to debug
             # PDF files
-            stream.write(b'<' + hexlify(raw) + b'>')
+            stream.write(b'<' + as_hex_bytes(raw) + b'>')
         else:
             stream.write(b'('+escape_pdf_string(raw)+b')')
 

src/calibre/gui2/book_details.py

@@ -4,7 +4,6 @@
 
 import os
 import re
-from binascii import unhexlify
 from collections import namedtuple
 from functools import partial
 
@@ -35,6 +34,8 @@ from calibre.utils.img import blend_image, image_from_x
 from calibre.utils.localization import is_rtl
 from calibre.utils.serialize import json_loads
 from polyglot.builtins import unicode_type
+from polyglot.binary import from_hex_bytes, from_hex_unicode
+
 
 _css = None
 InternetSearch = namedtuple('InternetSearch', 'author where')
@@ -286,7 +287,7 @@ def details_context_menu_event(view, ev, book_info):  # {{{
                                    lambda : book_info.search_requested('authors:"={}"'.format(author.replace('"', r'\"'))))
             if data:
                 try:
-                    field, value, book_id = json_loads(unhexlify(data))
+                    field, value, book_id = json_loads(from_hex_bytes(data))
                 except Exception:
                     field = value = book_id = None
                 if field:
@@ -874,7 +875,7 @@ class BookDetails(QWidget):  # {{{
         elif typ == 'devpath':
             self.view_device_book.emit(val)
         elif typ == 'search':
-            self.search_requested.emit(unhexlify(val).decode('utf-8'))
+            self.search_requested.emit(from_hex_unicode(val))
         else:
             try:
                 open_url(QUrl(link, QUrl.TolerantMode))

src/calibre/gui2/email.py

@@ -7,7 +7,6 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import os, socket, time, textwrap
-from binascii import unhexlify
 from functools import partial
 from threading import Thread
 from itertools import repeat
@@ -29,6 +28,7 @@ from calibre.utils.config import tweaks, prefs
 from calibre.utils.icu import primary_sort_key
 from calibre.gui2.threaded_jobs import ThreadedJob
 from polyglot.builtins import iteritems, itervalues, unicode_type
+from polyglot.binary import from_hex_unicode
 
 
 class Worker(Thread):
@@ -130,7 +130,7 @@ class Sendmail(object):
                         verbose=1,
                         relay=opts.relay_host,
                         username=opts.relay_username,
-                        password=unhexlify(opts.relay_password).decode('utf-8'), port=opts.relay_port,
+                        password=from_hex_unicode(opts.relay_password), port=opts.relay_port,
                         encryption=opts.encryption,
                         debug_output=safe_debug)
         finally:

src/calibre/gui2/main.py

@@ -28,7 +28,7 @@ from calibre.utils.config import dynamic, prefs
 from calibre.utils.ipc import RC, gui_socket_address
 from calibre.utils.lock import singleinstance
 from calibre.utils.monotonic import monotonic
-from polyglot.builtins import unicode_type, range
+from polyglot.builtins import unicode_type, range, environ_item
 
 if iswindows:
     winutil = plugins['winutil'][0]
@@ -173,19 +173,19 @@ def repair_library(library_path):
 
 
 def windows_repair(library_path=None):
-    from binascii import hexlify, unhexlify
     import subprocess
     from calibre.utils.serialize import json_dumps, json_loads
+    from polyglot.binary import as_hex_unicode, from_hex_bytes
     if library_path:
-        library_path = hexlify(json_dumps(library_path))
+        library_path = as_hex_unicode(json_dumps(library_path))
         winutil.prepare_for_restart()
-        os.environ['CALIBRE_REPAIR_CORRUPTED_DB'] = library_path
+        os.environ['CALIBRE_REPAIR_CORRUPTED_DB'] = environ_item(library_path)
         subprocess.Popen([sys.executable])
     else:
         try:
             app = Application([])
             from calibre.gui2.dialogs.restore_library import repair_library_at
-            library_path = json_loads(unhexlify(os.environ.pop('CALIBRE_REPAIR_CORRUPTED_DB')))
+            library_path = json_loads(from_hex_bytes(os.environ.pop('CALIBRE_REPAIR_CORRUPTED_DB')))
             done = repair_library_at(library_path, wait_time=4)
         except Exception:
             done = False

src/calibre/gui2/tweak_book/file_list.py

@@ -7,7 +7,6 @@ import os
 import posixpath
 import sys
 import textwrap
-from binascii import hexlify
 from collections import Counter, OrderedDict, defaultdict
 from functools import partial
 
@@ -40,6 +39,7 @@ from calibre.gui2.tweak_book.editor import syntax_from_mime
 from calibre.gui2.tweak_book.templates import template_for
 from calibre.utils.icu import numeric_sort_key
 from polyglot.builtins import iteritems, itervalues, unicode_type, range
+from polyglot.binary import as_hex_unicode
 
 try:
     from PyQt5 import sip
@@ -392,7 +392,7 @@ class FileList(QTreeWidget):
 
             seen[text] = item
             item.setText(0, text)
-            item.setText(1, hexlify(numeric_sort_key(text)))
+            item.setText(1, as_hex_unicode(numeric_sort_key(text)))
 
         def render_emblems(item, emblems):
             emblems = tuple(emblems)

src/calibre/gui2/update.py

@@ -1,7 +1,7 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
-import re, binascii, ssl, json
+import re, ssl, json
 from polyglot.builtins import map, unicode_type
 from threading import Thread, Event
 
@@ -17,6 +17,7 @@ from calibre.utils.https import get_https_resource_securely
 from calibre.gui2 import config, dynamic, open_url
 from calibre.gui2.dialogs.plugin_updater import get_plugin_updates_available
 from calibre.utils.serialize import msgpack_dumps, msgpack_loads
+from polyglot.binary import as_hex_unicode, from_hex_bytes
 
 URL = 'https://code.calibre-ebook.com/latest'
 # URL = 'http://localhost:8000/latest'
@@ -209,7 +210,7 @@ class UpdateMixin(object):
         has_calibre_update = calibre_version != NO_CALIBRE_UPDATE
         has_plugin_updates = number_of_plugin_updates > 0
         self.plugin_update_found(number_of_plugin_updates)
-        version_url = binascii.hexlify(msgpack_dumps((calibre_version, number_of_plugin_updates)))
+        version_url = as_hex_unicode(msgpack_dumps((calibre_version, number_of_plugin_updates)))
         calibre_version = u'.'.join(map(unicode_type, calibre_version))
 
         if not has_calibre_update and not has_plugin_updates:
@@ -263,7 +264,7 @@ class UpdateMixin(object):
     def update_link_clicked(self, url):
         url = unicode_type(url)
         if url.startswith('update:'):
-            calibre_version, number_of_plugin_updates = msgpack_loads(binascii.unhexlify(url[len('update:'):]))
+            calibre_version, number_of_plugin_updates = msgpack_loads(from_hex_bytes(url[len('update:'):]))
             self.update_found(calibre_version, number_of_plugin_updates, force=True)
 
 

src/calibre/gui2/wizard/send_email.py

@@ -7,7 +7,6 @@ __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import cStringIO, sys
-from binascii import hexlify, unhexlify
 from functools import partial
 from threading import Thread
 
@@ -21,6 +20,7 @@ from calibre.gui2.wizard.send_email_ui import Ui_Form
 from calibre.utils.smtp import config as smtp_prefs
 from calibre.gui2 import error_dialog, question_dialog
 from polyglot.builtins import unicode_type
+from polyglot.binary import as_hex_unicode, from_hex_unicode
 
 
 class TestEmail(QDialog):
@@ -47,7 +47,7 @@ class TestEmail(QDialog):
         l.addLayout(h)
         if opts.relay_host:
             self.la = la = QLabel(_('Using: %(un)s:%(pw)s@%(host)s:%(port)s and %(enc)s encryption')%
-                    dict(un=opts.relay_username, pw=unhexlify(opts.relay_password).decode('utf-8'),
+                    dict(un=opts.relay_username, pw=from_hex_unicode(opts.relay_password),
                         host=opts.relay_host, port=opts.relay_port, enc=opts.encryption))
             l.addWidget(la)
         self.log = QPlainTextEdit(self)
@@ -163,7 +163,7 @@ class SendEmail(QWidget, Ui_Form):
             self.relay_username.setText(opts.relay_username)
         self.relay_username.textChanged.connect(self.changed)
         if opts.relay_password:
-            self.relay_password.setText(unhexlify(opts.relay_password).decode('utf-8'))
+            self.relay_password.setText(from_hex_unicode(opts.relay_password))
         self.relay_password.textChanged.connect(self.changed)
         getattr(self, 'relay_'+opts.encryption.lower()).setChecked(True)
         self.relay_tls.toggled.connect(self.changed)
@@ -204,7 +204,7 @@ class SendEmail(QWidget, Ui_Form):
             sendmail(msg, from_=opts.from_, to=[to],
                 verbose=3, timeout=30, relay=opts.relay_host,
                 username=opts.relay_username, debug_output=debug_out,
-                password=unhexlify(opts.relay_password).decode('utf-8'),
+                password=from_hex_unicode(opts.relay_password),
                 encryption=opts.encryption, port=opts.relay_port)
         except:
             import traceback
@@ -291,6 +291,6 @@ class SendEmail(QWidget, Ui_Form):
         conf.set('relay_host', host if host else None)
         conf.set('relay_port', self.relay_port.value())
         conf.set('relay_username', username if username else None)
-        conf.set('relay_password', hexlify(password.encode('utf-8')))
+        conf.set('relay_password', as_hex_unicode(password))
         conf.set('encryption', enc_method)
         return True

src/calibre/library/coloring.py

@@ -1,16 +1,15 @@
 #!/usr/bin/env python2
 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import (unicode_literals, division, absolute_import,
+# License: GPLv3 Copyright: 2011, Kovid Goyal <kovid at kovidgoyal.net>
-                        print_function)
+from __future__ import absolute_import, division, print_function, unicode_literals
-from polyglot.builtins import map
 
-__license__   = 'GPL v3'
+import json
-__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+import re
-__docformat__ = 'restructuredtext en'
-
-import binascii, re, json
 from textwrap import dedent
 
+from polyglot.binary import as_hex_unicode, from_hex_bytes
+from polyglot.builtins import map
+
 color_row_key = '*row'
 
 
@@ -55,7 +54,7 @@ class Rule(object):  # {{{
     def signature(self):
         args = (self.color, self.conditions)
         sig = json.dumps(args, ensure_ascii=False)
-        return self.SIGNATURE + binascii.hexlify(sig.encode('utf-8'))
+        return self.SIGNATURE + as_hex_unicode(sig)
 
     @property
     def template(self):
@@ -213,7 +212,7 @@ def rule_from_template(fm, template):
         if line.startswith(Rule.SIGNATURE):
             raw = line[len(Rule.SIGNATURE):].strip()
             try:
-                color, conditions = json.loads(binascii.unhexlify(raw).decode('utf-8'))
+                color, conditions = json.loads(from_hex_bytes(raw))
             except:
                 continue
             r = Rule(fm)

src/calibre/ptempfile.py

@@ -105,10 +105,10 @@ def base_dir():
     if _base_dir is None:
         td = os.environ.get('CALIBRE_WORKER_TEMP_DIR', None)
         if td is not None:
-            import binascii
             from calibre.utils.serialize import msgpack_loads
+            from polyglot.binary import from_hex_bytes
             try:
-                td = msgpack_loads(binascii.unhexlify(td))
+                td = msgpack_loads(from_hex_bytes(td))
             except Exception:
                 td = None
         if td and os.path.exists(td):

src/calibre/srv/auth.py

@@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
 __license__ = 'GPL v3'
 __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
 
-import binascii, os, random, struct
+import os, random, struct
 from collections import OrderedDict
 from hashlib import md5, sha256
 from itertools import permutations
@@ -17,7 +17,7 @@ from calibre.srv.http_request import parse_uri
 from calibre.srv.utils import parse_http_dict, encode_path
 from calibre.utils.monotonic import monotonic
 from polyglot import http_client
-from polyglot.binary import from_base64_unicode
+from polyglot.binary import from_base64_unicode, from_hex_bytes, as_hex_unicode
 
 MAX_AGE_SECONDS = 3600
 nonce_counter, nonce_counter_lock = 0, Lock()
@@ -94,7 +94,7 @@ def synthesize_nonce(key_order, realm, secret, timestamp=None):
             # The resolution of monotonic() on windows is very low (10s of
             # milliseconds) so to ensure nonce values are not re-used, we have a
             # global counter
-            timestamp = binascii.hexlify(struct.pack(b'!dH', float(monotonic()), nonce_counter))
+            timestamp = as_hex_unicode(struct.pack(b'!dH', float(monotonic()), nonce_counter))
     h = sha256_hex(key_order.format(timestamp, realm, secret))
     nonce = ':'.join((timestamp, h))
     return nonce
@@ -108,7 +108,7 @@ def validate_nonce(key_order, nonce, realm, secret):
 
 def is_nonce_stale(nonce, max_age_seconds=MAX_AGE_SECONDS):
     try:
-        timestamp = struct.unpack(b'!dH', binascii.unhexlify(as_bytestring(nonce.partition(':')[0])))[0]
+        timestamp = struct.unpack(b'!dH', from_hex_bytes(as_bytestring(nonce.partition(':')[0])))[0]
         return timestamp + max_age_seconds < monotonic()
     except Exception:
         pass
@@ -243,7 +243,7 @@ class AuthController(object):
         self.user_credentials, self.prefer_basic_auth = user_credentials, prefer_basic_auth
         self.ban_list = BanList(ban_time_in_minutes=ban_time_in_minutes, max_failures_before_ban=ban_after)
         self.log = log
-        self.secret = binascii.hexlify(os.urandom(random.randint(20, 30))).decode('ascii')
+        self.secret = as_hex_unicode(os.urandom(random.randint(20, 30)))
         self.max_age_seconds = max_age_seconds
         self.key_order = '{%d}:{%d}:{%d}' % random.choice(tuple(permutations((0,1,2))))
         self.realm = realm

src/calibre/srv/content.py

@@ -7,7 +7,6 @@ __license__ = 'GPL v3'
 __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import os, errno
-from binascii import hexlify
 from io import BytesIO
 from threading import Lock
 from polyglot.builtins import map
@@ -30,6 +29,7 @@ from calibre.utils.img import scale_image, image_from_data
 from calibre.utils.filenames import ascii_filename, atomic_rename
 from calibre.utils.shared_file import share_open
 from polyglot.urllib import quote
+from polyglot.binary import as_hex_unicode
 
 plugboard_content_server_value = 'content_server'
 plugboard_content_server_formats = ['epub', 'mobi', 'azw3']
@@ -111,7 +111,7 @@ def create_file_copy(ctx, rd, prefix, library_id, book_id, ext, mtime, copy_func
                 ans.seek(0)
         if ctx.testing:
             rd.outheaders['Used-Cache'] = used_cache
-            rd.outheaders['Tempfile'] = hexlify(fname.encode('utf-8'))
+            rd.outheaders['Tempfile'] = as_hex_unicode(fname)
         return rd.filesystem_file_with_custom_etag(ans, prefix, library_id, book_id, mtime, extra_etag_data)
 
 

src/calibre/srv/opds.py

@@ -5,7 +5,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import hashlib, binascii
+import hashlib
 from functools import partial
 from collections import OrderedDict, namedtuple
 
@@ -28,16 +28,7 @@ from calibre.srv.routes import endpoint
 from calibre.srv.utils import get_library_data, http_date, Offsets
 from polyglot.builtins import iteritems, unicode_type
 from polyglot.urllib import urlencode
-
+from polyglot.binary import as_hex_unicode, from_hex_unicode
-
-def hexlify(x):
-    if isinstance(x, unicode_type):
-        x = x.encode('utf-8')
-    return binascii.hexlify(x)
-
-
-def unhexlify(x):
-    return binascii.unhexlify(x).decode('utf-8')
 
 
 def atom(ctx, rd, endpoint, output):
@@ -105,7 +96,7 @@ SUBTITLE = E.subtitle
 
 
 def NAVCATALOG_ENTRY(url_for, updated, title, description, query):
-    href = url_for('/opds/navcatalog', which=hexlify(query))
+    href = url_for('/opds/navcatalog', which=as_hex_unicode(query))
     id_ = 'calibre-navcatalog:'+str(hashlib.sha1(href).hexdigest())
     return E.entry(
         TITLE(title),
@@ -154,7 +145,7 @@ def CATALOG_ENTRY(item, item_kind, request_context, updated, catalog_name,
     if item.id is not None:
         iid = 'I' + str(item.id)
         iid += ':'+item_kind
-    href = request_context.url_for('/opds/category', category=hexlify(catalog_name), which=hexlify(iid))
+    href = request_context.url_for('/opds/category', category=as_hex_unicode(catalog_name), which=as_hex_unicode(iid))
     link = NAVLINK(href=href)
     if ignore_count:
         count = ''
@@ -176,7 +167,7 @@ def CATALOG_ENTRY(item, item_kind, request_context, updated, catalog_name,
 def CATALOG_GROUP_ENTRY(item, category, request_context, updated):
     id_ = 'calibre:category-group:'+category+':'+item.text
     iid = item.text
-    link = NAVLINK(href=request_context.url_for('/opds/categorygroup', category=hexlify(category), which=hexlify(iid)))
+    link = NAVLINK(href=request_context.url_for('/opds/categorygroup', category=as_hex_unicode(category), which=as_hex_unicode(iid)))
     return E.entry(
         TITLE(item.text),
         ID(id_),
@@ -519,7 +510,7 @@ def opds_navcatalog(ctx, rd, which):
 
     page_url = rc.url_for('/opds/navcatalog', which=which)
     up_url = rc.url_for('/opds')
-    which = unhexlify(which)
+    which = from_hex_unicode(which)
     type_ = which[0]
     which = which[1:]
     if type_ == 'O':
@@ -542,7 +533,7 @@ def opds_category(ctx, rd, category, which):
     page_url = rc.url_for('/opds/category', which=which, category=category)
     up_url = rc.url_for('/opds/navcatalog', which=category)
 
-    which, category = unhexlify(which), unhexlify(category)
+    which, category = from_hex_unicode(which), from_hex_unicode(category)
     type_ = which[0]
     which = which[1:]
     if type_ == 'I':
@@ -594,15 +585,15 @@ def opds_categorygroup(ctx, rd, category, which):
     categories = rc.get_categories()
     page_url = rc.url_for('/opds/categorygroup', category=category, which=which)
 
-    category = unhexlify(category)
+    category = from_hex_unicode(category)
     if category not in categories:
         raise HTTPNotFound('Category %r not found'%which)
     category_meta = rc.db.field_metadata
     meta = category_meta.get(category, {})
     category_name = meta.get('name', which)
-    which = unhexlify(which)
+    which = from_hex_unicode(which)
     feed_title = default_feed_title + ' :: ' + (_('By {0} :: {1}').format(category_name, which))
-    owhich = hexlify('N'+which)
+    owhich = as_hex_unicode('N'+which)
     up_url = rc.url_for('/opds/navcatalog', which=owhich)
     items = categories[category]
 

src/calibre/srv/tests/content.py

@@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
 __license__ = 'GPL v3'
 __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
 
-import zlib, json, binascii, time, os
+import zlib, json, time, os
 from io import BytesIO
 
 from calibre.ebooks.metadata.epub import get_metadata
@@ -15,6 +15,7 @@ from calibre.srv.tests.base import LibraryBaseTest
 from calibre.utils.imghdr import identify
 from calibre.utils.shared_file import share_open
 from polyglot import http_client
+from polyglot.binary import from_hex_unicode
 
 
 def setUpModule():
@@ -181,7 +182,7 @@ class ContentTest(LibraryBaseTest):
             self.ae(r.status, http_client.OK)
             self.ae(data, db.cover(2))
             self.ae(r.getheader('Used-Cache'), 'no')
-            path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8')
+            path = from_hex_unicode(r.getheader('Tempfile')).decode('utf-8')
             f, fdata = share_open(path, 'rb'), data
             # Now force an update
             change_cover(1)
@@ -189,7 +190,7 @@ class ContentTest(LibraryBaseTest):
             self.ae(r.status, http_client.OK)
             self.ae(data, db.cover(2))
             self.ae(r.getheader('Used-Cache'), 'no')
-            path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8')
+            path = from_hex_unicode(r.getheader('Tempfile')).decode('utf-8')
             f2, f2data = share_open(path, 'rb'), data
             # Do it again
             change_cover(2)

src/calibre/srv/utils.py

@@ -10,7 +10,6 @@ import errno, socket, select, os, time
 from contextlib import closing
 from email.utils import formatdate
 from operator import itemgetter
-from binascii import hexlify, unhexlify
 
 from calibre import prints
 from calibre.constants import iswindows, ispy3
@@ -20,14 +19,16 @@ from calibre.utils.localization import get_translator
 from calibre.utils.socket_inheritance import set_socket_inherit
 from calibre.utils.logging import ThreadSafeLog
 from calibre.utils.shared_file import share_open, raise_winerror
-from polyglot.builtins import iteritems, map, unicode_type, range
+from polyglot.builtins import iteritems, map, range
 from polyglot import reprlib
 from polyglot.http_cookie import SimpleCookie
 from polyglot.urllib import parse_qs, quote as urlquote
+from polyglot.binary import as_hex_unicode as encode_name, from_hex_unicode as decode_name
 
 HTTP1  = 'HTTP/1.0'
 HTTP11 = 'HTTP/1.1'
 DESIRED_SEND_BUFFER_SIZE = 16 * 1024  # windows 7 uses an 8KB sndbuf
+encode_name, decode_name
 
 
 def http_date(timeval=None):
@@ -286,17 +287,6 @@ def encode_path(*components):
     return '/' + '/'.join(urlquote(x.encode('utf-8'), '').decode('ascii') for x in components)
 
 
-def encode_name(name):
-    'Encode a name (arbitrary string) as URL safe characters. See decode_name() also.'
-    if isinstance(name, unicode_type):
-        name = name.encode('utf-8')
-    return hexlify(name)
-
-
-def decode_name(name):
-    return unhexlify(name).decode('utf-8')
-
-
 class Cookie(SimpleCookie):
 
     def _BaseCookie__set(self, key, real_value, coded_value):

src/calibre/utils/exim.py

@@ -5,7 +5,6 @@
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 import os, json, struct, hashlib, sys, errno, tempfile, time, shutil, uuid
-from binascii import hexlify
 from collections import Counter
 
 from calibre import prints
@@ -14,6 +13,7 @@ from calibre.utils.config_base import prefs, StringConfig, create_global_prefs
 from calibre.utils.config import JSONConfig
 from calibre.utils.filenames import samefile
 from polyglot.builtins import iteritems, raw_input
+from polyglot.binary import as_hex_unicode
 
 
 # Export {{{
@@ -133,7 +133,7 @@ class Exporter(object):
         return FileDest(key, self, mtime=mtime)
 
     def export_dir(self, path, dir_key):
-        pkey = hexlify(dir_key)
+        pkey = as_hex_unicode(dir_key)
         self.metadata[dir_key] = files = []
         for dirpath, dirnames, filenames in os.walk(path):
             for fname in filenames:

src/calibre/utils/ipc/launch.py

@@ -6,14 +6,15 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import subprocess, os, sys, time, binascii
+import subprocess, os, sys, time
 from functools import partial
 
-from calibre.constants import iswindows, isosx, isfrozen, filesystem_encoding
+from calibre.constants import iswindows, isosx, isfrozen, filesystem_encoding, ispy3
 from calibre.utils.config import prefs
 from calibre.ptempfile import PersistentTemporaryFile, base_dir
 from calibre.utils.serialize import msgpack_dumps
-from polyglot.builtins import iteritems, unicode_type, string_or_bytes
+from polyglot.builtins import iteritems, unicode_type, string_or_bytes, environ_item
+from polyglot.binary import as_hex_unicode
 
 if iswindows:
     import win32process
@@ -88,6 +89,9 @@ class Worker(object):
 
     @property
     def env(self):
+        if ispy3:
+            env = os.environ.copy()
+        else:
             # We use this inefficient method of copying the environment variables
             # because of non ascii env vars on windows. See https://bugs.launchpad.net/bugs/811191
             env = {}
@@ -105,9 +109,9 @@ class Worker(object):
                     env[key] = val
                 except:
                     pass
-        env[str('CALIBRE_WORKER')] = str('1')
+        env[str('CALIBRE_WORKER')] = environ_item('1')
-        td = binascii.hexlify(msgpack_dumps(base_dir())).decode('ascii')
+        td = as_hex_unicode(msgpack_dumps(base_dir()))
-        env[b'CALIBRE_WORKER_TEMP_DIR'] = str(td)
+        env[str('CALIBRE_WORKER_TEMP_DIR')] = environ_item(td)
         env.update(self._env)
         return env
 
@@ -181,7 +185,7 @@ class Worker(object):
         except EnvironmentError:
             # cwd no longer exists
             origwd = cwd or os.path.expanduser(u'~')
-        env[str('ORIGWD')] = binascii.hexlify(msgpack_dumps(origwd))
+        env[str('ORIGWD')] = environ_item(as_hex_unicode(msgpack_dumps(origwd)))
         _cwd = cwd
         if priority is None:
             priority = prefs['worker_process_priority']

src/calibre/utils/ipc/server.py

@@ -13,7 +13,6 @@ import os
 import sys
 import tempfile
 import time
-from binascii import hexlify
 from collections import deque
 from math import ceil
 from multiprocessing.connection import Listener, arbitrary_address
@@ -28,6 +27,7 @@ from calibre.utils.ipc.worker import PARALLEL_FUNCS
 from calibre.utils.serialize import msgpack_dumps, pickle_loads
 from polyglot.builtins import string_or_bytes, environ_item
 from polyglot.queue import Empty, Queue
+from polyglot.binary import as_hex_unicode
 
 
 _counter = 0
@@ -219,10 +219,10 @@ class Server(Thread):
             redirect_output = not gui
 
         env = {
-                'CALIBRE_WORKER_ADDRESS' : environ_item(hexlify(msgpack_dumps(
+                'CALIBRE_WORKER_ADDRESS' : environ_item(as_hex_unicode(msgpack_dumps(
                     self.listener.address))),
-                'CALIBRE_WORKER_KEY' : environ_item(hexlify(self.auth_key)),
+                'CALIBRE_WORKER_KEY' : environ_item(as_hex_unicode(self.auth_key)),
-                'CALIBRE_WORKER_RESULT' : environ_item(hexlify(rfile.encode('utf-8'))),
+                'CALIBRE_WORKER_RESULT' : environ_item(as_hex_unicode(rfile)),
               }
         cw = self.do_launch(env, gui, redirect_output, rfile, job_name=job_name)
         if isinstance(cw, string_or_bytes):

src/calibre/utils/ipc/simple_worker.py

@@ -8,7 +8,6 @@ __copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import os, traceback, time, importlib
-from binascii import hexlify, unhexlify
 from multiprocessing.connection import Client
 from threading import Thread
 from contextlib import closing
@@ -18,6 +17,7 @@ from calibre.utils.ipc import eintr_retry_call
 from calibre.utils.ipc.launch import Worker
 from calibre.utils.serialize import msgpack_loads, msgpack_dumps
 from polyglot.builtins import unicode_type, string_or_bytes, environ_item
+from polyglot.binary import as_hex_unicode, from_hex_bytes
 
 
 class WorkerError(Exception):
@@ -131,8 +131,8 @@ def create_worker(env, priority='normal', cwd=None, func='main'):
 
     env = dict(env)
     env.update({
-        'CALIBRE_WORKER_ADDRESS': environ_item(hexlify(msgpack_dumps(listener.address))),
+        'CALIBRE_WORKER_ADDRESS': environ_item(as_hex_unicode(msgpack_dumps(listener.address))),
-        'CALIBRE_WORKER_KEY': environ_item(hexlify(auth_key)),
+        'CALIBRE_WORKER_KEY': environ_item(as_hex_unicode(auth_key)),
         'CALIBRE_SIMPLE_WORKER': environ_item('calibre.utils.ipc.simple_worker:%s' % func),
     })
 
@@ -271,8 +271,8 @@ def compile_code(src):
 
 def main():
     # The entry point for the simple worker process
-    address = msgpack_loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS']))
+    address = msgpack_loads(from_hex_bytes(os.environ['CALIBRE_WORKER_ADDRESS']))
-    key     = unhexlify(os.environ['CALIBRE_WORKER_KEY'])
+    key     = from_hex_bytes(os.environ['CALIBRE_WORKER_KEY'])
     with closing(Client(address, authkey=key)) as conn:
         args = eintr_retry_call(conn.recv)
         try:
@@ -301,8 +301,8 @@ def main():
 
 def offload():
     # The entry point for the offload worker process
-    address = msgpack_loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS']))
+    address = msgpack_loads(from_hex_bytes(os.environ['CALIBRE_WORKER_ADDRESS']))
-    key     = unhexlify(os.environ['CALIBRE_WORKER_KEY'])
+    key     = from_hex_bytes(os.environ['CALIBRE_WORKER_KEY'])
     func_cache = {}
     with closing(Client(address, authkey=key)) as conn:
         while True:

src/calibre/utils/ipc/worker.py

@@ -11,7 +11,6 @@ import os, sys, importlib
 from multiprocessing.connection import Client
 from threading import Thread
 from contextlib import closing
-from binascii import unhexlify
 from zipimport import ZipImportError
 
 from calibre import prints
@@ -19,6 +18,7 @@ from calibre.constants import iswindows, isosx
 from calibre.utils.ipc import eintr_retry_call
 from calibre.utils.serialize import msgpack_loads, pickle_dumps
 from polyglot.queue import Queue
+from polyglot.binary import from_hex_bytes, from_hex_unicode
 
 PARALLEL_FUNCS = {
     'lrfviewer'    :
@@ -183,9 +183,9 @@ def main():
             print('Failed to run pipe worker with command:', sys.argv[-1])
             raise
         return
-    address = msgpack_loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS']))
+    address = msgpack_loads(from_hex_bytes(os.environ['CALIBRE_WORKER_ADDRESS']))
-    key     = unhexlify(os.environ['CALIBRE_WORKER_KEY'])
+    key     = from_hex_bytes(os.environ['CALIBRE_WORKER_KEY'])
-    resultf = unhexlify(os.environ['CALIBRE_WORKER_RESULT']).decode('utf-8')
+    resultf = from_hex_unicode(os.environ['CALIBRE_WORKER_RESULT'])
     with closing(Client(address, authkey=key)) as conn:
         name, args, kwargs, desc = eintr_retry_call(conn.recv)
         if desc:

src/polyglot/binary.py

@@ -4,7 +4,9 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-from base64 import standard_b64encode, standard_b64decode
+from base64 import standard_b64decode, standard_b64encode
+from binascii import hexlify, unhexlify
+
 from polyglot.builtins import unicode_type
 
 
@@ -30,3 +32,27 @@ def from_base64_bytes(x):
     if isinstance(x, unicode_type):
         x = x.encode('ascii')
     return standard_b64decode(x)
+
+
+def as_hex_bytes(x, enc='utf-8'):
+    if isinstance(x, unicode_type):
+        x = x.encode(enc)
+    return hexlify(x)
+
+
+def as_hex_unicode(x, enc='utf-8'):
+    if isinstance(x, unicode_type):
+        x = x.encode(enc)
+    return hexlify(x).decode('ascii')
+
+
+def from_hex_unicode(x, enc='utf-8'):
+    if isinstance(x, unicode_type):
+        x = x.encode('ascii')
+    return unhexlify(x).decode(enc)
+
+
+def from_hex_bytes(x):
+    if isinstance(x, unicode_type):
+        x = x.encode('ascii')
+    return unhexlify(x)

src/tinycss/decoding.py

@@ -12,10 +12,10 @@
 
 from __future__ import unicode_literals
 
-from binascii import unhexlify
 import operator
 import re
-import sys
+
+from polyglot.binary import from_hex_bytes
 
 
 __all__ = ['decode']  # Everything else is implementation detail
@@ -98,7 +98,7 @@ def try_encoding(css_bytes, encoding, fallback=True):
 
 
 def hex2re(hex_data):
-    return re.escape(unhexlify(hex_data.replace(' ', '').encode('ascii')))
+    return re.escape(from_hex_bytes(hex_data.replace(' ', '').encode('ascii')))
 
 
 class Slicer(object):
@@ -106,6 +106,7 @@ class Slicer(object):
     def __getitem__(self, slice_):
         return operator.itemgetter(slice_)
 
+
 Slice = Slicer()