diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index aa43785292..0414e878ea 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -2206,8 +2206,8 @@ class Cache(object): @read_api def export_library(self, library_key, exporter, progress=None, abort=None): - from binascii import hexlify - key_prefix = hexlify(library_key) + from polyglot.binary import as_hex_unicode + key_prefix = as_hex_unicode(library_key) book_ids = self._all_book_ids() total = len(book_ids) + 1 format_metadata = {} diff --git a/src/calibre/ebooks/conversion/plugins/epub_output.py b/src/calibre/ebooks/conversion/plugins/epub_output.py index 294c7afcfc..ff2b9742e7 100644 --- a/src/calibre/ebooks/conversion/plugins/epub_output.py +++ b/src/calibre/ebooks/conversion/plugins/epub_output.py @@ -311,13 +311,12 @@ class EPUBOutput(OutputFormatPlugin): pass def encrypt_fonts(self, uris, tdir, uuid): # {{{ - from binascii import unhexlify + from polyglot.binary import from_hex_bytes key = re.sub(r'[^a-fA-F0-9]', '', uuid) if len(key) < 16: raise ValueError('UUID identifier %r is invalid'%uuid) - key = unhexlify((key + key)[:32]) - key = tuple(map(ord, key)) + key = bytearray(from_hex_bytes((key + key)[:32])) paths = [] with CurrentDir(tdir): paths = [os.path.join(*x.split('/')) for x in uris] diff --git a/src/calibre/ebooks/metadata/book/render.py b/src/calibre/ebooks/metadata/book/render.py index 9b648356aa..2a19041903 100644 --- a/src/calibre/ebooks/metadata/book/render.py +++ b/src/calibre/ebooks/metadata/book/render.py @@ -8,7 +8,6 @@ __copyright__ = '2014, Kovid Goyal ' import os from functools import partial -from binascii import hexlify from calibre import prepare_string_for_xml, force_unicode from calibre.ebooks.metadata import fmt_sidx, rating_to_stars @@ -22,6 +21,7 @@ from calibre.utils.date import is_date_undefined from calibre.utils.localization import calibre_langcode_to_name from calibre.utils.serialize import json_dumps from polyglot.builtins import unicode_type +from polyglot.binary import as_hex_unicode default_sort = ('title', 'title_sort', 'authors', 'author_sort', 'series', 'rating', 'pubdate', 'tags', 'publisher', 'identifiers') @@ -55,7 +55,7 @@ def get_field_list(mi): def search_href(search_term, value): search = '%s:"=%s"' % (search_term, value.replace('"', '\\"')) - return prepare_string_for_xml('search:' + hexlify(search.encode('utf-8')), True) + return prepare_string_for_xml('search:' + as_hex_unicode(search.encode('utf-8')), True) DEFAULT_AUTHOR_LINK = 'search-{}'.format(DEFAULT_AUTHOR_SOURCE) @@ -80,7 +80,7 @@ def author_search_href(which, title=None, author=None): def item_data(field_name, value, book_id): - return hexlify(json_dumps((field_name, value, book_id))) + return as_hex_unicode(json_dumps((field_name, value, book_id))) def mi_to_html(mi, field_list=None, default_author_link=None, use_roman_numbers=True, rating_font='Liberation Serif', rtl=False): diff --git a/src/calibre/ebooks/pdf/render/common.py b/src/calibre/ebooks/pdf/render/common.py index eb53bb9031..b146b7faac 100644 --- a/src/calibre/ebooks/pdf/render/common.py +++ b/src/calibre/ebooks/pdf/render/common.py @@ -10,11 +10,11 @@ __docformat__ = 'restructuredtext en' import codecs, zlib, numbers from io import BytesIO from datetime import datetime -from binascii import hexlify from calibre.constants import plugins, ispy3 from calibre.utils.logging import default_log from polyglot.builtins import iteritems, unicode_type +from polyglot.binary import as_hex_bytes pdf_float = plugins['speedup'][0].pdf_float @@ -137,7 +137,7 @@ class UTF16String(unicode_type): if False: # Disabled as the parentheses based strings give easier to debug # PDF files - stream.write(b'<' + hexlify(raw) + b'>') + stream.write(b'<' + as_hex_bytes(raw) + b'>') else: stream.write(b'('+escape_pdf_string(raw)+b')') diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py index 4b97b1d772..031cbac7b4 100644 --- a/src/calibre/gui2/book_details.py +++ b/src/calibre/gui2/book_details.py @@ -4,7 +4,6 @@ import os import re -from binascii import unhexlify from collections import namedtuple from functools import partial @@ -35,6 +34,8 @@ from calibre.utils.img import blend_image, image_from_x from calibre.utils.localization import is_rtl from calibre.utils.serialize import json_loads from polyglot.builtins import unicode_type +from polyglot.binary import from_hex_bytes, from_hex_unicode + _css = None InternetSearch = namedtuple('InternetSearch', 'author where') @@ -286,7 +287,7 @@ def details_context_menu_event(view, ev, book_info): # {{{ lambda : book_info.search_requested('authors:"={}"'.format(author.replace('"', r'\"')))) if data: try: - field, value, book_id = json_loads(unhexlify(data)) + field, value, book_id = json_loads(from_hex_bytes(data)) except Exception: field = value = book_id = None if field: @@ -874,7 +875,7 @@ class BookDetails(QWidget): # {{{ elif typ == 'devpath': self.view_device_book.emit(val) elif typ == 'search': - self.search_requested.emit(unhexlify(val).decode('utf-8')) + self.search_requested.emit(from_hex_unicode(val)) else: try: open_url(QUrl(link, QUrl.TolerantMode)) diff --git a/src/calibre/gui2/email.py b/src/calibre/gui2/email.py index 3fcfc9696a..5040eb133c 100644 --- a/src/calibre/gui2/email.py +++ b/src/calibre/gui2/email.py @@ -7,7 +7,6 @@ __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' import os, socket, time, textwrap -from binascii import unhexlify from functools import partial from threading import Thread from itertools import repeat @@ -29,6 +28,7 @@ from calibre.utils.config import tweaks, prefs from calibre.utils.icu import primary_sort_key from calibre.gui2.threaded_jobs import ThreadedJob from polyglot.builtins import iteritems, itervalues, unicode_type +from polyglot.binary import from_hex_unicode class Worker(Thread): @@ -130,7 +130,7 @@ class Sendmail(object): verbose=1, relay=opts.relay_host, username=opts.relay_username, - password=unhexlify(opts.relay_password).decode('utf-8'), port=opts.relay_port, + password=from_hex_unicode(opts.relay_password), port=opts.relay_port, encryption=opts.encryption, debug_output=safe_debug) finally: diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index c97f887f08..e57bc06bc7 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -28,7 +28,7 @@ from calibre.utils.config import dynamic, prefs from calibre.utils.ipc import RC, gui_socket_address from calibre.utils.lock import singleinstance from calibre.utils.monotonic import monotonic -from polyglot.builtins import unicode_type, range +from polyglot.builtins import unicode_type, range, environ_item if iswindows: winutil = plugins['winutil'][0] @@ -173,19 +173,19 @@ def repair_library(library_path): def windows_repair(library_path=None): - from binascii import hexlify, unhexlify import subprocess from calibre.utils.serialize import json_dumps, json_loads + from polyglot.binary import as_hex_unicode, from_hex_bytes if library_path: - library_path = hexlify(json_dumps(library_path)) + library_path = as_hex_unicode(json_dumps(library_path)) winutil.prepare_for_restart() - os.environ['CALIBRE_REPAIR_CORRUPTED_DB'] = library_path + os.environ['CALIBRE_REPAIR_CORRUPTED_DB'] = environ_item(library_path) subprocess.Popen([sys.executable]) else: try: app = Application([]) from calibre.gui2.dialogs.restore_library import repair_library_at - library_path = json_loads(unhexlify(os.environ.pop('CALIBRE_REPAIR_CORRUPTED_DB'))) + library_path = json_loads(from_hex_bytes(os.environ.pop('CALIBRE_REPAIR_CORRUPTED_DB'))) done = repair_library_at(library_path, wait_time=4) except Exception: done = False diff --git a/src/calibre/gui2/tweak_book/file_list.py b/src/calibre/gui2/tweak_book/file_list.py index 674d37ceab..e2a53a35d1 100644 --- a/src/calibre/gui2/tweak_book/file_list.py +++ b/src/calibre/gui2/tweak_book/file_list.py @@ -7,7 +7,6 @@ import os import posixpath import sys import textwrap -from binascii import hexlify from collections import Counter, OrderedDict, defaultdict from functools import partial @@ -40,6 +39,7 @@ from calibre.gui2.tweak_book.editor import syntax_from_mime from calibre.gui2.tweak_book.templates import template_for from calibre.utils.icu import numeric_sort_key from polyglot.builtins import iteritems, itervalues, unicode_type, range +from polyglot.binary import as_hex_unicode try: from PyQt5 import sip @@ -392,7 +392,7 @@ class FileList(QTreeWidget): seen[text] = item item.setText(0, text) - item.setText(1, hexlify(numeric_sort_key(text))) + item.setText(1, as_hex_unicode(numeric_sort_key(text))) def render_emblems(item, emblems): emblems = tuple(emblems) diff --git a/src/calibre/gui2/update.py b/src/calibre/gui2/update.py index 3e878ba97a..dcf45bc586 100644 --- a/src/calibre/gui2/update.py +++ b/src/calibre/gui2/update.py @@ -1,7 +1,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' -import re, binascii, ssl, json +import re, ssl, json from polyglot.builtins import map, unicode_type from threading import Thread, Event @@ -17,6 +17,7 @@ from calibre.utils.https import get_https_resource_securely from calibre.gui2 import config, dynamic, open_url from calibre.gui2.dialogs.plugin_updater import get_plugin_updates_available from calibre.utils.serialize import msgpack_dumps, msgpack_loads +from polyglot.binary import as_hex_unicode, from_hex_bytes URL = 'https://code.calibre-ebook.com/latest' # URL = 'http://localhost:8000/latest' @@ -209,7 +210,7 @@ class UpdateMixin(object): has_calibre_update = calibre_version != NO_CALIBRE_UPDATE has_plugin_updates = number_of_plugin_updates > 0 self.plugin_update_found(number_of_plugin_updates) - version_url = binascii.hexlify(msgpack_dumps((calibre_version, number_of_plugin_updates))) + version_url = as_hex_unicode(msgpack_dumps((calibre_version, number_of_plugin_updates))) calibre_version = u'.'.join(map(unicode_type, calibre_version)) if not has_calibre_update and not has_plugin_updates: @@ -263,7 +264,7 @@ class UpdateMixin(object): def update_link_clicked(self, url): url = unicode_type(url) if url.startswith('update:'): - calibre_version, number_of_plugin_updates = msgpack_loads(binascii.unhexlify(url[len('update:'):])) + calibre_version, number_of_plugin_updates = msgpack_loads(from_hex_bytes(url[len('update:'):])) self.update_found(calibre_version, number_of_plugin_updates, force=True) diff --git a/src/calibre/gui2/wizard/send_email.py b/src/calibre/gui2/wizard/send_email.py index e3e60a0fd4..d41733b136 100644 --- a/src/calibre/gui2/wizard/send_email.py +++ b/src/calibre/gui2/wizard/send_email.py @@ -7,7 +7,6 @@ __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' import cStringIO, sys -from binascii import hexlify, unhexlify from functools import partial from threading import Thread @@ -21,6 +20,7 @@ from calibre.gui2.wizard.send_email_ui import Ui_Form from calibre.utils.smtp import config as smtp_prefs from calibre.gui2 import error_dialog, question_dialog from polyglot.builtins import unicode_type +from polyglot.binary import as_hex_unicode, from_hex_unicode class TestEmail(QDialog): @@ -47,7 +47,7 @@ class TestEmail(QDialog): l.addLayout(h) if opts.relay_host: self.la = la = QLabel(_('Using: %(un)s:%(pw)s@%(host)s:%(port)s and %(enc)s encryption')% - dict(un=opts.relay_username, pw=unhexlify(opts.relay_password).decode('utf-8'), + dict(un=opts.relay_username, pw=from_hex_unicode(opts.relay_password), host=opts.relay_host, port=opts.relay_port, enc=opts.encryption)) l.addWidget(la) self.log = QPlainTextEdit(self) @@ -163,7 +163,7 @@ class SendEmail(QWidget, Ui_Form): self.relay_username.setText(opts.relay_username) self.relay_username.textChanged.connect(self.changed) if opts.relay_password: - self.relay_password.setText(unhexlify(opts.relay_password).decode('utf-8')) + self.relay_password.setText(from_hex_unicode(opts.relay_password)) self.relay_password.textChanged.connect(self.changed) getattr(self, 'relay_'+opts.encryption.lower()).setChecked(True) self.relay_tls.toggled.connect(self.changed) @@ -204,7 +204,7 @@ class SendEmail(QWidget, Ui_Form): sendmail(msg, from_=opts.from_, to=[to], verbose=3, timeout=30, relay=opts.relay_host, username=opts.relay_username, debug_output=debug_out, - password=unhexlify(opts.relay_password).decode('utf-8'), + password=from_hex_unicode(opts.relay_password), encryption=opts.encryption, port=opts.relay_port) except: import traceback @@ -291,6 +291,6 @@ class SendEmail(QWidget, Ui_Form): conf.set('relay_host', host if host else None) conf.set('relay_port', self.relay_port.value()) conf.set('relay_username', username if username else None) - conf.set('relay_password', hexlify(password.encode('utf-8'))) + conf.set('relay_password', as_hex_unicode(password)) conf.set('encryption', enc_method) return True diff --git a/src/calibre/library/coloring.py b/src/calibre/library/coloring.py index b494ef9a5a..bd3b21f8b7 100644 --- a/src/calibre/library/coloring.py +++ b/src/calibre/library/coloring.py @@ -1,16 +1,15 @@ #!/usr/bin/env python2 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import (unicode_literals, division, absolute_import, - print_function) -from polyglot.builtins import map +# License: GPLv3 Copyright: 2011, Kovid Goyal +from __future__ import absolute_import, division, print_function, unicode_literals -__license__ = 'GPL v3' -__copyright__ = '2011, Kovid Goyal ' -__docformat__ = 'restructuredtext en' - -import binascii, re, json +import json +import re from textwrap import dedent +from polyglot.binary import as_hex_unicode, from_hex_bytes +from polyglot.builtins import map + color_row_key = '*row' @@ -55,7 +54,7 @@ class Rule(object): # {{{ def signature(self): args = (self.color, self.conditions) sig = json.dumps(args, ensure_ascii=False) - return self.SIGNATURE + binascii.hexlify(sig.encode('utf-8')) + return self.SIGNATURE + as_hex_unicode(sig) @property def template(self): @@ -213,7 +212,7 @@ def rule_from_template(fm, template): if line.startswith(Rule.SIGNATURE): raw = line[len(Rule.SIGNATURE):].strip() try: - color, conditions = json.loads(binascii.unhexlify(raw).decode('utf-8')) + color, conditions = json.loads(from_hex_bytes(raw)) except: continue r = Rule(fm) diff --git a/src/calibre/ptempfile.py b/src/calibre/ptempfile.py index 56ce1c5b4f..c2821e8343 100644 --- a/src/calibre/ptempfile.py +++ b/src/calibre/ptempfile.py @@ -105,10 +105,10 @@ def base_dir(): if _base_dir is None: td = os.environ.get('CALIBRE_WORKER_TEMP_DIR', None) if td is not None: - import binascii from calibre.utils.serialize import msgpack_loads + from polyglot.binary import from_hex_bytes try: - td = msgpack_loads(binascii.unhexlify(td)) + td = msgpack_loads(from_hex_bytes(td)) except Exception: td = None if td and os.path.exists(td): diff --git a/src/calibre/srv/auth.py b/src/calibre/srv/auth.py index 43d97278aa..97014515d2 100644 --- a/src/calibre/srv/auth.py +++ b/src/calibre/srv/auth.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import binascii, os, random, struct +import os, random, struct from collections import OrderedDict from hashlib import md5, sha256 from itertools import permutations @@ -17,7 +17,7 @@ from calibre.srv.http_request import parse_uri from calibre.srv.utils import parse_http_dict, encode_path from calibre.utils.monotonic import monotonic from polyglot import http_client -from polyglot.binary import from_base64_unicode +from polyglot.binary import from_base64_unicode, from_hex_bytes, as_hex_unicode MAX_AGE_SECONDS = 3600 nonce_counter, nonce_counter_lock = 0, Lock() @@ -94,7 +94,7 @@ def synthesize_nonce(key_order, realm, secret, timestamp=None): # The resolution of monotonic() on windows is very low (10s of # milliseconds) so to ensure nonce values are not re-used, we have a # global counter - timestamp = binascii.hexlify(struct.pack(b'!dH', float(monotonic()), nonce_counter)) + timestamp = as_hex_unicode(struct.pack(b'!dH', float(monotonic()), nonce_counter)) h = sha256_hex(key_order.format(timestamp, realm, secret)) nonce = ':'.join((timestamp, h)) return nonce @@ -108,7 +108,7 @@ def validate_nonce(key_order, nonce, realm, secret): def is_nonce_stale(nonce, max_age_seconds=MAX_AGE_SECONDS): try: - timestamp = struct.unpack(b'!dH', binascii.unhexlify(as_bytestring(nonce.partition(':')[0])))[0] + timestamp = struct.unpack(b'!dH', from_hex_bytes(as_bytestring(nonce.partition(':')[0])))[0] return timestamp + max_age_seconds < monotonic() except Exception: pass @@ -243,7 +243,7 @@ class AuthController(object): self.user_credentials, self.prefer_basic_auth = user_credentials, prefer_basic_auth self.ban_list = BanList(ban_time_in_minutes=ban_time_in_minutes, max_failures_before_ban=ban_after) self.log = log - self.secret = binascii.hexlify(os.urandom(random.randint(20, 30))).decode('ascii') + self.secret = as_hex_unicode(os.urandom(random.randint(20, 30))) self.max_age_seconds = max_age_seconds self.key_order = '{%d}:{%d}:{%d}' % random.choice(tuple(permutations((0,1,2)))) self.realm = realm diff --git a/src/calibre/srv/content.py b/src/calibre/srv/content.py index 8835720659..aa54b122a4 100644 --- a/src/calibre/srv/content.py +++ b/src/calibre/srv/content.py @@ -7,7 +7,6 @@ __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' import os, errno -from binascii import hexlify from io import BytesIO from threading import Lock from polyglot.builtins import map @@ -30,6 +29,7 @@ from calibre.utils.img import scale_image, image_from_data from calibre.utils.filenames import ascii_filename, atomic_rename from calibre.utils.shared_file import share_open from polyglot.urllib import quote +from polyglot.binary import as_hex_unicode plugboard_content_server_value = 'content_server' plugboard_content_server_formats = ['epub', 'mobi', 'azw3'] @@ -111,7 +111,7 @@ def create_file_copy(ctx, rd, prefix, library_id, book_id, ext, mtime, copy_func ans.seek(0) if ctx.testing: rd.outheaders['Used-Cache'] = used_cache - rd.outheaders['Tempfile'] = hexlify(fname.encode('utf-8')) + rd.outheaders['Tempfile'] = as_hex_unicode(fname) return rd.filesystem_file_with_custom_etag(ans, prefix, library_id, book_id, mtime, extra_etag_data) diff --git a/src/calibre/srv/opds.py b/src/calibre/srv/opds.py index d62112ec22..1166a151ed 100644 --- a/src/calibre/srv/opds.py +++ b/src/calibre/srv/opds.py @@ -5,7 +5,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import hashlib, binascii +import hashlib from functools import partial from collections import OrderedDict, namedtuple @@ -28,16 +28,7 @@ from calibre.srv.routes import endpoint from calibre.srv.utils import get_library_data, http_date, Offsets from polyglot.builtins import iteritems, unicode_type from polyglot.urllib import urlencode - - -def hexlify(x): - if isinstance(x, unicode_type): - x = x.encode('utf-8') - return binascii.hexlify(x) - - -def unhexlify(x): - return binascii.unhexlify(x).decode('utf-8') +from polyglot.binary import as_hex_unicode, from_hex_unicode def atom(ctx, rd, endpoint, output): @@ -105,7 +96,7 @@ SUBTITLE = E.subtitle def NAVCATALOG_ENTRY(url_for, updated, title, description, query): - href = url_for('/opds/navcatalog', which=hexlify(query)) + href = url_for('/opds/navcatalog', which=as_hex_unicode(query)) id_ = 'calibre-navcatalog:'+str(hashlib.sha1(href).hexdigest()) return E.entry( TITLE(title), @@ -154,7 +145,7 @@ def CATALOG_ENTRY(item, item_kind, request_context, updated, catalog_name, if item.id is not None: iid = 'I' + str(item.id) iid += ':'+item_kind - href = request_context.url_for('/opds/category', category=hexlify(catalog_name), which=hexlify(iid)) + href = request_context.url_for('/opds/category', category=as_hex_unicode(catalog_name), which=as_hex_unicode(iid)) link = NAVLINK(href=href) if ignore_count: count = '' @@ -176,7 +167,7 @@ def CATALOG_ENTRY(item, item_kind, request_context, updated, catalog_name, def CATALOG_GROUP_ENTRY(item, category, request_context, updated): id_ = 'calibre:category-group:'+category+':'+item.text iid = item.text - link = NAVLINK(href=request_context.url_for('/opds/categorygroup', category=hexlify(category), which=hexlify(iid))) + link = NAVLINK(href=request_context.url_for('/opds/categorygroup', category=as_hex_unicode(category), which=as_hex_unicode(iid))) return E.entry( TITLE(item.text), ID(id_), @@ -519,7 +510,7 @@ def opds_navcatalog(ctx, rd, which): page_url = rc.url_for('/opds/navcatalog', which=which) up_url = rc.url_for('/opds') - which = unhexlify(which) + which = from_hex_unicode(which) type_ = which[0] which = which[1:] if type_ == 'O': @@ -542,7 +533,7 @@ def opds_category(ctx, rd, category, which): page_url = rc.url_for('/opds/category', which=which, category=category) up_url = rc.url_for('/opds/navcatalog', which=category) - which, category = unhexlify(which), unhexlify(category) + which, category = from_hex_unicode(which), from_hex_unicode(category) type_ = which[0] which = which[1:] if type_ == 'I': @@ -594,15 +585,15 @@ def opds_categorygroup(ctx, rd, category, which): categories = rc.get_categories() page_url = rc.url_for('/opds/categorygroup', category=category, which=which) - category = unhexlify(category) + category = from_hex_unicode(category) if category not in categories: raise HTTPNotFound('Category %r not found'%which) category_meta = rc.db.field_metadata meta = category_meta.get(category, {}) category_name = meta.get('name', which) - which = unhexlify(which) + which = from_hex_unicode(which) feed_title = default_feed_title + ' :: ' + (_('By {0} :: {1}').format(category_name, which)) - owhich = hexlify('N'+which) + owhich = as_hex_unicode('N'+which) up_url = rc.url_for('/opds/navcatalog', which=owhich) items = categories[category] diff --git a/src/calibre/srv/tests/content.py b/src/calibre/srv/tests/content.py index fa9a12ccc7..bd3b19b3c1 100644 --- a/src/calibre/srv/tests/content.py +++ b/src/calibre/srv/tests/content.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import zlib, json, binascii, time, os +import zlib, json, time, os from io import BytesIO from calibre.ebooks.metadata.epub import get_metadata @@ -15,6 +15,7 @@ from calibre.srv.tests.base import LibraryBaseTest from calibre.utils.imghdr import identify from calibre.utils.shared_file import share_open from polyglot import http_client +from polyglot.binary import from_hex_unicode def setUpModule(): @@ -181,7 +182,7 @@ class ContentTest(LibraryBaseTest): self.ae(r.status, http_client.OK) self.ae(data, db.cover(2)) self.ae(r.getheader('Used-Cache'), 'no') - path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8') + path = from_hex_unicode(r.getheader('Tempfile')).decode('utf-8') f, fdata = share_open(path, 'rb'), data # Now force an update change_cover(1) @@ -189,7 +190,7 @@ class ContentTest(LibraryBaseTest): self.ae(r.status, http_client.OK) self.ae(data, db.cover(2)) self.ae(r.getheader('Used-Cache'), 'no') - path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8') + path = from_hex_unicode(r.getheader('Tempfile')).decode('utf-8') f2, f2data = share_open(path, 'rb'), data # Do it again change_cover(2) diff --git a/src/calibre/srv/utils.py b/src/calibre/srv/utils.py index 22f9d2b777..1e49628033 100644 --- a/src/calibre/srv/utils.py +++ b/src/calibre/srv/utils.py @@ -10,7 +10,6 @@ import errno, socket, select, os, time from contextlib import closing from email.utils import formatdate from operator import itemgetter -from binascii import hexlify, unhexlify from calibre import prints from calibre.constants import iswindows, ispy3 @@ -20,14 +19,16 @@ from calibre.utils.localization import get_translator from calibre.utils.socket_inheritance import set_socket_inherit from calibre.utils.logging import ThreadSafeLog from calibre.utils.shared_file import share_open, raise_winerror -from polyglot.builtins import iteritems, map, unicode_type, range +from polyglot.builtins import iteritems, map, range from polyglot import reprlib from polyglot.http_cookie import SimpleCookie from polyglot.urllib import parse_qs, quote as urlquote +from polyglot.binary import as_hex_unicode as encode_name, from_hex_unicode as decode_name HTTP1 = 'HTTP/1.0' HTTP11 = 'HTTP/1.1' DESIRED_SEND_BUFFER_SIZE = 16 * 1024 # windows 7 uses an 8KB sndbuf +encode_name, decode_name def http_date(timeval=None): @@ -286,17 +287,6 @@ def encode_path(*components): return '/' + '/'.join(urlquote(x.encode('utf-8'), '').decode('ascii') for x in components) -def encode_name(name): - 'Encode a name (arbitrary string) as URL safe characters. See decode_name() also.' - if isinstance(name, unicode_type): - name = name.encode('utf-8') - return hexlify(name) - - -def decode_name(name): - return unhexlify(name).decode('utf-8') - - class Cookie(SimpleCookie): def _BaseCookie__set(self, key, real_value, coded_value): diff --git a/src/calibre/utils/exim.py b/src/calibre/utils/exim.py index dd384f30d6..9011340f53 100644 --- a/src/calibre/utils/exim.py +++ b/src/calibre/utils/exim.py @@ -5,7 +5,6 @@ from __future__ import (unicode_literals, division, absolute_import, print_function) import os, json, struct, hashlib, sys, errno, tempfile, time, shutil, uuid -from binascii import hexlify from collections import Counter from calibre import prints @@ -14,6 +13,7 @@ from calibre.utils.config_base import prefs, StringConfig, create_global_prefs from calibre.utils.config import JSONConfig from calibre.utils.filenames import samefile from polyglot.builtins import iteritems, raw_input +from polyglot.binary import as_hex_unicode # Export {{{ @@ -133,7 +133,7 @@ class Exporter(object): return FileDest(key, self, mtime=mtime) def export_dir(self, path, dir_key): - pkey = hexlify(dir_key) + pkey = as_hex_unicode(dir_key) self.metadata[dir_key] = files = [] for dirpath, dirnames, filenames in os.walk(path): for fname in filenames: diff --git a/src/calibre/utils/ipc/launch.py b/src/calibre/utils/ipc/launch.py index 2c8641025b..a13869db95 100644 --- a/src/calibre/utils/ipc/launch.py +++ b/src/calibre/utils/ipc/launch.py @@ -6,14 +6,15 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import subprocess, os, sys, time, binascii +import subprocess, os, sys, time from functools import partial -from calibre.constants import iswindows, isosx, isfrozen, filesystem_encoding +from calibre.constants import iswindows, isosx, isfrozen, filesystem_encoding, ispy3 from calibre.utils.config import prefs from calibre.ptempfile import PersistentTemporaryFile, base_dir from calibre.utils.serialize import msgpack_dumps -from polyglot.builtins import iteritems, unicode_type, string_or_bytes +from polyglot.builtins import iteritems, unicode_type, string_or_bytes, environ_item +from polyglot.binary import as_hex_unicode if iswindows: import win32process @@ -88,26 +89,29 @@ class Worker(object): @property def env(self): - # We use this inefficient method of copying the environment variables - # because of non ascii env vars on windows. See https://bugs.launchpad.net/bugs/811191 - env = {} - for key in os.environ: - try: - val = os.environ[key] - if isinstance(val, unicode_type): - # On windows subprocess cannot handle unicode env vars - try: - val = val.encode(filesystem_encoding) - except ValueError: - val = val.encode('utf-8') - if isinstance(key, unicode_type): - key = key.encode('ascii') - env[key] = val - except: - pass - env[str('CALIBRE_WORKER')] = str('1') - td = binascii.hexlify(msgpack_dumps(base_dir())).decode('ascii') - env[b'CALIBRE_WORKER_TEMP_DIR'] = str(td) + if ispy3: + env = os.environ.copy() + else: + # We use this inefficient method of copying the environment variables + # because of non ascii env vars on windows. See https://bugs.launchpad.net/bugs/811191 + env = {} + for key in os.environ: + try: + val = os.environ[key] + if isinstance(val, unicode_type): + # On windows subprocess cannot handle unicode env vars + try: + val = val.encode(filesystem_encoding) + except ValueError: + val = val.encode('utf-8') + if isinstance(key, unicode_type): + key = key.encode('ascii') + env[key] = val + except: + pass + env[str('CALIBRE_WORKER')] = environ_item('1') + td = as_hex_unicode(msgpack_dumps(base_dir())) + env[str('CALIBRE_WORKER_TEMP_DIR')] = environ_item(td) env.update(self._env) return env @@ -181,7 +185,7 @@ class Worker(object): except EnvironmentError: # cwd no longer exists origwd = cwd or os.path.expanduser(u'~') - env[str('ORIGWD')] = binascii.hexlify(msgpack_dumps(origwd)) + env[str('ORIGWD')] = environ_item(as_hex_unicode(msgpack_dumps(origwd))) _cwd = cwd if priority is None: priority = prefs['worker_process_priority'] diff --git a/src/calibre/utils/ipc/server.py b/src/calibre/utils/ipc/server.py index dab19e00f0..a32a04dee4 100644 --- a/src/calibre/utils/ipc/server.py +++ b/src/calibre/utils/ipc/server.py @@ -13,7 +13,6 @@ import os import sys import tempfile import time -from binascii import hexlify from collections import deque from math import ceil from multiprocessing.connection import Listener, arbitrary_address @@ -28,6 +27,7 @@ from calibre.utils.ipc.worker import PARALLEL_FUNCS from calibre.utils.serialize import msgpack_dumps, pickle_loads from polyglot.builtins import string_or_bytes, environ_item from polyglot.queue import Empty, Queue +from polyglot.binary import as_hex_unicode _counter = 0 @@ -219,10 +219,10 @@ class Server(Thread): redirect_output = not gui env = { - 'CALIBRE_WORKER_ADDRESS' : environ_item(hexlify(msgpack_dumps( + 'CALIBRE_WORKER_ADDRESS' : environ_item(as_hex_unicode(msgpack_dumps( self.listener.address))), - 'CALIBRE_WORKER_KEY' : environ_item(hexlify(self.auth_key)), - 'CALIBRE_WORKER_RESULT' : environ_item(hexlify(rfile.encode('utf-8'))), + 'CALIBRE_WORKER_KEY' : environ_item(as_hex_unicode(self.auth_key)), + 'CALIBRE_WORKER_RESULT' : environ_item(as_hex_unicode(rfile)), } cw = self.do_launch(env, gui, redirect_output, rfile, job_name=job_name) if isinstance(cw, string_or_bytes): diff --git a/src/calibre/utils/ipc/simple_worker.py b/src/calibre/utils/ipc/simple_worker.py index 1a8b91ab4b..26e4173653 100644 --- a/src/calibre/utils/ipc/simple_worker.py +++ b/src/calibre/utils/ipc/simple_worker.py @@ -8,7 +8,6 @@ __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' import os, traceback, time, importlib -from binascii import hexlify, unhexlify from multiprocessing.connection import Client from threading import Thread from contextlib import closing @@ -18,6 +17,7 @@ from calibre.utils.ipc import eintr_retry_call from calibre.utils.ipc.launch import Worker from calibre.utils.serialize import msgpack_loads, msgpack_dumps from polyglot.builtins import unicode_type, string_or_bytes, environ_item +from polyglot.binary import as_hex_unicode, from_hex_bytes class WorkerError(Exception): @@ -131,8 +131,8 @@ def create_worker(env, priority='normal', cwd=None, func='main'): env = dict(env) env.update({ - 'CALIBRE_WORKER_ADDRESS': environ_item(hexlify(msgpack_dumps(listener.address))), - 'CALIBRE_WORKER_KEY': environ_item(hexlify(auth_key)), + 'CALIBRE_WORKER_ADDRESS': environ_item(as_hex_unicode(msgpack_dumps(listener.address))), + 'CALIBRE_WORKER_KEY': environ_item(as_hex_unicode(auth_key)), 'CALIBRE_SIMPLE_WORKER': environ_item('calibre.utils.ipc.simple_worker:%s' % func), }) @@ -271,8 +271,8 @@ def compile_code(src): def main(): # The entry point for the simple worker process - address = msgpack_loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS'])) - key = unhexlify(os.environ['CALIBRE_WORKER_KEY']) + address = msgpack_loads(from_hex_bytes(os.environ['CALIBRE_WORKER_ADDRESS'])) + key = from_hex_bytes(os.environ['CALIBRE_WORKER_KEY']) with closing(Client(address, authkey=key)) as conn: args = eintr_retry_call(conn.recv) try: @@ -301,8 +301,8 @@ def main(): def offload(): # The entry point for the offload worker process - address = msgpack_loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS'])) - key = unhexlify(os.environ['CALIBRE_WORKER_KEY']) + address = msgpack_loads(from_hex_bytes(os.environ['CALIBRE_WORKER_ADDRESS'])) + key = from_hex_bytes(os.environ['CALIBRE_WORKER_KEY']) func_cache = {} with closing(Client(address, authkey=key)) as conn: while True: diff --git a/src/calibre/utils/ipc/worker.py b/src/calibre/utils/ipc/worker.py index a244b5e3f5..66f1cf8452 100644 --- a/src/calibre/utils/ipc/worker.py +++ b/src/calibre/utils/ipc/worker.py @@ -11,7 +11,6 @@ import os, sys, importlib from multiprocessing.connection import Client from threading import Thread from contextlib import closing -from binascii import unhexlify from zipimport import ZipImportError from calibre import prints @@ -19,6 +18,7 @@ from calibre.constants import iswindows, isosx from calibre.utils.ipc import eintr_retry_call from calibre.utils.serialize import msgpack_loads, pickle_dumps from polyglot.queue import Queue +from polyglot.binary import from_hex_bytes, from_hex_unicode PARALLEL_FUNCS = { 'lrfviewer' : @@ -183,9 +183,9 @@ def main(): print('Failed to run pipe worker with command:', sys.argv[-1]) raise return - address = msgpack_loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS'])) - key = unhexlify(os.environ['CALIBRE_WORKER_KEY']) - resultf = unhexlify(os.environ['CALIBRE_WORKER_RESULT']).decode('utf-8') + address = msgpack_loads(from_hex_bytes(os.environ['CALIBRE_WORKER_ADDRESS'])) + key = from_hex_bytes(os.environ['CALIBRE_WORKER_KEY']) + resultf = from_hex_unicode(os.environ['CALIBRE_WORKER_RESULT']) with closing(Client(address, authkey=key)) as conn: name, args, kwargs, desc = eintr_retry_call(conn.recv) if desc: diff --git a/src/polyglot/binary.py b/src/polyglot/binary.py index 7631b13bc8..360ac0b3ac 100644 --- a/src/polyglot/binary.py +++ b/src/polyglot/binary.py @@ -4,7 +4,9 @@ from __future__ import absolute_import, division, print_function, unicode_literals -from base64 import standard_b64encode, standard_b64decode +from base64 import standard_b64decode, standard_b64encode +from binascii import hexlify, unhexlify + from polyglot.builtins import unicode_type @@ -30,3 +32,27 @@ def from_base64_bytes(x): if isinstance(x, unicode_type): x = x.encode('ascii') return standard_b64decode(x) + + +def as_hex_bytes(x, enc='utf-8'): + if isinstance(x, unicode_type): + x = x.encode(enc) + return hexlify(x) + + +def as_hex_unicode(x, enc='utf-8'): + if isinstance(x, unicode_type): + x = x.encode(enc) + return hexlify(x).decode('ascii') + + +def from_hex_unicode(x, enc='utf-8'): + if isinstance(x, unicode_type): + x = x.encode('ascii') + return unhexlify(x).decode(enc) + + +def from_hex_bytes(x): + if isinstance(x, unicode_type): + x = x.encode('ascii') + return unhexlify(x) diff --git a/src/tinycss/decoding.py b/src/tinycss/decoding.py index 6303e1afda..32e1799c48 100644 --- a/src/tinycss/decoding.py +++ b/src/tinycss/decoding.py @@ -12,10 +12,10 @@ from __future__ import unicode_literals -from binascii import unhexlify import operator import re -import sys + +from polyglot.binary import from_hex_bytes __all__ = ['decode'] # Everything else is implementation detail @@ -98,7 +98,7 @@ def try_encoding(css_bytes, encoding, fallback=True): def hex2re(hex_data): - return re.escape(unhexlify(hex_data.replace(' ', '').encode('ascii'))) + return re.escape(from_hex_bytes(hex_data.replace(' ', '').encode('ascii'))) class Slicer(object): @@ -106,6 +106,7 @@ class Slicer(object): def __getitem__(self, slice_): return operator.itemgetter(slice_) + Slice = Slicer()