Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

py3: Port use of (un)hexlify()

Browse Source
This commit is contained in:
Kovid Goyal 2019-03-30 14:34:48 +05:30
parent fcd5700306
commit f149427104
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
24 changed files with 142 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/calibre/db/cache.py
View File

@ -2206,8 +2206,8 @@ class Cache(object):
@read_api
def export_library(self, library_key, exporter, progress=None, abort=None):
from binascii import hexlify
key_prefix = hexlify(library_key)
from polyglot.binary import as_hex_unicode
key_prefix = as_hex_unicode(library_key)
book_ids = self._all_book_ids()
total = len(book_ids) + 1
format_metadata = {}

5
src/calibre/ebooks/conversion/plugins/epub_output.py
View File

@ -311,13 +311,12 @@ class EPUBOutput(OutputFormatPlugin):
pass
def encrypt_fonts(self, uris, tdir, uuid): # {{{
from binascii import unhexlify
from polyglot.binary import from_hex_bytes
key = re.sub(r'[^a-fA-F0-9]', '', uuid)
if len(key) < 16:
raise ValueError('UUID identifier %r is invalid'%uuid)
key = unhexlify((key + key)[:32])
key = tuple(map(ord, key))
key = bytearray(from_hex_bytes((key + key)[:32]))
paths = []
with CurrentDir(tdir):
paths = [os.path.join(*x.split('/')) for x in uris]

6
src/calibre/ebooks/metadata/book/render.py
View File

@ -8,7 +8,6 @@ __copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
import os
from functools import partial
from binascii import hexlify
from calibre import prepare_string_for_xml, force_unicode
from calibre.ebooks.metadata import fmt_sidx, rating_to_stars
@ -22,6 +21,7 @@ from calibre.utils.date import is_date_undefined
from calibre.utils.localization import calibre_langcode_to_name
from calibre.utils.serialize import json_dumps
from polyglot.builtins import unicode_type
from polyglot.binary import as_hex_unicode
default_sort = ('title', 'title_sort', 'authors', 'author_sort', 'series', 'rating', 'pubdate', 'tags', 'publisher', 'identifiers')
@ -55,7 +55,7 @@ def get_field_list(mi):
def search_href(search_term, value):
search = '%s:"=%s"' % (search_term, value.replace('"', '\\"'))
return prepare_string_for_xml('search:' + hexlify(search.encode('utf-8')), True)
return prepare_string_for_xml('search:' + as_hex_unicode(search.encode('utf-8')), True)
DEFAULT_AUTHOR_LINK = 'search-{}'.format(DEFAULT_AUTHOR_SOURCE)
@ -80,7 +80,7 @@ def author_search_href(which, title=None, author=None):
def item_data(field_name, value, book_id):
return hexlify(json_dumps((field_name, value, book_id)))
return as_hex_unicode(json_dumps((field_name, value, book_id)))
def mi_to_html(mi, field_list=None, default_author_link=None, use_roman_numbers=True, rating_font='Liberation Serif', rtl=False):

4
src/calibre/ebooks/pdf/render/common.py
View File

@ -10,11 +10,11 @@ __docformat__ = 'restructuredtext en'
import codecs, zlib, numbers
from io import BytesIO
from datetime import datetime
from binascii import hexlify
from calibre.constants import plugins, ispy3
from calibre.utils.logging import default_log
from polyglot.builtins import iteritems, unicode_type
from polyglot.binary import as_hex_bytes
pdf_float = plugins['speedup'][0].pdf_float
@ -137,7 +137,7 @@ class UTF16String(unicode_type):
if False:
# Disabled as the parentheses based strings give easier to debug
# PDF files
stream.write(b'<' + hexlify(raw) + b'>')
stream.write(b'<' + as_hex_bytes(raw) + b'>')
else:
stream.write(b'('+escape_pdf_string(raw)+b')')

7
src/calibre/gui2/book_details.py
View File

@ -4,7 +4,6 @@
import os
import re
from binascii import unhexlify
from collections import namedtuple
from functools import partial
@ -35,6 +34,8 @@ from calibre.utils.img import blend_image, image_from_x
from calibre.utils.localization import is_rtl
from calibre.utils.serialize import json_loads
from polyglot.builtins import unicode_type
from polyglot.binary import from_hex_bytes, from_hex_unicode
_css = None
InternetSearch = namedtuple('InternetSearch', 'author where')
@ -286,7 +287,7 @@ def details_context_menu_event(view, ev, book_info): # {{{
lambda : book_info.search_requested('authors:"={}"'.format(author.replace('"', r'\"'))))
if data:
try:
field, value, book_id = json_loads(unhexlify(data))
field, value, book_id = json_loads(from_hex_bytes(data))
except Exception:
field = value = book_id = None
if field:
@ -874,7 +875,7 @@ class BookDetails(QWidget): # {{{
elif typ == 'devpath':
self.view_device_book.emit(val)
elif typ == 'search':
self.search_requested.emit(unhexlify(val).decode('utf-8'))
self.search_requested.emit(from_hex_unicode(val))
else:
try:
open_url(QUrl(link, QUrl.TolerantMode))

4
src/calibre/gui2/email.py
View File

@ -7,7 +7,6 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import os, socket, time, textwrap
from binascii import unhexlify
from functools import partial
from threading import Thread
from itertools import repeat
@ -29,6 +28,7 @@ from calibre.utils.config import tweaks, prefs
from calibre.utils.icu import primary_sort_key
from calibre.gui2.threaded_jobs import ThreadedJob
from polyglot.builtins import iteritems, itervalues, unicode_type
from polyglot.binary import from_hex_unicode
class Worker(Thread):
@ -130,7 +130,7 @@ class Sendmail(object):
verbose=1,
relay=opts.relay_host,
username=opts.relay_username,
password=unhexlify(opts.relay_password).decode('utf-8'), port=opts.relay_port,
password=from_hex_unicode(opts.relay_password), port=opts.relay_port,
encryption=opts.encryption,
debug_output=safe_debug)
finally:

10
src/calibre/gui2/main.py
View File

@ -28,7 +28,7 @@ from calibre.utils.config import dynamic, prefs
from calibre.utils.ipc import RC, gui_socket_address
from calibre.utils.lock import singleinstance
from calibre.utils.monotonic import monotonic
from polyglot.builtins import unicode_type, range
from polyglot.builtins import unicode_type, range, environ_item
if iswindows:
winutil = plugins['winutil'][0]
@ -173,19 +173,19 @@ def repair_library(library_path):
def windows_repair(library_path=None):
from binascii import hexlify, unhexlify
import subprocess
from calibre.utils.serialize import json_dumps, json_loads
from polyglot.binary import as_hex_unicode, from_hex_bytes
if library_path:
library_path = hexlify(json_dumps(library_path))
library_path = as_hex_unicode(json_dumps(library_path))
winutil.prepare_for_restart()
os.environ['CALIBRE_REPAIR_CORRUPTED_DB'] = library_path
os.environ['CALIBRE_REPAIR_CORRUPTED_DB'] = environ_item(library_path)
subprocess.Popen([sys.executable])
else:
try:
app = Application([])
from calibre.gui2.dialogs.restore_library import repair_library_at
library_path = json_loads(unhexlify(os.environ.pop('CALIBRE_REPAIR_CORRUPTED_DB')))
library_path = json_loads(from_hex_bytes(os.environ.pop('CALIBRE_REPAIR_CORRUPTED_DB')))
done = repair_library_at(library_path, wait_time=4)
except Exception:
done = False

4
src/calibre/gui2/tweak_book/file_list.py
View File

@ -7,7 +7,6 @@ import os
import posixpath
import sys
import textwrap
from binascii import hexlify
from collections import Counter, OrderedDict, defaultdict
from functools import partial
@ -40,6 +39,7 @@ from calibre.gui2.tweak_book.editor import syntax_from_mime
from calibre.gui2.tweak_book.templates import template_for
from calibre.utils.icu import numeric_sort_key
from polyglot.builtins import iteritems, itervalues, unicode_type, range
from polyglot.binary import as_hex_unicode
try:
from PyQt5 import sip
@ -392,7 +392,7 @@ class FileList(QTreeWidget):
seen[text] = item
item.setText(0, text)
item.setText(1, hexlify(numeric_sort_key(text)))
item.setText(1, as_hex_unicode(numeric_sort_key(text)))
def render_emblems(item, emblems):
emblems = tuple(emblems)

7
src/calibre/gui2/update.py
View File

@ -1,7 +1,7 @@
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
import re, binascii, ssl, json
import re, ssl, json
from polyglot.builtins import map, unicode_type
from threading import Thread, Event
@ -17,6 +17,7 @@ from calibre.utils.https import get_https_resource_securely
from calibre.gui2 import config, dynamic, open_url
from calibre.gui2.dialogs.plugin_updater import get_plugin_updates_available
from calibre.utils.serialize import msgpack_dumps, msgpack_loads
from polyglot.binary import as_hex_unicode, from_hex_bytes
URL = 'https://code.calibre-ebook.com/latest'
# URL = 'http://localhost:8000/latest'
@ -209,7 +210,7 @@ class UpdateMixin(object):
has_calibre_update = calibre_version != NO_CALIBRE_UPDATE
has_plugin_updates = number_of_plugin_updates > 0
self.plugin_update_found(number_of_plugin_updates)
version_url = binascii.hexlify(msgpack_dumps((calibre_version, number_of_plugin_updates)))
version_url = as_hex_unicode(msgpack_dumps((calibre_version, number_of_plugin_updates)))
calibre_version = u'.'.join(map(unicode_type, calibre_version))
if not has_calibre_update and not has_plugin_updates:
@ -263,7 +264,7 @@ class UpdateMixin(object):
def update_link_clicked(self, url):
url = unicode_type(url)
if url.startswith('update:'):
calibre_version, number_of_plugin_updates = msgpack_loads(binascii.unhexlify(url[len('update:'):]))
calibre_version, number_of_plugin_updates = msgpack_loads(from_hex_bytes(url[len('update:'):]))
self.update_found(calibre_version, number_of_plugin_updates, force=True)

10
src/calibre/gui2/wizard/send_email.py
View File

@ -7,7 +7,6 @@ __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import cStringIO, sys
from binascii import hexlify, unhexlify
from functools import partial
from threading import Thread
@ -21,6 +20,7 @@ from calibre.gui2.wizard.send_email_ui import Ui_Form
from calibre.utils.smtp import config as smtp_prefs
from calibre.gui2 import error_dialog, question_dialog
from polyglot.builtins import unicode_type
from polyglot.binary import as_hex_unicode, from_hex_unicode
class TestEmail(QDialog):
@ -47,7 +47,7 @@ class TestEmail(QDialog):
l.addLayout(h)
if opts.relay_host:
self.la = la = QLabel(_('Using: %(un)s:%(pw)s@%(host)s:%(port)s and %(enc)s encryption')%
dict(un=opts.relay_username, pw=unhexlify(opts.relay_password).decode('utf-8'),
dict(un=opts.relay_username, pw=from_hex_unicode(opts.relay_password),
host=opts.relay_host, port=opts.relay_port, enc=opts.encryption))
l.addWidget(la)
self.log = QPlainTextEdit(self)
@ -163,7 +163,7 @@ class SendEmail(QWidget, Ui_Form):
self.relay_username.setText(opts.relay_username)
self.relay_username.textChanged.connect(self.changed)
if opts.relay_password:
self.relay_password.setText(unhexlify(opts.relay_password).decode('utf-8'))
self.relay_password.setText(from_hex_unicode(opts.relay_password))
self.relay_password.textChanged.connect(self.changed)
getattr(self, 'relay_'+opts.encryption.lower()).setChecked(True)
self.relay_tls.toggled.connect(self.changed)
@ -204,7 +204,7 @@ class SendEmail(QWidget, Ui_Form):
sendmail(msg, from_=opts.from_, to=[to],
verbose=3, timeout=30, relay=opts.relay_host,
username=opts.relay_username, debug_output=debug_out,
password=unhexlify(opts.relay_password).decode('utf-8'),
password=from_hex_unicode(opts.relay_password),
encryption=opts.encryption, port=opts.relay_port)
except:
import traceback
@ -291,6 +291,6 @@ class SendEmail(QWidget, Ui_Form):
conf.set('relay_host', host if host else None)
conf.set('relay_port', self.relay_port.value())
conf.set('relay_username', username if username else None)
conf.set('relay_password', hexlify(password.encode('utf-8')))
conf.set('relay_password', as_hex_unicode(password))
conf.set('encryption', enc_method)
return True

19
src/calibre/library/coloring.py
View File

@ -1,16 +1,15 @@
#!/usr/bin/env python2
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
from __future__ import (unicode_literals, division, absolute_import,
print_function)
from polyglot.builtins import map
# License: GPLv3 Copyright: 2011, Kovid Goyal <kovid at kovidgoyal.net>
from __future__ import absolute_import, division, print_function, unicode_literals
__license__ = 'GPL v3'
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import binascii, re, json
import json
import re
from textwrap import dedent
from polyglot.binary import as_hex_unicode, from_hex_bytes
from polyglot.builtins import map
color_row_key = '*row'
@ -55,7 +54,7 @@ class Rule(object): # {{{
def signature(self):
args = (self.color, self.conditions)
sig = json.dumps(args, ensure_ascii=False)
return self.SIGNATURE + binascii.hexlify(sig.encode('utf-8'))
return self.SIGNATURE + as_hex_unicode(sig)
@property
def template(self):
@ -213,7 +212,7 @@ def rule_from_template(fm, template):
if line.startswith(Rule.SIGNATURE):
raw = line[len(Rule.SIGNATURE):].strip()
try:
color, conditions = json.loads(binascii.unhexlify(raw).decode('utf-8'))
color, conditions = json.loads(from_hex_bytes(raw))
except:
continue
r = Rule(fm)

4
src/calibre/ptempfile.py
View File

@ -105,10 +105,10 @@ def base_dir():
if _base_dir is None:
td = os.environ.get('CALIBRE_WORKER_TEMP_DIR', None)
if td is not None:
import binascii
from calibre.utils.serialize import msgpack_loads
from polyglot.binary import from_hex_bytes
try:
td = msgpack_loads(binascii.unhexlify(td))
td = msgpack_loads(from_hex_bytes(td))
except Exception:
td = None
if td and os.path.exists(td):

10
src/calibre/srv/auth.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import binascii, os, random, struct
import os, random, struct
from collections import OrderedDict
from hashlib import md5, sha256
from itertools import permutations
@ -17,7 +17,7 @@ from calibre.srv.http_request import parse_uri
from calibre.srv.utils import parse_http_dict, encode_path
from calibre.utils.monotonic import monotonic
from polyglot import http_client
from polyglot.binary import from_base64_unicode
from polyglot.binary import from_base64_unicode, from_hex_bytes, as_hex_unicode
MAX_AGE_SECONDS = 3600
nonce_counter, nonce_counter_lock = 0, Lock()
@ -94,7 +94,7 @@ def synthesize_nonce(key_order, realm, secret, timestamp=None):
# The resolution of monotonic() on windows is very low (10s of
# milliseconds) so to ensure nonce values are not re-used, we have a
# global counter
timestamp = binascii.hexlify(struct.pack(b'!dH', float(monotonic()), nonce_counter))
timestamp = as_hex_unicode(struct.pack(b'!dH', float(monotonic()), nonce_counter))
h = sha256_hex(key_order.format(timestamp, realm, secret))
nonce = ':'.join((timestamp, h))
return nonce
@ -108,7 +108,7 @@ def validate_nonce(key_order, nonce, realm, secret):
def is_nonce_stale(nonce, max_age_seconds=MAX_AGE_SECONDS):
try:
timestamp = struct.unpack(b'!dH', binascii.unhexlify(as_bytestring(nonce.partition(':')[0])))[0]
timestamp = struct.unpack(b'!dH', from_hex_bytes(as_bytestring(nonce.partition(':')[0])))[0]
return timestamp + max_age_seconds < monotonic()
except Exception:
pass
@ -243,7 +243,7 @@ class AuthController(object):
self.user_credentials, self.prefer_basic_auth = user_credentials, prefer_basic_auth
self.ban_list = BanList(ban_time_in_minutes=ban_time_in_minutes, max_failures_before_ban=ban_after)
self.log = log
self.secret = binascii.hexlify(os.urandom(random.randint(20, 30))).decode('ascii')
self.secret = as_hex_unicode(os.urandom(random.randint(20, 30)))
self.max_age_seconds = max_age_seconds
self.key_order = '{%d}:{%d}:{%d}' % random.choice(tuple(permutations((0,1,2))))
self.realm = realm

4
src/calibre/srv/content.py
View File

@ -7,7 +7,6 @@ __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import os, errno
from binascii import hexlify
from io import BytesIO
from threading import Lock
from polyglot.builtins import map
@ -30,6 +29,7 @@ from calibre.utils.img import scale_image, image_from_data
from calibre.utils.filenames import ascii_filename, atomic_rename
from calibre.utils.shared_file import share_open
from polyglot.urllib import quote
from polyglot.binary import as_hex_unicode
plugboard_content_server_value = 'content_server'
plugboard_content_server_formats = ['epub', 'mobi', 'azw3']
@ -111,7 +111,7 @@ def create_file_copy(ctx, rd, prefix, library_id, book_id, ext, mtime, copy_func
ans.seek(0)
if ctx.testing:
rd.outheaders['Used-Cache'] = used_cache
rd.outheaders['Tempfile'] = hexlify(fname.encode('utf-8'))
rd.outheaders['Tempfile'] = as_hex_unicode(fname)
return rd.filesystem_file_with_custom_etag(ans, prefix, library_id, book_id, mtime, extra_etag_data)

29
src/calibre/srv/opds.py
View File

@ -5,7 +5,7 @@ __license__ = 'GPL v3'
__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import hashlib, binascii
import hashlib
from functools import partial
from collections import OrderedDict, namedtuple
@ -28,16 +28,7 @@ from calibre.srv.routes import endpoint
from calibre.srv.utils import get_library_data, http_date, Offsets
from polyglot.builtins import iteritems, unicode_type
from polyglot.urllib import urlencode
def hexlify(x):
if isinstance(x, unicode_type):
x = x.encode('utf-8')
return binascii.hexlify(x)
def unhexlify(x):
return binascii.unhexlify(x).decode('utf-8')
from polyglot.binary import as_hex_unicode, from_hex_unicode
def atom(ctx, rd, endpoint, output):
@ -105,7 +96,7 @@ SUBTITLE = E.subtitle
def NAVCATALOG_ENTRY(url_for, updated, title, description, query):
href = url_for('/opds/navcatalog', which=hexlify(query))
href = url_for('/opds/navcatalog', which=as_hex_unicode(query))
id_ = 'calibre-navcatalog:'+str(hashlib.sha1(href).hexdigest())
return E.entry(
TITLE(title),
@ -154,7 +145,7 @@ def CATALOG_ENTRY(item, item_kind, request_context, updated, catalog_name,
if item.id is not None:
iid = 'I' + str(item.id)
iid += ':'+item_kind
href = request_context.url_for('/opds/category', category=hexlify(catalog_name), which=hexlify(iid))
href = request_context.url_for('/opds/category', category=as_hex_unicode(catalog_name), which=as_hex_unicode(iid))
link = NAVLINK(href=href)
if ignore_count:
count = ''
@ -176,7 +167,7 @@ def CATALOG_ENTRY(item, item_kind, request_context, updated, catalog_name,
def CATALOG_GROUP_ENTRY(item, category, request_context, updated):
id_ = 'calibre:category-group:'+category+':'+item.text
iid = item.text
link = NAVLINK(href=request_context.url_for('/opds/categorygroup', category=hexlify(category), which=hexlify(iid)))
link = NAVLINK(href=request_context.url_for('/opds/categorygroup', category=as_hex_unicode(category), which=as_hex_unicode(iid)))
return E.entry(
TITLE(item.text),
ID(id_),
@ -519,7 +510,7 @@ def opds_navcatalog(ctx, rd, which):
page_url = rc.url_for('/opds/navcatalog', which=which)
up_url = rc.url_for('/opds')
which = unhexlify(which)
which = from_hex_unicode(which)
type_ = which[0]
which = which[1:]
if type_ == 'O':
@ -542,7 +533,7 @@ def opds_category(ctx, rd, category, which):
page_url = rc.url_for('/opds/category', which=which, category=category)
up_url = rc.url_for('/opds/navcatalog', which=category)
which, category = unhexlify(which), unhexlify(category)
which, category = from_hex_unicode(which), from_hex_unicode(category)
type_ = which[0]
which = which[1:]
if type_ == 'I':
@ -594,15 +585,15 @@ def opds_categorygroup(ctx, rd, category, which):
categories = rc.get_categories()
page_url = rc.url_for('/opds/categorygroup', category=category, which=which)
category = unhexlify(category)
category = from_hex_unicode(category)
if category not in categories:
raise HTTPNotFound('Category %r not found'%which)
category_meta = rc.db.field_metadata
meta = category_meta.get(category, {})
category_name = meta.get('name', which)
which = unhexlify(which)
which = from_hex_unicode(which)
feed_title = default_feed_title + ' :: ' + (_('By {0} :: {1}').format(category_name, which))
owhich = hexlify('N'+which)
owhich = as_hex_unicode('N'+which)
up_url = rc.url_for('/opds/navcatalog', which=owhich)
items = categories[category]

7
src/calibre/srv/tests/content.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import zlib, json, binascii, time, os
import zlib, json, time, os
from io import BytesIO
from calibre.ebooks.metadata.epub import get_metadata
@ -15,6 +15,7 @@ from calibre.srv.tests.base import LibraryBaseTest
from calibre.utils.imghdr import identify
from calibre.utils.shared_file import share_open
from polyglot import http_client
from polyglot.binary import from_hex_unicode
def setUpModule():
@ -181,7 +182,7 @@ class ContentTest(LibraryBaseTest):
self.ae(r.status, http_client.OK)
self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no')
path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8')
path = from_hex_unicode(r.getheader('Tempfile')).decode('utf-8')
f, fdata = share_open(path, 'rb'), data
# Now force an update
change_cover(1)
@ -189,7 +190,7 @@ class ContentTest(LibraryBaseTest):
self.ae(r.status, http_client.OK)
self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no')
path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8')
path = from_hex_unicode(r.getheader('Tempfile')).decode('utf-8')
f2, f2data = share_open(path, 'rb'), data
# Do it again
change_cover(2)

16
src/calibre/srv/utils.py
View File

@ -10,7 +10,6 @@ import errno, socket, select, os, time
from contextlib import closing
from email.utils import formatdate
from operator import itemgetter
from binascii import hexlify, unhexlify
from calibre import prints
from calibre.constants import iswindows, ispy3
@ -20,14 +19,16 @@ from calibre.utils.localization import get_translator
from calibre.utils.socket_inheritance import set_socket_inherit
from calibre.utils.logging import ThreadSafeLog
from calibre.utils.shared_file import share_open, raise_winerror
from polyglot.builtins import iteritems, map, unicode_type, range
from polyglot.builtins import iteritems, map, range
from polyglot import reprlib
from polyglot.http_cookie import SimpleCookie
from polyglot.urllib import parse_qs, quote as urlquote
from polyglot.binary import as_hex_unicode as encode_name, from_hex_unicode as decode_name
HTTP1 = 'HTTP/1.0'
HTTP11 = 'HTTP/1.1'
DESIRED_SEND_BUFFER_SIZE = 16 * 1024 # windows 7 uses an 8KB sndbuf
encode_name, decode_name
def http_date(timeval=None):
@ -286,17 +287,6 @@ def encode_path(*components):
return '/' + '/'.join(urlquote(x.encode('utf-8'), '').decode('ascii') for x in components)
def encode_name(name):
'Encode a name (arbitrary string) as URL safe characters. See decode_name() also.'
if isinstance(name, unicode_type):
name = name.encode('utf-8')
return hexlify(name)
def decode_name(name):
return unhexlify(name).decode('utf-8')
class Cookie(SimpleCookie):
def _BaseCookie__set(self, key, real_value, coded_value):

4
src/calibre/utils/exim.py
View File

@ -5,7 +5,6 @@
from __future__ import (unicode_literals, division, absolute_import,
print_function)
import os, json, struct, hashlib, sys, errno, tempfile, time, shutil, uuid
from binascii import hexlify
from collections import Counter
from calibre import prints
@ -14,6 +13,7 @@ from calibre.utils.config_base import prefs, StringConfig, create_global_prefs
from calibre.utils.config import JSONConfig
from calibre.utils.filenames import samefile
from polyglot.builtins import iteritems, raw_input
from polyglot.binary import as_hex_unicode
# Export {{{
@ -133,7 +133,7 @@ class Exporter(object):
return FileDest(key, self, mtime=mtime)
def export_dir(self, path, dir_key):
pkey = hexlify(dir_key)
pkey = as_hex_unicode(dir_key)
self.metadata[dir_key] = files = []
for dirpath, dirnames, filenames in os.walk(path):
for fname in filenames:

52
src/calibre/utils/ipc/launch.py
View File

@ -6,14 +6,15 @@ __license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import subprocess, os, sys, time, binascii
import subprocess, os, sys, time
from functools import partial
from calibre.constants import iswindows, isosx, isfrozen, filesystem_encoding
from calibre.constants import iswindows, isosx, isfrozen, filesystem_encoding, ispy3
from calibre.utils.config import prefs
from calibre.ptempfile import PersistentTemporaryFile, base_dir
from calibre.utils.serialize import msgpack_dumps
from polyglot.builtins import iteritems, unicode_type, string_or_bytes
from polyglot.builtins import iteritems, unicode_type, string_or_bytes, environ_item
from polyglot.binary import as_hex_unicode
if iswindows:
import win32process
@ -88,26 +89,29 @@ class Worker(object):
@property
def env(self):
# We use this inefficient method of copying the environment variables
# because of non ascii env vars on windows. See https://bugs.launchpad.net/bugs/811191
env = {}
for key in os.environ:
try:
val = os.environ[key]
if isinstance(val, unicode_type):
# On windows subprocess cannot handle unicode env vars
try:
val = val.encode(filesystem_encoding)
except ValueError:
val = val.encode('utf-8')
if isinstance(key, unicode_type):
key = key.encode('ascii')
env[key] = val
except:
pass
env[str('CALIBRE_WORKER')] = str('1')
td = binascii.hexlify(msgpack_dumps(base_dir())).decode('ascii')
env[b'CALIBRE_WORKER_TEMP_DIR'] = str(td)
if ispy3:
env = os.environ.copy()
else:
# We use this inefficient method of copying the environment variables
# because of non ascii env vars on windows. See https://bugs.launchpad.net/bugs/811191
env = {}
for key in os.environ:
try:
val = os.environ[key]
if isinstance(val, unicode_type):
# On windows subprocess cannot handle unicode env vars
try:
val = val.encode(filesystem_encoding)
except ValueError:
val = val.encode('utf-8')
if isinstance(key, unicode_type):
key = key.encode('ascii')
env[key] = val
except:
pass
env[str('CALIBRE_WORKER')] = environ_item('1')
td = as_hex_unicode(msgpack_dumps(base_dir()))
env[str('CALIBRE_WORKER_TEMP_DIR')] = environ_item(td)
env.update(self._env)
return env
@ -181,7 +185,7 @@ class Worker(object):
except EnvironmentError:
# cwd no longer exists
origwd = cwd or os.path.expanduser(u'~')
env[str('ORIGWD')] = binascii.hexlify(msgpack_dumps(origwd))
env[str('ORIGWD')] = environ_item(as_hex_unicode(msgpack_dumps(origwd)))
_cwd = cwd
if priority is None:
priority = prefs['worker_process_priority']

8
src/calibre/utils/ipc/server.py
View File

@ -13,7 +13,6 @@ import os
import sys
import tempfile
import time
from binascii import hexlify
from collections import deque
from math import ceil
from multiprocessing.connection import Listener, arbitrary_address
@ -28,6 +27,7 @@ from calibre.utils.ipc.worker import PARALLEL_FUNCS
from calibre.utils.serialize import msgpack_dumps, pickle_loads
from polyglot.builtins import string_or_bytes, environ_item
from polyglot.queue import Empty, Queue
from polyglot.binary import as_hex_unicode
_counter = 0
@ -219,10 +219,10 @@ class Server(Thread):
redirect_output = not gui
env = {
'CALIBRE_WORKER_ADDRESS' : environ_item(hexlify(msgpack_dumps(
'CALIBRE_WORKER_ADDRESS' : environ_item(as_hex_unicode(msgpack_dumps(
self.listener.address))),
'CALIBRE_WORKER_KEY' : environ_item(hexlify(self.auth_key)),
'CALIBRE_WORKER_RESULT' : environ_item(hexlify(rfile.encode('utf-8'))),
'CALIBRE_WORKER_KEY' : environ_item(as_hex_unicode(self.auth_key)),
'CALIBRE_WORKER_RESULT' : environ_item(as_hex_unicode(rfile)),
}
cw = self.do_launch(env, gui, redirect_output, rfile, job_name=job_name)
if isinstance(cw, string_or_bytes):

14
src/calibre/utils/ipc/simple_worker.py
View File

@ -8,7 +8,6 @@ __copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import os, traceback, time, importlib
from binascii import hexlify, unhexlify
from multiprocessing.connection import Client
from threading import Thread
from contextlib import closing
@ -18,6 +17,7 @@ from calibre.utils.ipc import eintr_retry_call
from calibre.utils.ipc.launch import Worker
from calibre.utils.serialize import msgpack_loads, msgpack_dumps
from polyglot.builtins import unicode_type, string_or_bytes, environ_item
from polyglot.binary import as_hex_unicode, from_hex_bytes
class WorkerError(Exception):
@ -131,8 +131,8 @@ def create_worker(env, priority='normal', cwd=None, func='main'):
env = dict(env)
env.update({
'CALIBRE_WORKER_ADDRESS': environ_item(hexlify(msgpack_dumps(listener.address))),
'CALIBRE_WORKER_KEY': environ_item(hexlify(auth_key)),
'CALIBRE_WORKER_ADDRESS': environ_item(as_hex_unicode(msgpack_dumps(listener.address))),
'CALIBRE_WORKER_KEY': environ_item(as_hex_unicode(auth_key)),
'CALIBRE_SIMPLE_WORKER': environ_item('calibre.utils.ipc.simple_worker:%s' % func),
})
@ -271,8 +271,8 @@ def compile_code(src):
def main():
# The entry point for the simple worker process
address = msgpack_loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS']))
key = unhexlify(os.environ['CALIBRE_WORKER_KEY'])
address = msgpack_loads(from_hex_bytes(os.environ['CALIBRE_WORKER_ADDRESS']))
key = from_hex_bytes(os.environ['CALIBRE_WORKER_KEY'])
with closing(Client(address, authkey=key)) as conn:
args = eintr_retry_call(conn.recv)
try:
@ -301,8 +301,8 @@ def main():
def offload():
# The entry point for the offload worker process
address = msgpack_loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS']))
key = unhexlify(os.environ['CALIBRE_WORKER_KEY'])
address = msgpack_loads(from_hex_bytes(os.environ['CALIBRE_WORKER_ADDRESS']))
key = from_hex_bytes(os.environ['CALIBRE_WORKER_KEY'])
func_cache = {}
with closing(Client(address, authkey=key)) as conn:
while True:

8
src/calibre/utils/ipc/worker.py
View File

@ -11,7 +11,6 @@ import os, sys, importlib
from multiprocessing.connection import Client
from threading import Thread
from contextlib import closing
from binascii import unhexlify
from zipimport import ZipImportError
from calibre import prints
@ -19,6 +18,7 @@ from calibre.constants import iswindows, isosx
from calibre.utils.ipc import eintr_retry_call
from calibre.utils.serialize import msgpack_loads, pickle_dumps
from polyglot.queue import Queue
from polyglot.binary import from_hex_bytes, from_hex_unicode
PARALLEL_FUNCS = {
'lrfviewer' :
@ -183,9 +183,9 @@ def main():
print('Failed to run pipe worker with command:', sys.argv[-1])
raise
return
address = msgpack_loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS']))
key = unhexlify(os.environ['CALIBRE_WORKER_KEY'])
resultf = unhexlify(os.environ['CALIBRE_WORKER_RESULT']).decode('utf-8')
address = msgpack_loads(from_hex_bytes(os.environ['CALIBRE_WORKER_ADDRESS']))
key = from_hex_bytes(os.environ['CALIBRE_WORKER_KEY'])
resultf = from_hex_unicode(os.environ['CALIBRE_WORKER_RESULT'])
with closing(Client(address, authkey=key)) as conn:
name, args, kwargs, desc = eintr_retry_call(conn.recv)
if desc:

28
src/polyglot/binary.py
View File

@ -4,7 +4,9 @@
from __future__ import absolute_import, division, print_function, unicode_literals
from base64 import standard_b64encode, standard_b64decode
from base64 import standard_b64decode, standard_b64encode
from binascii import hexlify, unhexlify
from polyglot.builtins import unicode_type
@ -30,3 +32,27 @@ def from_base64_bytes(x):
if isinstance(x, unicode_type):
x = x.encode('ascii')
return standard_b64decode(x)
def as_hex_bytes(x, enc='utf-8'):
if isinstance(x, unicode_type):
x = x.encode(enc)
return hexlify(x)
def as_hex_unicode(x, enc='utf-8'):
if isinstance(x, unicode_type):
x = x.encode(enc)
return hexlify(x).decode('ascii')
def from_hex_unicode(x, enc='utf-8'):
if isinstance(x, unicode_type):
x = x.encode('ascii')
return unhexlify(x).decode(enc)
def from_hex_bytes(x):
if isinstance(x, unicode_type):
x = x.encode('ascii')
return unhexlify(x)

7
src/tinycss/decoding.py
View File

@ -12,10 +12,10 @@
from __future__ import unicode_literals
from binascii import unhexlify
import operator
import re
import sys
from polyglot.binary import from_hex_bytes
__all__ = ['decode'] # Everything else is implementation detail
@ -98,7 +98,7 @@ def try_encoding(css_bytes, encoding, fallback=True):
def hex2re(hex_data):
return re.escape(unhexlify(hex_data.replace(' ', '').encode('ascii')))
return re.escape(from_hex_bytes(hex_data.replace(' ', '').encode('ascii')))
class Slicer(object):
@ -106,6 +106,7 @@ class Slicer(object):
def __getitem__(self, slice_):
return operator.itemgetter(slice_)
Slice = Slicer()