From 03df438219404de3411154affb33bdbee034a163 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 15 Apr 2019 19:48:42 +0530 Subject: [PATCH] py3: Port bytes() --- src/calibre/devices/mtp/windows/remote.py | 5 +++-- .../devices/smart_device_app/driver.py | 2 +- .../ebooks/conversion/plugins/epub_input.py | 6 +++--- .../ebooks/conversion/plugins/pdf_input.py | 3 ++- .../ebooks/metadata/book/json_codec.py | 4 ++-- src/calibre/ebooks/metadata/docx.py | 2 +- src/calibre/ebooks/metadata/fb2.py | 2 +- src/calibre/ebooks/metadata/odt.py | 4 ++-- src/calibre/ebooks/mobi/debug/headers.py | 4 ++-- src/calibre/ebooks/mobi/debug/mobi6.py | 4 ++-- src/calibre/ebooks/mobi/utils.py | 4 ++-- src/calibre/ebooks/mobi/writer8/header.py | 4 ++-- src/calibre/ebooks/oeb/polish/container.py | 8 +++---- src/calibre/ebooks/pdf/render/fonts.py | 12 +++++++---- src/calibre/gui2/dnd.py | 10 ++++----- src/calibre/gui2/tweak_book/char_select.py | 2 +- src/calibre/gui2/tweak_book/diff/view.py | 4 ++-- src/calibre/gui2/tweak_book/editor/canvas.py | 3 ++- src/calibre/gui2/tweak_book/editor/text.py | 4 ++-- src/calibre/srv/legacy.py | 4 ++-- src/calibre/utils/__init__.py | 3 ++- src/calibre/utils/fonts/utils.py | 4 ++-- src/polyglot/builtins.py | 21 +++++++++++++++++++ 23 files changed, 74 insertions(+), 45 deletions(-) diff --git a/src/calibre/devices/mtp/windows/remote.py b/src/calibre/devices/mtp/windows/remote.py index fc0b78879c..e42f16b4a2 100644 --- a/src/calibre/devices/mtp/windows/remote.py +++ b/src/calibre/devices/mtp/windows/remote.py @@ -9,6 +9,7 @@ __docformat__ = 'restructuredtext en' import subprocess, sys, os, pprint, signal, time, glob, io pprint, io +from polyglot.builtins import environ_item def build(mod='wpd'): @@ -42,9 +43,9 @@ def build(mod='wpd'): def main(): fp, d = os.path.abspath(__file__), os.path.dirname - if b'CALIBRE_DEVELOP_FROM' not in os.environ: + if 'CALIBRE_DEVELOP_FROM' not in os.environ: env = os.environ.copy() - env[b'CALIBRE_DEVELOP_FROM'] = bytes(d(d(d(d(d(fp)))))) + env['CALIBRE_DEVELOP_FROM'] = environ_item(d(d(d(d(d(fp)))))) subprocess.call(['calibre-debug', '-e', fp], env=env) return diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index 03e15227e5..6dd7fdb250 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -569,7 +569,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): raise def _read_string_from_net(self): - data = bytes(0) + data = b'\0' while True: dex = data.find(b'[') if dex >= 0: diff --git a/src/calibre/ebooks/conversion/plugins/epub_input.py b/src/calibre/ebooks/conversion/plugins/epub_input.py index a72571cf12..2bd81761fd 100644 --- a/src/calibre/ebooks/conversion/plugins/epub_input.py +++ b/src/calibre/ebooks/conversion/plugins/epub_input.py @@ -7,7 +7,7 @@ import os, re, posixpath from itertools import cycle from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation -from polyglot.builtins import unicode_type +from polyglot.builtins import unicode_type, as_bytes ADOBE_OBFUSCATION = 'http://ns.adobe.com/pdf/enc#RC' IDPF_OBFUSCATION = 'http://www.idpf.org/2008/embedding' @@ -55,8 +55,8 @@ class EPUBInput(InputFormatPlugin): if (scheme and scheme.lower() == 'uuid') or \ (item.text and item.text.startswith('urn:uuid:')): try: - key = bytes(item.text).rpartition(':')[-1] - key = uuid.UUID(key).bytes + key = item.text.rpartition(':')[-1] + key = uuid.UUID(as_bytes(key)).bytes except: import traceback traceback.print_exc() diff --git a/src/calibre/ebooks/conversion/plugins/pdf_input.py b/src/calibre/ebooks/conversion/plugins/pdf_input.py index 979bf6c451..f1e7127db3 100644 --- a/src/calibre/ebooks/conversion/plugins/pdf_input.py +++ b/src/calibre/ebooks/conversion/plugins/pdf_input.py @@ -7,6 +7,7 @@ __docformat__ = 'restructuredtext en' import os from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation +from polyglot.builtins import as_bytes class PDFInput(InputFormatPlugin): @@ -73,7 +74,7 @@ class PDFInput(InputFormatPlugin): ncxid = opf.manifest.id_for_path('toc.ncx') if ncxid: with open(u'metadata.opf', 'r+b') as f: - raw = f.read().replace(b'L', self.data) elif self.type in {209, 300}: - self.data = bytes(self.data.encode('hex')) + self.data = as_bytes(self.data.encode('hex')) def __str__(self): return '%s (%d): %r'%(self.name, self.type, self.data) diff --git a/src/calibre/ebooks/mobi/debug/mobi6.py b/src/calibre/ebooks/mobi/debug/mobi6.py index 5884fc1069..6ac1368359 100644 --- a/src/calibre/ebooks/mobi/debug/mobi6.py +++ b/src/calibre/ebooks/mobi/debug/mobi6.py @@ -20,7 +20,7 @@ from calibre.ebooks.mobi.utils import (decode_hex_number, decint, from calibre.utils.imghdr import what from calibre.ebooks.mobi.debug import format_bytes from calibre.ebooks.mobi.debug.headers import TextRecord -from polyglot.builtins import unicode_type, range, iteritems +from polyglot.builtins import unicode_type, range, iteritems, as_bytes class TagX(object): # {{{ @@ -606,7 +606,7 @@ class TBSIndexing(object): # {{{ def bin4(num): ans = bin(num)[2:] - return bytes('0'*(4-len(ans)) + ans) + return as_bytes('0'*(4-len(ans)) + ans) def repr_extra(x): return str({bin4(k):v for k, v in iteritems(extra)}) diff --git a/src/calibre/ebooks/mobi/utils.py b/src/calibre/ebooks/mobi/utils.py index 74c76f721d..854c2b4395 100644 --- a/src/calibre/ebooks/mobi/utils.py +++ b/src/calibre/ebooks/mobi/utils.py @@ -14,7 +14,7 @@ from io import BytesIO from calibre.utils.img import save_cover_data_to, scale_image, image_to_data, image_from_data, resize_image from calibre.utils.imghdr import what from calibre.ebooks import normalize -from polyglot.builtins import unicode_type, range +from polyglot.builtins import unicode_type, range, as_bytes from tinycss.color3 import parse_color_string IMAGE_MAX_SIZE = 10 * 1024 * 1024 @@ -65,7 +65,7 @@ def decode_hex_number(raw, codec='utf-8'): def encode_string(raw): - ans = bytearray(bytes(raw)) + ans = bytearray(as_bytes(raw)) ans.insert(0, len(ans)) return bytes(ans) diff --git a/src/calibre/ebooks/mobi/writer8/header.py b/src/calibre/ebooks/mobi/writer8/header.py index 6ac9f42076..3599cb2e0a 100644 --- a/src/calibre/ebooks/mobi/writer8/header.py +++ b/src/calibre/ebooks/mobi/writer8/header.py @@ -13,7 +13,7 @@ from collections import OrderedDict from struct import pack from calibre.ebooks.mobi.utils import align_block -from polyglot.builtins import iteritems +from polyglot.builtins import iteritems, as_bytes NULL = 0xffffffff zeroes = lambda x: b'\0'*x @@ -62,7 +62,7 @@ class Header(OrderedDict): self[name] = val buf = BytesIO() - buf.write(bytes(self.HEADER_NAME)) + buf.write(as_bytes(self.HEADER_NAME)) for name, val in iteritems(self): val = self.format_value(name, val) positions[name] = buf.tell() diff --git a/src/calibre/ebooks/oeb/polish/container.py b/src/calibre/ebooks/oeb/polish/container.py index 01db43481c..c889a9c5df 100644 --- a/src/calibre/ebooks/oeb/polish/container.py +++ b/src/calibre/ebooks/oeb/polish/container.py @@ -14,7 +14,7 @@ import time import unicodedata import uuid from collections import defaultdict -from polyglot.builtins import iteritems, unicode_type, zip +from polyglot.builtins import iteritems, unicode_type, zip, as_bytes from io import BytesIO from itertools import count @@ -1289,9 +1289,9 @@ class EpubContainer(Container): if (scheme and scheme.lower() == 'uuid') or \ (item.text and item.text.startswith('urn:uuid:')): try: - key = bytes(item.text).rpartition(':')[-1] - key = uuid.UUID(key).bytes - except: + key = item.text.rpartition(':')[-1] + key = uuid.UUID(as_bytes(key)).bytes + except Exception: self.log.exception('Failed to parse obfuscation key') key = None diff --git a/src/calibre/ebooks/pdf/render/fonts.py b/src/calibre/ebooks/pdf/render/fonts.py index 1853ba5eb2..d8e8515246 100644 --- a/src/calibre/ebooks/pdf/render/fonts.py +++ b/src/calibre/ebooks/pdf/render/fonts.py @@ -11,7 +11,7 @@ import re from itertools import groupby from operator import itemgetter from collections import Counter, OrderedDict -from polyglot.builtins import iteritems, map, zip +from polyglot.builtins import iteritems, map, zip, unicode_type from calibre import as_unicode from calibre.ebooks.pdf.render.common import (Array, String, Stream, @@ -62,7 +62,10 @@ class FontStream(Stream): def to_hex_string(c): - return bytes(hex(int(c))[2:]).rjust(4, b'0').decode('ascii') + ans = hex(int(c))[2:].rjust(4, '0') + if isinstance(ans, bytes): + ans = ans.decode('ascii') + return ans class CMap(Stream): @@ -120,8 +123,9 @@ class Font(object): def __init__(self, metrics, num, objects, compress): self.metrics, self.compress = metrics, compress self.is_otf = self.metrics.is_otf - self.subset_tag = bytes(re.sub('.', lambda m: chr(int(m.group())+ord('A')), - oct(num))).rjust(6, b'A').decode('ascii') + self.subset_tag = unicode_type( + re.sub('.', lambda m: chr(int(m.group())+ord('A')), oct(num).replace('o', '') + )).rjust(6, 'A') self.font_stream = FontStream(metrics.is_otf, compress=compress) try: psname = metrics.postscript_name diff --git a/src/calibre/gui2/dnd.py b/src/calibre/gui2/dnd.py index 2b6201792f..d082ae7760 100644 --- a/src/calibre/gui2/dnd.py +++ b/src/calibre/gui2/dnd.py @@ -18,14 +18,14 @@ from calibre.ptempfile import PersistentTemporaryFile from calibre import browser, as_unicode, prints from calibre.gui2 import error_dialog from calibre.utils.imghdr import what -from polyglot.builtins import unicode_type +from polyglot.builtins import unicode_type, as_unicode as as_unicode_polyglot from polyglot.urllib import unquote, urlparse from polyglot.queue import Queue, Empty def image_extensions(): if not hasattr(image_extensions, 'ans'): - image_extensions.ans = [bytes(x).decode('utf-8') for x in QImageReader.supportedImageFormats()] + image_extensions.ans = [as_unicode_polyglot(x) for x in QImageReader.supportedImageFormats()] return image_extensions.ans @@ -162,7 +162,7 @@ def path_from_qurl(qurl): raw = bytes(qurl.toEncoded( QUrl.PreferLocalFile | QUrl.RemoveScheme | QUrl.RemovePassword | QUrl.RemoveUserInfo | QUrl.RemovePort | QUrl.RemoveAuthority | QUrl.RemoveQuery | QUrl.RemoveFragment)) - ans = unquote(raw).decode('utf-8', 'replace') + ans = as_unicode_polyglot(unquote(raw), errors='replace') if iswindows and ans.startswith('/'): ans = ans[1:] return ans @@ -172,7 +172,7 @@ def remote_urls_from_qurl(qurls, allowed_exts): for qurl in qurls: if qurl.scheme() in {'http', 'https', 'ftp'} and posixpath.splitext( qurl.path())[1][1:].lower() in allowed_exts: - yield bytes(qurl.toEncoded()), posixpath.basename(qurl.path()) + yield bytes(qurl.toEncoded()).decode('utf-8'), posixpath.basename(qurl.path()) def dnd_has_extension(md, extensions, allow_all_extensions=False): @@ -189,7 +189,7 @@ def dnd_has_extension(md, extensions, allow_all_extensions=False): paths = [path_from_qurl(u) for u in urls] exts = frozenset([posixpath.splitext(u)[1][1:].lower() for u in paths if u]) if DEBUG: - repr_urls = [bytes(u.toEncoded()) for u in urls] + repr_urls = [bytes(u.toEncoded()).decode('utf-8') for u in urls] prints('URLS:', repr(repr_urls)) prints('Paths:', paths) prints('Extensions:', exts) diff --git a/src/calibre/gui2/tweak_book/char_select.py b/src/calibre/gui2/tweak_book/char_select.py index d8204abe69..f3528a950d 100644 --- a/src/calibre/gui2/tweak_book/char_select.py +++ b/src/calibre/gui2/tweak_book/char_select.py @@ -529,7 +529,7 @@ class CharModel(QAbstractListModel): def dropMimeData(self, md, action, row, column, parent): if action != Qt.MoveAction or not md.hasFormat('application/calibre_charcode_indices') or row < 0 or column != 0: return False - indices = map(int, bytes(md.data('application/calibre_charcode_indices')).split(',')) + indices = map(int, bytes(md.data('application/calibre_charcode_indices')).decode('ascii').split(',')) codes = [self.chars[x] for x in indices] for x in indices: self.chars[x] = None diff --git a/src/calibre/gui2/tweak_book/diff/view.py b/src/calibre/gui2/tweak_book/diff/view.py index 9fd6ee4be1..ad84e45825 100644 --- a/src/calibre/gui2/tweak_book/diff/view.py +++ b/src/calibre/gui2/tweak_book/diff/view.py @@ -12,7 +12,7 @@ from math import ceil from functools import partial from collections import namedtuple, OrderedDict from difflib import SequenceMatcher -from polyglot.builtins import iteritems, unicode_type, zip, range +from polyglot.builtins import iteritems, unicode_type, zip, range, as_bytes import regex from PyQt5.Qt import ( @@ -589,7 +589,7 @@ class DiffSplit(QSplitter): # {{{ def add_image_diff(self, left_data, right_data): def load(data): p = QPixmap() - p.loadFromData(bytes(data)) + p.loadFromData(as_bytes(data)) try: dpr = self.devicePixelRatioF() except AttributeError: diff --git a/src/calibre/gui2/tweak_book/editor/canvas.py b/src/calibre/gui2/tweak_book/editor/canvas.py index 5b76457784..a29bac1389 100644 --- a/src/calibre/gui2/tweak_book/editor/canvas.py +++ b/src/calibre/gui2/tweak_book/editor/canvas.py @@ -25,6 +25,7 @@ from calibre.utils.img import ( remove_borders_from_image, gaussian_sharpen_image, gaussian_blur_image, image_to_data, despeckle_image, normalize_image, oil_paint_image ) +from polyglot.builtins import as_unicode def painter(func): @@ -325,7 +326,7 @@ class Canvas(QWidget): if not self.is_modified: return self.original_image_data fmt = self.original_image_format or 'JPEG' - if fmt.lower() not in set(map(lambda x:bytes(x).decode('ascii'), QImageWriter.supportedImageFormats())): + if fmt.lower() not in set(map(as_unicode, QImageWriter.supportedImageFormats())): if fmt.lower() == 'gif': data = image_to_data(self.current_image, fmt='PNG', png_compression_level=0) from PIL import Image diff --git a/src/calibre/gui2/tweak_book/editor/text.py b/src/calibre/gui2/tweak_book/editor/text.py index 9432b3db55..8d6dd1023f 100644 --- a/src/calibre/gui2/tweak_book/editor/text.py +++ b/src/calibre/gui2/tweak_book/editor/text.py @@ -8,7 +8,7 @@ import os import re import textwrap import unicodedata -from polyglot.builtins import unicode_type, map, range +from polyglot.builtins import unicode_type, map, range, as_unicode from PyQt5.Qt import ( QColor, QColorDialog, QFont, QFontDatabase, QKeySequence, QPainter, QPalette, @@ -131,7 +131,7 @@ class TextEdit(PlainTextEdit): ) if md.hasFormat(CONTAINER_DND_MIMETYPE): - for line in bytes(md.data(CONTAINER_DND_MIMETYPE)).decode('utf-8').splitlines(): + for line in as_unicode(bytes(md.data(CONTAINER_DND_MIMETYPE))).splitlines(): mt = current_container().mime_map.get(line, 'application/octet-stream') if is_mt_ok(mt): yield line, mt, True diff --git a/src/calibre/srv/legacy.py b/src/calibre/srv/legacy.py index 3fd1f82afa..196b42974e 100644 --- a/src/calibre/srv/legacy.py +++ b/src/calibre/srv/legacy.py @@ -19,7 +19,7 @@ from calibre.srv.routes import endpoint from calibre.srv.utils import get_library_data, http_date from calibre.utils.cleantext import clean_xml_chars from calibre.utils.date import dt_as_local, is_date_undefined, timestampfromdt -from polyglot.builtins import iteritems, string_or_bytes, filter +from polyglot.builtins import iteritems, string_or_bytes, filter, as_bytes from polyglot.urllib import urlencode # /mobile {{{ @@ -245,7 +245,7 @@ def mobile(ctx, rd): books = [db.get_metadata(book_id) for book_id in book_ids[(start-1):(start-1)+num]] rd.outheaders['Last-Modified'] = http_date(timestampfromdt(db.last_modified())) order = 'ascending' if ascending else 'descending' - q = {b'search':search.encode('utf-8'), b'order':bytes(order), b'sort':sort_by.encode('utf-8'), b'num':bytes(num), 'library_id':library_id} + q = {b'search':search.encode('utf-8'), b'order':order.encode('ascii'), b'sort':sort_by.encode('utf-8'), b'num':as_bytes(num), 'library_id':library_id} url_base = ctx.url_for('/mobile') + '?' + urlencode(q) lm = {k:v for k, v in iteritems(library_map) if k != library_id} return build_index(rd, books, num, search, sort_by, order, start, total, url_base, db.field_metadata, ctx, lm, library_id) diff --git a/src/calibre/utils/__init__.py b/src/calibre/utils/__init__.py index eb28ba7102..d019555817 100644 --- a/src/calibre/utils/__init__.py +++ b/src/calibre/utils/__init__.py @@ -8,6 +8,7 @@ Miscelleaneous utilities. ''' from time import time +from polyglot.builtins import as_bytes def join_with_timeout(q, timeout=2): @@ -48,5 +49,5 @@ def pickle_binary_string(data): # Maintains compatibility with python's pickle module protocol version 2 import struct PROTO, STOP, BINSTRING = b'\x80', b'.', b'T' - data = bytes(data) + data = as_bytes(data) return PROTO + b'\x02' + BINSTRING + struct.pack(b'