From fcd570030660bd07a0644f6f7d2dedd12f054121 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 30 Mar 2019 12:52:32 +0530 Subject: [PATCH] py3: port use as base64 module --- src/calibre/devices/prs505/sony_cache.py | 6 +-- src/calibre/ebooks/fb2/__init__.py | 8 ++-- src/calibre/ebooks/fb2/fb2ml.py | 6 +-- .../ebooks/metadata/book/json_codec.py | 6 +-- src/calibre/ebooks/metadata/book/serialize.py | 4 +- src/calibre/ebooks/metadata/fb2.py | 4 +- src/calibre/ebooks/metadata/kfx.py | 9 +++-- src/calibre/ebooks/oeb/polish/download.py | 4 +- src/calibre/ebooks/oeb/transforms/data_url.py | 4 +- .../gui2/store/search/download_thread.py | 5 ++- .../gui2/store/stores/ebookpoint_plugin.py | 13 +++++- src/calibre/gui2/store/stores/empik_plugin.py | 14 +++++-- .../gui2/store/stores/gutenberg_plugin.py | 8 +++- .../gui2/store/stores/legimi_plugin.py | 13 +++++- .../gui2/store/stores/publio_plugin.py | 13 +++++- .../gui2/store/stores/swiatebookow_plugin.py | 13 +++++- .../gui2/store/stores/virtualo_plugin.py | 15 +++++-- .../gui2/store/stores/woblink_plugin.py | 13 +++++- src/calibre/gui2/toc/location.py | 4 +- src/calibre/gui2/tweak_book/preview.py | 4 +- src/calibre/gui2/viewer/documentview.py | 4 +- src/calibre/srv/auth.py | 5 ++- src/calibre/srv/cdb.py | 4 +- src/calibre/srv/render_book.py | 40 +++++++++---------- src/calibre/srv/tests/ajax.py | 5 ++- src/calibre/srv/tests/auth.py | 9 +++-- src/calibre/srv/tests/web_sockets.py | 6 +-- src/calibre/srv/web_socket.py | 20 ++++++---- src/calibre/utils/config_base.py | 8 ++-- src/calibre/utils/serialize.py | 6 +-- src/calibre/web/fetch/simple.py | 6 +-- src/polyglot/binary.py | 32 +++++++++++++++ 32 files changed, 205 insertions(+), 106 deletions(-) create mode 100644 src/polyglot/binary.py diff --git a/src/calibre/devices/prs505/sony_cache.py b/src/calibre/devices/prs505/sony_cache.py index 0e5a90aaee..9984b1316f 100644 --- a/src/calibre/devices/prs505/sony_cache.py +++ b/src/calibre/devices/prs505/sony_cache.py @@ -6,7 +6,6 @@ __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' import os, time -from base64 import b64decode from datetime import date from calibre import prints, guess_type, isbytestring, fsync @@ -16,6 +15,7 @@ from calibre.constants import DEBUG, preferred_encoding from calibre.ebooks.chardet import xml_to_unicode from calibre.ebooks.metadata import authors_to_string, title_sort, \ authors_to_sort_string +from polyglot.binary import from_base64_bytes ''' cahceExt.xml @@ -380,8 +380,8 @@ class XMLCache(object): 'descendant::*[local-name()="png"]'): if img.text: try: - raw = b64decode(img.text.strip()) - except: + raw = from_base64_bytes(img.text.strip()) + except Exception: continue book.thumbnail = raw break diff --git a/src/calibre/ebooks/fb2/__init__.py b/src/calibre/ebooks/fb2/__init__.py index 4d9887e6a5..94b17b104a 100644 --- a/src/calibre/ebooks/fb2/__init__.py +++ b/src/calibre/ebooks/fb2/__init__.py @@ -10,12 +10,12 @@ __docformat__ = 'restructuredtext en' def base64_decode(raw): from io import BytesIO - from base64 import b64decode + from polyglot.binary import from_base64_bytes # First try the python implementation as it is faster try: - return b64decode(raw) - except TypeError: + return from_base64_bytes(raw) + except Exception: pass # Try a more robust version (adapted from FBReader sources) @@ -49,5 +49,3 @@ def base64_decode(raw): tot >>= 8 out.write(bytes(triple)) return out.getvalue() - - diff --git a/src/calibre/ebooks/fb2/fb2ml.py b/src/calibre/ebooks/fb2/fb2ml.py index 0f2225e665..5dede1505e 100644 --- a/src/calibre/ebooks/fb2/fb2ml.py +++ b/src/calibre/ebooks/fb2/fb2ml.py @@ -9,7 +9,6 @@ Transform OEB content into FB2 markup ''' import re, textwrap, uuid -from base64 import b64encode from datetime import datetime from lxml import etree @@ -20,6 +19,7 @@ from calibre.utils.localization import lang_as_iso639_1 from calibre.utils.img import save_cover_data_to from calibre.ebooks.oeb.base import urlnormalize from polyglot.builtins import unicode_type, string_or_bytes +from polyglot.binary import as_base64_unicode class FB2MLizer(object): @@ -308,9 +308,9 @@ class FB2MLizer(object): try: if item.media_type != 'image/jpeg': imdata = save_cover_data_to(item.data, compression_quality=70) - raw_data = b64encode(imdata) + raw_data = as_base64_unicode(imdata) else: - raw_data = b64encode(item.data) + raw_data = as_base64_unicode(item.data) # Don't put the encoded image on a single line. data = '' col = 1 diff --git a/src/calibre/ebooks/metadata/book/json_codec.py b/src/calibre/ebooks/metadata/book/json_codec.py index 4a411b15a2..46027c03c0 100644 --- a/src/calibre/ebooks/metadata/book/json_codec.py +++ b/src/calibre/ebooks/metadata/book/json_codec.py @@ -5,7 +5,6 @@ Created on 4 Jun 2010 ''' from __future__ import print_function -from base64 import b64encode, b64decode import json, traceback from datetime import datetime, time @@ -14,6 +13,7 @@ from calibre.constants import filesystem_encoding, preferred_encoding from calibre.library.field_metadata import FieldMetadata from calibre import isbytestring from polyglot.builtins import iteritems, itervalues +from polyglot.binary import as_base64_unicode, from_base64_bytes # Translate datetimes to and from strings. The string form is the datetime in # UTC. The returned date is also UTC @@ -57,7 +57,7 @@ def encode_thumbnail(thumbnail): thumbnail = (width, height, thumbnail) except Exception: return None - return (thumbnail[0], thumbnail[1], b64encode(str(thumbnail[2]))) + return (thumbnail[0], thumbnail[1], as_base64_unicode(thumbnail[2])) def decode_thumbnail(tup): @@ -66,7 +66,7 @@ def decode_thumbnail(tup): ''' if tup is None: return None - return (tup[0], tup[1], b64decode(tup[2])) + return (tup[0], tup[1], from_base64_bytes(tup[2])) def object_to_unicode(obj, enc=preferred_encoding): diff --git a/src/calibre/ebooks/metadata/book/serialize.py b/src/calibre/ebooks/metadata/book/serialize.py index 8c741cc70f..f5dbfe257c 100644 --- a/src/calibre/ebooks/metadata/book/serialize.py +++ b/src/calibre/ebooks/metadata/book/serialize.py @@ -4,13 +4,13 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import base64 from calibre.constants import preferred_encoding from calibre.ebooks.metadata.book import SERIALIZABLE_FIELDS from calibre.ebooks.metadata.book.base import Metadata from calibre.utils.imghdr import what from polyglot.builtins import iteritems, unicode_type +from polyglot.binary import as_base64_unicode def ensure_unicode(obj, enc=preferred_encoding): @@ -52,7 +52,7 @@ def metadata_as_dict(mi, encode_cover_data=False): ans[field] = ensure_unicode(val) if mi.cover_data and mi.cover_data[1]: if encode_cover_data: - ans['cover_data'] = [mi.cover_data[0], base64.standard_b64encode(bytes(mi.cover_data[1]))] + ans['cover_data'] = [mi.cover_data[0], as_base64_unicode(mi.cover_data[1])] else: ans['cover_data'] = mi.cover_data um = mi.get_all_user_metadata(False) diff --git a/src/calibre/ebooks/metadata/fb2.py b/src/calibre/ebooks/metadata/fb2.py index 8889333944..4e8f19e2fb 100644 --- a/src/calibre/ebooks/metadata/fb2.py +++ b/src/calibre/ebooks/metadata/fb2.py @@ -8,7 +8,6 @@ __copyright__ = '2011, Roman Mukhin , '\ import os, random from functools import partial from string import ascii_letters, digits -from base64 import b64encode from lxml import etree @@ -19,6 +18,7 @@ from calibre import guess_type, guess_all_extensions, prints, force_unicode from calibre.ebooks.metadata import MetaInformation, check_isbn from calibre.ebooks.chardet import xml_to_unicode from polyglot.builtins import unicode_type +from polyglot.binary import as_base64_unicode NAMESPACES = { @@ -383,7 +383,7 @@ def _rnd_pic_file_name(prefix='calibre_cover_', size=32, ext='jpg'): def _encode_into_jpeg(data): data = save_cover_data_to(data) - return b64encode(data) + return as_base64_unicode(data) def _set_cover(title_info, mi, ctx): diff --git a/src/calibre/ebooks/metadata/kfx.py b/src/calibre/ebooks/metadata/kfx.py index d427c0a111..17ccf21dbf 100644 --- a/src/calibre/ebooks/metadata/kfx.py +++ b/src/calibre/ebooks/metadata/kfx.py @@ -8,7 +8,7 @@ from __future__ import (unicode_literals, division, absolute_import, # Based on work of John Howell reversing the KFX format # https://www.mobileread.com/forums/showpost.php?p=3176029&postcount=89 -import struct, sys, base64, re +import struct, sys, re from collections import defaultdict from calibre.ebooks.metadata.book.base import Metadata @@ -19,6 +19,7 @@ from calibre.utils.date import parse_only_date from calibre.utils.localization import canonicalize_lang from calibre.utils.imghdr import identify from polyglot.builtins import unicode_type +from polyglot.binary import as_base64_bytes, from_base64_bytes class InvalidKFX(ValueError): @@ -155,7 +156,7 @@ class Entity(PackedBlock): if PackedData(self.entity_data).unpack_one('4s') == ION_MAGIC: entity_value = PackedIon(self.entity_data).decode() else: - entity_value = base64.b64encode(self.entity_data) + entity_value = as_base64_bytes(self.entity_data) return (property_name(self.entity_type), property_name(self.entity_id), entity_value) @@ -343,8 +344,8 @@ def read_metadata_kfx(stream, read_cover=True): mi.publisher = get('publisher') if read_cover and m[COVER_KEY]: try: - data = base64.standard_b64decode(m[COVER_KEY]) - fmt, w, h = identify(bytes(data)) + data = from_base64_bytes(m[COVER_KEY]) + fmt, w, h = identify(data) except Exception: w, h, fmt = 0, 0, None if fmt and w > -1 and h > -1: diff --git a/src/calibre/ebooks/oeb/polish/download.py b/src/calibre/ebooks/oeb/polish/download.py index 46fc78f22c..ce624b8d0c 100644 --- a/src/calibre/ebooks/oeb/polish/download.py +++ b/src/calibre/ebooks/oeb/polish/download.py @@ -10,7 +10,6 @@ import os import posixpath import re import shutil -from base64 import standard_b64decode from collections import defaultdict from contextlib import closing from functools import partial @@ -25,6 +24,7 @@ from calibre.ptempfile import TemporaryDirectory from calibre.web import get_download_filename_from_response from polyglot.builtins import iteritems from polyglot.urllib import urlopen, urlparse +from polyglot.binary import from_base64_bytes def is_external(url): @@ -117,7 +117,7 @@ def download_one(tdir, timeout, progress_report, data_uri_map, url): parts = prefix.split(';') if parts and parts[-1].lower() == 'base64': payload = re.sub(r'\s+', '', payload) - payload = standard_b64decode(payload) + payload = from_base64_bytes(payload) else: payload = payload.encode('utf-8') seen_before = data_uri_map.get(payload) diff --git a/src/calibre/ebooks/oeb/transforms/data_url.py b/src/calibre/ebooks/oeb/transforms/data_url.py index fc62a8ceda..81ba781af9 100644 --- a/src/calibre/ebooks/oeb/transforms/data_url.py +++ b/src/calibre/ebooks/oeb/transforms/data_url.py @@ -28,9 +28,9 @@ class DataURL(object): continue if ';base64' in header: data = re.sub(r'\s+', '', data) - from base64 import b64decode + from polyglot.binary import from_base64_bytes try: - data = b64decode(data) + data = from_base64_bytes(data) except Exception: self.log.error('Found invalid base64 encoded data URI, ignoring it') continue diff --git a/src/calibre/gui2/store/search/download_thread.py b/src/calibre/gui2/store/search/download_thread.py index b41c4fefe1..70bbfc1dbe 100644 --- a/src/calibre/gui2/store/search/download_thread.py +++ b/src/calibre/gui2/store/search/download_thread.py @@ -6,7 +6,7 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' -import traceback, base64 +import traceback from contextlib import closing from threading import Thread @@ -15,6 +15,7 @@ from calibre.constants import DEBUG from calibre.utils.img import scale_image from polyglot.builtins import range from polyglot.queue import Queue +from polyglot.binary import from_base64_bytes class GenericDownloadThreadPool(object): @@ -143,7 +144,7 @@ class CoverThreadPool(GenericDownloadThreadPool): def decode_data_url(url): - return base64.standard_b64decode(url.partition(',')[2]) + return from_base64_bytes(url.partition(',')[2]) class CoverThread(Thread): diff --git a/src/calibre/gui2/store/stores/ebookpoint_plugin.py b/src/calibre/gui2/store/stores/ebookpoint_plugin.py index a5b8202753..d2cf0e557e 100644 --- a/src/calibre/gui2/store/stores/ebookpoint_plugin.py +++ b/src/calibre/gui2/store/stores/ebookpoint_plugin.py @@ -24,6 +24,15 @@ from calibre.gui2.store.search_result import SearchResult from calibre.gui2.store.web_store_dialog import WebStoreDialog +def as_base64(data): + if not isinstance(data, bytes): + data = data.encode('utf-8') + ans = b64encode(data) + if isinstance(ans, bytes): + ans = ans.decode('ascii') + return ans + + class EbookpointStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): @@ -31,11 +40,11 @@ class EbookpointStore(BasicStoreConfig, StorePlugin): url = 'http://ebookpoint.pl/' - aff_url = aff_root + str(b64encode(url)) + aff_url = aff_root + as_base64(url) detail_url = None if detail_item: - detail_url = aff_root + str(b64encode(detail_item)) + detail_url = aff_root + as_base64(detail_item) if external or self.config.get('open_external', False): open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url))) diff --git a/src/calibre/gui2/store/stores/empik_plugin.py b/src/calibre/gui2/store/stores/empik_plugin.py index ef9192899e..6f2e6d0568 100644 --- a/src/calibre/gui2/store/stores/empik_plugin.py +++ b/src/calibre/gui2/store/stores/empik_plugin.py @@ -24,6 +24,15 @@ from calibre.gui2.store.search_result import SearchResult from calibre.gui2.store.web_store_dialog import WebStoreDialog +def as_base64(data): + if not isinstance(data, bytes): + data = data.encode('utf-8') + ans = b64encode(data) + if isinstance(ans, bytes): + ans = ans.decode('ascii') + return ans + + class EmpikStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): @@ -31,11 +40,11 @@ class EmpikStore(BasicStoreConfig, StorePlugin): url = 'http://www.empik.com/ebooki' - aff_url = aff_root + str(b64encode(url)) + aff_url = aff_root + as_base64(url) detail_url = None if detail_item: - detail_url = aff_root + str(b64encode(detail_item)) + detail_url = aff_root + as_base64(detail_item) if external or self.config.get('open_external', False): open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url))) @@ -82,4 +91,3 @@ class EmpikStore(BasicStoreConfig, StorePlugin): s.formats = formats.upper().strip() yield s - diff --git a/src/calibre/gui2/store/stores/gutenberg_plugin.py b/src/calibre/gui2/store/stores/gutenberg_plugin.py index dab8c7a61e..dc58626d01 100644 --- a/src/calibre/gui2/store/stores/gutenberg_plugin.py +++ b/src/calibre/gui2/store/stores/gutenberg_plugin.py @@ -52,7 +52,7 @@ def search(query, max_results=10, timeout=60, write_raw_to=None): # We could use the tag from the # detail odps page but this is easier. id = fix_url(''.join(data.xpath('./*[local-name() = "id"]/text()')).strip()) - s.detail_item = url_slash_cleaner('%s/ebooks/%s' % (web_url, re.sub('[^\d]', '', id))) + s.detail_item = url_slash_cleaner('%s/ebooks/%s' % (web_url, re.sub(r'[^\d]', '', id))) s.title = ' '.join(data.xpath('./*[local-name() = "title"]//text()')).strip() s.author = ', '.join(data.xpath('./*[local-name() = "content"]//text()')).strip() if not s.title or not s.author: @@ -83,7 +83,10 @@ def search(query, max_results=10, timeout=60, write_raw_to=None): href = fix_url(href) if rel in ('http://opds-spec.org/thumbnail', 'http://opds-spec.org/image/thumbnail'): if href.startswith('data:image/png;base64,'): - s.cover_data = base64.b64decode(href.replace('data:image/png;base64,', '')) + cdata = href.replace('data:image/png;base64,', '') + if not isinstance(cdata, bytes): + cdata = cdata.encode('ascii') + s.cover_data = base64.b64decode(cdata) yield s @@ -123,6 +126,7 @@ class GutenbergStore(BasicStoreConfig, OpenSearchOPDSStore): for result in search(query, max_results, timeout): yield result + if __name__ == '__main__': import sys for result in search(' '.join(sys.argv[1:]), write_raw_to='/t/gutenberg.html'): diff --git a/src/calibre/gui2/store/stores/legimi_plugin.py b/src/calibre/gui2/store/stores/legimi_plugin.py index 8ce7998fa2..3c8c80ac80 100644 --- a/src/calibre/gui2/store/stores/legimi_plugin.py +++ b/src/calibre/gui2/store/stores/legimi_plugin.py @@ -24,6 +24,15 @@ from calibre.gui2.store.search_result import SearchResult from calibre.gui2.store.web_store_dialog import WebStoreDialog +def as_base64(data): + if not isinstance(data, bytes): + data = data.encode('utf-8') + ans = b64encode(data) + if isinstance(ans, bytes): + ans = ans.decode('ascii') + return ans + + class LegimiStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): @@ -31,11 +40,11 @@ class LegimiStore(BasicStoreConfig, StorePlugin): url = 'https://www.legimi.pl/ebooki/' - aff_url = aff_root + str(b64encode(url)) + aff_url = aff_root + as_base64(url) detail_url = None if detail_item: - detail_url = aff_root + str(b64encode(detail_item)) + detail_url = aff_root + as_base64(detail_item) if external or self.config.get('open_external', False): open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url))) diff --git a/src/calibre/gui2/store/stores/publio_plugin.py b/src/calibre/gui2/store/stores/publio_plugin.py index 7ab060112a..bf8913c529 100644 --- a/src/calibre/gui2/store/stores/publio_plugin.py +++ b/src/calibre/gui2/store/stores/publio_plugin.py @@ -23,17 +23,26 @@ from calibre.gui2.store.search_result import SearchResult from calibre.gui2.store.web_store_dialog import WebStoreDialog +def as_base64(data): + if not isinstance(data, bytes): + data = data.encode('utf-8') + ans = b64encode(data) + if isinstance(ans, bytes): + ans = ans.decode('ascii') + return ans + + class PublioStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): aff_root = 'https://www.a4b-tracking.com/pl/stat-click-text-link/29/58/' url = 'http://www.publio.pl/' - aff_url = aff_root + str(b64encode(url)) + aff_url = aff_root + as_base64(url) detail_url = None if detail_item: - detail_url = aff_root + str(b64encode(detail_item)) + detail_url = aff_root + as_base64(detail_item) if external or self.config.get('open_external', False): open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url))) diff --git a/src/calibre/gui2/store/stores/swiatebookow_plugin.py b/src/calibre/gui2/store/stores/swiatebookow_plugin.py index c45ab0f91a..f680eed78a 100644 --- a/src/calibre/gui2/store/stores/swiatebookow_plugin.py +++ b/src/calibre/gui2/store/stores/swiatebookow_plugin.py @@ -23,6 +23,15 @@ from calibre.gui2.store.search_result import SearchResult from calibre.gui2.store.web_store_dialog import WebStoreDialog +def as_base64(data): + if not isinstance(data, bytes): + data = data.encode('utf-8') + ans = b64encode(data) + if isinstance(ans, bytes): + ans = ans.decode('ascii') + return ans + + class SwiatEbookowStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): @@ -30,11 +39,11 @@ class SwiatEbookowStore(BasicStoreConfig, StorePlugin): url = 'https://www.swiatebookow.pl/' - aff_url = aff_root + str(b64encode(url)) + aff_url = aff_root + as_base64(url) detail_url = None if detail_item: - detail_url = aff_root + str(b64encode(detail_item)) + detail_url = aff_root + as_base64(detail_item) if external or self.config.get('open_external', False): open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url))) diff --git a/src/calibre/gui2/store/stores/virtualo_plugin.py b/src/calibre/gui2/store/stores/virtualo_plugin.py index 212b56e1e2..bdd484e722 100644 --- a/src/calibre/gui2/store/stores/virtualo_plugin.py +++ b/src/calibre/gui2/store/stores/virtualo_plugin.py @@ -24,6 +24,15 @@ from calibre.gui2.store.search_result import SearchResult from calibre.gui2.store.web_store_dialog import WebStoreDialog +def as_base64(data): + if not isinstance(data, bytes): + data = data.encode('utf-8') + ans = b64encode(data) + if isinstance(ans, bytes): + ans = ans.decode('ascii') + return ans + + class VirtualoStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): @@ -31,11 +40,11 @@ class VirtualoStore(BasicStoreConfig, StorePlugin): url = 'http://virtualo.pl/ebook/c2/' - aff_url = aff_root + str(b64encode(url)) + aff_url = aff_root + as_base64(url) detail_url = None if detail_item: - detail_url = aff_root + str(b64encode(detail_item)) + detail_url = aff_root + as_base64(detail_item) if external or self.config.get('open_external', False): open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url))) @@ -75,7 +84,7 @@ class VirtualoStore(BasicStoreConfig, StorePlugin): s.cover_url = cover_url s.title = title.strip() s.author = author.strip() - s.price = re.sub('\.',',',price.strip()) + s.price = re.sub(r'\.',',',price.strip()) s.detail_item = id s.formats = ', '.join(formats).upper() s.drm = SearchResult.DRM_UNLOCKED if nodrm else SearchResult.DRM_LOCKED diff --git a/src/calibre/gui2/store/stores/woblink_plugin.py b/src/calibre/gui2/store/stores/woblink_plugin.py index cc832ff72b..bdcd95a8d9 100644 --- a/src/calibre/gui2/store/stores/woblink_plugin.py +++ b/src/calibre/gui2/store/stores/woblink_plugin.py @@ -23,6 +23,15 @@ from calibre.gui2.store.search_result import SearchResult from calibre.gui2.store.web_store_dialog import WebStoreDialog +def as_base64(data): + if not isinstance(data, bytes): + data = data.encode('utf-8') + ans = b64encode(data) + if isinstance(ans, bytes): + ans = ans.decode('ascii') + return ans + + def search(query, max_results=10, timeout=60): url = 'http://woblink.com/publication/ajax?mode=none&query=' + urllib.quote_plus(query.encode('utf-8')) if max_results > 10: @@ -79,11 +88,11 @@ class WoblinkStore(BasicStoreConfig, StorePlugin): aff_root = 'https://www.a4b-tracking.com/pl/stat-click-text-link/16/58/' url = 'http://woblink.com/publication' - aff_url = aff_root + str(b64encode(url)) + aff_url = aff_root + as_base64(url) detail_url = None if detail_item: - detail_url = aff_root + str(b64encode('http://woblink.com' + detail_item)) + detail_url = aff_root + as_base64('http://woblink.com' + detail_item) if external or self.config.get('open_external', False): open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url))) diff --git a/src/calibre/gui2/toc/location.py b/src/calibre/gui2/toc/location.py index 8dd91685a6..f6fdd214bb 100644 --- a/src/calibre/gui2/toc/location.py +++ b/src/calibre/gui2/toc/location.py @@ -8,7 +8,6 @@ __copyright__ = '2013, Kovid Goyal ' __docformat__ = 'restructuredtext en' import json -from base64 import b64encode from PyQt5.Qt import (QWidget, QGridLayout, QListWidget, QSize, Qt, QUrl, pyqtSlot, pyqtSignal, QVBoxLayout, QFrame, QLabel, @@ -20,6 +19,7 @@ from calibre.ebooks.oeb.display.webview import load_html from calibre.gui2 import error_dialog, question_dialog, gprefs, secure_web_page from calibre.utils.logging import default_log from polyglot.builtins import unicode_type, range +from polyglot.binary import as_base64_unicode class Page(QWebPage): # {{{ @@ -77,7 +77,7 @@ class WebView(QWebView): # {{{ ''' raw = '::selection {background:#ffff00; color:#000;}\n'+raw data = 'data:text/css;charset=utf-8;base64,' - data += b64encode(raw.encode('utf-8')) + data += as_base64_unicode(raw) self.settings().setUserStyleSheetUrl(QUrl(data)) def load_js(self): diff --git a/src/calibre/gui2/tweak_book/preview.py b/src/calibre/gui2/tweak_book/preview.py index aa655d7682..45024ee6be 100644 --- a/src/calibre/gui2/tweak_book/preview.py +++ b/src/calibre/gui2/tweak_book/preview.py @@ -8,7 +8,6 @@ __copyright__ = '2013, Kovid Goyal ' import time, textwrap, json from bisect import bisect_right -from base64 import b64encode from polyglot.builtins import map, unicode_type from threading import Thread from functools import partial @@ -31,6 +30,7 @@ from calibre.gui2.widgets2 import HistoryLineEdit2 from calibre.utils.ipc.simple_worker import offload_worker from polyglot.urllib import urlparse from polyglot.queue import Queue, Empty +from polyglot.binary import as_base64_unicode shutdown = object() @@ -269,7 +269,7 @@ class WebPage(QWebPage): settings.setDefaultTextEncoding('utf-8') data = 'data:text/css;charset=utf-8;base64,' css = '[data-in-split-mode="1"] [data-is-block="1"]:hover { cursor: pointer !important; border-top: solid 5px green !important }' - data += b64encode(css.encode('utf-8')) + data += as_base64_unicode(css) settings.setUserStyleSheetUrl(QUrl(data)) self.setNetworkAccessManager(NetworkAccessManager(self)) diff --git a/src/calibre/gui2/viewer/documentview.py b/src/calibre/gui2/viewer/documentview.py index 9430df1d7a..91232c0cd7 100644 --- a/src/calibre/gui2/viewer/documentview.py +++ b/src/calibre/gui2/viewer/documentview.py @@ -5,7 +5,6 @@ __docformat__ = 'restructuredtext en' # Imports {{{ import math, json -from base64 import b64encode from functools import partial from polyglot.builtins import iteritems, map, unicode_type, string_or_bytes @@ -33,6 +32,7 @@ from calibre.gui2.viewer.footnote import Footnotes from calibre.gui2.viewer.fake_net import NetworkAccessManager from calibre.ebooks.oeb.display.webview import load_html from calibre.constants import isxp, iswindows, DEBUG, __version__ +from polyglot.binary import as_base64_unicode # }}} @@ -144,7 +144,7 @@ class Document(QWebPage): # {{{ raw = prefix + opts.user_css raw = '::selection {background:#ffff00; color:#000;}\n'+raw data = 'data:text/css;charset=utf-8;base64,' - data += b64encode(raw.encode('utf-8')) + data += as_base64_unicode(raw) self.settings().setUserStyleSheetUrl(QUrl(data)) def findText(self, q, flags): diff --git a/src/calibre/srv/auth.py b/src/calibre/srv/auth.py index 84fd260219..43d97278aa 100644 --- a/src/calibre/srv/auth.py +++ b/src/calibre/srv/auth.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import binascii, os, random, struct, base64 +import binascii, os, random, struct from collections import OrderedDict from hashlib import md5, sha256 from itertools import permutations @@ -17,6 +17,7 @@ from calibre.srv.http_request import parse_uri from calibre.srv.utils import parse_http_dict, encode_path from calibre.utils.monotonic import monotonic from polyglot import http_client +from polyglot.binary import from_base64_unicode MAX_AGE_SECONDS = 3600 nonce_counter, nonce_counter_lock = 0, Lock() @@ -76,7 +77,7 @@ def sha256_hex(s): def base64_decode(s): - return base64.standard_b64decode(as_bytestring(s)).decode('utf-8') + return from_base64_unicode(s) def synthesize_nonce(key_order, realm, secret, timestamp=None): diff --git a/src/calibre/srv/cdb.py b/src/calibre/srv/cdb.py index 7ed12d5d7d..0f79010be2 100644 --- a/src/calibre/srv/cdb.py +++ b/src/calibre/srv/cdb.py @@ -5,7 +5,6 @@ from __future__ import absolute_import, division, print_function, unicode_literals import os -from base64 import standard_b64decode from functools import partial from io import BytesIO @@ -20,6 +19,7 @@ from calibre.srv.utils import get_db, get_library_data from calibre.utils.imghdr import what from calibre.utils.serialize import MSGPACK_MIME, json_loads, msgpack_loads from polyglot.builtins import iteritems +from polyglot.binary import from_base64_bytes receive_data_methods = {'GET', 'POST'} @@ -160,7 +160,7 @@ def cdb_set_fields(ctx, rd, book_id, library_id): if cdata is not False: if cdata is not None: try: - cdata = standard_b64decode(cdata.split(',', 1)[-1].encode('ascii')) + cdata = from_base64_bytes(cdata.split(',', 1)[-1]) except Exception: raise HTTPBadRequest('Cover data is not valid base64 encoded data') try: diff --git a/src/calibre/srv/render_book.py b/src/calibre/srv/render_book.py index c689a1c0de..1bac7c4082 100644 --- a/src/calibre/srv/render_book.py +++ b/src/calibre/srv/render_book.py @@ -2,31 +2,35 @@ # vim:fileencoding=utf-8 # License: GPLv3 Copyright: 2016, Kovid Goyal -from __future__ import (unicode_literals, division, absolute_import, - print_function) -import sys, os, json, re -from base64 import standard_b64encode, standard_b64decode -from collections import defaultdict, OrderedDict -from itertools import count +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import os +import re +import sys +from collections import OrderedDict, defaultdict from functools import partial +from itertools import count from css_parser import replaceUrls from css_parser.css import CSSRule -from calibre import prepare_string_for_xml, force_unicode +from calibre import force_unicode, prepare_string_for_xml from calibre.ebooks import parse_css_length +from calibre.ebooks.css_transform_rules import StyleDeclaration from calibre.ebooks.oeb.base import ( - OEB_DOCS, OEB_STYLES, rewrite_links, XPath, urlunquote, XLINK, XHTML_NS, OPF, XHTML, EPUB_NS) + EPUB_NS, OEB_DOCS, OEB_STYLES, OPF, XHTML, XHTML_NS, XLINK, XPath, rewrite_links, + urlunquote +) from calibre.ebooks.oeb.iterator.book import extract_book from calibre.ebooks.oeb.polish.container import Container as ContainerBase -from calibre.ebooks.oeb.polish.cover import set_epub_cover, find_cover_image +from calibre.ebooks.oeb.polish.cover import find_cover_image, set_epub_cover from calibre.ebooks.oeb.polish.css import transform_css -from calibre.ebooks.oeb.polish.utils import extract -from calibre.ebooks.css_transform_rules import StyleDeclaration -from calibre.ebooks.oeb.polish.toc import get_toc, get_landmarks -from calibre.ebooks.oeb.polish.utils import guess_type -from calibre.utils.short_uuid import uuid4 +from calibre.ebooks.oeb.polish.toc import get_landmarks, get_toc +from calibre.ebooks.oeb.polish.utils import extract, guess_type from calibre.utils.logging import default_log +from calibre.utils.short_uuid import uuid4 +from polyglot.binary import as_base64_unicode as encode_component, from_base64_unicode as decode_component from polyglot.builtins import iteritems, map, unicode_type from polyglot.urllib import quote, urlparse @@ -35,14 +39,6 @@ RENDER_VERSION = 1 BLANK_JPEG = b'\xff\xd8\xff\xdb\x00C\x00\x03\x02\x02\x02\x02\x02\x03\x02\x02\x02\x03\x03\x03\x03\x04\x06\x04\x04\x04\x04\x04\x08\x06\x06\x05\x06\t\x08\n\n\t\x08\t\t\n\x0c\x0f\x0c\n\x0b\x0e\x0b\t\t\r\x11\r\x0e\x0f\x10\x10\x11\x10\n\x0c\x12\x13\x12\x10\x13\x0f\x10\x10\x10\xff\xc9\x00\x0b\x08\x00\x01\x00\x01\x01\x01\x11\x00\xff\xcc\x00\x06\x00\x10\x10\x05\xff\xda\x00\x08\x01\x01\x00\x00?\x00\xd2\xcf \xff\xd9' # noqa -def encode_component(x): - return standard_b64encode(x.encode('utf-8')).decode('ascii') - - -def decode_component(x): - return standard_b64decode(x).decode('utf-8') - - def encode_url(name, frag=''): name = encode_component(name) if frag: diff --git a/src/calibre/srv/tests/ajax.py b/src/calibre/srv/tests/ajax.py index 0fade7e9d0..0b50a96b10 100644 --- a/src/calibre/srv/tests/ajax.py +++ b/src/calibre/srv/tests/ajax.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import zlib, json, base64, os +import zlib, json, os from io import BytesIO from functools import partial @@ -14,11 +14,12 @@ from calibre.ebooks.metadata.meta import get_metadata from calibre.srv.tests.base import LibraryBaseTest from polyglot.http_client import OK, NOT_FOUND, FORBIDDEN from polyglot.urllib import urlencode, quote +from polyglot.binary import as_base64_bytes def make_request(conn, url, headers={}, prefix='/ajax', username=None, password=None, method='GET', data=None): if username and password: - headers[b'Authorization'] = b'Basic ' + base64.standard_b64encode((username + ':' + password).encode('utf-8')) + headers[b'Authorization'] = b'Basic ' + as_base64_bytes((username + ':' + password)) conn.request(method, prefix + url, headers=headers, body=data) r = conn.getresponse() data = r.read() diff --git a/src/calibre/srv/tests/auth.py b/src/calibre/srv/tests/auth.py index 45764253e6..231a600b26 100644 --- a/src/calibre/srv/tests/auth.py +++ b/src/calibre/srv/tests/auth.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import base64, subprocess, os, time +import subprocess, os, time from collections import namedtuple try: from distutils.spawn import find_executable @@ -22,6 +22,7 @@ from polyglot import http_client from polyglot.http_cookie import CookieJar from polyglot.urllib import (build_opener, HTTPBasicAuthHandler, HTTPCookieProcessor, HTTPDigestAuthHandler, HTTPError) +from polyglot.binary import as_base64_bytes REALM = 'calibre-test' @@ -101,7 +102,7 @@ class TestAuth(BaseTest): self.ae(r.status, http_client.UNAUTHORIZED) self.ae(r.getheader('WWW-Authenticate'), b'Basic realm="%s"' % bytes(REALM)) self.assertFalse(r.read()) - conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + base64.standard_b64encode(b'testuser:testpw')}) + conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + as_base64_bytes(b'testuser:testpw')}) r = conn.getresponse() self.ae(r.read(), b'closed') self.ae(r.status, http_client.OK) @@ -109,7 +110,7 @@ class TestAuth(BaseTest): self.ae(b'closed', urlopen(server, un='!@#$%^&*()-=_+', pw='!@#$%^&*()-=_+', method='basic').read()) def request(un='testuser', pw='testpw'): - conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + base64.standard_b64encode(bytes('%s:%s' % (un, pw)))}) + conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + as_base64_bytes('%s:%s' % (un, pw))}) r = conn.getresponse() return r.status, r.read() @@ -254,7 +255,7 @@ class TestAuth(BaseTest): conn = server.connect() def request(un='testuser', pw='testpw'): - conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + base64.standard_b64encode(bytes('%s:%s' % (un, pw)))}) + conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + as_base64_bytes('%s:%s' % (un, pw))}) r = conn.getresponse() return r.status, r.read() diff --git a/src/calibre/srv/tests/web_sockets.py b/src/calibre/srv/tests/web_sockets.py index c456d532ff..062b2cfcff 100644 --- a/src/calibre/srv/tests/web_sockets.py +++ b/src/calibre/srv/tests/web_sockets.py @@ -5,7 +5,6 @@ from __future__ import (unicode_literals, division, absolute_import, print_function) import socket, os, struct, errno, numbers -from base64 import standard_b64encode from collections import deque, namedtuple from functools import partial from hashlib import sha1 @@ -17,6 +16,7 @@ from calibre.srv.web_socket import ( from calibre.utils.monotonic import monotonic from calibre.utils.socket_inheritance import set_socket_inherit from polyglot.builtins import range, unicode_type +from polyglot.binary import as_base64_bytes, as_base64_unicode HANDSHAKE_STR = '''\ GET / HTTP/1.1\r @@ -35,7 +35,7 @@ class WSClient(object): self.timeout = timeout self.socket = socket.create_connection(('localhost', port), timeout) set_socket_inherit(self.socket, False) - self.key = standard_b64encode(os.urandom(8)) + self.key = as_base64_bytes(os.urandom(8)) self.socket.sendall(HANDSHAKE_STR.format(self.key).encode('ascii')) self.read_buf = deque() self.read_upgrade_response() @@ -64,7 +64,7 @@ class WSClient(object): if rl != b'HTTP/1.1 101 Switching Protocols\r\n': raise ValueError('Server did not respond with correct switching protocols line') headers = read_headers(partial(next, lines)) - key = standard_b64encode(sha1(self.key + GUID_STR).digest()) + key = as_base64_unicode(sha1(self.key + GUID_STR).digest()) if headers.get('Sec-WebSocket-Accept') != key: raise ValueError('Server did not respond with correct key in Sec-WebSocket-Accept') diff --git a/src/calibre/srv/web_socket.py b/src/calibre/srv/web_socket.py index 59a5cfc9f8..0f76576fd2 100644 --- a/src/calibre/srv/web_socket.py +++ b/src/calibre/srv/web_socket.py @@ -2,24 +2,28 @@ # vim:fileencoding=utf-8 # License: GPLv3 Copyright: 2015, Kovid Goyal -from __future__ import (unicode_literals, division, absolute_import, - print_function) +from __future__ import absolute_import, division, print_function, unicode_literals -import os, weakref, socket -from base64 import standard_b64encode +import os +import socket +import weakref from collections import deque from hashlib import sha1 -from struct import unpack_from, pack, error as struct_error +from struct import error as struct_error, pack, unpack_from from threading import Lock from calibre import as_unicode from calibre.constants import plugins -from calibre.srv.loop import ServerLoop, HandleInterrupt, WRITE, READ, RDWR, Connection from calibre.srv.http_response import HTTPConnection, create_http_handler +from calibre.srv.loop import ( + RDWR, READ, WRITE, Connection, HandleInterrupt, ServerLoop +) from calibre.srv.utils import DESIRED_SEND_BUFFER_SIZE from calibre.utils.speedups import ReadOnlyFileBuffer -from polyglot.queue import Queue, Empty from polyglot import http_client +from polyglot.binary import as_base64_unicode +from polyglot.queue import Empty, Queue + speedup, err = plugins['speedup'] if not speedup: raise RuntimeError('Failed to load speedup module with error: ' + err) @@ -291,7 +295,7 @@ class WebSocketConnection(HTTPConnection): if self.method != 'GET': return self.simple_response(http_client.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method) - response = HANDSHAKE_STR % standard_b64encode(sha1(key + GUID_STR).digest()) + response = HANDSHAKE_STR % as_base64_unicode(sha1(key + GUID_STR).digest()) self.optimize_for_sending_packet() self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) self.set_state(WRITE, self.upgrade_connection_to_ws, ReadOnlyFileBuffer(response.encode('ascii')), inheaders) diff --git a/src/calibre/utils/config_base.py b/src/calibre/utils/config_base.py index 0434abfc0e..6eae6234bb 100644 --- a/src/calibre/utils/config_base.py +++ b/src/calibre/utils/config_base.py @@ -21,9 +21,9 @@ plugin_dir = os.path.join(config_dir, 'plugins') def to_json(obj): import datetime if isinstance(obj, bytearray): - import base64 + from base64 import standard_b64encode return {'__class__': 'bytearray', - '__value__': base64.standard_b64encode(bytes(obj)).decode('ascii')} + '__value__': standard_b64encode(bytes(obj)).decode('ascii')} if isinstance(obj, datetime.datetime): from calibre.utils.date import isoformat return {'__class__': 'datetime.datetime', @@ -42,8 +42,8 @@ def from_json(obj): custom = obj.get('__class__') if custom is not None: if custom == 'bytearray': - import base64 - return bytearray(base64.standard_b64decode(obj['__value__'])) + from base64 import standard_b64decode + return bytearray(standard_b64decode(obj['__value__'].encode('ascii'))) if custom == 'datetime.datetime': from calibre.utils.iso8601 import parse_iso8601 return parse_iso8601(obj['__value__'], assume_utc=True) diff --git a/src/calibre/utils/serialize.py b/src/calibre/utils/serialize.py index cd5cd28a1b..d54d6d6885 100644 --- a/src/calibre/utils/serialize.py +++ b/src/calibre/utils/serialize.py @@ -66,13 +66,11 @@ def json_dumps(data, **kw): def decode_metadata(x, for_json): - import base64 + from polyglot.binary import from_base64_bytes from calibre.ebooks.metadata.book.serialize import metadata_from_dict obj = metadata_from_dict(x) if for_json and obj.cover_data and obj.cover_data[1]: - obj.cover_data = obj.cover_data[0], base64.standard_b64decode( - obj.cover_data[1] - ) + obj.cover_data = obj.cover_data[0], from_base64_bytes(obj.cover_data[1]) return obj diff --git a/src/calibre/web/fetch/simple.py b/src/calibre/web/fetch/simple.py index 07ac9b4a93..971f52a4e5 100644 --- a/src/calibre/web/fetch/simple.py +++ b/src/calibre/web/fetch/simple.py @@ -17,7 +17,6 @@ import sys import threading import time import traceback -from base64 import b64decode from calibre import browser, relpath, unicode_path from calibre.constants import filesystem_encoding, iswindows @@ -35,6 +34,7 @@ from polyglot.urllib import ( URLError, quote, url2pathname, urljoin, urlparse, urlsplit, urlunparse, urlunsplit ) +from polyglot.binary import from_base64_bytes class AbortArticle(Exception): @@ -391,8 +391,8 @@ class RecursiveFetcher(object): iurl = tag['src'] if iurl.startswith('data:image/'): try: - data = b64decode(iurl.partition(',')[-1]) - except: + data = from_base64_bytes(iurl.partition(',')[-1]) + except Exception: self.log.exception('Failed to decode embedded image') continue else: diff --git a/src/polyglot/binary.py b/src/polyglot/binary.py new file mode 100644 index 0000000000..7631b13bc8 --- /dev/null +++ b/src/polyglot/binary.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +# License: GPL v3 Copyright: 2019, Kovid Goyal + +from __future__ import absolute_import, division, print_function, unicode_literals + +from base64 import standard_b64encode, standard_b64decode +from polyglot.builtins import unicode_type + + +def as_base64_bytes(x, enc='utf-8'): + if isinstance(x, unicode_type): + x = x.encode(enc) + return standard_b64encode(x) + + +def as_base64_unicode(x, enc='utf-8'): + if isinstance(x, unicode_type): + x = x.encode(enc) + return standard_b64encode(x).decode('ascii') + + +def from_base64_unicode(x, enc='utf-8'): + if isinstance(x, unicode_type): + x = x.encode('ascii') + return standard_b64decode(x).decode(enc) + + +def from_base64_bytes(x): + if isinstance(x, unicode_type): + x = x.encode('ascii') + return standard_b64decode(x)