py3: port use as base64 module

src/calibre/devices/prs505/sony_cache.py

@@ -6,7 +6,6 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import os, time
-from base64 import b64decode
 from datetime import date
 
 from calibre import prints, guess_type, isbytestring, fsync
@@ -16,6 +15,7 @@ from calibre.constants import DEBUG, preferred_encoding
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.ebooks.metadata import authors_to_string, title_sort, \
                                     authors_to_sort_string
+from polyglot.binary import from_base64_bytes
 
 '''
 cahceExt.xml
@@ -380,8 +380,8 @@ class XMLCache(object):
                             'descendant::*[local-name()="png"]'):
                         if img.text:
                             try:
-                                raw = b64decode(img.text.strip())
+                                raw = from_base64_bytes(img.text.strip())
-                            except:
+                            except Exception:
                                 continue
                             book.thumbnail = raw
                             break

src/calibre/ebooks/fb2/__init__.py

@@ -10,12 +10,12 @@ __docformat__ = 'restructuredtext en'
 
 def base64_decode(raw):
     from io import BytesIO
-    from base64 import b64decode
+    from polyglot.binary import from_base64_bytes
 
     # First try the python implementation as it is faster
     try:
-        return b64decode(raw)
+        return from_base64_bytes(raw)
-    except TypeError:
+    except Exception:
         pass
 
     # Try a more robust version (adapted from FBReader sources)
@@ -49,5 +49,3 @@ def base64_decode(raw):
             tot >>= 8
         out.write(bytes(triple))
     return out.getvalue()
-
-

src/calibre/ebooks/fb2/fb2ml.py

@@ -9,7 +9,6 @@ Transform OEB content into FB2 markup
 '''
 
 import re, textwrap, uuid
-from base64 import b64encode
 from datetime import datetime
 
 from lxml import etree
@@ -20,6 +19,7 @@ from calibre.utils.localization import lang_as_iso639_1
 from calibre.utils.img import save_cover_data_to
 from calibre.ebooks.oeb.base import urlnormalize
 from polyglot.builtins import unicode_type, string_or_bytes
+from polyglot.binary import as_base64_unicode
 
 
 class FB2MLizer(object):
@@ -308,9 +308,9 @@ class FB2MLizer(object):
                 try:
                     if item.media_type != 'image/jpeg':
                         imdata = save_cover_data_to(item.data, compression_quality=70)
-                        raw_data = b64encode(imdata)
+                        raw_data = as_base64_unicode(imdata)
                     else:
-                        raw_data = b64encode(item.data)
+                        raw_data = as_base64_unicode(item.data)
                     # Don't put the encoded image on a single line.
                     data = ''
                     col = 1

src/calibre/ebooks/metadata/book/json_codec.py

@@ -5,7 +5,6 @@ Created on 4 Jun 2010
 '''
 from __future__ import print_function
 
-from base64 import b64encode, b64decode
 import json, traceback
 from datetime import datetime, time
 
@@ -14,6 +13,7 @@ from calibre.constants import filesystem_encoding, preferred_encoding
 from calibre.library.field_metadata import FieldMetadata
 from calibre import isbytestring
 from polyglot.builtins import iteritems, itervalues
+from polyglot.binary import as_base64_unicode, from_base64_bytes
 
 # Translate datetimes to and from strings. The string form is the datetime in
 # UTC. The returned date is also UTC
@@ -57,7 +57,7 @@ def encode_thumbnail(thumbnail):
             thumbnail = (width, height, thumbnail)
         except Exception:
             return None
-    return (thumbnail[0], thumbnail[1], b64encode(str(thumbnail[2])))
+    return (thumbnail[0], thumbnail[1], as_base64_unicode(thumbnail[2]))
 
 
 def decode_thumbnail(tup):
@@ -66,7 +66,7 @@ def decode_thumbnail(tup):
     '''
     if tup is None:
         return None
-    return (tup[0], tup[1], b64decode(tup[2]))
+    return (tup[0], tup[1], from_base64_bytes(tup[2]))
 
 
 def object_to_unicode(obj, enc=preferred_encoding):

src/calibre/ebooks/metadata/book/serialize.py

@@ -4,13 +4,13 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-import base64
 
 from calibre.constants import preferred_encoding
 from calibre.ebooks.metadata.book import SERIALIZABLE_FIELDS
 from calibre.ebooks.metadata.book.base import Metadata
 from calibre.utils.imghdr import what
 from polyglot.builtins import iteritems, unicode_type
+from polyglot.binary import as_base64_unicode
 
 
 def ensure_unicode(obj, enc=preferred_encoding):
@@ -52,7 +52,7 @@ def metadata_as_dict(mi, encode_cover_data=False):
             ans[field] = ensure_unicode(val)
     if mi.cover_data and mi.cover_data[1]:
         if encode_cover_data:
-            ans['cover_data'] = [mi.cover_data[0], base64.standard_b64encode(bytes(mi.cover_data[1]))]
+            ans['cover_data'] = [mi.cover_data[0], as_base64_unicode(mi.cover_data[1])]
         else:
             ans['cover_data'] = mi.cover_data
     um = mi.get_all_user_metadata(False)

src/calibre/ebooks/metadata/fb2.py

@@ -8,7 +8,6 @@ __copyright__ = '2011, Roman Mukhin <ramses_ru at hotmail.com>, '\
 import os, random
 from functools import partial
 from string import ascii_letters, digits
-from base64 import b64encode
 
 from lxml import etree
 
@@ -19,6 +18,7 @@ from calibre import guess_type, guess_all_extensions, prints, force_unicode
 from calibre.ebooks.metadata import MetaInformation, check_isbn
 from calibre.ebooks.chardet import xml_to_unicode
 from polyglot.builtins import unicode_type
+from polyglot.binary import as_base64_unicode
 
 
 NAMESPACES = {
@@ -383,7 +383,7 @@ def _rnd_pic_file_name(prefix='calibre_cover_', size=32, ext='jpg'):
 
 def _encode_into_jpeg(data):
     data = save_cover_data_to(data)
-    return b64encode(data)
+    return as_base64_unicode(data)
 
 
 def _set_cover(title_info, mi, ctx):

src/calibre/ebooks/metadata/kfx.py

@@ -8,7 +8,7 @@ from __future__ import (unicode_literals, division, absolute_import,
 # Based on work of John Howell reversing the KFX format
 # https://www.mobileread.com/forums/showpost.php?p=3176029&postcount=89
 
-import struct, sys, base64, re
+import struct, sys, re
 from collections import defaultdict
 
 from calibre.ebooks.metadata.book.base import Metadata
@@ -19,6 +19,7 @@ from calibre.utils.date import parse_only_date
 from calibre.utils.localization import canonicalize_lang
 from calibre.utils.imghdr import identify
 from polyglot.builtins import unicode_type
+from polyglot.binary import as_base64_bytes, from_base64_bytes
 
 
 class InvalidKFX(ValueError):
@@ -155,7 +156,7 @@ class Entity(PackedBlock):
         if PackedData(self.entity_data).unpack_one('4s') == ION_MAGIC:
             entity_value = PackedIon(self.entity_data).decode()
         else:
-            entity_value = base64.b64encode(self.entity_data)
+            entity_value = as_base64_bytes(self.entity_data)
 
         return (property_name(self.entity_type), property_name(self.entity_id), entity_value)
 
@@ -343,8 +344,8 @@ def read_metadata_kfx(stream, read_cover=True):
         mi.publisher = get('publisher')
     if read_cover and m[COVER_KEY]:
         try:
-            data = base64.standard_b64decode(m[COVER_KEY])
+            data = from_base64_bytes(m[COVER_KEY])
-            fmt, w, h = identify(bytes(data))
+            fmt, w, h = identify(data)
         except Exception:
             w, h, fmt = 0, 0, None
         if fmt and w > -1 and h > -1:

src/calibre/ebooks/oeb/polish/download.py

@@ -10,7 +10,6 @@ import os
 import posixpath
 import re
 import shutil
-from base64 import standard_b64decode
 from collections import defaultdict
 from contextlib import closing
 from functools import partial
@@ -25,6 +24,7 @@ from calibre.ptempfile import TemporaryDirectory
 from calibre.web import get_download_filename_from_response
 from polyglot.builtins import iteritems
 from polyglot.urllib import urlopen, urlparse
+from polyglot.binary import from_base64_bytes
 
 
 def is_external(url):
@@ -117,7 +117,7 @@ def download_one(tdir, timeout, progress_report, data_uri_map, url):
                 parts = prefix.split(';')
                 if parts and parts[-1].lower() == 'base64':
                     payload = re.sub(r'\s+', '', payload)
-                    payload = standard_b64decode(payload)
+                    payload = from_base64_bytes(payload)
                 else:
                     payload = payload.encode('utf-8')
                 seen_before = data_uri_map.get(payload)

src/calibre/ebooks/oeb/transforms/data_url.py

@@ -28,9 +28,9 @@ class DataURL(object):
                     continue
                 if ';base64' in header:
                     data = re.sub(r'\s+', '', data)
-                    from base64 import b64decode
+                    from polyglot.binary import from_base64_bytes
                     try:
-                        data = b64decode(data)
+                        data = from_base64_bytes(data)
                     except Exception:
                         self.log.error('Found invalid base64 encoded data URI, ignoring it')
                         continue

src/calibre/gui2/store/search/download_thread.py

@@ -6,7 +6,7 @@ __license__ = 'GPL 3'
 __copyright__ = '2011, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
-import traceback, base64
+import traceback
 from contextlib import closing
 from threading import Thread
 
@@ -15,6 +15,7 @@ from calibre.constants import DEBUG
 from calibre.utils.img import scale_image
 from polyglot.builtins import range
 from polyglot.queue import Queue
+from polyglot.binary import from_base64_bytes
 
 
 class GenericDownloadThreadPool(object):
@@ -143,7 +144,7 @@ class CoverThreadPool(GenericDownloadThreadPool):
 
 
 def decode_data_url(url):
-    return base64.standard_b64decode(url.partition(',')[2])
+    return from_base64_bytes(url.partition(',')[2])
 
 
 class CoverThread(Thread):

src/calibre/gui2/store/stores/ebookpoint_plugin.py

@@ -24,6 +24,15 @@ from calibre.gui2.store.search_result import SearchResult
 from calibre.gui2.store.web_store_dialog import WebStoreDialog
 
 
+def as_base64(data):
+    if not isinstance(data, bytes):
+        data = data.encode('utf-8')
+    ans = b64encode(data)
+    if isinstance(ans, bytes):
+        ans = ans.decode('ascii')
+    return ans
+
+
 class EbookpointStore(BasicStoreConfig, StorePlugin):
 
     def open(self, parent=None, detail_item=None, external=False):
@@ -31,11 +40,11 @@ class EbookpointStore(BasicStoreConfig, StorePlugin):
 
         url = 'http://ebookpoint.pl/'
 
-        aff_url = aff_root + str(b64encode(url))
+        aff_url = aff_root + as_base64(url)
 
         detail_url = None
         if detail_item:
-            detail_url = aff_root + str(b64encode(detail_item))
+            detail_url = aff_root + as_base64(detail_item)
 
         if external or self.config.get('open_external', False):
             open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url)))

src/calibre/gui2/store/stores/empik_plugin.py

@@ -24,6 +24,15 @@ from calibre.gui2.store.search_result import SearchResult
 from calibre.gui2.store.web_store_dialog import WebStoreDialog
 
 
+def as_base64(data):
+    if not isinstance(data, bytes):
+        data = data.encode('utf-8')
+    ans = b64encode(data)
+    if isinstance(ans, bytes):
+        ans = ans.decode('ascii')
+    return ans
+
+
 class EmpikStore(BasicStoreConfig, StorePlugin):
 
     def open(self, parent=None, detail_item=None, external=False):
@@ -31,11 +40,11 @@ class EmpikStore(BasicStoreConfig, StorePlugin):
 
         url = 'http://www.empik.com/ebooki'
 
-        aff_url = aff_root + str(b64encode(url))
+        aff_url = aff_root + as_base64(url)
 
         detail_url = None
         if detail_item:
-            detail_url = aff_root + str(b64encode(detail_item))
+            detail_url = aff_root + as_base64(detail_item)
 
         if external or self.config.get('open_external', False):
             open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url)))
@@ -82,4 +91,3 @@ class EmpikStore(BasicStoreConfig, StorePlugin):
                 s.formats = formats.upper().strip()
 
                 yield s
-

src/calibre/gui2/store/stores/gutenberg_plugin.py

@@ -52,7 +52,7 @@ def search(query, max_results=10, timeout=60, write_raw_to=None):
             # We could use the <link rel="alternate" type="text/html" ...> tag from the
             # detail odps page but this is easier.
             id = fix_url(''.join(data.xpath('./*[local-name() = "id"]/text()')).strip())
-            s.detail_item = url_slash_cleaner('%s/ebooks/%s' % (web_url, re.sub('[^\d]', '', id)))
+            s.detail_item = url_slash_cleaner('%s/ebooks/%s' % (web_url, re.sub(r'[^\d]', '', id)))
             s.title = ' '.join(data.xpath('./*[local-name() = "title"]//text()')).strip()
             s.author = ', '.join(data.xpath('./*[local-name() = "content"]//text()')).strip()
             if not s.title or not s.author:
@@ -83,7 +83,10 @@ def search(query, max_results=10, timeout=60, write_raw_to=None):
                     href = fix_url(href)
                     if rel in ('http://opds-spec.org/thumbnail', 'http://opds-spec.org/image/thumbnail'):
                         if href.startswith('data:image/png;base64,'):
-                            s.cover_data = base64.b64decode(href.replace('data:image/png;base64,', ''))
+                            cdata = href.replace('data:image/png;base64,', '')
+                            if not isinstance(cdata, bytes):
+                                cdata = cdata.encode('ascii')
+                            s.cover_data = base64.b64decode(cdata)
 
             yield s
 
@@ -123,6 +126,7 @@ class GutenbergStore(BasicStoreConfig, OpenSearchOPDSStore):
         for result in search(query, max_results, timeout):
             yield result
 
+
 if __name__ == '__main__':
     import sys
     for result in search(' '.join(sys.argv[1:]), write_raw_to='/t/gutenberg.html'):

src/calibre/gui2/store/stores/legimi_plugin.py

@@ -24,6 +24,15 @@ from calibre.gui2.store.search_result import SearchResult
 from calibre.gui2.store.web_store_dialog import WebStoreDialog
 
 
+def as_base64(data):
+    if not isinstance(data, bytes):
+        data = data.encode('utf-8')
+    ans = b64encode(data)
+    if isinstance(ans, bytes):
+        ans = ans.decode('ascii')
+    return ans
+
+
 class LegimiStore(BasicStoreConfig, StorePlugin):
 
     def open(self, parent=None, detail_item=None, external=False):
@@ -31,11 +40,11 @@ class LegimiStore(BasicStoreConfig, StorePlugin):
 
         url = 'https://www.legimi.pl/ebooki/'
 
-        aff_url = aff_root + str(b64encode(url))
+        aff_url = aff_root + as_base64(url)
 
         detail_url = None
         if detail_item:
-            detail_url = aff_root + str(b64encode(detail_item))
+            detail_url = aff_root + as_base64(detail_item)
 
         if external or self.config.get('open_external', False):
             open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url)))

src/calibre/gui2/store/stores/publio_plugin.py

@@ -23,17 +23,26 @@ from calibre.gui2.store.search_result import SearchResult
 from calibre.gui2.store.web_store_dialog import WebStoreDialog
 
 
+def as_base64(data):
+    if not isinstance(data, bytes):
+        data = data.encode('utf-8')
+    ans = b64encode(data)
+    if isinstance(ans, bytes):
+        ans = ans.decode('ascii')
+    return ans
+
+
 class PublioStore(BasicStoreConfig, StorePlugin):
 
     def open(self, parent=None, detail_item=None, external=False):
         aff_root = 'https://www.a4b-tracking.com/pl/stat-click-text-link/29/58/'
         url = 'http://www.publio.pl/'
 
-        aff_url = aff_root + str(b64encode(url))
+        aff_url = aff_root + as_base64(url)
 
         detail_url = None
         if detail_item:
-            detail_url = aff_root + str(b64encode(detail_item))
+            detail_url = aff_root + as_base64(detail_item)
 
         if external or self.config.get('open_external', False):
             open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url)))

src/calibre/gui2/store/stores/swiatebookow_plugin.py

@@ -23,6 +23,15 @@ from calibre.gui2.store.search_result import SearchResult
 from calibre.gui2.store.web_store_dialog import WebStoreDialog
 
 
+def as_base64(data):
+    if not isinstance(data, bytes):
+        data = data.encode('utf-8')
+    ans = b64encode(data)
+    if isinstance(ans, bytes):
+        ans = ans.decode('ascii')
+    return ans
+
+
 class SwiatEbookowStore(BasicStoreConfig, StorePlugin):
 
     def open(self, parent=None, detail_item=None, external=False):
@@ -30,11 +39,11 @@ class SwiatEbookowStore(BasicStoreConfig, StorePlugin):
 
         url = 'https://www.swiatebookow.pl/'
 
-        aff_url = aff_root + str(b64encode(url))
+        aff_url = aff_root + as_base64(url)
 
         detail_url = None
         if detail_item:
-            detail_url = aff_root + str(b64encode(detail_item))
+            detail_url = aff_root + as_base64(detail_item)
 
         if external or self.config.get('open_external', False):
             open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url)))

src/calibre/gui2/store/stores/virtualo_plugin.py

@@ -24,6 +24,15 @@ from calibre.gui2.store.search_result import SearchResult
 from calibre.gui2.store.web_store_dialog import WebStoreDialog
 
 
+def as_base64(data):
+    if not isinstance(data, bytes):
+        data = data.encode('utf-8')
+    ans = b64encode(data)
+    if isinstance(ans, bytes):
+        ans = ans.decode('ascii')
+    return ans
+
+
 class VirtualoStore(BasicStoreConfig, StorePlugin):
 
     def open(self, parent=None, detail_item=None, external=False):
@@ -31,11 +40,11 @@ class VirtualoStore(BasicStoreConfig, StorePlugin):
 
         url = 'http://virtualo.pl/ebook/c2/'
 
-        aff_url = aff_root + str(b64encode(url))
+        aff_url = aff_root + as_base64(url)
 
         detail_url = None
         if detail_item:
-            detail_url = aff_root + str(b64encode(detail_item))
+            detail_url = aff_root + as_base64(detail_item)
 
         if external or self.config.get('open_external', False):
             open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url)))
@@ -75,7 +84,7 @@ class VirtualoStore(BasicStoreConfig, StorePlugin):
                 s.cover_url = cover_url
                 s.title = title.strip()
                 s.author = author.strip()
-                s.price = re.sub('\.',',',price.strip())
+                s.price = re.sub(r'\.',',',price.strip())
                 s.detail_item = id
                 s.formats = ', '.join(formats).upper()
                 s.drm = SearchResult.DRM_UNLOCKED if nodrm else SearchResult.DRM_LOCKED

src/calibre/gui2/store/stores/woblink_plugin.py

@@ -23,6 +23,15 @@ from calibre.gui2.store.search_result import SearchResult
 from calibre.gui2.store.web_store_dialog import WebStoreDialog
 
 
+def as_base64(data):
+    if not isinstance(data, bytes):
+        data = data.encode('utf-8')
+    ans = b64encode(data)
+    if isinstance(ans, bytes):
+        ans = ans.decode('ascii')
+    return ans
+
+
 def search(query, max_results=10, timeout=60):
     url = 'http://woblink.com/publication/ajax?mode=none&query=' + urllib.quote_plus(query.encode('utf-8'))
     if max_results > 10:
@@ -79,11 +88,11 @@ class WoblinkStore(BasicStoreConfig, StorePlugin):
         aff_root = 'https://www.a4b-tracking.com/pl/stat-click-text-link/16/58/'
         url = 'http://woblink.com/publication'
 
-        aff_url = aff_root + str(b64encode(url))
+        aff_url = aff_root + as_base64(url)
         detail_url = None
 
         if detail_item:
-            detail_url = aff_root + str(b64encode('http://woblink.com' + detail_item))
+            detail_url = aff_root + as_base64('http://woblink.com' + detail_item)
 
         if external or self.config.get('open_external', False):
             open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url)))

src/calibre/gui2/toc/location.py

@@ -8,7 +8,6 @@ __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import json
-from base64 import b64encode
 
 from PyQt5.Qt import (QWidget, QGridLayout, QListWidget, QSize, Qt, QUrl,
                       pyqtSlot, pyqtSignal, QVBoxLayout, QFrame, QLabel,
@@ -20,6 +19,7 @@ from calibre.ebooks.oeb.display.webview import load_html
 from calibre.gui2 import error_dialog, question_dialog, gprefs, secure_web_page
 from calibre.utils.logging import default_log
 from polyglot.builtins import unicode_type, range
+from polyglot.binary import as_base64_unicode
 
 
 class Page(QWebPage):  # {{{
@@ -77,7 +77,7 @@ class WebView(QWebView):  # {{{
         '''
         raw = '::selection {background:#ffff00; color:#000;}\n'+raw
         data = 'data:text/css;charset=utf-8;base64,'
-        data += b64encode(raw.encode('utf-8'))
+        data += as_base64_unicode(raw)
         self.settings().setUserStyleSheetUrl(QUrl(data))
 
     def load_js(self):

src/calibre/gui2/tweak_book/preview.py

@@ -8,7 +8,6 @@ __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import time, textwrap, json
 from bisect import bisect_right
-from base64 import b64encode
 from polyglot.builtins import map, unicode_type
 from threading import Thread
 from functools import partial
@@ -31,6 +30,7 @@ from calibre.gui2.widgets2 import HistoryLineEdit2
 from calibre.utils.ipc.simple_worker import offload_worker
 from polyglot.urllib import urlparse
 from polyglot.queue import Queue, Empty
+from polyglot.binary import as_base64_unicode
 
 shutdown = object()
 
@@ -269,7 +269,7 @@ class WebPage(QWebPage):
         settings.setDefaultTextEncoding('utf-8')
         data = 'data:text/css;charset=utf-8;base64,'
         css = '[data-in-split-mode="1"] [data-is-block="1"]:hover { cursor: pointer !important; border-top: solid 5px green !important }'
-        data += b64encode(css.encode('utf-8'))
+        data += as_base64_unicode(css)
         settings.setUserStyleSheetUrl(QUrl(data))
 
         self.setNetworkAccessManager(NetworkAccessManager(self))

src/calibre/gui2/viewer/documentview.py

@@ -5,7 +5,6 @@ __docformat__ = 'restructuredtext en'
 
 # Imports {{{
 import math, json
-from base64 import b64encode
 from functools import partial
 from polyglot.builtins import iteritems, map, unicode_type, string_or_bytes
 
@@ -33,6 +32,7 @@ from calibre.gui2.viewer.footnote import Footnotes
 from calibre.gui2.viewer.fake_net import NetworkAccessManager
 from calibre.ebooks.oeb.display.webview import load_html
 from calibre.constants import isxp, iswindows, DEBUG, __version__
+from polyglot.binary import as_base64_unicode
 # }}}
 
 
@@ -144,7 +144,7 @@ class Document(QWebPage):  # {{{
         raw = prefix + opts.user_css
         raw = '::selection {background:#ffff00; color:#000;}\n'+raw
         data = 'data:text/css;charset=utf-8;base64,'
-        data += b64encode(raw.encode('utf-8'))
+        data += as_base64_unicode(raw)
         self.settings().setUserStyleSheetUrl(QUrl(data))
 
     def findText(self, q, flags):

src/calibre/srv/auth.py

@@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
 __license__ = 'GPL v3'
 __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
 
-import binascii, os, random, struct, base64
+import binascii, os, random, struct
 from collections import OrderedDict
 from hashlib import md5, sha256
 from itertools import permutations
@@ -17,6 +17,7 @@ from calibre.srv.http_request import parse_uri
 from calibre.srv.utils import parse_http_dict, encode_path
 from calibre.utils.monotonic import monotonic
 from polyglot import http_client
+from polyglot.binary import from_base64_unicode
 
 MAX_AGE_SECONDS = 3600
 nonce_counter, nonce_counter_lock = 0, Lock()
@@ -76,7 +77,7 @@ def sha256_hex(s):
 
 
 def base64_decode(s):
-    return base64.standard_b64decode(as_bytestring(s)).decode('utf-8')
+    return from_base64_unicode(s)
 
 
 def synthesize_nonce(key_order, realm, secret, timestamp=None):

src/calibre/srv/cdb.py

@@ -5,7 +5,6 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import os
-from base64 import standard_b64decode
 from functools import partial
 from io import BytesIO
 
@@ -20,6 +19,7 @@ from calibre.srv.utils import get_db, get_library_data
 from calibre.utils.imghdr import what
 from calibre.utils.serialize import MSGPACK_MIME, json_loads, msgpack_loads
 from polyglot.builtins import iteritems
+from polyglot.binary import from_base64_bytes
 
 receive_data_methods = {'GET', 'POST'}
 
@@ -160,7 +160,7 @@ def cdb_set_fields(ctx, rd, book_id, library_id):
     if cdata is not False:
         if cdata is not None:
             try:
-                cdata = standard_b64decode(cdata.split(',', 1)[-1].encode('ascii'))
+                cdata = from_base64_bytes(cdata.split(',', 1)[-1])
             except Exception:
                 raise HTTPBadRequest('Cover data is not valid base64 encoded data')
             try:

src/calibre/srv/render_book.py

@@ -2,31 +2,35 @@
 # vim:fileencoding=utf-8
 # License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
-from __future__ import (unicode_literals, division, absolute_import,
+from __future__ import absolute_import, division, print_function, unicode_literals
-                        print_function)
+
-import sys, os, json, re
+import json
-from base64 import standard_b64encode, standard_b64decode
+import os
-from collections import defaultdict, OrderedDict
+import re
-from itertools import count
+import sys
+from collections import OrderedDict, defaultdict
 from functools import partial
+from itertools import count
 
 from css_parser import replaceUrls
 from css_parser.css import CSSRule
 
-from calibre import prepare_string_for_xml, force_unicode
+from calibre import force_unicode, prepare_string_for_xml
 from calibre.ebooks import parse_css_length
+from calibre.ebooks.css_transform_rules import StyleDeclaration
 from calibre.ebooks.oeb.base import (
-    OEB_DOCS, OEB_STYLES, rewrite_links, XPath, urlunquote, XLINK, XHTML_NS, OPF, XHTML, EPUB_NS)
+    EPUB_NS, OEB_DOCS, OEB_STYLES, OPF, XHTML, XHTML_NS, XLINK, XPath, rewrite_links,
+    urlunquote
+)
 from calibre.ebooks.oeb.iterator.book import extract_book
 from calibre.ebooks.oeb.polish.container import Container as ContainerBase
-from calibre.ebooks.oeb.polish.cover import set_epub_cover, find_cover_image
+from calibre.ebooks.oeb.polish.cover import find_cover_image, set_epub_cover
 from calibre.ebooks.oeb.polish.css import transform_css
-from calibre.ebooks.oeb.polish.utils import extract
+from calibre.ebooks.oeb.polish.toc import get_landmarks, get_toc
-from calibre.ebooks.css_transform_rules import StyleDeclaration
+from calibre.ebooks.oeb.polish.utils import extract, guess_type
-from calibre.ebooks.oeb.polish.toc import get_toc, get_landmarks
-from calibre.ebooks.oeb.polish.utils import guess_type
-from calibre.utils.short_uuid import uuid4
 from calibre.utils.logging import default_log
+from calibre.utils.short_uuid import uuid4
+from polyglot.binary import as_base64_unicode as encode_component, from_base64_unicode as decode_component
 from polyglot.builtins import iteritems, map, unicode_type
 from polyglot.urllib import quote, urlparse
 
@@ -35,14 +39,6 @@ RENDER_VERSION = 1
 BLANK_JPEG = b'\xff\xd8\xff\xdb\x00C\x00\x03\x02\x02\x02\x02\x02\x03\x02\x02\x02\x03\x03\x03\x03\x04\x06\x04\x04\x04\x04\x04\x08\x06\x06\x05\x06\t\x08\n\n\t\x08\t\t\n\x0c\x0f\x0c\n\x0b\x0e\x0b\t\t\r\x11\r\x0e\x0f\x10\x10\x11\x10\n\x0c\x12\x13\x12\x10\x13\x0f\x10\x10\x10\xff\xc9\x00\x0b\x08\x00\x01\x00\x01\x01\x01\x11\x00\xff\xcc\x00\x06\x00\x10\x10\x05\xff\xda\x00\x08\x01\x01\x00\x00?\x00\xd2\xcf \xff\xd9'  # noqa
 
 
-def encode_component(x):
-    return standard_b64encode(x.encode('utf-8')).decode('ascii')
-
-
-def decode_component(x):
-    return standard_b64decode(x).decode('utf-8')
-
-
 def encode_url(name, frag=''):
     name = encode_component(name)
     if frag:

src/calibre/srv/tests/ajax.py

@@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
 __license__ = 'GPL v3'
 __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
 
-import zlib, json, base64, os
+import zlib, json, os
 from io import BytesIO
 from functools import partial
 
@@ -14,11 +14,12 @@ from calibre.ebooks.metadata.meta import get_metadata
 from calibre.srv.tests.base import LibraryBaseTest
 from polyglot.http_client import OK, NOT_FOUND, FORBIDDEN
 from polyglot.urllib import urlencode, quote
+from polyglot.binary import as_base64_bytes
 
 
 def make_request(conn, url, headers={}, prefix='/ajax', username=None, password=None, method='GET', data=None):
     if username and password:
-        headers[b'Authorization'] = b'Basic ' + base64.standard_b64encode((username + ':' + password).encode('utf-8'))
+        headers[b'Authorization'] = b'Basic ' + as_base64_bytes((username + ':' + password))
     conn.request(method, prefix + url, headers=headers, body=data)
     r = conn.getresponse()
     data = r.read()

src/calibre/srv/tests/auth.py

@@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
 __license__ = 'GPL v3'
 __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
 
-import base64, subprocess, os, time
+import subprocess, os, time
 from collections import namedtuple
 try:
     from distutils.spawn import find_executable
@@ -22,6 +22,7 @@ from polyglot import http_client
 from polyglot.http_cookie import CookieJar
 from polyglot.urllib import (build_opener, HTTPBasicAuthHandler,
         HTTPCookieProcessor, HTTPDigestAuthHandler, HTTPError)
+from polyglot.binary import as_base64_bytes
 
 REALM = 'calibre-test'
 
@@ -101,7 +102,7 @@ class TestAuth(BaseTest):
             self.ae(r.status, http_client.UNAUTHORIZED)
             self.ae(r.getheader('WWW-Authenticate'), b'Basic realm="%s"' % bytes(REALM))
             self.assertFalse(r.read())
-            conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + base64.standard_b64encode(b'testuser:testpw')})
+            conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + as_base64_bytes(b'testuser:testpw')})
             r = conn.getresponse()
             self.ae(r.read(), b'closed')
             self.ae(r.status, http_client.OK)
@@ -109,7 +110,7 @@ class TestAuth(BaseTest):
             self.ae(b'closed', urlopen(server, un='!@#$%^&*()-=_+', pw='!@#$%^&*()-=_+', method='basic').read())
 
             def request(un='testuser', pw='testpw'):
-                conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + base64.standard_b64encode(bytes('%s:%s' % (un, pw)))})
+                conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + as_base64_bytes('%s:%s' % (un, pw))})
                 r = conn.getresponse()
                 return r.status, r.read()
 
@@ -254,7 +255,7 @@ class TestAuth(BaseTest):
             conn = server.connect()
 
             def request(un='testuser', pw='testpw'):
-                conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + base64.standard_b64encode(bytes('%s:%s' % (un, pw)))})
+                conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + as_base64_bytes('%s:%s' % (un, pw))})
                 r = conn.getresponse()
                 return r.status, r.read()
 

src/calibre/srv/tests/web_sockets.py

@@ -5,7 +5,6 @@
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 import socket, os, struct, errno, numbers
-from base64 import standard_b64encode
 from collections import deque, namedtuple
 from functools import partial
 from hashlib import sha1
@@ -17,6 +16,7 @@ from calibre.srv.web_socket import (
 from calibre.utils.monotonic import monotonic
 from calibre.utils.socket_inheritance import set_socket_inherit
 from polyglot.builtins import range, unicode_type
+from polyglot.binary import as_base64_bytes, as_base64_unicode
 
 HANDSHAKE_STR = '''\
 GET / HTTP/1.1\r
@@ -35,7 +35,7 @@ class WSClient(object):
         self.timeout = timeout
         self.socket = socket.create_connection(('localhost', port), timeout)
         set_socket_inherit(self.socket, False)
-        self.key = standard_b64encode(os.urandom(8))
+        self.key = as_base64_bytes(os.urandom(8))
         self.socket.sendall(HANDSHAKE_STR.format(self.key).encode('ascii'))
         self.read_buf = deque()
         self.read_upgrade_response()
@@ -64,7 +64,7 @@ class WSClient(object):
         if rl != b'HTTP/1.1 101 Switching Protocols\r\n':
             raise ValueError('Server did not respond with correct switching protocols line')
         headers = read_headers(partial(next, lines))
-        key = standard_b64encode(sha1(self.key + GUID_STR).digest())
+        key = as_base64_unicode(sha1(self.key + GUID_STR).digest())
         if headers.get('Sec-WebSocket-Accept') != key:
             raise ValueError('Server did not respond with correct key in Sec-WebSocket-Accept')
 

src/calibre/srv/web_socket.py

@@ -2,24 +2,28 @@
 # vim:fileencoding=utf-8
 # License: GPLv3 Copyright: 2015, Kovid Goyal <kovid at kovidgoyal.net>
 
-from __future__ import (unicode_literals, division, absolute_import,
+from __future__ import absolute_import, division, print_function, unicode_literals
-                        print_function)
 
-import os, weakref, socket
+import os
-from base64 import standard_b64encode
+import socket
+import weakref
 from collections import deque
 from hashlib import sha1
-from struct import unpack_from, pack, error as struct_error
+from struct import error as struct_error, pack, unpack_from
 from threading import Lock
 
 from calibre import as_unicode
 from calibre.constants import plugins
-from calibre.srv.loop import ServerLoop, HandleInterrupt, WRITE, READ, RDWR, Connection
 from calibre.srv.http_response import HTTPConnection, create_http_handler
+from calibre.srv.loop import (
+    RDWR, READ, WRITE, Connection, HandleInterrupt, ServerLoop
+)
 from calibre.srv.utils import DESIRED_SEND_BUFFER_SIZE
 from calibre.utils.speedups import ReadOnlyFileBuffer
-from polyglot.queue import Queue, Empty
 from polyglot import http_client
+from polyglot.binary import as_base64_unicode
+from polyglot.queue import Empty, Queue
+
 speedup, err = plugins['speedup']
 if not speedup:
     raise RuntimeError('Failed to load speedup module with error: ' + err)
@@ -291,7 +295,7 @@ class WebSocketConnection(HTTPConnection):
         if self.method != 'GET':
             return self.simple_response(http_client.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method)
 
-        response = HANDSHAKE_STR % standard_b64encode(sha1(key + GUID_STR).digest())
+        response = HANDSHAKE_STR % as_base64_unicode(sha1(key + GUID_STR).digest())
         self.optimize_for_sending_packet()
         self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
         self.set_state(WRITE, self.upgrade_connection_to_ws, ReadOnlyFileBuffer(response.encode('ascii')), inheaders)

src/calibre/utils/config_base.py

@@ -21,9 +21,9 @@ plugin_dir = os.path.join(config_dir, 'plugins')
 def to_json(obj):
     import datetime
     if isinstance(obj, bytearray):
-        import base64
+        from base64 import standard_b64encode
         return {'__class__': 'bytearray',
-                '__value__': base64.standard_b64encode(bytes(obj)).decode('ascii')}
+                '__value__': standard_b64encode(bytes(obj)).decode('ascii')}
     if isinstance(obj, datetime.datetime):
         from calibre.utils.date import isoformat
         return {'__class__': 'datetime.datetime',
@@ -42,8 +42,8 @@ def from_json(obj):
     custom = obj.get('__class__')
     if custom is not None:
         if custom == 'bytearray':
-            import base64
+            from base64 import standard_b64decode
-            return bytearray(base64.standard_b64decode(obj['__value__']))
+            return bytearray(standard_b64decode(obj['__value__'].encode('ascii')))
         if custom == 'datetime.datetime':
             from calibre.utils.iso8601 import parse_iso8601
             return parse_iso8601(obj['__value__'], assume_utc=True)

src/calibre/utils/serialize.py

@@ -66,13 +66,11 @@ def json_dumps(data, **kw):
 
 
 def decode_metadata(x, for_json):
-    import base64
+    from polyglot.binary import from_base64_bytes
     from calibre.ebooks.metadata.book.serialize import metadata_from_dict
     obj = metadata_from_dict(x)
     if for_json and obj.cover_data and obj.cover_data[1]:
-        obj.cover_data = obj.cover_data[0], base64.standard_b64decode(
+        obj.cover_data = obj.cover_data[0], from_base64_bytes(obj.cover_data[1])
-            obj.cover_data[1]
-        )
     return obj
 
 

src/calibre/web/fetch/simple.py

@@ -17,7 +17,6 @@ import sys
 import threading
 import time
 import traceback
-from base64 import b64decode
 
 from calibre import browser, relpath, unicode_path
 from calibre.constants import filesystem_encoding, iswindows
@@ -35,6 +34,7 @@ from polyglot.urllib import (
     URLError, quote, url2pathname, urljoin, urlparse, urlsplit, urlunparse,
     urlunsplit
 )
+from polyglot.binary import from_base64_bytes
 
 
 class AbortArticle(Exception):
@@ -391,8 +391,8 @@ class RecursiveFetcher(object):
             iurl = tag['src']
             if iurl.startswith('data:image/'):
                 try:
-                    data = b64decode(iurl.partition(',')[-1])
+                    data = from_base64_bytes(iurl.partition(',')[-1])
-                except:
+                except Exception:
                     self.log.exception('Failed to decode embedded image')
                     continue
             else:

src/polyglot/binary.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python2
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2019, Kovid Goyal <kovid at kovidgoyal.net>
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from base64 import standard_b64encode, standard_b64decode
+from polyglot.builtins import unicode_type
+
+
+def as_base64_bytes(x, enc='utf-8'):
+    if isinstance(x, unicode_type):
+        x = x.encode(enc)
+    return standard_b64encode(x)
+
+
+def as_base64_unicode(x, enc='utf-8'):
+    if isinstance(x, unicode_type):
+        x = x.encode(enc)
+    return standard_b64encode(x).decode('ascii')
+
+
+def from_base64_unicode(x, enc='utf-8'):
+    if isinstance(x, unicode_type):
+        x = x.encode('ascii')
+    return standard_b64decode(x).decode(enc)
+
+
+def from_base64_bytes(x):
+    if isinstance(x, unicode_type):
+        x = x.encode('ascii')
+    return standard_b64decode(x)