Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

py3: port use as base64 module

Browse Source
This commit is contained in:
Kovid Goyal 2019-03-30 12:52:32 +05:30
parent a0afe775ba
commit fcd5700306
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
32 changed files with 205 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/calibre/devices/prs505/sony_cache.py
View File

@ -6,7 +6,6 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import os, time
from base64 import b64decode
from datetime import date
from calibre import prints, guess_type, isbytestring, fsync
@ -16,6 +15,7 @@ from calibre.constants import DEBUG, preferred_encoding
from calibre.ebooks.chardet import xml_to_unicode
from calibre.ebooks.metadata import authors_to_string, title_sort, \
authors_to_sort_string
from polyglot.binary import from_base64_bytes
'''
cahceExt.xml
@ -380,8 +380,8 @@ class XMLCache(object):
'descendant::*[local-name()="png"]'):
if img.text:
try:
raw = b64decode(img.text.strip())
except:
raw = from_base64_bytes(img.text.strip())
except Exception:
continue
book.thumbnail = raw
break

8
src/calibre/ebooks/fb2/__init__.py
View File

@ -10,12 +10,12 @@ __docformat__ = 'restructuredtext en'
def base64_decode(raw):
from io import BytesIO
from base64 import b64decode
from polyglot.binary import from_base64_bytes
# First try the python implementation as it is faster
try:
return b64decode(raw)
except TypeError:
return from_base64_bytes(raw)
except Exception:
pass
# Try a more robust version (adapted from FBReader sources)
@ -49,5 +49,3 @@ def base64_decode(raw):
tot >>= 8
out.write(bytes(triple))
return out.getvalue()

6
src/calibre/ebooks/fb2/fb2ml.py
View File

@ -9,7 +9,6 @@ Transform OEB content into FB2 markup
'''
import re, textwrap, uuid
from base64 import b64encode
from datetime import datetime
from lxml import etree
@ -20,6 +19,7 @@ from calibre.utils.localization import lang_as_iso639_1
from calibre.utils.img import save_cover_data_to
from calibre.ebooks.oeb.base import urlnormalize
from polyglot.builtins import unicode_type, string_or_bytes
from polyglot.binary import as_base64_unicode
class FB2MLizer(object):
@ -308,9 +308,9 @@ class FB2MLizer(object):
try:
if item.media_type != 'image/jpeg':
imdata = save_cover_data_to(item.data, compression_quality=70)
raw_data = b64encode(imdata)
raw_data = as_base64_unicode(imdata)
else:
raw_data = b64encode(item.data)
raw_data = as_base64_unicode(item.data)
# Don't put the encoded image on a single line.
data = ''
col = 1

6
src/calibre/ebooks/metadata/book/json_codec.py
View File

@ -5,7 +5,6 @@ Created on 4 Jun 2010
'''
from __future__ import print_function
from base64 import b64encode, b64decode
import json, traceback
from datetime import datetime, time
@ -14,6 +13,7 @@ from calibre.constants import filesystem_encoding, preferred_encoding
from calibre.library.field_metadata import FieldMetadata
from calibre import isbytestring
from polyglot.builtins import iteritems, itervalues
from polyglot.binary import as_base64_unicode, from_base64_bytes
# Translate datetimes to and from strings. The string form is the datetime in
# UTC. The returned date is also UTC
@ -57,7 +57,7 @@ def encode_thumbnail(thumbnail):
thumbnail = (width, height, thumbnail)
except Exception:
return None
return (thumbnail[0], thumbnail[1], b64encode(str(thumbnail[2])))
return (thumbnail[0], thumbnail[1], as_base64_unicode(thumbnail[2]))
def decode_thumbnail(tup):
@ -66,7 +66,7 @@ def decode_thumbnail(tup):
'''
if tup is None:
return None
return (tup[0], tup[1], b64decode(tup[2]))
return (tup[0], tup[1], from_base64_bytes(tup[2]))
def object_to_unicode(obj, enc=preferred_encoding):

4
src/calibre/ebooks/metadata/book/serialize.py
View File

@ -4,13 +4,13 @@
from __future__ import absolute_import, division, print_function, unicode_literals
import base64
from calibre.constants import preferred_encoding
from calibre.ebooks.metadata.book import SERIALIZABLE_FIELDS
from calibre.ebooks.metadata.book.base import Metadata
from calibre.utils.imghdr import what
from polyglot.builtins import iteritems, unicode_type
from polyglot.binary import as_base64_unicode
def ensure_unicode(obj, enc=preferred_encoding):
@ -52,7 +52,7 @@ def metadata_as_dict(mi, encode_cover_data=False):
ans[field] = ensure_unicode(val)
if mi.cover_data and mi.cover_data[1]:
if encode_cover_data:
ans['cover_data'] = [mi.cover_data[0], base64.standard_b64encode(bytes(mi.cover_data[1]))]
ans['cover_data'] = [mi.cover_data[0], as_base64_unicode(mi.cover_data[1])]
else:
ans['cover_data'] = mi.cover_data
um = mi.get_all_user_metadata(False)

4
src/calibre/ebooks/metadata/fb2.py
View File

@ -8,7 +8,6 @@ __copyright__ = '2011, Roman Mukhin <ramses_ru at hotmail.com>, '\
import os, random
from functools import partial
from string import ascii_letters, digits
from base64 import b64encode
from lxml import etree
@ -19,6 +18,7 @@ from calibre import guess_type, guess_all_extensions, prints, force_unicode
from calibre.ebooks.metadata import MetaInformation, check_isbn
from calibre.ebooks.chardet import xml_to_unicode
from polyglot.builtins import unicode_type
from polyglot.binary import as_base64_unicode
NAMESPACES = {
@ -383,7 +383,7 @@ def _rnd_pic_file_name(prefix='calibre_cover_', size=32, ext='jpg'):
def _encode_into_jpeg(data):
data = save_cover_data_to(data)
return b64encode(data)
return as_base64_unicode(data)
def _set_cover(title_info, mi, ctx):

9
src/calibre/ebooks/metadata/kfx.py
View File

@ -8,7 +8,7 @@ from __future__ import (unicode_literals, division, absolute_import,
# Based on work of John Howell reversing the KFX format
# https://www.mobileread.com/forums/showpost.php?p=3176029&postcount=89
import struct, sys, base64, re
import struct, sys, re
from collections import defaultdict
from calibre.ebooks.metadata.book.base import Metadata
@ -19,6 +19,7 @@ from calibre.utils.date import parse_only_date
from calibre.utils.localization import canonicalize_lang
from calibre.utils.imghdr import identify
from polyglot.builtins import unicode_type
from polyglot.binary import as_base64_bytes, from_base64_bytes
class InvalidKFX(ValueError):
@ -155,7 +156,7 @@ class Entity(PackedBlock):
if PackedData(self.entity_data).unpack_one('4s') == ION_MAGIC:
entity_value = PackedIon(self.entity_data).decode()
else:
entity_value = base64.b64encode(self.entity_data)
entity_value = as_base64_bytes(self.entity_data)
return (property_name(self.entity_type), property_name(self.entity_id), entity_value)
@ -343,8 +344,8 @@ def read_metadata_kfx(stream, read_cover=True):
mi.publisher = get('publisher')
if read_cover and m[COVER_KEY]:
try:
data = base64.standard_b64decode(m[COVER_KEY])
fmt, w, h = identify(bytes(data))
data = from_base64_bytes(m[COVER_KEY])
fmt, w, h = identify(data)
except Exception:
w, h, fmt = 0, 0, None
if fmt and w > -1 and h > -1:

4
src/calibre/ebooks/oeb/polish/download.py
View File

@ -10,7 +10,6 @@ import os
import posixpath
import re
import shutil
from base64 import standard_b64decode
from collections import defaultdict
from contextlib import closing
from functools import partial
@ -25,6 +24,7 @@ from calibre.ptempfile import TemporaryDirectory
from calibre.web import get_download_filename_from_response
from polyglot.builtins import iteritems
from polyglot.urllib import urlopen, urlparse
from polyglot.binary import from_base64_bytes
def is_external(url):
@ -117,7 +117,7 @@ def download_one(tdir, timeout, progress_report, data_uri_map, url):
parts = prefix.split(';')
if parts and parts[-1].lower() == 'base64':
payload = re.sub(r'\s+', '', payload)
payload = standard_b64decode(payload)
payload = from_base64_bytes(payload)
else:
payload = payload.encode('utf-8')
seen_before = data_uri_map.get(payload)

4
src/calibre/ebooks/oeb/transforms/data_url.py
View File

@ -28,9 +28,9 @@ class DataURL(object):
continue
if ';base64' in header:
data = re.sub(r'\s+', '', data)
from base64 import b64decode
from polyglot.binary import from_base64_bytes
try:
data = b64decode(data)
data = from_base64_bytes(data)
except Exception:
self.log.error('Found invalid base64 encoded data URI, ignoring it')
continue

5
src/calibre/gui2/store/search/download_thread.py
View File

@ -6,7 +6,7 @@ __license__ = 'GPL 3'
__copyright__ = '2011, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en'
import traceback, base64
import traceback
from contextlib import closing
from threading import Thread
@ -15,6 +15,7 @@ from calibre.constants import DEBUG
from calibre.utils.img import scale_image
from polyglot.builtins import range
from polyglot.queue import Queue
from polyglot.binary import from_base64_bytes
class GenericDownloadThreadPool(object):
@ -143,7 +144,7 @@ class CoverThreadPool(GenericDownloadThreadPool):
def decode_data_url(url):
return base64.standard_b64decode(url.partition(',')[2])
return from_base64_bytes(url.partition(',')[2])
class CoverThread(Thread):

13
src/calibre/gui2/store/stores/ebookpoint_plugin.py
View File

@ -24,6 +24,15 @@ from calibre.gui2.store.search_result import SearchResult
from calibre.gui2.store.web_store_dialog import WebStoreDialog
def as_base64(data):
if not isinstance(data, bytes):
data = data.encode('utf-8')
ans = b64encode(data)
if isinstance(ans, bytes):
ans = ans.decode('ascii')
return ans
class EbookpointStore(BasicStoreConfig, StorePlugin):
def open(self, parent=None, detail_item=None, external=False):
@ -31,11 +40,11 @@ class EbookpointStore(BasicStoreConfig, StorePlugin):
url = 'http://ebookpoint.pl/'
aff_url = aff_root + str(b64encode(url))
aff_url = aff_root + as_base64(url)
detail_url = None
if detail_item:
detail_url = aff_root + str(b64encode(detail_item))
detail_url = aff_root + as_base64(detail_item)
if external or self.config.get('open_external', False):
open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url)))

14
src/calibre/gui2/store/stores/empik_plugin.py
View File

@ -24,6 +24,15 @@ from calibre.gui2.store.search_result import SearchResult
from calibre.gui2.store.web_store_dialog import WebStoreDialog
def as_base64(data):
if not isinstance(data, bytes):
data = data.encode('utf-8')
ans = b64encode(data)
if isinstance(ans, bytes):
ans = ans.decode('ascii')
return ans
class EmpikStore(BasicStoreConfig, StorePlugin):
def open(self, parent=None, detail_item=None, external=False):
@ -31,11 +40,11 @@ class EmpikStore(BasicStoreConfig, StorePlugin):
url = 'http://www.empik.com/ebooki'
aff_url = aff_root + str(b64encode(url))
aff_url = aff_root + as_base64(url)
detail_url = None
if detail_item:
detail_url = aff_root + str(b64encode(detail_item))
detail_url = aff_root + as_base64(detail_item)
if external or self.config.get('open_external', False):
open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url)))
@ -82,4 +91,3 @@ class EmpikStore(BasicStoreConfig, StorePlugin):
s.formats = formats.upper().strip()
yield s

8
src/calibre/gui2/store/stores/gutenberg_plugin.py
View File

@ -52,7 +52,7 @@ def search(query, max_results=10, timeout=60, write_raw_to=None):
# We could use the <link rel="alternate" type="text/html" ...> tag from the
# detail odps page but this is easier.
id = fix_url(''.join(data.xpath('./*[local-name() = "id"]/text()')).strip())
s.detail_item = url_slash_cleaner('%s/ebooks/%s' % (web_url, re.sub('[^\d]', '', id)))
s.detail_item = url_slash_cleaner('%s/ebooks/%s' % (web_url, re.sub(r'[^\d]', '', id)))
s.title = ' '.join(data.xpath('./*[local-name() = "title"]//text()')).strip()
s.author = ', '.join(data.xpath('./*[local-name() = "content"]//text()')).strip()
if not s.title or not s.author:
@ -83,7 +83,10 @@ def search(query, max_results=10, timeout=60, write_raw_to=None):
href = fix_url(href)
if rel in ('http://opds-spec.org/thumbnail', 'http://opds-spec.org/image/thumbnail'):
if href.startswith('data:image/png;base64,'):
s.cover_data = base64.b64decode(href.replace('data:image/png;base64,', ''))
cdata = href.replace('data:image/png;base64,', '')
if not isinstance(cdata, bytes):
cdata = cdata.encode('ascii')
s.cover_data = base64.b64decode(cdata)
yield s
@ -123,6 +126,7 @@ class GutenbergStore(BasicStoreConfig, OpenSearchOPDSStore):
for result in search(query, max_results, timeout):
yield result
if __name__ == '__main__':
import sys
for result in search(' '.join(sys.argv[1:]), write_raw_to='/t/gutenberg.html'):

13
src/calibre/gui2/store/stores/legimi_plugin.py
View File

@ -24,6 +24,15 @@ from calibre.gui2.store.search_result import SearchResult
from calibre.gui2.store.web_store_dialog import WebStoreDialog
def as_base64(data):
if not isinstance(data, bytes):
data = data.encode('utf-8')
ans = b64encode(data)
if isinstance(ans, bytes):
ans = ans.decode('ascii')
return ans
class LegimiStore(BasicStoreConfig, StorePlugin):
def open(self, parent=None, detail_item=None, external=False):
@ -31,11 +40,11 @@ class LegimiStore(BasicStoreConfig, StorePlugin):
url = 'https://www.legimi.pl/ebooki/'
aff_url = aff_root + str(b64encode(url))
aff_url = aff_root + as_base64(url)
detail_url = None
if detail_item:
detail_url = aff_root + str(b64encode(detail_item))
detail_url = aff_root + as_base64(detail_item)
if external or self.config.get('open_external', False):
open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url)))

13
src/calibre/gui2/store/stores/publio_plugin.py
View File

@ -23,17 +23,26 @@ from calibre.gui2.store.search_result import SearchResult
from calibre.gui2.store.web_store_dialog import WebStoreDialog
def as_base64(data):
if not isinstance(data, bytes):
data = data.encode('utf-8')
ans = b64encode(data)
if isinstance(ans, bytes):
ans = ans.decode('ascii')
return ans
class PublioStore(BasicStoreConfig, StorePlugin):
def open(self, parent=None, detail_item=None, external=False):
aff_root = 'https://www.a4b-tracking.com/pl/stat-click-text-link/29/58/'
url = 'http://www.publio.pl/'
aff_url = aff_root + str(b64encode(url))
aff_url = aff_root + as_base64(url)
detail_url = None
if detail_item:
detail_url = aff_root + str(b64encode(detail_item))
detail_url = aff_root + as_base64(detail_item)
if external or self.config.get('open_external', False):
open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url)))

13
src/calibre/gui2/store/stores/swiatebookow_plugin.py
View File

@ -23,6 +23,15 @@ from calibre.gui2.store.search_result import SearchResult
from calibre.gui2.store.web_store_dialog import WebStoreDialog
def as_base64(data):
if not isinstance(data, bytes):
data = data.encode('utf-8')
ans = b64encode(data)
if isinstance(ans, bytes):
ans = ans.decode('ascii')
return ans
class SwiatEbookowStore(BasicStoreConfig, StorePlugin):
def open(self, parent=None, detail_item=None, external=False):
@ -30,11 +39,11 @@ class SwiatEbookowStore(BasicStoreConfig, StorePlugin):
url = 'https://www.swiatebookow.pl/'
aff_url = aff_root + str(b64encode(url))
aff_url = aff_root + as_base64(url)
detail_url = None
if detail_item:
detail_url = aff_root + str(b64encode(detail_item))
detail_url = aff_root + as_base64(detail_item)
if external or self.config.get('open_external', False):
open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url)))

15
src/calibre/gui2/store/stores/virtualo_plugin.py
View File

@ -24,6 +24,15 @@ from calibre.gui2.store.search_result import SearchResult
from calibre.gui2.store.web_store_dialog import WebStoreDialog
def as_base64(data):
if not isinstance(data, bytes):
data = data.encode('utf-8')
ans = b64encode(data)
if isinstance(ans, bytes):
ans = ans.decode('ascii')
return ans
class VirtualoStore(BasicStoreConfig, StorePlugin):
def open(self, parent=None, detail_item=None, external=False):
@ -31,11 +40,11 @@ class VirtualoStore(BasicStoreConfig, StorePlugin):
url = 'http://virtualo.pl/ebook/c2/'
aff_url = aff_root + str(b64encode(url))
aff_url = aff_root + as_base64(url)
detail_url = None
if detail_item:
detail_url = aff_root + str(b64encode(detail_item))
detail_url = aff_root + as_base64(detail_item)
if external or self.config.get('open_external', False):
open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url)))
@ -75,7 +84,7 @@ class VirtualoStore(BasicStoreConfig, StorePlugin):
s.cover_url = cover_url
s.title = title.strip()
s.author = author.strip()
s.price = re.sub('\.',',',price.strip())
s.price = re.sub(r'\.',',',price.strip())
s.detail_item = id
s.formats = ', '.join(formats).upper()
s.drm = SearchResult.DRM_UNLOCKED if nodrm else SearchResult.DRM_LOCKED

13
src/calibre/gui2/store/stores/woblink_plugin.py
View File

@ -23,6 +23,15 @@ from calibre.gui2.store.search_result import SearchResult
from calibre.gui2.store.web_store_dialog import WebStoreDialog
def as_base64(data):
if not isinstance(data, bytes):
data = data.encode('utf-8')
ans = b64encode(data)
if isinstance(ans, bytes):
ans = ans.decode('ascii')
return ans
def search(query, max_results=10, timeout=60):
url = 'http://woblink.com/publication/ajax?mode=none&query=' + urllib.quote_plus(query.encode('utf-8'))
if max_results > 10:
@ -79,11 +88,11 @@ class WoblinkStore(BasicStoreConfig, StorePlugin):
aff_root = 'https://www.a4b-tracking.com/pl/stat-click-text-link/16/58/'
url = 'http://woblink.com/publication'
aff_url = aff_root + str(b64encode(url))
aff_url = aff_root + as_base64(url)
detail_url = None
if detail_item:
detail_url = aff_root + str(b64encode('http://woblink.com' + detail_item))
detail_url = aff_root + as_base64('http://woblink.com' + detail_item)
if external or self.config.get('open_external', False):
open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url)))

4
src/calibre/gui2/toc/location.py
View File

@ -8,7 +8,6 @@ __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import json
from base64 import b64encode
from PyQt5.Qt import (QWidget, QGridLayout, QListWidget, QSize, Qt, QUrl,
pyqtSlot, pyqtSignal, QVBoxLayout, QFrame, QLabel,
@ -20,6 +19,7 @@ from calibre.ebooks.oeb.display.webview import load_html
from calibre.gui2 import error_dialog, question_dialog, gprefs, secure_web_page
from calibre.utils.logging import default_log
from polyglot.builtins import unicode_type, range
from polyglot.binary import as_base64_unicode
class Page(QWebPage): # {{{
@ -77,7 +77,7 @@ class WebView(QWebView): # {{{
'''
raw = '::selection {background:#ffff00; color:#000;}\n'+raw
data = 'data:text/css;charset=utf-8;base64,'
data += b64encode(raw.encode('utf-8'))
data += as_base64_unicode(raw)
self.settings().setUserStyleSheetUrl(QUrl(data))
def load_js(self):

4
src/calibre/gui2/tweak_book/preview.py
View File

@ -8,7 +8,6 @@ __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
import time, textwrap, json
from bisect import bisect_right
from base64 import b64encode
from polyglot.builtins import map, unicode_type
from threading import Thread
from functools import partial
@ -31,6 +30,7 @@ from calibre.gui2.widgets2 import HistoryLineEdit2
from calibre.utils.ipc.simple_worker import offload_worker
from polyglot.urllib import urlparse
from polyglot.queue import Queue, Empty
from polyglot.binary import as_base64_unicode
shutdown = object()
@ -269,7 +269,7 @@ class WebPage(QWebPage):
settings.setDefaultTextEncoding('utf-8')
data = 'data:text/css;charset=utf-8;base64,'
css = '[data-in-split-mode="1"] [data-is-block="1"]:hover { cursor: pointer !important; border-top: solid 5px green !important }'
data += b64encode(css.encode('utf-8'))
data += as_base64_unicode(css)
settings.setUserStyleSheetUrl(QUrl(data))
self.setNetworkAccessManager(NetworkAccessManager(self))

4
src/calibre/gui2/viewer/documentview.py
View File

@ -5,7 +5,6 @@ __docformat__ = 'restructuredtext en'
# Imports {{{
import math, json
from base64 import b64encode
from functools import partial
from polyglot.builtins import iteritems, map, unicode_type, string_or_bytes
@ -33,6 +32,7 @@ from calibre.gui2.viewer.footnote import Footnotes
from calibre.gui2.viewer.fake_net import NetworkAccessManager
from calibre.ebooks.oeb.display.webview import load_html
from calibre.constants import isxp, iswindows, DEBUG, __version__
from polyglot.binary import as_base64_unicode
# }}}
@ -144,7 +144,7 @@ class Document(QWebPage): # {{{
raw = prefix + opts.user_css
raw = '::selection {background:#ffff00; color:#000;}\n'+raw
data = 'data:text/css;charset=utf-8;base64,'
data += b64encode(raw.encode('utf-8'))
data += as_base64_unicode(raw)
self.settings().setUserStyleSheetUrl(QUrl(data))
def findText(self, q, flags):

5
src/calibre/srv/auth.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import binascii, os, random, struct, base64
import binascii, os, random, struct
from collections import OrderedDict
from hashlib import md5, sha256
from itertools import permutations
@ -17,6 +17,7 @@ from calibre.srv.http_request import parse_uri
from calibre.srv.utils import parse_http_dict, encode_path
from calibre.utils.monotonic import monotonic
from polyglot import http_client
from polyglot.binary import from_base64_unicode
MAX_AGE_SECONDS = 3600
nonce_counter, nonce_counter_lock = 0, Lock()
@ -76,7 +77,7 @@ def sha256_hex(s):
def base64_decode(s):
return base64.standard_b64decode(as_bytestring(s)).decode('utf-8')
return from_base64_unicode(s)
def synthesize_nonce(key_order, realm, secret, timestamp=None):

4
src/calibre/srv/cdb.py
View File

@ -5,7 +5,6 @@
from __future__ import absolute_import, division, print_function, unicode_literals
import os
from base64 import standard_b64decode
from functools import partial
from io import BytesIO
@ -20,6 +19,7 @@ from calibre.srv.utils import get_db, get_library_data
from calibre.utils.imghdr import what
from calibre.utils.serialize import MSGPACK_MIME, json_loads, msgpack_loads
from polyglot.builtins import iteritems
from polyglot.binary import from_base64_bytes
receive_data_methods = {'GET', 'POST'}
@ -160,7 +160,7 @@ def cdb_set_fields(ctx, rd, book_id, library_id):
if cdata is not False:
if cdata is not None:
try:
cdata = standard_b64decode(cdata.split(',', 1)[-1].encode('ascii'))
cdata = from_base64_bytes(cdata.split(',', 1)[-1])
except Exception:
raise HTTPBadRequest('Cover data is not valid base64 encoded data')
try:

40
src/calibre/srv/render_book.py
View File

@ -2,31 +2,35 @@
# vim:fileencoding=utf-8
# License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
from __future__ import (unicode_literals, division, absolute_import,
print_function)
import sys, os, json, re
from base64 import standard_b64encode, standard_b64decode
from collections import defaultdict, OrderedDict
from itertools import count
from __future__ import absolute_import, division, print_function, unicode_literals
import json
import os
import re
import sys
from collections import OrderedDict, defaultdict
from functools import partial
from itertools import count
from css_parser import replaceUrls
from css_parser.css import CSSRule
from calibre import prepare_string_for_xml, force_unicode
from calibre import force_unicode, prepare_string_for_xml
from calibre.ebooks import parse_css_length
from calibre.ebooks.css_transform_rules import StyleDeclaration
from calibre.ebooks.oeb.base import (
OEB_DOCS, OEB_STYLES, rewrite_links, XPath, urlunquote, XLINK, XHTML_NS, OPF, XHTML, EPUB_NS)
EPUB_NS, OEB_DOCS, OEB_STYLES, OPF, XHTML, XHTML_NS, XLINK, XPath, rewrite_links,
urlunquote
)
from calibre.ebooks.oeb.iterator.book import extract_book
from calibre.ebooks.oeb.polish.container import Container as ContainerBase
from calibre.ebooks.oeb.polish.cover import set_epub_cover, find_cover_image
from calibre.ebooks.oeb.polish.cover import find_cover_image, set_epub_cover
from calibre.ebooks.oeb.polish.css import transform_css
from calibre.ebooks.oeb.polish.utils import extract
from calibre.ebooks.css_transform_rules import StyleDeclaration
from calibre.ebooks.oeb.polish.toc import get_toc, get_landmarks
from calibre.ebooks.oeb.polish.utils import guess_type
from calibre.utils.short_uuid import uuid4
from calibre.ebooks.oeb.polish.toc import get_landmarks, get_toc
from calibre.ebooks.oeb.polish.utils import extract, guess_type
from calibre.utils.logging import default_log
from calibre.utils.short_uuid import uuid4
from polyglot.binary import as_base64_unicode as encode_component, from_base64_unicode as decode_component
from polyglot.builtins import iteritems, map, unicode_type
from polyglot.urllib import quote, urlparse
@ -35,14 +39,6 @@ RENDER_VERSION = 1
BLANK_JPEG = b'\xff\xd8\xff\xdb\x00C\x00\x03\x02\x02\x02\x02\x02\x03\x02\x02\x02\x03\x03\x03\x03\x04\x06\x04\x04\x04\x04\x04\x08\x06\x06\x05\x06\t\x08\n\n\t\x08\t\t\n\x0c\x0f\x0c\n\x0b\x0e\x0b\t\t\r\x11\r\x0e\x0f\x10\x10\x11\x10\n\x0c\x12\x13\x12\x10\x13\x0f\x10\x10\x10\xff\xc9\x00\x0b\x08\x00\x01\x00\x01\x01\x01\x11\x00\xff\xcc\x00\x06\x00\x10\x10\x05\xff\xda\x00\x08\x01\x01\x00\x00?\x00\xd2\xcf \xff\xd9' # noqa
def encode_component(x):
return standard_b64encode(x.encode('utf-8')).decode('ascii')
def decode_component(x):
return standard_b64decode(x).decode('utf-8')
def encode_url(name, frag=''):
name = encode_component(name)
if frag:

5
src/calibre/srv/tests/ajax.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import zlib, json, base64, os
import zlib, json, os
from io import BytesIO
from functools import partial
@ -14,11 +14,12 @@ from calibre.ebooks.metadata.meta import get_metadata
from calibre.srv.tests.base import LibraryBaseTest
from polyglot.http_client import OK, NOT_FOUND, FORBIDDEN
from polyglot.urllib import urlencode, quote
from polyglot.binary import as_base64_bytes
def make_request(conn, url, headers={}, prefix='/ajax', username=None, password=None, method='GET', data=None):
if username and password:
headers[b'Authorization'] = b'Basic ' + base64.standard_b64encode((username + ':' + password).encode('utf-8'))
headers[b'Authorization'] = b'Basic ' + as_base64_bytes((username + ':' + password))
conn.request(method, prefix + url, headers=headers, body=data)
r = conn.getresponse()
data = r.read()

9
src/calibre/srv/tests/auth.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import base64, subprocess, os, time
import subprocess, os, time
from collections import namedtuple
try:
from distutils.spawn import find_executable
@ -22,6 +22,7 @@ from polyglot import http_client
from polyglot.http_cookie import CookieJar
from polyglot.urllib import (build_opener, HTTPBasicAuthHandler,
HTTPCookieProcessor, HTTPDigestAuthHandler, HTTPError)
from polyglot.binary import as_base64_bytes
REALM = 'calibre-test'
@ -101,7 +102,7 @@ class TestAuth(BaseTest):
self.ae(r.status, http_client.UNAUTHORIZED)
self.ae(r.getheader('WWW-Authenticate'), b'Basic realm="%s"' % bytes(REALM))
self.assertFalse(r.read())
conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + base64.standard_b64encode(b'testuser:testpw')})
conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + as_base64_bytes(b'testuser:testpw')})
r = conn.getresponse()
self.ae(r.read(), b'closed')
self.ae(r.status, http_client.OK)
@ -109,7 +110,7 @@ class TestAuth(BaseTest):
self.ae(b'closed', urlopen(server, un='!@#$%^&*()-=_+', pw='!@#$%^&*()-=_+', method='basic').read())
def request(un='testuser', pw='testpw'):
conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + base64.standard_b64encode(bytes('%s:%s' % (un, pw)))})
conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + as_base64_bytes('%s:%s' % (un, pw))})
r = conn.getresponse()
return r.status, r.read()
@ -254,7 +255,7 @@ class TestAuth(BaseTest):
conn = server.connect()
def request(un='testuser', pw='testpw'):
conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + base64.standard_b64encode(bytes('%s:%s' % (un, pw)))})
conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + as_base64_bytes('%s:%s' % (un, pw))})
r = conn.getresponse()
return r.status, r.read()

6
src/calibre/srv/tests/web_sockets.py
View File

@ -5,7 +5,6 @@
from __future__ import (unicode_literals, division, absolute_import,
print_function)
import socket, os, struct, errno, numbers
from base64 import standard_b64encode
from collections import deque, namedtuple
from functools import partial
from hashlib import sha1
@ -17,6 +16,7 @@ from calibre.srv.web_socket import (
from calibre.utils.monotonic import monotonic
from calibre.utils.socket_inheritance import set_socket_inherit
from polyglot.builtins import range, unicode_type
from polyglot.binary import as_base64_bytes, as_base64_unicode
HANDSHAKE_STR = '''\
GET / HTTP/1.1\r
@ -35,7 +35,7 @@ class WSClient(object):
self.timeout = timeout
self.socket = socket.create_connection(('localhost', port), timeout)
set_socket_inherit(self.socket, False)
self.key = standard_b64encode(os.urandom(8))
self.key = as_base64_bytes(os.urandom(8))
self.socket.sendall(HANDSHAKE_STR.format(self.key).encode('ascii'))
self.read_buf = deque()
self.read_upgrade_response()
@ -64,7 +64,7 @@ class WSClient(object):
if rl != b'HTTP/1.1 101 Switching Protocols\r\n':
raise ValueError('Server did not respond with correct switching protocols line')
headers = read_headers(partial(next, lines))
key = standard_b64encode(sha1(self.key + GUID_STR).digest())
key = as_base64_unicode(sha1(self.key + GUID_STR).digest())
if headers.get('Sec-WebSocket-Accept') != key:
raise ValueError('Server did not respond with correct key in Sec-WebSocket-Accept')

20
src/calibre/srv/web_socket.py
View File

@ -2,24 +2,28 @@
# vim:fileencoding=utf-8
# License: GPLv3 Copyright: 2015, Kovid Goyal <kovid at kovidgoyal.net>
from __future__ import (unicode_literals, division, absolute_import,
print_function)
from __future__ import absolute_import, division, print_function, unicode_literals
import os, weakref, socket
from base64 import standard_b64encode
import os
import socket
import weakref
from collections import deque
from hashlib import sha1
from struct import unpack_from, pack, error as struct_error
from struct import error as struct_error, pack, unpack_from
from threading import Lock
from calibre import as_unicode
from calibre.constants import plugins
from calibre.srv.loop import ServerLoop, HandleInterrupt, WRITE, READ, RDWR, Connection
from calibre.srv.http_response import HTTPConnection, create_http_handler
from calibre.srv.loop import (
RDWR, READ, WRITE, Connection, HandleInterrupt, ServerLoop
)
from calibre.srv.utils import DESIRED_SEND_BUFFER_SIZE
from calibre.utils.speedups import ReadOnlyFileBuffer
from polyglot.queue import Queue, Empty
from polyglot import http_client
from polyglot.binary import as_base64_unicode
from polyglot.queue import Empty, Queue
speedup, err = plugins['speedup']
if not speedup:
raise RuntimeError('Failed to load speedup module with error: ' + err)
@ -291,7 +295,7 @@ class WebSocketConnection(HTTPConnection):
if self.method != 'GET':
return self.simple_response(http_client.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method)
response = HANDSHAKE_STR % standard_b64encode(sha1(key + GUID_STR).digest())
response = HANDSHAKE_STR % as_base64_unicode(sha1(key + GUID_STR).digest())
self.optimize_for_sending_packet()
self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.set_state(WRITE, self.upgrade_connection_to_ws, ReadOnlyFileBuffer(response.encode('ascii')), inheaders)

8
src/calibre/utils/config_base.py
View File

@ -21,9 +21,9 @@ plugin_dir = os.path.join(config_dir, 'plugins')
def to_json(obj):
import datetime
if isinstance(obj, bytearray):
import base64
from base64 import standard_b64encode
return {'__class__': 'bytearray',
'__value__': base64.standard_b64encode(bytes(obj)).decode('ascii')}
'__value__': standard_b64encode(bytes(obj)).decode('ascii')}
if isinstance(obj, datetime.datetime):
from calibre.utils.date import isoformat
return {'__class__': 'datetime.datetime',
@ -42,8 +42,8 @@ def from_json(obj):
custom = obj.get('__class__')
if custom is not None:
if custom == 'bytearray':
import base64
return bytearray(base64.standard_b64decode(obj['__value__']))
from base64 import standard_b64decode
return bytearray(standard_b64decode(obj['__value__'].encode('ascii')))
if custom == 'datetime.datetime':
from calibre.utils.iso8601 import parse_iso8601
return parse_iso8601(obj['__value__'], assume_utc=True)

6
src/calibre/utils/serialize.py
View File

@ -66,13 +66,11 @@ def json_dumps(data, **kw):
def decode_metadata(x, for_json):
import base64
from polyglot.binary import from_base64_bytes
from calibre.ebooks.metadata.book.serialize import metadata_from_dict
obj = metadata_from_dict(x)
if for_json and obj.cover_data and obj.cover_data[1]:
obj.cover_data = obj.cover_data[0], base64.standard_b64decode(
obj.cover_data[1]
)
obj.cover_data = obj.cover_data[0], from_base64_bytes(obj.cover_data[1])
return obj

6
src/calibre/web/fetch/simple.py
View File

@ -17,7 +17,6 @@ import sys
import threading
import time
import traceback
from base64 import b64decode
from calibre import browser, relpath, unicode_path
from calibre.constants import filesystem_encoding, iswindows
@ -35,6 +34,7 @@ from polyglot.urllib import (
URLError, quote, url2pathname, urljoin, urlparse, urlsplit, urlunparse,
urlunsplit
)
from polyglot.binary import from_base64_bytes
class AbortArticle(Exception):
@ -391,8 +391,8 @@ class RecursiveFetcher(object):
iurl = tag['src']
if iurl.startswith('data:image/'):
try:
data = b64decode(iurl.partition(',')[-1])
except:
data = from_base64_bytes(iurl.partition(',')[-1])
except Exception:
self.log.exception('Failed to decode embedded image')
continue
else:

32
src/polyglot/binary.py Normal file
View File

@ -0,0 +1,32 @@
#!/usr/bin/env python2
# vim:fileencoding=utf-8
# License: GPL v3 Copyright: 2019, Kovid Goyal <kovid at kovidgoyal.net>
from __future__ import absolute_import, division, print_function, unicode_literals
from base64 import standard_b64encode, standard_b64decode
from polyglot.builtins import unicode_type
def as_base64_bytes(x, enc='utf-8'):
if isinstance(x, unicode_type):
x = x.encode(enc)
return standard_b64encode(x)
def as_base64_unicode(x, enc='utf-8'):
if isinstance(x, unicode_type):
x = x.encode(enc)
return standard_b64encode(x).decode('ascii')
def from_base64_unicode(x, enc='utf-8'):
if isinstance(x, unicode_type):
x = x.encode('ascii')
return standard_b64decode(x).decode(enc)
def from_base64_bytes(x):
if isinstance(x, unicode_type):
x = x.encode('ascii')
return standard_b64decode(x)