Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

python3: add httplib/http.client polyglot wrapper

Browse Source
This commit is contained in:
Eli Schwartz 2019-03-25 12:07:48 -04:00
parent 92c621c718
commit bfcb301fe3
No known key found for this signature in database
GPG Key ID: CEB167EFB5722BD6
17 changed files with 198 additions and 190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/calibre/db/cli/main.py
View File

@ -4,7 +4,6 @@
from __future__ import absolute_import, division, print_function, unicode_literals from __future__ import absolute_import, division, print_function, unicode_literals
import httplib
import json import json
import os import os
import sys import sys
@ -17,6 +16,7 @@ from calibre.utils.config import OptionParser, prefs
from calibre.utils.localization import localize_user_manual_link from calibre.utils.localization import localize_user_manual_link
from calibre.utils.lock import singleinstance from calibre.utils.lock import singleinstance
from calibre.utils.serialize import MSGPACK_MIME from calibre.utils.serialize import MSGPACK_MIME
from polyglot import http_client
from polyglot.urllib import urlencode, urlparse, urlunparse from polyglot.urllib import urlencode, urlparse, urlunparse
COMMANDS = ( COMMANDS = (
@ -191,13 +191,13 @@ class DBCtx(object):
return m.implementation(self.db.new_api, None, *args) return m.implementation(self.db.new_api, None, *args)
def interpret_http_error(self, err): def interpret_http_error(self, err):
if err.code == httplib.UNAUTHORIZED: if err.code == http_client.UNAUTHORIZED:
if self.has_credentials: if self.has_credentials:
raise SystemExit('The username/password combination is incorrect') raise SystemExit('The username/password combination is incorrect')
raise SystemExit('A username and password is required to access this server') raise SystemExit('A username and password is required to access this server')
if err.code == httplib.FORBIDDEN: if err.code == http_client.FORBIDDEN:
raise SystemExit(err.reason) raise SystemExit(err.reason)
if err.code == httplib.NOT_FOUND: if err.code == http_client.NOT_FOUND:
raise SystemExit(err.reason) raise SystemExit(err.reason)
def remote_run(self, name, m, *args): def remote_run(self, name, m, *args):

7
src/calibre/gui2/icon_theme.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import os, errno, json, importlib, math, httplib, bz2, shutil, sys import os, errno, json, importlib, math, bz2, shutil, sys
from itertools import count from itertools import count
from io import BytesIO from io import BytesIO
from threading import Thread, Event from threading import Thread, Event
@ -35,8 +35,9 @@ from calibre.utils.img import image_from_data, Canvas, optimize_png, optimize_jp
from calibre.utils.zipfile import ZipFile, ZIP_STORED from calibre.utils.zipfile import ZipFile, ZIP_STORED
from calibre.utils.filenames import atomic_rename from calibre.utils.filenames import atomic_rename
from lzma.xz import compress, decompress from lzma.xz import compress, decompress
from polyglot.queue import Queue, Empty
from polyglot.builtins import iteritems, map, range, reraise from polyglot.builtins import iteritems, map, range, reraise
from polyglot import http_client
from polyglot.queue import Queue, Empty
IMAGE_EXTENSIONS = {'png', 'jpg', 'jpeg'} IMAGE_EXTENSIONS = {'png', 'jpg', 'jpeg'}
THEME_COVER = 'icon-theme-cover.jpg' THEME_COVER = 'icon-theme-cover.jpg'
@ -439,7 +440,7 @@ def download_cover(cover_url, etag=None, cached=b''):
etag = response.getheader('ETag', None) or None etag = response.getheader('ETag', None) or None
return cached, etag return cached, etag
except HTTPError as e: except HTTPError as e:
if etag and e.code == httplib.NOT_MODIFIED: if etag and e.code == http_client.NOT_MODIFIED:
return cached, etag return cached, etag
raise raise

21
src/calibre/srv/auth.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import binascii, os, random, struct, base64, httplib import binascii, os, random, struct, base64
from collections import OrderedDict from collections import OrderedDict
from hashlib import md5, sha256 from hashlib import md5, sha256
from itertools import permutations from itertools import permutations
@ -16,6 +16,7 @@ from calibre.srv.errors import HTTPAuthRequired, HTTPSimpleResponse, HTTPForbidd
from calibre.srv.http_request import parse_uri from calibre.srv.http_request import parse_uri
from calibre.srv.utils import parse_http_dict, encode_path from calibre.srv.utils import parse_http_dict, encode_path
from calibre.utils.monotonic import monotonic from calibre.utils.monotonic import monotonic
from polyglot import http_client
MAX_AGE_SECONDS = 3600 MAX_AGE_SECONDS = 3600
nonce_counter, nonce_counter_lock = 0, Lock() nonce_counter, nonce_counter_lock = 0, Lock()
@ -133,19 +134,19 @@ class DigestAuth(object): # {{{
self.nonce_count = data.get('nc') self.nonce_count = data.get('nc')
if self.algorithm not in self.valid_algorithms: if self.algorithm not in self.valid_algorithms:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported digest algorithm') raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported digest algorithm')
if not (self.username and self.realm and self.nonce and self.uri and self.response): if not (self.username and self.realm and self.nonce and self.uri and self.response):
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Digest algorithm required fields missing') raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Digest algorithm required fields missing')
if self.qop: if self.qop:
if self.qop not in self.valid_qops: if self.qop not in self.valid_qops:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported digest qop') raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported digest qop')
if not (self.cnonce and self.nonce_count): if not (self.cnonce and self.nonce_count):
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'qop present, but cnonce and nonce_count absent') raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'qop present, but cnonce and nonce_count absent')
else: else:
if self.cnonce or self.nonce_count: if self.cnonce or self.nonce_count:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'qop missing') raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'qop missing')
def H(self, val): def H(self, val):
return md5_hex(val) return md5_hex(val)
@ -201,7 +202,7 @@ class DigestAuth(object): # {{{
if log is not None: if log is not None:
log.warn('Authorization URI mismatch: %s != %s from client: %s' % ( log.warn('Authorization URI mismatch: %s != %s from client: %s' % (
data.path, path, data.remote_addr)) data.path, path, data.remote_addr))
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The uri in the Request Line and the Authorization header do not match') raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The uri in the Request Line and the Authorization header do not match')
return self.response is not None and data.path == path and self.request_digest(pw, data) == self.response return self.response is not None and data.path == path and self.request_digest(pw, data) == self.response
# }}} # }}}
@ -290,16 +291,16 @@ class AuthController(object):
try: try:
un, pw = base64_decode(rest.strip()).partition(':')[::2] un, pw = base64_decode(rest.strip()).partition(':')[::2]
except ValueError: except ValueError:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The username or password contained non-UTF8 encoded characters') raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The username or password contained non-UTF8 encoded characters')
if not un or not pw: if not un or not pw:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The username or password was empty') raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The username or password was empty')
if self.check(un, pw): if self.check(un, pw):
data.username = un data.username = un
return return
log_msg = 'Failed login attempt from: %s' % data.remote_addr log_msg = 'Failed login attempt from: %s' % data.remote_addr
self.ban_list.failed(ban_key) self.ban_list.failed(ban_key)
else: else:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported authentication method') raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported authentication method')
if self.prefer_basic_auth: if self.prefer_basic_auth:
raise HTTPAuthRequired('Basic realm="%s"' % self.realm, log=log_msg) raise HTTPAuthRequired('Basic realm="%s"' % self.realm, log=log_msg)

14
src/calibre/srv/errors.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib from polyglot import http_client
class JobQueueFull(Exception): class JobQueueFull(Exception):
@ -30,38 +30,38 @@ class HTTPSimpleResponse(Exception):
class HTTPRedirect(HTTPSimpleResponse): class HTTPRedirect(HTTPSimpleResponse):
def __init__(self, location, http_code=httplib.MOVED_PERMANENTLY, http_message='', close_connection=False): def __init__(self, location, http_code=http_client.MOVED_PERMANENTLY, http_message='', close_connection=False):
HTTPSimpleResponse.__init__(self, http_code, http_message, close_connection, location) HTTPSimpleResponse.__init__(self, http_code, http_message, close_connection, location)
class HTTPNotFound(HTTPSimpleResponse): class HTTPNotFound(HTTPSimpleResponse):
def __init__(self, http_message='', close_connection=False): def __init__(self, http_message='', close_connection=False):
HTTPSimpleResponse.__init__(self, httplib.NOT_FOUND, http_message, close_connection) HTTPSimpleResponse.__init__(self, http_client.NOT_FOUND, http_message, close_connection)
class HTTPAuthRequired(HTTPSimpleResponse): class HTTPAuthRequired(HTTPSimpleResponse):
def __init__(self, payload, log=None): def __init__(self, payload, log=None):
HTTPSimpleResponse.__init__(self, httplib.UNAUTHORIZED, authenticate=payload, log=log) HTTPSimpleResponse.__init__(self, http_client.UNAUTHORIZED, authenticate=payload, log=log)
class HTTPBadRequest(HTTPSimpleResponse): class HTTPBadRequest(HTTPSimpleResponse):
def __init__(self, message, close_connection=False): def __init__(self, message, close_connection=False):
HTTPSimpleResponse.__init__(self, httplib.BAD_REQUEST, message, close_connection) HTTPSimpleResponse.__init__(self, http_client.BAD_REQUEST, message, close_connection)
class HTTPForbidden(HTTPSimpleResponse): class HTTPForbidden(HTTPSimpleResponse):
def __init__(self, http_message='', close_connection=True, log=None): def __init__(self, http_message='', close_connection=True, log=None):
HTTPSimpleResponse.__init__(self, httplib.FORBIDDEN, http_message, close_connection, log=log) HTTPSimpleResponse.__init__(self, http_client.FORBIDDEN, http_message, close_connection, log=log)
class HTTPInternalServerError(HTTPSimpleResponse): class HTTPInternalServerError(HTTPSimpleResponse):
def __init__(self, http_message='', close_connection=True, log=None): def __init__(self, http_message='', close_connection=True, log=None):
HTTPSimpleResponse.__init__(self, httplib.INTERNAL_SERVER_ERROR, http_message, close_connection, log=log) HTTPSimpleResponse.__init__(self, http_client.INTERNAL_SERVER_ERROR, http_message, close_connection, log=log)
class BookNotFound(HTTPNotFound): class BookNotFound(HTTPNotFound):

43
src/calibre/srv/http_request.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import re, httplib, repr as reprlib import re, repr as reprlib
from io import BytesIO, DEFAULT_BUFFER_SIZE from io import BytesIO, DEFAULT_BUFFER_SIZE
from calibre import as_unicode, force_unicode from calibre import as_unicode, force_unicode
@ -14,6 +14,7 @@ from calibre.ptempfile import SpooledTemporaryFile
from calibre.srv.errors import HTTPSimpleResponse from calibre.srv.errors import HTTPSimpleResponse
from calibre.srv.loop import Connection, READ, WRITE from calibre.srv.loop import Connection, READ, WRITE
from calibre.srv.utils import MultiDict, HTTP1, HTTP11, Accumulator from calibre.srv.utils import MultiDict, HTTP1, HTTP11, Accumulator
from polyglot import http_client
from polyglot.urllib import unquote from polyglot.urllib import unquote
protocol_map = {(1, 0):HTTP1, (1, 1):HTTP11} protocol_map = {(1, 0):HTTP1, (1, 1):HTTP11}
@ -68,29 +69,29 @@ def parse_request_uri(uri):
def parse_uri(uri, parse_query=True): def parse_uri(uri, parse_query=True):
scheme, authority, path = parse_request_uri(uri) scheme, authority, path = parse_request_uri(uri)
if path is None: if path is None:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, "No path component") raise HTTPSimpleResponse(http_client.BAD_REQUEST, "No path component")
if b'#' in path: if b'#' in path:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, "Illegal #fragment in Request-URI.") raise HTTPSimpleResponse(http_client.BAD_REQUEST, "Illegal #fragment in Request-URI.")
if scheme: if scheme:
try: try:
scheme = scheme.decode('ascii') scheme = scheme.decode('ascii')
except ValueError: except ValueError:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Un-decodeable scheme') raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Un-decodeable scheme')
path, qs = path.partition(b'?')[::2] path, qs = path.partition(b'?')[::2]
if parse_query: if parse_query:
try: try:
query = MultiDict.create_from_query_string(qs) query = MultiDict.create_from_query_string(qs)
except Exception: except Exception:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unparseable query string') raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unparseable query string')
else: else:
query = None query = None
try: try:
path = '%2F'.join(unquote(x).decode('utf-8') for x in quoted_slash.split(path)) path = '%2F'.join(unquote(x).decode('utf-8') for x in quoted_slash.split(path))
except ValueError as e: except ValueError as e:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, as_unicode(e)) raise HTTPSimpleResponse(http_client.BAD_REQUEST, as_unicode(e))
path = tuple(filter(None, (x.replace('%2F', '/') for x in path.split('/')))) path = tuple(filter(None, (x.replace('%2F', '/') for x in path.split('/'))))
return scheme, path, query return scheme, path, query
@ -233,7 +234,7 @@ class HTTPRequest(Connection):
if line.endswith(b'\n'): if line.endswith(b'\n'):
line = buf.getvalue() line = buf.getvalue()
if not line.endswith(b'\r\n'): if not line.endswith(b'\r\n'):
self.simple_response(httplib.BAD_REQUEST, 'HTTP requires CRLF line terminators') self.simple_response(http_client.BAD_REQUEST, 'HTTP requires CRLF line terminators')
return return
return line return line
if not line: if not line:
@ -247,7 +248,7 @@ class HTTPRequest(Connection):
self.forwarded_for = None self.forwarded_for = None
self.path = self.query = None self.path = self.query = None
self.close_after_response = False self.close_after_response = False
self.header_line_too_long_error_code = httplib.REQUEST_URI_TOO_LONG self.header_line_too_long_error_code = http_client.REQUEST_URI_TOO_LONG
self.response_started = False self.response_started = False
self.set_state(READ, self.parse_request_line, Accumulator(), first=True) self.set_state(READ, self.parse_request_line, Accumulator(), first=True)
@ -260,28 +261,28 @@ class HTTPRequest(Connection):
# Ignore a single leading empty line, as per RFC 2616 sec 4.1 # Ignore a single leading empty line, as per RFC 2616 sec 4.1
if first: if first:
return self.set_state(READ, self.parse_request_line, Accumulator()) return self.set_state(READ, self.parse_request_line, Accumulator())
return self.simple_response(httplib.BAD_REQUEST, 'Multiple leading empty lines not allowed') return self.simple_response(http_client.BAD_REQUEST, 'Multiple leading empty lines not allowed')
try: try:
method, uri, req_protocol = line.strip().split(b' ', 2) method, uri, req_protocol = line.strip().split(b' ', 2)
rp = int(req_protocol[5]), int(req_protocol[7]) rp = int(req_protocol[5]), int(req_protocol[7])
self.method = method.decode('ascii').upper() self.method = method.decode('ascii').upper()
except Exception: except Exception:
return self.simple_response(httplib.BAD_REQUEST, "Malformed Request-Line") return self.simple_response(http_client.BAD_REQUEST, "Malformed Request-Line")
if self.method not in HTTP_METHODS: if self.method not in HTTP_METHODS:
return self.simple_response(httplib.BAD_REQUEST, "Unknown HTTP method") return self.simple_response(http_client.BAD_REQUEST, "Unknown HTTP method")
try: try:
self.request_protocol = protocol_map[rp] self.request_protocol = protocol_map[rp]
except KeyError: except KeyError:
return self.simple_response(httplib.HTTP_VERSION_NOT_SUPPORTED) return self.simple_response(http_client.HTTP_VERSION_NOT_SUPPORTED)
self.response_protocol = protocol_map[min((1, 1), rp)] self.response_protocol = protocol_map[min((1, 1), rp)]
try: try:
self.scheme, self.path, self.query = parse_uri(uri) self.scheme, self.path, self.query = parse_uri(uri)
except HTTPSimpleResponse as e: except HTTPSimpleResponse as e:
return self.simple_response(e.http_code, e.message, close_after_response=False) return self.simple_response(e.http_code, e.message, close_after_response=False)
self.header_line_too_long_error_code = httplib.REQUEST_ENTITY_TOO_LARGE self.header_line_too_long_error_code = http_client.REQUEST_ENTITY_TOO_LARGE
self.set_state(READ, self.parse_header_line, HTTPHeaderParser(), Accumulator()) self.set_state(READ, self.parse_header_line, HTTPHeaderParser(), Accumulator())
# }}} # }}}
@ -299,7 +300,7 @@ class HTTPRequest(Connection):
try: try:
parser(line) parser(line)
except ValueError: except ValueError:
self.simple_response(httplib.BAD_REQUEST, 'Failed to parse header line') self.simple_response(http_client.BAD_REQUEST, 'Failed to parse header line')
return return
if parser.finished: if parser.finished:
self.finalize_headers(parser.hdict) self.finalize_headers(parser.hdict)
@ -307,7 +308,7 @@ class HTTPRequest(Connection):
def finalize_headers(self, inheaders): def finalize_headers(self, inheaders):
request_content_length = int(inheaders.get('Content-Length', 0)) request_content_length = int(inheaders.get('Content-Length', 0))
if request_content_length > self.max_request_body_size: if request_content_length > self.max_request_body_size:
return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE, return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE,
"The entity sent with the request exceeds the maximum " "The entity sent with the request exceeds the maximum "
"allowed bytes (%d)." % self.max_request_body_size) "allowed bytes (%d)." % self.max_request_body_size)
# Persistent connection support # Persistent connection support
@ -334,7 +335,7 @@ class HTTPRequest(Connection):
else: else:
# Note that, even if we see "chunked", we must reject # Note that, even if we see "chunked", we must reject
# if there is an extension we don't recognize. # if there is an extension we don't recognize.
return self.simple_response(httplib.NOT_IMPLEMENTED, "Unknown transfer encoding: %r" % enc) return self.simple_response(http_client.NOT_IMPLEMENTED, "Unknown transfer encoding: %r" % enc)
if inheaders.get("Expect", '').lower() == "100-continue": if inheaders.get("Expect", '').lower() == "100-continue":
buf = BytesIO((HTTP11 + " 100 Continue\r\n\r\n").encode('ascii')) buf = BytesIO((HTTP11 + " 100 Continue\r\n\r\n").encode('ascii'))
@ -369,9 +370,9 @@ class HTTPRequest(Connection):
try: try:
chunk_size = int(line.strip(), 16) chunk_size = int(line.strip(), 16)
except Exception: except Exception:
return self.simple_response(httplib.BAD_REQUEST, '%s is not a valid chunk size' % reprlib.repr(line.strip())) return self.simple_response(http_client.BAD_REQUEST, '%s is not a valid chunk size' % reprlib.repr(line.strip()))
if bytes_read[0] + chunk_size + 2 > self.max_request_body_size: if bytes_read[0] + chunk_size + 2 > self.max_request_body_size:
return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE, return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE,
'Chunked request is larger than %d bytes' % self.max_request_body_size) 'Chunked request is larger than %d bytes' % self.max_request_body_size)
if chunk_size == 0: if chunk_size == 0:
self.set_state(READ, self.read_chunk_separator, inheaders, Accumulator(), buf, bytes_read, last=True) self.set_state(READ, self.read_chunk_separator, inheaders, Accumulator(), buf, bytes_read, last=True)
@ -389,10 +390,10 @@ class HTTPRequest(Connection):
if line is None: if line is None:
return return
if line != b'\r\n': if line != b'\r\n':
return self.simple_response(httplib.BAD_REQUEST, 'Chunk does not have trailing CRLF') return self.simple_response(http_client.BAD_REQUEST, 'Chunk does not have trailing CRLF')
bytes_read[0] += len(line) bytes_read[0] += len(line)
if bytes_read[0] > self.max_request_body_size: if bytes_read[0] > self.max_request_body_size:
return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE, return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE,
'Chunked request is larger than %d bytes' % self.max_request_body_size) 'Chunked request is larger than %d bytes' % self.max_request_body_size)
if last: if last:
self.prepare_response(inheaders, buf) self.prepare_response(inheaders, buf)
@ -402,7 +403,7 @@ class HTTPRequest(Connection):
def handle_timeout(self): def handle_timeout(self):
if self.response_started: if self.response_started:
return False return False
self.simple_response(httplib.REQUEST_TIMEOUT) self.simple_response(http_client.REQUEST_TIMEOUT)
return True return True
def write(self, buf, end=None): def write(self, buf, end=None):

41
src/calibre/srv/http_response.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import os, httplib, hashlib, uuid, struct, repr as reprlib import os, hashlib, uuid, struct, repr as reprlib
from collections import namedtuple from collections import namedtuple
from io import BytesIO, DEFAULT_BUFFER_SIZE from io import BytesIO, DEFAULT_BUFFER_SIZE
from itertools import chain, repeat from itertools import chain, repeat
@ -26,6 +26,7 @@ from calibre.srv.utils import (
sort_q_values, get_translator_for_lang, Cookie, fast_now_strftime) sort_q_values, get_translator_for_lang, Cookie, fast_now_strftime)
from calibre.utils.speedups import ReadOnlyFileBuffer from calibre.utils.speedups import ReadOnlyFileBuffer
from calibre.utils.monotonic import monotonic from calibre.utils.monotonic import monotonic
from polyglot import http_client
Range = namedtuple('Range', 'start stop size') Range = namedtuple('Range', 'start stop size')
MULTIPART_SEPARATOR = uuid.uuid4().hex.decode('ascii') MULTIPART_SEPARATOR = uuid.uuid4().hex.decode('ascii')
@ -226,7 +227,7 @@ class RequestData(object): # {{{
self.remote_addr, self.remote_port, self.is_local_connection = remote_addr, remote_port, is_local_connection self.remote_addr, self.remote_port, self.is_local_connection = remote_addr, remote_port, is_local_connection
self.forwarded_for = forwarded_for self.forwarded_for = forwarded_for
self.opts = opts self.opts = opts
self.status_code = httplib.OK self.status_code = http_client.OK
self.outcookie = Cookie() self.outcookie = Cookie()
self.lang_code = self.gettext_func = self.ngettext_func = None self.lang_code = self.gettext_func = self.ngettext_func = None
self.set_translator(self.get_preferred_language()) self.set_translator(self.get_preferred_language())
@ -402,16 +403,16 @@ class HTTPConnection(HTTPRequest):
if self.response_protocol is HTTP1: if self.response_protocol is HTTP1:
# HTTP/1.0 has no 413/414/303 codes # HTTP/1.0 has no 413/414/303 codes
status_code = { status_code = {
httplib.REQUEST_ENTITY_TOO_LARGE:httplib.BAD_REQUEST, http_client.REQUEST_ENTITY_TOO_LARGE:http_client.BAD_REQUEST,
httplib.REQUEST_URI_TOO_LONG:httplib.BAD_REQUEST, http_client.REQUEST_URI_TOO_LONG:http_client.BAD_REQUEST,
httplib.SEE_OTHER:httplib.FOUND http_client.SEE_OTHER:http_client.FOUND
}.get(status_code, status_code) }.get(status_code, status_code)
self.close_after_response = close_after_response self.close_after_response = close_after_response
msg = msg.encode('utf-8') msg = msg.encode('utf-8')
ct = 'http' if self.method == 'TRACE' else 'plain' ct = 'http' if self.method == 'TRACE' else 'plain'
buf = [ buf = [
'%s %d %s' % (self.response_protocol, status_code, httplib.responses[status_code]), '%s %d %s' % (self.response_protocol, status_code, http_client.responses[status_code]),
"Content-Length: %s" % len(msg), "Content-Length: %s" % len(msg),
"Content-Type: text/%s; charset=UTF-8" % ct, "Content-Type: text/%s; charset=UTF-8" % ct,
"Date: " + http_date(), "Date: " + http_date(),
@ -432,7 +433,7 @@ class HTTPConnection(HTTPRequest):
def prepare_response(self, inheaders, request_body_file): def prepare_response(self, inheaders, request_body_file):
if self.method == 'TRACE': if self.method == 'TRACE':
msg = force_unicode(self.request_line, 'utf-8') + '\n' + inheaders.pretty() msg = force_unicode(self.request_line, 'utf-8') + '\n' + inheaders.pretty()
return self.simple_response(httplib.OK, msg, close_after_response=False) return self.simple_response(http_client.OK, msg, close_after_response=False)
request_body_file.seek(0) request_body_file.seek(0)
outheaders = MultiDict() outheaders = MultiDict()
data = RequestData( data = RequestData(
@ -449,28 +450,28 @@ class HTTPConnection(HTTPRequest):
def send_range_not_satisfiable(self, content_length): def send_range_not_satisfiable(self, content_length):
buf = [ buf = [
'%s %d %s' % (self.response_protocol, httplib.REQUESTED_RANGE_NOT_SATISFIABLE, httplib.responses[httplib.REQUESTED_RANGE_NOT_SATISFIABLE]), '%s %d %s' % (self.response_protocol, http_client.REQUESTED_RANGE_NOT_SATISFIABLE, http_client.responses[http_client.REQUESTED_RANGE_NOT_SATISFIABLE]),
"Date: " + http_date(), "Date: " + http_date(),
"Content-Range: bytes */%d" % content_length, "Content-Range: bytes */%d" % content_length,
] ]
response_data = header_list_to_file(buf) response_data = header_list_to_file(buf)
self.log_access(status_code=httplib.REQUESTED_RANGE_NOT_SATISFIABLE, response_size=response_data.sz) self.log_access(status_code=http_client.REQUESTED_RANGE_NOT_SATISFIABLE, response_size=response_data.sz)
self.response_ready(response_data) self.response_ready(response_data)
def send_not_modified(self, etag=None): def send_not_modified(self, etag=None):
buf = [ buf = [
'%s %d %s' % (self.response_protocol, httplib.NOT_MODIFIED, httplib.responses[httplib.NOT_MODIFIED]), '%s %d %s' % (self.response_protocol, http_client.NOT_MODIFIED, http_client.responses[http_client.NOT_MODIFIED]),
"Content-Length: 0", "Content-Length: 0",
"Date: " + http_date(), "Date: " + http_date(),
] ]
if etag is not None: if etag is not None:
buf.append('ETag: ' + etag) buf.append('ETag: ' + etag)
response_data = header_list_to_file(buf) response_data = header_list_to_file(buf)
self.log_access(status_code=httplib.NOT_MODIFIED, response_size=response_data.sz) self.log_access(status_code=http_client.NOT_MODIFIED, response_size=response_data.sz)
self.response_ready(response_data) self.response_ready(response_data)
def report_busy(self): def report_busy(self):
self.simple_response(httplib.SERVICE_UNAVAILABLE) self.simple_response(http_client.SERVICE_UNAVAILABLE)
def job_done(self, ok, result): def job_done(self, ok, result):
if not ok: if not ok:
@ -509,7 +510,7 @@ class HTTPConnection(HTTPRequest):
if ct.startswith('text/') and 'charset=' not in ct: if ct.startswith('text/') and 'charset=' not in ct:
outheaders.set('Content-Type', ct + '; charset=UTF-8', replace_all=True) outheaders.set('Content-Type', ct + '; charset=UTF-8', replace_all=True)
buf = [HTTP11 + (' %d ' % data.status_code) + httplib.responses[data.status_code]] buf = [HTTP11 + (' %d ' % data.status_code) + http_client.responses[data.status_code]]
for header, value in sorted(iteritems(outheaders), key=itemgetter(0)): for header, value in sorted(iteritems(outheaders), key=itemgetter(0)):
buf.append('%s: %s' % (header, value)) buf.append('%s: %s' % (header, value))
for morsel in itervalues(data.outcookie): for morsel in itervalues(data.outcookie):
@ -530,7 +531,7 @@ class HTTPConnection(HTTPRequest):
def log_access(self, status_code, response_size=None, username=None): def log_access(self, status_code, response_size=None, username=None):
if self.access_log is None: if self.access_log is None:
return return
if not self.opts.log_not_found and status_code == httplib.NOT_FOUND: if not self.opts.log_not_found and status_code == http_client.NOT_FOUND:
return return
ff = self.forwarded_for ff = self.forwarded_for
if ff: if ff:
@ -623,7 +624,7 @@ class HTTPConnection(HTTPRequest):
self.ready = ready self.ready = ready
def report_unhandled_exception(self, e, formatted_traceback): def report_unhandled_exception(self, e, formatted_traceback):
self.simple_response(httplib.INTERNAL_SERVER_ERROR) self.simple_response(http_client.INTERNAL_SERVER_ERROR)
def finalize_output(self, output, request, is_http1): def finalize_output(self, output, request, is_http1):
none_match = parse_if_none_match(request.inheaders.get('If-None-Match', '')) none_match = parse_if_none_match(request.inheaders.get('If-None-Match', ''))
@ -633,7 +634,7 @@ class HTTPConnection(HTTPRequest):
if self.method in ('GET', 'HEAD'): if self.method in ('GET', 'HEAD'):
self.send_not_modified(output.etag) self.send_not_modified(output.etag)
else: else:
self.simple_response(httplib.PRECONDITION_FAILED) self.simple_response(http_client.PRECONDITION_FAILED)
return return
opts = self.opts opts = self.opts
@ -660,10 +661,10 @@ class HTTPConnection(HTTPRequest):
ct = outheaders.get('Content-Type', '').partition(';')[0] ct = outheaders.get('Content-Type', '').partition(';')[0]
compressible = (not ct or ct.startswith('text/') or ct.startswith('image/svg') or compressible = (not ct or ct.startswith('text/') or ct.startswith('image/svg') or
ct.partition(';')[0] in COMPRESSIBLE_TYPES) ct.partition(';')[0] in COMPRESSIBLE_TYPES)
compressible = (compressible and request.status_code == httplib.OK and compressible = (compressible and request.status_code == http_client.OK and
(opts.compress_min_size > -1 and output.content_length >= opts.compress_min_size) and (opts.compress_min_size > -1 and output.content_length >= opts.compress_min_size) and
acceptable_encoding(request.inheaders.get('Accept-Encoding', '')) and not is_http1) acceptable_encoding(request.inheaders.get('Accept-Encoding', '')) and not is_http1)
accept_ranges = (not compressible and output.accept_ranges is not None and request.status_code == httplib.OK and accept_ranges = (not compressible and output.accept_ranges is not None and request.status_code == http_client.OK and
not is_http1) not is_http1)
ranges = get_ranges(request.inheaders.get('Range'), output.content_length) if output.accept_ranges and self.method in ('GET', 'HEAD') else None ranges = get_ranges(request.inheaders.get('Range'), output.content_length) if output.accept_ranges and self.method in ('GET', 'HEAD') else None
if_range = (request.inheaders.get('If-Range') or '').strip() if_range = (request.inheaders.get('If-Range') or '').strip()
@ -680,7 +681,7 @@ class HTTPConnection(HTTPRequest):
if self.method in ('GET', 'HEAD'): if self.method in ('GET', 'HEAD'):
self.send_not_modified(output.etag) self.send_not_modified(output.etag)
else: else:
self.simple_response(httplib.PRECONDITION_FAILED) self.simple_response(http_client.PRECONDITION_FAILED)
return return
output.ranges = None output.ranges = None
@ -712,7 +713,7 @@ class HTTPConnection(HTTPRequest):
outheaders.set('Content-Length', '%d' % size, replace_all=True) outheaders.set('Content-Length', '%d' % size, replace_all=True)
outheaders.set('Content-Type', 'multipart/byteranges; boundary=' + MULTIPART_SEPARATOR, replace_all=True) outheaders.set('Content-Type', 'multipart/byteranges; boundary=' + MULTIPART_SEPARATOR, replace_all=True)
output.ranges = zip_longest(ranges, range_parts) output.ranges = zip_longest(ranges, range_parts)
request.status_code = httplib.PARTIAL_CONTENT request.status_code = http_client.PARTIAL_CONTENT
return output return output

5
src/calibre/srv/routes.py
View File

@ -6,13 +6,14 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, sys, inspect, re, time, numbers, json as jsonlib, textwrap import sys, inspect, re, time, numbers, json as jsonlib, textwrap
from operator import attrgetter from operator import attrgetter
from calibre.srv.errors import HTTPSimpleResponse, HTTPNotFound, RouteError from calibre.srv.errors import HTTPSimpleResponse, HTTPNotFound, RouteError
from calibre.srv.utils import http_date from calibre.srv.utils import http_date
from calibre.utils.serialize import msgpack_dumps, json_dumps, MSGPACK_MIME from calibre.utils.serialize import msgpack_dumps, json_dumps, MSGPACK_MIME
from polyglot.builtins import iteritems, itervalues, unicode_type, range, zip from polyglot.builtins import iteritems, itervalues, unicode_type, range, zip
from polyglot import http_client
from polyglot.urllib import quote as urlquote from polyglot.urllib import quote as urlquote
default_methods = frozenset(('HEAD', 'GET')) default_methods = frozenset(('HEAD', 'GET'))
@ -297,7 +298,7 @@ class Router(object):
def dispatch(self, data): def dispatch(self, data):
endpoint_, args = self.find_route(data.path) endpoint_, args = self.find_route(data.path)
if data.method not in endpoint_.methods: if data.method not in endpoint_.methods:
raise HTTPSimpleResponse(httplib.METHOD_NOT_ALLOWED) raise HTTPSimpleResponse(http_client.METHOD_NOT_ALLOWED)
self.read_cookies(data) self.read_cookies(data)

20
src/calibre/srv/tests/ajax.py
View File

@ -6,13 +6,13 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, zlib, json, base64, os import zlib, json, base64, os
from io import BytesIO from io import BytesIO
from functools import partial from functools import partial
from httplib import OK, NOT_FOUND, FORBIDDEN
from calibre.ebooks.metadata.meta import get_metadata from calibre.ebooks.metadata.meta import get_metadata
from calibre.srv.tests.base import LibraryBaseTest from calibre.srv.tests.base import LibraryBaseTest
from polyglot.http_client import OK, NOT_FOUND, FORBIDDEN
from polyglot.urllib import urlencode, quote from polyglot.urllib import urlencode, quote
@ -22,7 +22,7 @@ def make_request(conn, url, headers={}, prefix='/ajax', username=None, password=
conn.request(method, prefix + url, headers=headers, body=data) conn.request(method, prefix + url, headers=headers, body=data)
r = conn.getresponse() r = conn.getresponse()
data = r.read() data = r.read()
if r.status == httplib.OK and data and data[0] in b'{[': if r.status == OK and data and data[0] in b'{[':
data = json.loads(data) data = json.loads(data)
return r, data return r, data
@ -37,10 +37,10 @@ class ContentTest(LibraryBaseTest):
request = partial(make_request, conn, prefix='/ajax/book') request = partial(make_request, conn, prefix='/ajax/book')
r, data = request('/x') r, data = request('/x')
self.ae(r.status, httplib.NOT_FOUND) self.ae(r.status, NOT_FOUND)
r, onedata = request('/1') r, onedata = request('/1')
self.ae(r.status, httplib.OK) self.ae(r.status, OK)
self.ae(request('/1/' + db.server_library_id)[1], onedata) self.ae(request('/1/' + db.server_library_id)[1], onedata)
self.ae(request('/%s?id_is_uuid=true' % db.field_for('uuid', 1))[1], onedata) self.ae(request('/%s?id_is_uuid=true' % db.field_for('uuid', 1))[1], onedata)
@ -63,22 +63,22 @@ class ContentTest(LibraryBaseTest):
request = partial(make_request, conn) request = partial(make_request, conn)
r, data = request('/categories') r, data = request('/categories')
self.ae(r.status, httplib.OK) self.ae(r.status, OK)
r, xdata = request('/categories/' + db.server_library_id) r, xdata = request('/categories/' + db.server_library_id)
self.ae(r.status, httplib.OK) self.ae(r.status, OK)
self.ae(data, xdata) self.ae(data, xdata)
names = {x['name']:x['url'] for x in data} names = {x['name']:x['url'] for x in data}
for q in ('Newest', 'All books', 'Tags', 'Series', 'Authors', 'Enum', 'Composite Tags'): for q in ('Newest', 'All books', 'Tags', 'Series', 'Authors', 'Enum', 'Composite Tags'):
self.assertIn(q, names) self.assertIn(q, names)
r, data = request(names['Tags'], prefix='') r, data = request(names['Tags'], prefix='')
self.ae(r.status, httplib.OK) self.ae(r.status, OK)
names = {x['name']:x['url'] for x in data['items']} names = {x['name']:x['url'] for x in data['items']}
self.ae(set(names), set('Tag One,Tag Two,News'.split(','))) self.ae(set(names), set('Tag One,Tag Two,News'.split(',')))
r, data = request(names['Tag One'], prefix='') r, data = request(names['Tag One'], prefix='')
self.ae(r.status, httplib.OK) self.ae(r.status, OK)
self.ae(set(data['book_ids']), {1, 2}) self.ae(set(data['book_ids']), {1, 2})
r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"'})) r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"'}))
self.ae(r.status, httplib.OK) self.ae(r.status, OK)
self.ae(set(data['book_ids']), {1, 2}) self.ae(set(data['book_ids']), {1, 2})
r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"', 'vl':'1'})) r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"', 'vl':'1'}))
self.ae(set(data['book_ids']), {2}) self.ae(set(data['book_ids']), {2})

53
src/calibre/srv/tests/auth.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, base64, subprocess, os, cookielib, time import base64, subprocess, os, cookielib, time
from collections import namedtuple from collections import namedtuple
try: try:
from distutils.spawn import find_executable from distutils.spawn import find_executable
@ -18,6 +18,7 @@ from calibre.srv.errors import HTTPForbidden
from calibre.srv.tests.base import BaseTest, TestServer from calibre.srv.tests.base import BaseTest, TestServer
from calibre.srv.routes import endpoint, Router from calibre.srv.routes import endpoint, Router
from polyglot.builtins import iteritems, itervalues from polyglot.builtins import iteritems, itervalues
from polyglot import http_client
from polyglot.urllib import (build_opener, HTTPBasicAuthHandler, from polyglot.urllib import (build_opener, HTTPBasicAuthHandler,
HTTPCookieProcessor, HTTPDigestAuthHandler, HTTPError) HTTPCookieProcessor, HTTPDigestAuthHandler, HTTPError)
@ -91,18 +92,18 @@ class TestAuth(BaseTest):
conn = server.connect() conn = server.connect()
conn.request('GET', '/open') conn.request('GET', '/open')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(r.read(), b'open') self.ae(r.read(), b'open')
conn.request('GET', '/closed') conn.request('GET', '/closed')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.UNAUTHORIZED) self.ae(r.status, http_client.UNAUTHORIZED)
self.ae(r.getheader('WWW-Authenticate'), b'Basic realm="%s"' % bytes(REALM)) self.ae(r.getheader('WWW-Authenticate'), b'Basic realm="%s"' % bytes(REALM))
self.assertFalse(r.read()) self.assertFalse(r.read())
conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + base64.standard_b64encode(b'testuser:testpw')}) conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + base64.standard_b64encode(b'testuser:testpw')})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.read(), b'closed') self.ae(r.read(), b'closed')
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(b'closed', urlopen(server, method='basic').read()) self.ae(b'closed', urlopen(server, method='basic').read())
self.ae(b'closed', urlopen(server, un='!@#$%^&*()-=_+', pw='!@#$%^&*()-=_+', method='basic').read()) self.ae(b'closed', urlopen(server, un='!@#$%^&*()-=_+', pw='!@#$%^&*()-=_+', method='basic').read())
@ -113,14 +114,14 @@ class TestAuth(BaseTest):
warnings = [] warnings = []
server.loop.log.warn = lambda *args, **kwargs: warnings.append(' '.join(args)) server.loop.log.warn = lambda *args, **kwargs: warnings.append(' '.join(args))
self.ae((httplib.OK, b'closed'), request()) self.ae((http_client.OK, b'closed'), request())
self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y')) self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y'))
self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('', '')) self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request('', ''))
self.ae(1, len(warnings)) self.ae(1, len(warnings))
self.ae((httplib.UNAUTHORIZED, b''), request('testuser', 'y')) self.ae((http_client.UNAUTHORIZED, b''), request('testuser', 'y'))
self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('testuser', '')) self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request('testuser', ''))
self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('')) self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request(''))
self.ae((httplib.UNAUTHORIZED, b''), request('asf', 'testpw')) self.ae((http_client.UNAUTHORIZED, b''), request('asf', 'testpw'))
# }}} # }}}
def test_library_restrictions(self): # {{{ def test_library_restrictions(self): # {{{
@ -169,7 +170,7 @@ class TestAuth(BaseTest):
with TestServer(r.dispatch) as server: with TestServer(r.dispatch) as server:
r.auth_controller.log = server.log r.auth_controller.log = server.log
def test(conn, path, headers={}, status=httplib.OK, body=b'', request_body=b''): def test(conn, path, headers={}, status=http_client.OK, body=b'', request_body=b''):
conn.request('GET', path, request_body, headers) conn.request('GET', path, request_body, headers)
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, status) self.ae(r.status, status)
@ -177,9 +178,9 @@ class TestAuth(BaseTest):
return {normalize_header_name(k):v for k, v in r.getheaders()} return {normalize_header_name(k):v for k, v in r.getheaders()}
conn = server.connect() conn = server.connect()
test(conn, '/open', body=b'open') test(conn, '/open', body=b'open')
auth = parse_http_dict(test(conn, '/closed', status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) auth = parse_http_dict(test(conn, '/closed', status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
nonce = auth['nonce'] nonce = auth['nonce']
auth = parse_http_dict(test(conn, '/closed', status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) auth = parse_http_dict(test(conn, '/closed', status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
self.assertNotEqual(nonce, auth['nonce'], 'nonce was re-used') self.assertNotEqual(nonce, auth['nonce'], 'nonce was re-used')
self.ae(auth[b'realm'], bytes(REALM)), self.ae(auth[b'algorithm'], b'MD5'), self.ae(auth[b'qop'], b'auth') self.ae(auth[b'realm'], bytes(REALM)), self.ae(auth[b'algorithm'], b'MD5'), self.ae(auth[b'qop'], b'auth')
self.assertNotIn('stale', auth) self.assertNotIn('stale', auth)
@ -199,14 +200,14 @@ class TestAuth(BaseTest):
# Check stale nonces # Check stale nonces
orig, r.auth_controller.max_age_seconds = r.auth_controller.max_age_seconds, -1 orig, r.auth_controller.max_age_seconds = r.auth_controller.max_age_seconds, -1
auth = parse_http_dict(test(conn, '/closed', headers={ auth = parse_http_dict(test(conn, '/closed', headers={
'Authorization':digest(**args)},status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) 'Authorization':digest(**args)},status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
self.assertIn('stale', auth) self.assertIn('stale', auth)
r.auth_controller.max_age_seconds = orig r.auth_controller.max_age_seconds = orig
ok_test(conn, digest(**args)) ok_test(conn, digest(**args))
def fail_test(conn, modify, **kw): def fail_test(conn, modify, **kw):
kw['body'] = kw.get('body', b'') kw['body'] = kw.get('body', b'')
kw['status'] = kw.get('status', httplib.UNAUTHORIZED) kw['status'] = kw.get('status', http_client.UNAUTHORIZED)
args['modify'] = modify args['modify'] = modify
return test(conn, '/closed', headers={'Authorization':digest(**args)}, **kw) return test(conn, '/closed', headers={'Authorization':digest(**args)}, **kw)
@ -258,13 +259,13 @@ class TestAuth(BaseTest):
warnings = [] warnings = []
server.loop.log.warn = lambda *args, **kwargs: warnings.append(' '.join(args)) server.loop.log.warn = lambda *args, **kwargs: warnings.append(' '.join(args))
self.ae((httplib.OK, b'closed'), request()) self.ae((http_client.OK, b'closed'), request())
self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y')) self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y'))
self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y')) self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y'))
self.ae(httplib.FORBIDDEN, request('x', 'y')[0]) self.ae(http_client.FORBIDDEN, request('x', 'y')[0])
self.ae(httplib.FORBIDDEN, request()[0]) self.ae(http_client.FORBIDDEN, request()[0])
time.sleep(ban_for * 60 + 0.01) time.sleep(ban_for * 60 + 0.01)
self.ae((httplib.OK, b'closed'), request()) self.ae((http_client.OK, b'closed'), request())
# }}} # }}}
def test_android_auth_workaround(self): # {{{ def test_android_auth_workaround(self): # {{{
@ -277,7 +278,7 @@ class TestAuth(BaseTest):
# First check that unauth access fails # First check that unauth access fails
conn.request('GET', '/android') conn.request('GET', '/android')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.UNAUTHORIZED) self.ae(r.status, http_client.UNAUTHORIZED)
auth_handler = HTTPDigestAuthHandler() auth_handler = HTTPDigestAuthHandler()
url = 'http://localhost:%d%s' % (server.address[1], '/android') url = 'http://localhost:%d%s' % (server.address[1], '/android')
@ -285,20 +286,20 @@ class TestAuth(BaseTest):
cj = cookielib.CookieJar() cj = cookielib.CookieJar()
cookie_handler = HTTPCookieProcessor(cj) cookie_handler = HTTPCookieProcessor(cj)
r = build_opener(auth_handler, cookie_handler).open(url) r = build_opener(auth_handler, cookie_handler).open(url)
self.ae(r.getcode(), httplib.OK) self.ae(r.getcode(), http_client.OK)
cookies = tuple(cj) cookies = tuple(cj)
self.ae(len(cookies), 1) self.ae(len(cookies), 1)
cookie = cookies[0] cookie = cookies[0]
self.assertIn(b':', cookie.value) self.assertIn(b':', cookie.value)
self.ae(cookie.path, b'/android') self.ae(cookie.path, b'/android')
r = build_opener(cookie_handler).open(url) r = build_opener(cookie_handler).open(url)
self.ae(r.getcode(), httplib.OK) self.ae(r.getcode(), http_client.OK)
self.ae(r.read(), b'android') self.ae(r.read(), b'android')
# Test that a replay attack against a different URL does not work # Test that a replay attack against a different URL does not work
try: try:
build_opener(cookie_handler).open(url+'2') build_opener(cookie_handler).open(url+'2')
assert ('Replay attack succeeded') assert ('Replay attack succeeded')
except HTTPError as e: except HTTPError as e:
self.ae(e.code, httplib.UNAUTHORIZED) self.ae(e.code, http_client.UNAUTHORIZED)
# }}} # }}}

5
src/calibre/srv/tests/base.py
View File

@ -7,12 +7,13 @@ __license__ = 'GPL v3'
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>' __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
import unittest, time, httplib, shutil, gc, tempfile, atexit, os import unittest, time, shutil, gc, tempfile, atexit, os
from io import BytesIO from io import BytesIO
from functools import partial from functools import partial
from threading import Thread from threading import Thread
from calibre.srv.utils import ServerLog from calibre.srv.utils import ServerLog
from polyglot import http_client
rmtree = partial(shutil.rmtree, ignore_errors=True) rmtree = partial(shutil.rmtree, ignore_errors=True)
@ -120,7 +121,7 @@ class TestServer(Thread):
timeout = self.loop.opts.timeout timeout = self.loop.opts.timeout
if interface is None: if interface is None:
interface = self.address[0] interface = self.address[0]
return httplib.HTTPConnection(interface, self.address[1], strict=True, timeout=timeout) return http_client.HTTPConnection(interface, self.address[1], strict=True, timeout=timeout)
def change_handler(self, handler): def change_handler(self, handler):
from calibre.srv.http_response import create_http_handler from calibre.srv.http_response import create_http_handler

43
src/calibre/srv/tests/content.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, zlib, json, binascii, time, os import zlib, json, binascii, time, os
from io import BytesIO from io import BytesIO
from calibre.ebooks.metadata.epub import get_metadata from calibre.ebooks.metadata.epub import get_metadata
@ -14,6 +14,7 @@ from calibre.ebooks.metadata.opf2 import OPF
from calibre.srv.tests.base import LibraryBaseTest from calibre.srv.tests.base import LibraryBaseTest
from calibre.utils.imghdr import identify from calibre.utils.imghdr import identify
from calibre.utils.shared_file import share_open from calibre.utils.shared_file import share_open
from polyglot import http_client
def setUpModule(): def setUpModule():
@ -32,7 +33,7 @@ class ContentTest(LibraryBaseTest):
def missing(url, body=b''): def missing(url, body=b''):
conn.request('GET', url) conn.request('GET', url)
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.NOT_FOUND) self.ae(r.status, http_client.NOT_FOUND)
self.ae(r.read(), body) self.ae(r.read(), body)
for prefix in ('static', 'icon'): for prefix in ('static', 'icon'):
@ -51,7 +52,7 @@ class ContentTest(LibraryBaseTest):
raw = P(src, data=True) raw = P(src, data=True)
conn.request('GET', url) conn.request('GET', url)
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
data = r.read() data = r.read()
if sz is None: if sz is None:
self.ae(data, raw) self.ae(data, raw)
@ -60,7 +61,7 @@ class ContentTest(LibraryBaseTest):
test_response(r) test_response(r)
conn.request('GET', url, headers={'If-None-Match':r.getheader('ETag')}) conn.request('GET', url, headers={'If-None-Match':r.getheader('ETag')})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.NOT_MODIFIED) self.ae(r.status, http_client.NOT_MODIFIED)
self.ae(b'', r.read()) self.ae(b'', r.read())
test('content-server/empty.html', '/static/empty.html') test('content-server/empty.html', '/static/empty.html')
@ -85,7 +86,7 @@ class ContentTest(LibraryBaseTest):
# Test various invalid parameters # Test various invalid parameters
def bad(*args): def bad(*args):
r, data = get(*args) r, data = get(*args)
self.ae(r.status, httplib.NOT_FOUND) self.ae(r.status, http_client.NOT_FOUND)
bad('xxx', 1) bad('xxx', 1)
bad('fmt1', 10) bad('fmt1', 10)
bad('fmt1', 1, 'zzzz') bad('fmt1', 1, 'zzzz')
@ -103,7 +104,7 @@ class ContentTest(LibraryBaseTest):
# Test fetching of format with metadata update # Test fetching of format with metadata update
raw = P('quick_start/eng.epub', data=True) raw = P('quick_start/eng.epub', data=True)
r, data = get('epub', 1) r, data = get('epub', 1)
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
etag = r.getheader('ETag') etag = r.getheader('ETag')
self.assertIsNotNone(etag) self.assertIsNotNone(etag)
self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Used-Cache'), 'no')
@ -145,39 +146,39 @@ class ContentTest(LibraryBaseTest):
os.utime(cpath, (t, t)) os.utime(cpath, (t, t))
r, data = get('cover', 1) r, data = get('cover', 1)
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(data, db.cover(1)) self.ae(data, db.cover(1))
self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Used-Cache'), 'no')
self.ae(r.getheader('Content-Type'), 'image/jpeg') self.ae(r.getheader('Content-Type'), 'image/jpeg')
r, data = get('cover', 1) r, data = get('cover', 1)
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(data, db.cover(1)) self.ae(data, db.cover(1))
self.ae(r.getheader('Used-Cache'), 'yes') self.ae(r.getheader('Used-Cache'), 'yes')
r, data = get('cover', 3) r, data = get('cover', 3)
self.ae(r.status, httplib.OK) # Auto generated cover self.ae(r.status, http_client.OK) # Auto generated cover
r, data = get('thumb', 1) r, data = get('thumb', 1)
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(identify(data), ('jpeg', 60, 60)) self.ae(identify(data), ('jpeg', 60, 60))
self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Used-Cache'), 'no')
r, data = get('thumb', 1) r, data = get('thumb', 1)
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(r.getheader('Used-Cache'), 'yes') self.ae(r.getheader('Used-Cache'), 'yes')
r, data = get('thumb', 1, q='sz=100') r, data = get('thumb', 1, q='sz=100')
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(identify(data), ('jpeg', 100, 100)) self.ae(identify(data), ('jpeg', 100, 100))
self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Used-Cache'), 'no')
r, data = get('thumb', 1, q='sz=100x100') r, data = get('thumb', 1, q='sz=100x100')
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(r.getheader('Used-Cache'), 'yes') self.ae(r.getheader('Used-Cache'), 'yes')
change_cover(1, 1) change_cover(1, 1)
r, data = get('thumb', 1, q='sz=100') r, data = get('thumb', 1, q='sz=100')
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(identify(data), ('jpeg', 100, 100)) self.ae(identify(data), ('jpeg', 100, 100))
self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Used-Cache'), 'no')
# Test file sharing in cache # Test file sharing in cache
r, data = get('cover', 2) r, data = get('cover', 2)
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(data, db.cover(2)) self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Used-Cache'), 'no')
path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8') path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8')
@ -185,7 +186,7 @@ class ContentTest(LibraryBaseTest):
# Now force an update # Now force an update
change_cover(1) change_cover(1)
r, data = get('cover', 2) r, data = get('cover', 2)
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(data, db.cover(2)) self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Used-Cache'), 'no')
path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8') path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8')
@ -193,7 +194,7 @@ class ContentTest(LibraryBaseTest):
# Do it again # Do it again
change_cover(2) change_cover(2)
r, data = get('cover', 2) r, data = get('cover', 2)
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(data, db.cover(2)) self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Used-Cache'), 'no')
self.ae(f.read(), fdata) self.ae(f.read(), fdata)
@ -201,7 +202,7 @@ class ContentTest(LibraryBaseTest):
# Test serving of metadata as opf # Test serving of metadata as opf
r, data = get('opf', 1) r, data = get('opf', 1)
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(r.getheader('Content-Type'), 'application/oebps-package+xml; charset=UTF-8') self.ae(r.getheader('Content-Type'), 'application/oebps-package+xml; charset=UTF-8')
self.assertIsNotNone(r.getheader('Last-Modified')) self.assertIsNotNone(r.getheader('Last-Modified'))
opf = OPF(BytesIO(data), populate_spine=False, try_to_guess_cover=False) opf = OPF(BytesIO(data), populate_spine=False, try_to_guess_cover=False)
@ -209,17 +210,17 @@ class ContentTest(LibraryBaseTest):
self.ae(db.field_for('authors', 1), tuple(opf.authors)) self.ae(db.field_for('authors', 1), tuple(opf.authors))
conn.request('GET', '/get/opf/1', headers={'Accept-Encoding':'gzip'}) conn.request('GET', '/get/opf/1', headers={'Accept-Encoding':'gzip'})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') self.ae(r.status, http_client.OK), self.ae(r.getheader('Content-Encoding'), 'gzip')
raw = r.read() raw = r.read()
self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data) self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data)
# Test serving metadata as json # Test serving metadata as json
r, data = get('json', 1) r, data = get('json', 1)
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(db.field_for('title', 1), json.loads(data)['title']) self.ae(db.field_for('title', 1), json.loads(data)['title'])
conn.request('GET', '/get/json/1', headers={'Accept-Encoding':'gzip'}) conn.request('GET', '/get/json/1', headers={'Accept-Encoding':'gzip'})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') self.ae(r.status, http_client.OK), self.ae(r.getheader('Content-Encoding'), 'gzip')
raw = r.read() raw = r.read()
self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data) self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data)

79
src/calibre/srv/tests/http.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, hashlib, zlib, string, time, os import hashlib, zlib, string, time, os
from io import BytesIO from io import BytesIO
from tempfile import NamedTemporaryFile from tempfile import NamedTemporaryFile
@ -15,6 +15,7 @@ from calibre.srv.tests.base import BaseTest, TestServer
from calibre.srv.utils import eintr_retry_call from calibre.srv.utils import eintr_retry_call
from calibre.utils.monotonic import monotonic from calibre.utils.monotonic import monotonic
from polyglot.builtins import iteritems, range from polyglot.builtins import iteritems, range
from polyglot import http_client
is_ci = os.environ.get('CI', '').lower() == 'true' is_ci = os.environ.get('CI', '').lower() == 'true'
@ -94,7 +95,7 @@ class TestHTTP(BaseTest):
def test(al, q): def test(al, q):
conn.request('GET', '/', headers={'Accept-Language': al}) conn.request('GET', '/', headers={'Accept-Language': al})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
q += get_translator(q)[-1].ugettext('Unknown') q += get_translator(q)[-1].ugettext('Unknown')
self.ae(r.read(), q) self.ae(r.read(), q)
@ -136,7 +137,7 @@ class TestHTTP(BaseTest):
def raw_send(conn, raw): def raw_send(conn, raw):
conn.send(raw) conn.send(raw)
conn._HTTPConnection__state = httplib._CS_REQ_SENT conn._HTTPConnection__state = http_client._CS_REQ_SENT
return conn.getresponse() return conn.getresponse()
base_timeout = 0.5 if is_ci else 0.1 base_timeout = 0.5 if is_ci else 0.1
@ -144,31 +145,31 @@ class TestHTTP(BaseTest):
with TestServer(handler, timeout=base_timeout, max_header_line_size=100./1024, max_request_body_size=100./(1024*1024)) as server: with TestServer(handler, timeout=base_timeout, max_header_line_size=100./1024, max_request_body_size=100./(1024*1024)) as server:
conn = server.connect() conn = server.connect()
r = raw_send(conn, b'hello\n') r = raw_send(conn, b'hello\n')
self.ae(r.status, httplib.BAD_REQUEST) self.ae(r.status, http_client.BAD_REQUEST)
self.ae(r.read(), b'HTTP requires CRLF line terminators') self.ae(r.read(), b'HTTP requires CRLF line terminators')
r = raw_send(conn, b'\r\nGET /index.html HTTP/1.1\r\n\r\n') r = raw_send(conn, b'\r\nGET /index.html HTTP/1.1\r\n\r\n')
self.ae(r.status, httplib.NOT_FOUND), self.ae(r.read(), b'Requested resource not found') self.ae(r.status, http_client.NOT_FOUND), self.ae(r.read(), b'Requested resource not found')
r = raw_send(conn, b'\r\n\r\nGET /index.html HTTP/1.1\r\n\r\n') r = raw_send(conn, b'\r\n\r\nGET /index.html HTTP/1.1\r\n\r\n')
self.ae(r.status, httplib.BAD_REQUEST) self.ae(r.status, http_client.BAD_REQUEST)
self.ae(r.read(), b'Multiple leading empty lines not allowed') self.ae(r.read(), b'Multiple leading empty lines not allowed')
r = raw_send(conn, b'hello world\r\n') r = raw_send(conn, b'hello world\r\n')
self.ae(r.status, httplib.BAD_REQUEST) self.ae(r.status, http_client.BAD_REQUEST)
self.ae(r.read(), b'Malformed Request-Line') self.ae(r.read(), b'Malformed Request-Line')
r = raw_send(conn, b'x' * 200) r = raw_send(conn, b'x' * 200)
self.ae(r.status, httplib.BAD_REQUEST) self.ae(r.status, http_client.BAD_REQUEST)
self.ae(r.read(), b'') self.ae(r.read(), b'')
r = raw_send(conn, b'XXX /index.html HTTP/1.1\r\n\r\n') r = raw_send(conn, b'XXX /index.html HTTP/1.1\r\n\r\n')
self.ae(r.status, httplib.BAD_REQUEST), self.ae(r.read(), b'Unknown HTTP method') self.ae(r.status, http_client.BAD_REQUEST), self.ae(r.read(), b'Unknown HTTP method')
# Test 404 # Test 404
conn.request('HEAD', '/moose') conn.request('HEAD', '/moose')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.NOT_FOUND) self.ae(r.status, http_client.NOT_FOUND)
self.assertIsNotNone(r.getheader('Date', None)) self.assertIsNotNone(r.getheader('Date', None))
self.ae(r.getheader('Content-Length'), str(len(body))) self.ae(r.getheader('Content-Length'), str(len(body)))
self.ae(r.getheader('Content-Type'), 'text/plain; charset=UTF-8') self.ae(r.getheader('Content-Type'), 'text/plain; charset=UTF-8')
@ -176,7 +177,7 @@ class TestHTTP(BaseTest):
self.ae(r.read(), '') self.ae(r.read(), '')
conn.request('GET', '/choose') conn.request('GET', '/choose')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.NOT_FOUND) self.ae(r.status, http_client.NOT_FOUND)
self.ae(r.read(), b'Requested resource not found') self.ae(r.read(), b'Requested resource not found')
# Test 500 # Test 500
@ -186,7 +187,7 @@ class TestHTTP(BaseTest):
conn = server.connect() conn = server.connect()
conn.request('GET', '/test/') conn.request('GET', '/test/')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.INTERNAL_SERVER_ERROR) self.ae(r.status, http_client.INTERNAL_SERVER_ERROR)
server.loop.log.filter_level = orig server.loop.log.filter_level = orig
# Test 301 # Test 301
@ -196,7 +197,7 @@ class TestHTTP(BaseTest):
conn = server.connect() conn = server.connect()
conn.request('GET', '/') conn.request('GET', '/')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.MOVED_PERMANENTLY) self.ae(r.status, http_client.MOVED_PERMANENTLY)
self.ae(r.getheader('Location'), '/somewhere-else') self.ae(r.getheader('Location'), '/somewhere-else')
self.ae('', r.read()) self.ae('', r.read())
@ -206,26 +207,26 @@ class TestHTTP(BaseTest):
# Test simple GET # Test simple GET
conn.request('GET', '/test/') conn.request('GET', '/test/')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(r.read(), b'test') self.ae(r.read(), b'test')
# Test TRACE # Test TRACE
lines = ['TRACE /xxx HTTP/1.1', 'Test: value', 'Xyz: abc, def', '', ''] lines = ['TRACE /xxx HTTP/1.1', 'Test: value', 'Xyz: abc, def', '', '']
r = raw_send(conn, ('\r\n'.join(lines)).encode('ascii')) r = raw_send(conn, ('\r\n'.join(lines)).encode('ascii'))
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(r.read().decode('utf-8'), '\n'.join(lines[:-2])) self.ae(r.read().decode('utf-8'), '\n'.join(lines[:-2]))
# Test POST with simple body # Test POST with simple body
conn.request('POST', '/test', 'body') conn.request('POST', '/test', 'body')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody') self.ae(r.read(), b'testbody')
# Test POST with chunked transfer encoding # Test POST with chunked transfer encoding
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'4\r\nbody\r\na\r\n1234567890\r\n0\r\n\r\n') conn.send(b'4\r\nbody\r\na\r\n1234567890\r\n0\r\n\r\n')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody1234567890') self.ae(r.read(), b'testbody1234567890')
# Test various incorrect input # Test various incorrect input
@ -233,39 +234,39 @@ class TestHTTP(BaseTest):
conn.request('GET', '/test' + ('a' * 200)) conn.request('GET', '/test' + ('a' * 200))
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.BAD_REQUEST) self.ae(r.status, http_client.BAD_REQUEST)
conn = server.connect() conn = server.connect()
conn.request('GET', '/test', ('a' * 200)) conn.request('GET', '/test', ('a' * 200))
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE) self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE)
conn = server.connect() conn = server.connect()
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'x\r\nbody\r\n0\r\n\r\n') conn.send(b'x\r\nbody\r\n0\r\n\r\n')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.BAD_REQUEST) self.ae(r.status, http_client.BAD_REQUEST)
self.assertIn(b'not a valid chunk size', r.read()) self.assertIn(b'not a valid chunk size', r.read())
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'4\r\nbody\r\n200\r\n\r\n') conn.send(b'4\r\nbody\r\n200\r\n\r\n')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE) self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE)
conn.request('POST', '/test', body='a'*200) conn.request('POST', '/test', body='a'*200)
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE) self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE)
conn = server.connect() conn = server.connect()
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'3\r\nbody\r\n0\r\n\r\n') conn.send(b'3\r\nbody\r\n0\r\n\r\n')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.BAD_REQUEST), self.ae(r.read(), b'Chunk does not have trailing CRLF') self.ae(r.status, http_client.BAD_REQUEST), self.ae(r.read(), b'Chunk does not have trailing CRLF')
conn = server.connect(timeout=base_timeout * 5) conn = server.connect(timeout=base_timeout * 5)
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'30\r\nbody\r\n0\r\n\r\n') conn.send(b'30\r\nbody\r\n0\r\n\r\n')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.REQUEST_TIMEOUT) self.ae(r.status, http_client.REQUEST_TIMEOUT)
self.assertIn(b'', r.read()) self.assertIn(b'', r.read())
server.log.filter_level = orig_level server.log.filter_level = orig_level
@ -273,14 +274,14 @@ class TestHTTP(BaseTest):
# Test pipelining # Test pipelining
responses = [] responses = []
for i in range(10): for i in range(10):
conn._HTTPConnection__state = httplib._CS_IDLE conn._HTTPConnection__state = http_client._CS_IDLE
conn.request('GET', '/%d'%i) conn.request('GET', '/%d'%i)
responses.append(conn.response_class(conn.sock, strict=conn.strict, method=conn._method)) responses.append(conn.response_class(conn.sock, strict=conn.strict, method=conn._method))
for i in range(10): for i in range(10):
r = responses[i] r = responses[i]
r.begin() r.begin()
self.ae(r.read(), ('%d' % i).encode('ascii')) self.ae(r.read(), ('%d' % i).encode('ascii'))
conn._HTTPConnection__state = httplib._CS_IDLE conn._HTTPConnection__state = http_client._CS_IDLE
# Test closing # Test closing
server.loop.opts.timeout = 10 # ensure socket is not closed because of timeout server.loop.opts.timeout = 10 # ensure socket is not closed because of timeout
@ -319,12 +320,12 @@ class TestHTTP(BaseTest):
conn = server.connect() conn = server.connect()
conn.request('GET', '/an_etagged_path') conn.request('GET', '/an_etagged_path')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.read(), b'an_etagged_path') self.ae(r.status, http_client.OK), self.ae(r.read(), b'an_etagged_path')
etag = r.getheader('ETag') etag = r.getheader('ETag')
self.ae(etag, '"%s"' % hashlib.sha1('an_etagged_path').hexdigest()) self.ae(etag, '"%s"' % hashlib.sha1('an_etagged_path').hexdigest())
conn.request('GET', '/an_etagged_path', headers={'If-None-Match':etag}) conn.request('GET', '/an_etagged_path', headers={'If-None-Match':etag})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.NOT_MODIFIED) self.ae(r.status, http_client.NOT_MODIFIED)
self.ae(r.read(), b'') self.ae(r.read(), b'')
# Test gzip # Test gzip
@ -334,7 +335,7 @@ class TestHTTP(BaseTest):
conn.request('GET', '/an_etagged_path', headers={'Accept-Encoding':'gzip'}) conn.request('GET', '/an_etagged_path', headers={'Accept-Encoding':'gzip'})
r = conn.getresponse() r = conn.getresponse()
self.ae(str(len(raw)), r.getheader('Calibre-Uncompressed-Length')) self.ae(str(len(raw)), r.getheader('Calibre-Uncompressed-Length'))
self.ae(r.status, httplib.OK), self.ae(zlib.decompress(r.read(), 16+zlib.MAX_WBITS), raw) self.ae(r.status, http_client.OK), self.ae(zlib.decompress(r.read(), 16+zlib.MAX_WBITS), raw)
# Test dynamic etagged content # Test dynamic etagged content
num_calls = [0] num_calls = [0]
@ -346,13 +347,13 @@ class TestHTTP(BaseTest):
conn = server.connect() conn = server.connect()
conn.request('GET', '/an_etagged_path') conn.request('GET', '/an_etagged_path')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.read(), b'data') self.ae(r.status, http_client.OK), self.ae(r.read(), b'data')
etag = r.getheader('ETag') etag = r.getheader('ETag')
self.ae(etag, b'"xxx"') self.ae(etag, b'"xxx"')
self.ae(r.getheader('Content-Length'), '4') self.ae(r.getheader('Content-Length'), '4')
conn.request('GET', '/an_etagged_path', headers={'If-None-Match':etag}) conn.request('GET', '/an_etagged_path', headers={'If-None-Match':etag})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.NOT_MODIFIED) self.ae(r.status, http_client.NOT_MODIFIED)
self.ae(r.read(), b'') self.ae(r.read(), b'')
self.ae(num_calls[0], 1) self.ae(num_calls[0], 1)
@ -368,11 +369,11 @@ class TestHTTP(BaseTest):
self.ae(r.getheader('Content-Type'), guess_type(f.name)[0]) self.ae(r.getheader('Content-Type'), guess_type(f.name)[0])
self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes') self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes')
self.ae(int(r.getheader('Content-Length')), len(fdata)) self.ae(int(r.getheader('Content-Length')), len(fdata))
self.ae(r.status, httplib.OK), self.ae(r.read(), fdata) self.ae(r.status, http_client.OK), self.ae(r.read(), fdata)
conn.request('GET', '/test', headers={'Range':'bytes=2-25'}) conn.request('GET', '/test', headers={'Range':'bytes=2-25'})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.PARTIAL_CONTENT) self.ae(r.status, http_client.PARTIAL_CONTENT)
self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes') self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes')
self.ae(type('')(r.getheader('Content-Range')), 'bytes 2-25/%d' % len(fdata)) self.ae(type('')(r.getheader('Content-Range')), 'bytes 2-25/%d' % len(fdata))
self.ae(int(r.getheader('Content-Length')), 24) self.ae(int(r.getheader('Content-Length')), 24)
@ -380,27 +381,27 @@ class TestHTTP(BaseTest):
conn.request('GET', '/test', headers={'Range':'bytes=100000-'}) conn.request('GET', '/test', headers={'Range':'bytes=100000-'})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.REQUESTED_RANGE_NOT_SATISFIABLE) self.ae(r.status, http_client.REQUESTED_RANGE_NOT_SATISFIABLE)
self.ae(type('')(r.getheader('Content-Range')), 'bytes */%d' % len(fdata)) self.ae(type('')(r.getheader('Content-Range')), 'bytes */%d' % len(fdata))
conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':etag}) conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':etag})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.PARTIAL_CONTENT), self.ae(r.read(), fdata[25:51]) self.ae(r.status, http_client.PARTIAL_CONTENT), self.ae(r.read(), fdata[25:51])
self.ae(int(r.getheader('Content-Length')), 26) self.ae(int(r.getheader('Content-Length')), 26)
conn.request('GET', '/test', headers={'Range':'bytes=0-1000000'}) conn.request('GET', '/test', headers={'Range':'bytes=0-1000000'})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.PARTIAL_CONTENT), self.ae(r.read(), fdata) self.ae(r.status, http_client.PARTIAL_CONTENT), self.ae(r.read(), fdata)
conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':'"nomatch"'}) conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':'"nomatch"'})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.read(), fdata) self.ae(r.status, http_client.OK), self.ae(r.read(), fdata)
self.assertFalse(r.getheader('Content-Range')) self.assertFalse(r.getheader('Content-Range'))
self.ae(int(r.getheader('Content-Length')), len(fdata)) self.ae(int(r.getheader('Content-Length')), len(fdata))
conn.request('GET', '/test', headers={'Range':'bytes=0-25,26-50'}) conn.request('GET', '/test', headers={'Range':'bytes=0-25,26-50'})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.PARTIAL_CONTENT) self.ae(r.status, http_client.PARTIAL_CONTENT)
clen = int(r.getheader('Content-Length')) clen = int(r.getheader('Content-Length'))
data = r.read() data = r.read()
self.ae(clen, len(data)) self.ae(clen, len(data))
@ -415,7 +416,7 @@ class TestHTTP(BaseTest):
conn = server.connect(timeout=1) conn = server.connect(timeout=1)
conn.request('GET', '/test') conn.request('GET', '/test')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
rdata = r.read() rdata = r.read()
self.ae(len(data), len(rdata)) self.ae(len(data), len(rdata))
self.ae(hashlib.sha1(data).hexdigest(), hashlib.sha1(rdata).hexdigest()) self.ae(hashlib.sha1(data).hexdigest(), hashlib.sha1(rdata).hexdigest())

13
src/calibre/srv/tests/loop.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, ssl, os, socket, time import ssl, os, socket, time
from collections import namedtuple from collections import namedtuple
from unittest import skipIf from unittest import skipIf
from glob import glob from glob import glob
@ -18,6 +18,7 @@ from calibre.ptempfile import TemporaryDirectory
from calibre.utils.certgen import create_server_cert from calibre.utils.certgen import create_server_cert
from calibre.utils.monotonic import monotonic from calibre.utils.monotonic import monotonic
from polyglot.builtins import range from polyglot.builtins import range
from polyglot import http_client
is_ci = os.environ.get('CI', '').lower() == 'true' is_ci = os.environ.get('CI', '').lower() == 'true'
@ -92,7 +93,7 @@ class LoopTest(BaseTest):
conn.request('GET', '/') conn.request('GET', '/')
with self.assertRaises(socket.timeout): with self.assertRaises(socket.timeout):
res = conn.getresponse() res = conn.getresponse()
if str(res.status) == str(httplib.REQUEST_TIMEOUT): if str(res.status) == str(http_client.REQUEST_TIMEOUT):
raise socket.timeout('Timeout') raise socket.timeout('Timeout')
raise Exception('Got unexpected response: code: %s %s headers: %r data: %r' % ( raise Exception('Got unexpected response: code: %s %s headers: %r data: %r' % (
res.status, res.reason, res.getheaders(), res.read())) res.status, res.reason, res.getheaders(), res.read()))
@ -135,7 +136,7 @@ class LoopTest(BaseTest):
conn = server.connect(interface='127.0.0.1') conn = server.connect(interface='127.0.0.1')
conn.request('GET', '/test', 'body') conn.request('GET', '/test', 'body')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody') self.ae(r.read(), b'testbody')
def test_ring_buffer(self): def test_ring_buffer(self):
@ -203,10 +204,10 @@ class LoopTest(BaseTest):
create_server_cert(address, ca_file, cert_file, key_file, key_size=1024) create_server_cert(address, ca_file, cert_file, key_file, key_size=1024)
ctx = ssl.create_default_context(cafile=ca_file) ctx = ssl.create_default_context(cafile=ca_file)
with TestServer(lambda data:(data.path[0] + data.read()), ssl_certfile=cert_file, ssl_keyfile=key_file, listen_on=address, port=0) as server: with TestServer(lambda data:(data.path[0] + data.read()), ssl_certfile=cert_file, ssl_keyfile=key_file, listen_on=address, port=0) as server:
conn = httplib.HTTPSConnection(address, server.address[1], strict=True, context=ctx) conn = http_client.HTTPSConnection(address, server.address[1], strict=True, context=ctx)
conn.request('GET', '/test', 'body') conn.request('GET', '/test', 'body')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody') self.ae(r.read(), b'testbody')
cert = conn.sock.getpeercert() cert = conn.sock.getpeercert()
subject = dict(x[0] for x in cert['subject']) subject = dict(x[0] for x in cert['subject'])
@ -226,7 +227,7 @@ class LoopTest(BaseTest):
conn = server.connect() conn = server.connect()
conn.request('GET', '/test', 'body') conn.request('GET', '/test', 'body')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody') self.ae(r.read(), b'testbody')
self.ae(server.loop.bound_address[1], port) self.ae(server.loop.bound_address[1], port)

7
src/calibre/srv/web_socket.py
View File

@ -5,7 +5,7 @@
from __future__ import (unicode_literals, division, absolute_import, from __future__ import (unicode_literals, division, absolute_import,
print_function) print_function)
import httplib, os, weakref, socket import os, weakref, socket
from base64 import standard_b64encode from base64 import standard_b64encode
from collections import deque from collections import deque
from hashlib import sha1 from hashlib import sha1
@ -19,6 +19,7 @@ from calibre.srv.http_response import HTTPConnection, create_http_handler
from calibre.srv.utils import DESIRED_SEND_BUFFER_SIZE from calibre.srv.utils import DESIRED_SEND_BUFFER_SIZE
from calibre.utils.speedups import ReadOnlyFileBuffer from calibre.utils.speedups import ReadOnlyFileBuffer
from polyglot.queue import Queue, Empty from polyglot.queue import Queue, Empty
from polyglot import http_client
speedup, err = plugins['speedup'] speedup, err = plugins['speedup']
if not speedup: if not speedup:
raise RuntimeError('Failed to load speedup module with error: ' + err) raise RuntimeError('Failed to load speedup module with error: ' + err)
@ -286,9 +287,9 @@ class WebSocketConnection(HTTPConnection):
except Exception: except Exception:
ver_ok = False ver_ok = False
if not ver_ok: if not ver_ok:
return self.simple_response(httplib.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver) return self.simple_response(http_client.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver)
if self.method != 'GET': if self.method != 'GET':
return self.simple_response(httplib.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method) return self.simple_response(http_client.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method)
response = HANDSHAKE_STR % standard_b64encode(sha1(key + GUID_STR).digest()) response = HANDSHAKE_STR % standard_b64encode(sha1(key + GUID_STR).digest())
self.optimize_for_sending_packet() self.optimize_for_sending_packet()

6
src/calibre/utils/browser.py
View File

@ -5,11 +5,13 @@ __license__ = 'GPL v3'
__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>' __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
import copy, httplib, ssl import copy, ssl
from cookielib import CookieJar, Cookie from cookielib import CookieJar, Cookie
from mechanize import Browser as B, HTTPSHandler from mechanize import Browser as B, HTTPSHandler
from polyglot import http_client
class ModernHTTPSHandler(HTTPSHandler): class ModernHTTPSHandler(HTTPSHandler):
@ -24,7 +26,7 @@ class ModernHTTPSHandler(HTTPSHandler):
def conn_factory(hostport, **kw): def conn_factory(hostport, **kw):
kw['context'] = self.ssl_context kw['context'] = self.ssl_context
return httplib.HTTPSConnection(hostport, **kw) return http_client.HTTPSConnection(hostport, **kw)
return self.do_open(conn_factory, req) return self.do_open(conn_factory, req)

21
src/calibre/utils/https.py
View File

@ -10,7 +10,7 @@ import ssl, socket, re
from contextlib import closing from contextlib import closing
from calibre import get_proxies from calibre import get_proxies
from calibre.constants import ispy3 from polyglot import http_client
from polyglot.urllib import urlsplit from polyglot.urllib import urlsplit
has_ssl_verify = hasattr(ssl, 'create_default_context') and hasattr(ssl, '_create_unverified_context') has_ssl_verify = hasattr(ssl, 'create_default_context') and hasattr(ssl, '_create_unverified_context')
@ -19,19 +19,14 @@ class HTTPError(ValueError):
def __init__(self, url, code): def __init__(self, url, code):
msg = '%s returned an unsupported http response code: %d (%s)' % ( msg = '%s returned an unsupported http response code: %d (%s)' % (
url, code, httplib.responses.get(code, None)) url, code, http_client.responses.get(code, None))
ValueError.__init__(self, msg) ValueError.__init__(self, msg)
self.code = code self.code = code
self.url = url self.url = url
if ispy3:
import http.client as httplib
else:
import httplib
if has_ssl_verify: if has_ssl_verify:
class HTTPSConnection(httplib.HTTPSConnection): class HTTPSConnection(http_client.HTTPSConnection):
def __init__(self, ssl_version, *args, **kwargs): def __init__(self, ssl_version, *args, **kwargs):
cafile = kwargs.pop('cert_file', None) cafile = kwargs.pop('cert_file', None)
@ -39,7 +34,7 @@ if has_ssl_verify:
kwargs['context'] = ssl._create_unverified_context() kwargs['context'] = ssl._create_unverified_context()
else: else:
kwargs['context'] = ssl.create_default_context(cafile=cafile) kwargs['context'] = ssl.create_default_context(cafile=cafile)
httplib.HTTPSConnection.__init__(self, *args, **kwargs) http_client.HTTPSConnection.__init__(self, *args, **kwargs)
else: else:
# Check certificate hostname {{{ # Check certificate hostname {{{
# Implementation taken from python 3 # Implementation taken from python 3
@ -136,10 +131,10 @@ else:
"subjectAltName fields were found") "subjectAltName fields were found")
# }}} # }}}
class HTTPSConnection(httplib.HTTPSConnection): class HTTPSConnection(http_client.HTTPSConnection):
def __init__(self, ssl_version, *args, **kwargs): def __init__(self, ssl_version, *args, **kwargs):
httplib.HTTPSConnection.__init__(self, *args, **kwargs) http_client.HTTPSConnection.__init__(self, *args, **kwargs)
self.calibre_ssl_version = ssl_version self.calibre_ssl_version = ssl_version
def connect(self): def connect(self):
@ -204,7 +199,7 @@ def get_https_resource_securely(
path += '?' + p.query path += '?' + p.query
c.request('GET', path, headers=headers or {}) c.request('GET', path, headers=headers or {})
response = c.getresponse() response = c.getresponse()
if response.status in (httplib.MOVED_PERMANENTLY, httplib.FOUND, httplib.SEE_OTHER): if response.status in (http_client.MOVED_PERMANENTLY, http_client.FOUND, http_client.SEE_OTHER):
if max_redirects <= 0: if max_redirects <= 0:
raise ValueError('Too many redirects, giving up') raise ValueError('Too many redirects, giving up')
newurl = response.getheader('Location', None) newurl = response.getheader('Location', None)
@ -212,7 +207,7 @@ def get_https_resource_securely(
raise ValueError('%s returned a redirect response with no Location header' % url) raise ValueError('%s returned a redirect response with no Location header' % url)
return get_https_resource_securely( return get_https_resource_securely(
newurl, cacerts=cacerts, timeout=timeout, max_redirects=max_redirects-1, ssl_version=ssl_version, get_response=get_response) newurl, cacerts=cacerts, timeout=timeout, max_redirects=max_redirects-1, ssl_version=ssl_version, get_response=get_response)
if response.status != httplib.OK: if response.status != http_client.OK:
raise HTTPError(url, response.status) raise HTTPError(url, response.status)
if get_response: if get_response:
return response return response

2
src/calibre/web/fetch/simple.py
View File

@ -18,7 +18,6 @@ import threading
import time import time
import traceback import traceback
from base64 import b64decode from base64 import b64decode
from httplib import responses
from calibre import browser, relpath, unicode_path from calibre import browser, relpath, unicode_path
from calibre.constants import filesystem_encoding, iswindows from calibre.constants import filesystem_encoding, iswindows
@ -31,6 +30,7 @@ from calibre.utils.imghdr import what
from calibre.utils.logging import Log from calibre.utils.logging import Log
from calibre.web.fetch.utils import rescale_image from calibre.web.fetch.utils import rescale_image
from polyglot.builtins import unicode_type from polyglot.builtins import unicode_type
from polyglot.http_client import responses
from polyglot.urllib import ( from polyglot.urllib import (
URLError, quote, url2pathname, urljoin, urlparse, urlsplit, urlunparse, URLError, quote, url2pathname, urljoin, urlparse, urlsplit, urlunparse,
urlunsplit urlunsplit