Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

python3: add httplib/http.client polyglot wrapper

Browse Source
This commit is contained in:
Eli Schwartz 2019-03-25 12:07:48 -04:00
parent 92c621c718
commit bfcb301fe3
No known key found for this signature in database
GPG Key ID: CEB167EFB5722BD6
17 changed files with 198 additions and 190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/calibre/db/cli/main.py
View File

@ -4,7 +4,6 @@
from __future__ import absolute_import, division, print_function, unicode_literals
import httplib
import json
import os
import sys
@ -17,6 +16,7 @@ from calibre.utils.config import OptionParser, prefs
from calibre.utils.localization import localize_user_manual_link
from calibre.utils.lock import singleinstance
from calibre.utils.serialize import MSGPACK_MIME
from polyglot import http_client
from polyglot.urllib import urlencode, urlparse, urlunparse
COMMANDS = (
@ -191,13 +191,13 @@ class DBCtx(object):
return m.implementation(self.db.new_api, None, *args)
def interpret_http_error(self, err):
if err.code == httplib.UNAUTHORIZED:
if err.code == http_client.UNAUTHORIZED:
if self.has_credentials:
raise SystemExit('The username/password combination is incorrect')
raise SystemExit('A username and password is required to access this server')
if err.code == httplib.FORBIDDEN:
if err.code == http_client.FORBIDDEN:
raise SystemExit(err.reason)
if err.code == httplib.NOT_FOUND:
if err.code == http_client.NOT_FOUND:
raise SystemExit(err.reason)
def remote_run(self, name, m, *args):

7
src/calibre/gui2/icon_theme.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import os, errno, json, importlib, math, httplib, bz2, shutil, sys
import os, errno, json, importlib, math, bz2, shutil, sys
from itertools import count
from io import BytesIO
from threading import Thread, Event
@ -35,8 +35,9 @@ from calibre.utils.img import image_from_data, Canvas, optimize_png, optimize_jp
from calibre.utils.zipfile import ZipFile, ZIP_STORED
from calibre.utils.filenames import atomic_rename
from lzma.xz import compress, decompress
from polyglot.queue import Queue, Empty
from polyglot.builtins import iteritems, map, range, reraise
from polyglot import http_client
from polyglot.queue import Queue, Empty
IMAGE_EXTENSIONS = {'png', 'jpg', 'jpeg'}
THEME_COVER = 'icon-theme-cover.jpg'
@ -439,7 +440,7 @@ def download_cover(cover_url, etag=None, cached=b''):
etag = response.getheader('ETag', None) or None
return cached, etag
except HTTPError as e:
if etag and e.code == httplib.NOT_MODIFIED:
if etag and e.code == http_client.NOT_MODIFIED:
return cached, etag
raise

21
src/calibre/srv/auth.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import binascii, os, random, struct, base64, httplib
import binascii, os, random, struct, base64
from collections import OrderedDict
from hashlib import md5, sha256
from itertools import permutations
@ -16,6 +16,7 @@ from calibre.srv.errors import HTTPAuthRequired, HTTPSimpleResponse, HTTPForbidd
from calibre.srv.http_request import parse_uri
from calibre.srv.utils import parse_http_dict, encode_path
from calibre.utils.monotonic import monotonic
from polyglot import http_client
MAX_AGE_SECONDS = 3600
nonce_counter, nonce_counter_lock = 0, Lock()
@ -133,19 +134,19 @@ class DigestAuth(object): # {{{
self.nonce_count = data.get('nc')
if self.algorithm not in self.valid_algorithms:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported digest algorithm')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported digest algorithm')
if not (self.username and self.realm and self.nonce and self.uri and self.response):
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Digest algorithm required fields missing')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Digest algorithm required fields missing')
if self.qop:
if self.qop not in self.valid_qops:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported digest qop')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported digest qop')
if not (self.cnonce and self.nonce_count):
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'qop present, but cnonce and nonce_count absent')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'qop present, but cnonce and nonce_count absent')
else:
if self.cnonce or self.nonce_count:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'qop missing')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'qop missing')
def H(self, val):
return md5_hex(val)
@ -201,7 +202,7 @@ class DigestAuth(object): # {{{
if log is not None:
log.warn('Authorization URI mismatch: %s != %s from client: %s' % (
data.path, path, data.remote_addr))
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The uri in the Request Line and the Authorization header do not match')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The uri in the Request Line and the Authorization header do not match')
return self.response is not None and data.path == path and self.request_digest(pw, data) == self.response
# }}}
@ -290,16 +291,16 @@ class AuthController(object):
try:
un, pw = base64_decode(rest.strip()).partition(':')[::2]
except ValueError:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The username or password contained non-UTF8 encoded characters')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The username or password contained non-UTF8 encoded characters')
if not un or not pw:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The username or password was empty')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The username or password was empty')
if self.check(un, pw):
data.username = un
return
log_msg = 'Failed login attempt from: %s' % data.remote_addr
self.ban_list.failed(ban_key)
else:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported authentication method')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported authentication method')
if self.prefer_basic_auth:
raise HTTPAuthRequired('Basic realm="%s"' % self.realm, log=log_msg)

14
src/calibre/srv/errors.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib
from polyglot import http_client
class JobQueueFull(Exception):
@ -30,38 +30,38 @@ class HTTPSimpleResponse(Exception):
class HTTPRedirect(HTTPSimpleResponse):
def __init__(self, location, http_code=httplib.MOVED_PERMANENTLY, http_message='', close_connection=False):
def __init__(self, location, http_code=http_client.MOVED_PERMANENTLY, http_message='', close_connection=False):
HTTPSimpleResponse.__init__(self, http_code, http_message, close_connection, location)
class HTTPNotFound(HTTPSimpleResponse):
def __init__(self, http_message='', close_connection=False):
HTTPSimpleResponse.__init__(self, httplib.NOT_FOUND, http_message, close_connection)
HTTPSimpleResponse.__init__(self, http_client.NOT_FOUND, http_message, close_connection)
class HTTPAuthRequired(HTTPSimpleResponse):
def __init__(self, payload, log=None):
HTTPSimpleResponse.__init__(self, httplib.UNAUTHORIZED, authenticate=payload, log=log)
HTTPSimpleResponse.__init__(self, http_client.UNAUTHORIZED, authenticate=payload, log=log)
class HTTPBadRequest(HTTPSimpleResponse):
def __init__(self, message, close_connection=False):
HTTPSimpleResponse.__init__(self, httplib.BAD_REQUEST, message, close_connection)
HTTPSimpleResponse.__init__(self, http_client.BAD_REQUEST, message, close_connection)
class HTTPForbidden(HTTPSimpleResponse):
def __init__(self, http_message='', close_connection=True, log=None):
HTTPSimpleResponse.__init__(self, httplib.FORBIDDEN, http_message, close_connection, log=log)
HTTPSimpleResponse.__init__(self, http_client.FORBIDDEN, http_message, close_connection, log=log)
class HTTPInternalServerError(HTTPSimpleResponse):
def __init__(self, http_message='', close_connection=True, log=None):
HTTPSimpleResponse.__init__(self, httplib.INTERNAL_SERVER_ERROR, http_message, close_connection, log=log)
HTTPSimpleResponse.__init__(self, http_client.INTERNAL_SERVER_ERROR, http_message, close_connection, log=log)
class BookNotFound(HTTPNotFound):

43
src/calibre/srv/http_request.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import re, httplib, repr as reprlib
import re, repr as reprlib
from io import BytesIO, DEFAULT_BUFFER_SIZE
from calibre import as_unicode, force_unicode
@ -14,6 +14,7 @@ from calibre.ptempfile import SpooledTemporaryFile
from calibre.srv.errors import HTTPSimpleResponse
from calibre.srv.loop import Connection, READ, WRITE
from calibre.srv.utils import MultiDict, HTTP1, HTTP11, Accumulator
from polyglot import http_client
from polyglot.urllib import unquote
protocol_map = {(1, 0):HTTP1, (1, 1):HTTP11}
@ -68,29 +69,29 @@ def parse_request_uri(uri):
def parse_uri(uri, parse_query=True):
scheme, authority, path = parse_request_uri(uri)
if path is None:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, "No path component")
raise HTTPSimpleResponse(http_client.BAD_REQUEST, "No path component")
if b'#' in path:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, "Illegal #fragment in Request-URI.")
raise HTTPSimpleResponse(http_client.BAD_REQUEST, "Illegal #fragment in Request-URI.")
if scheme:
try:
scheme = scheme.decode('ascii')
except ValueError:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Un-decodeable scheme')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Un-decodeable scheme')
path, qs = path.partition(b'?')[::2]
if parse_query:
try:
query = MultiDict.create_from_query_string(qs)
except Exception:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unparseable query string')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unparseable query string')
else:
query = None
try:
path = '%2F'.join(unquote(x).decode('utf-8') for x in quoted_slash.split(path))
except ValueError as e:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, as_unicode(e))
raise HTTPSimpleResponse(http_client.BAD_REQUEST, as_unicode(e))
path = tuple(filter(None, (x.replace('%2F', '/') for x in path.split('/'))))
return scheme, path, query
@ -233,7 +234,7 @@ class HTTPRequest(Connection):
if line.endswith(b'\n'):
line = buf.getvalue()
if not line.endswith(b'\r\n'):
self.simple_response(httplib.BAD_REQUEST, 'HTTP requires CRLF line terminators')
self.simple_response(http_client.BAD_REQUEST, 'HTTP requires CRLF line terminators')
return
return line
if not line:
@ -247,7 +248,7 @@ class HTTPRequest(Connection):
self.forwarded_for = None
self.path = self.query = None
self.close_after_response = False
self.header_line_too_long_error_code = httplib.REQUEST_URI_TOO_LONG
self.header_line_too_long_error_code = http_client.REQUEST_URI_TOO_LONG
self.response_started = False
self.set_state(READ, self.parse_request_line, Accumulator(), first=True)
@ -260,28 +261,28 @@ class HTTPRequest(Connection):
# Ignore a single leading empty line, as per RFC 2616 sec 4.1
if first:
return self.set_state(READ, self.parse_request_line, Accumulator())
return self.simple_response(httplib.BAD_REQUEST, 'Multiple leading empty lines not allowed')
return self.simple_response(http_client.BAD_REQUEST, 'Multiple leading empty lines not allowed')
try:
method, uri, req_protocol = line.strip().split(b' ', 2)
rp = int(req_protocol[5]), int(req_protocol[7])
self.method = method.decode('ascii').upper()
except Exception:
return self.simple_response(httplib.BAD_REQUEST, "Malformed Request-Line")
return self.simple_response(http_client.BAD_REQUEST, "Malformed Request-Line")
if self.method not in HTTP_METHODS:
return self.simple_response(httplib.BAD_REQUEST, "Unknown HTTP method")
return self.simple_response(http_client.BAD_REQUEST, "Unknown HTTP method")
try:
self.request_protocol = protocol_map[rp]
except KeyError:
return self.simple_response(httplib.HTTP_VERSION_NOT_SUPPORTED)
return self.simple_response(http_client.HTTP_VERSION_NOT_SUPPORTED)
self.response_protocol = protocol_map[min((1, 1), rp)]
try:
self.scheme, self.path, self.query = parse_uri(uri)
except HTTPSimpleResponse as e:
return self.simple_response(e.http_code, e.message, close_after_response=False)
self.header_line_too_long_error_code = httplib.REQUEST_ENTITY_TOO_LARGE
self.header_line_too_long_error_code = http_client.REQUEST_ENTITY_TOO_LARGE
self.set_state(READ, self.parse_header_line, HTTPHeaderParser(), Accumulator())
# }}}
@ -299,7 +300,7 @@ class HTTPRequest(Connection):
try:
parser(line)
except ValueError:
self.simple_response(httplib.BAD_REQUEST, 'Failed to parse header line')
self.simple_response(http_client.BAD_REQUEST, 'Failed to parse header line')
return
if parser.finished:
self.finalize_headers(parser.hdict)
@ -307,7 +308,7 @@ class HTTPRequest(Connection):
def finalize_headers(self, inheaders):
request_content_length = int(inheaders.get('Content-Length', 0))
if request_content_length > self.max_request_body_size:
return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE,
return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE,
"The entity sent with the request exceeds the maximum "
"allowed bytes (%d)." % self.max_request_body_size)
# Persistent connection support
@ -334,7 +335,7 @@ class HTTPRequest(Connection):
else:
# Note that, even if we see "chunked", we must reject
# if there is an extension we don't recognize.
return self.simple_response(httplib.NOT_IMPLEMENTED, "Unknown transfer encoding: %r" % enc)
return self.simple_response(http_client.NOT_IMPLEMENTED, "Unknown transfer encoding: %r" % enc)
if inheaders.get("Expect", '').lower() == "100-continue":
buf = BytesIO((HTTP11 + " 100 Continue\r\n\r\n").encode('ascii'))
@ -369,9 +370,9 @@ class HTTPRequest(Connection):
try:
chunk_size = int(line.strip(), 16)
except Exception:
return self.simple_response(httplib.BAD_REQUEST, '%s is not a valid chunk size' % reprlib.repr(line.strip()))
return self.simple_response(http_client.BAD_REQUEST, '%s is not a valid chunk size' % reprlib.repr(line.strip()))
if bytes_read[0] + chunk_size + 2 > self.max_request_body_size:
return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE,
return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE,
'Chunked request is larger than %d bytes' % self.max_request_body_size)
if chunk_size == 0:
self.set_state(READ, self.read_chunk_separator, inheaders, Accumulator(), buf, bytes_read, last=True)
@ -389,10 +390,10 @@ class HTTPRequest(Connection):
if line is None:
return
if line != b'\r\n':
return self.simple_response(httplib.BAD_REQUEST, 'Chunk does not have trailing CRLF')
return self.simple_response(http_client.BAD_REQUEST, 'Chunk does not have trailing CRLF')
bytes_read[0] += len(line)
if bytes_read[0] > self.max_request_body_size:
return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE,
return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE,
'Chunked request is larger than %d bytes' % self.max_request_body_size)
if last:
self.prepare_response(inheaders, buf)
@ -402,7 +403,7 @@ class HTTPRequest(Connection):
def handle_timeout(self):
if self.response_started:
return False
self.simple_response(httplib.REQUEST_TIMEOUT)
self.simple_response(http_client.REQUEST_TIMEOUT)
return True
def write(self, buf, end=None):

41
src/calibre/srv/http_response.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import os, httplib, hashlib, uuid, struct, repr as reprlib
import os, hashlib, uuid, struct, repr as reprlib
from collections import namedtuple
from io import BytesIO, DEFAULT_BUFFER_SIZE
from itertools import chain, repeat
@ -26,6 +26,7 @@ from calibre.srv.utils import (
sort_q_values, get_translator_for_lang, Cookie, fast_now_strftime)
from calibre.utils.speedups import ReadOnlyFileBuffer
from calibre.utils.monotonic import monotonic
from polyglot import http_client
Range = namedtuple('Range', 'start stop size')
MULTIPART_SEPARATOR = uuid.uuid4().hex.decode('ascii')
@ -226,7 +227,7 @@ class RequestData(object): # {{{
self.remote_addr, self.remote_port, self.is_local_connection = remote_addr, remote_port, is_local_connection
self.forwarded_for = forwarded_for
self.opts = opts
self.status_code = httplib.OK
self.status_code = http_client.OK
self.outcookie = Cookie()
self.lang_code = self.gettext_func = self.ngettext_func = None
self.set_translator(self.get_preferred_language())
@ -402,16 +403,16 @@ class HTTPConnection(HTTPRequest):
if self.response_protocol is HTTP1:
# HTTP/1.0 has no 413/414/303 codes
status_code = {
httplib.REQUEST_ENTITY_TOO_LARGE:httplib.BAD_REQUEST,
httplib.REQUEST_URI_TOO_LONG:httplib.BAD_REQUEST,
httplib.SEE_OTHER:httplib.FOUND
http_client.REQUEST_ENTITY_TOO_LARGE:http_client.BAD_REQUEST,
http_client.REQUEST_URI_TOO_LONG:http_client.BAD_REQUEST,
http_client.SEE_OTHER:http_client.FOUND
}.get(status_code, status_code)
self.close_after_response = close_after_response
msg = msg.encode('utf-8')
ct = 'http' if self.method == 'TRACE' else 'plain'
buf = [
'%s %d %s' % (self.response_protocol, status_code, httplib.responses[status_code]),
'%s %d %s' % (self.response_protocol, status_code, http_client.responses[status_code]),
"Content-Length: %s" % len(msg),
"Content-Type: text/%s; charset=UTF-8" % ct,
"Date: " + http_date(),
@ -432,7 +433,7 @@ class HTTPConnection(HTTPRequest):
def prepare_response(self, inheaders, request_body_file):
if self.method == 'TRACE':
msg = force_unicode(self.request_line, 'utf-8') + '\n' + inheaders.pretty()
return self.simple_response(httplib.OK, msg, close_after_response=False)
return self.simple_response(http_client.OK, msg, close_after_response=False)
request_body_file.seek(0)
outheaders = MultiDict()
data = RequestData(
@ -449,28 +450,28 @@ class HTTPConnection(HTTPRequest):
def send_range_not_satisfiable(self, content_length):
buf = [
'%s %d %s' % (self.response_protocol, httplib.REQUESTED_RANGE_NOT_SATISFIABLE, httplib.responses[httplib.REQUESTED_RANGE_NOT_SATISFIABLE]),
'%s %d %s' % (self.response_protocol, http_client.REQUESTED_RANGE_NOT_SATISFIABLE, http_client.responses[http_client.REQUESTED_RANGE_NOT_SATISFIABLE]),
"Date: " + http_date(),
"Content-Range: bytes */%d" % content_length,
]
response_data = header_list_to_file(buf)
self.log_access(status_code=httplib.REQUESTED_RANGE_NOT_SATISFIABLE, response_size=response_data.sz)
self.log_access(status_code=http_client.REQUESTED_RANGE_NOT_SATISFIABLE, response_size=response_data.sz)
self.response_ready(response_data)
def send_not_modified(self, etag=None):
buf = [
'%s %d %s' % (self.response_protocol, httplib.NOT_MODIFIED, httplib.responses[httplib.NOT_MODIFIED]),
'%s %d %s' % (self.response_protocol, http_client.NOT_MODIFIED, http_client.responses[http_client.NOT_MODIFIED]),
"Content-Length: 0",
"Date: " + http_date(),
]
if etag is not None:
buf.append('ETag: ' + etag)
response_data = header_list_to_file(buf)
self.log_access(status_code=httplib.NOT_MODIFIED, response_size=response_data.sz)
self.log_access(status_code=http_client.NOT_MODIFIED, response_size=response_data.sz)
self.response_ready(response_data)
def report_busy(self):
self.simple_response(httplib.SERVICE_UNAVAILABLE)
self.simple_response(http_client.SERVICE_UNAVAILABLE)
def job_done(self, ok, result):
if not ok:
@ -509,7 +510,7 @@ class HTTPConnection(HTTPRequest):
if ct.startswith('text/') and 'charset=' not in ct:
outheaders.set('Content-Type', ct + '; charset=UTF-8', replace_all=True)
buf = [HTTP11 + (' %d ' % data.status_code) + httplib.responses[data.status_code]]
buf = [HTTP11 + (' %d ' % data.status_code) + http_client.responses[data.status_code]]
for header, value in sorted(iteritems(outheaders), key=itemgetter(0)):
buf.append('%s: %s' % (header, value))
for morsel in itervalues(data.outcookie):
@ -530,7 +531,7 @@ class HTTPConnection(HTTPRequest):
def log_access(self, status_code, response_size=None, username=None):
if self.access_log is None:
return
if not self.opts.log_not_found and status_code == httplib.NOT_FOUND:
if not self.opts.log_not_found and status_code == http_client.NOT_FOUND:
return
ff = self.forwarded_for
if ff:
@ -623,7 +624,7 @@ class HTTPConnection(HTTPRequest):
self.ready = ready
def report_unhandled_exception(self, e, formatted_traceback):
self.simple_response(httplib.INTERNAL_SERVER_ERROR)
self.simple_response(http_client.INTERNAL_SERVER_ERROR)
def finalize_output(self, output, request, is_http1):
none_match = parse_if_none_match(request.inheaders.get('If-None-Match', ''))
@ -633,7 +634,7 @@ class HTTPConnection(HTTPRequest):
if self.method in ('GET', 'HEAD'):
self.send_not_modified(output.etag)
else:
self.simple_response(httplib.PRECONDITION_FAILED)
self.simple_response(http_client.PRECONDITION_FAILED)
return
opts = self.opts
@ -660,10 +661,10 @@ class HTTPConnection(HTTPRequest):
ct = outheaders.get('Content-Type', '').partition(';')[0]
compressible = (not ct or ct.startswith('text/') or ct.startswith('image/svg') or
ct.partition(';')[0] in COMPRESSIBLE_TYPES)
compressible = (compressible and request.status_code == httplib.OK and
compressible = (compressible and request.status_code == http_client.OK and
(opts.compress_min_size > -1 and output.content_length >= opts.compress_min_size) and
acceptable_encoding(request.inheaders.get('Accept-Encoding', '')) and not is_http1)
accept_ranges = (not compressible and output.accept_ranges is not None and request.status_code == httplib.OK and
accept_ranges = (not compressible and output.accept_ranges is not None and request.status_code == http_client.OK and
not is_http1)
ranges = get_ranges(request.inheaders.get('Range'), output.content_length) if output.accept_ranges and self.method in ('GET', 'HEAD') else None
if_range = (request.inheaders.get('If-Range') or '').strip()
@ -680,7 +681,7 @@ class HTTPConnection(HTTPRequest):
if self.method in ('GET', 'HEAD'):
self.send_not_modified(output.etag)
else:
self.simple_response(httplib.PRECONDITION_FAILED)
self.simple_response(http_client.PRECONDITION_FAILED)
return
output.ranges = None
@ -712,7 +713,7 @@ class HTTPConnection(HTTPRequest):
outheaders.set('Content-Length', '%d' % size, replace_all=True)
outheaders.set('Content-Type', 'multipart/byteranges; boundary=' + MULTIPART_SEPARATOR, replace_all=True)
output.ranges = zip_longest(ranges, range_parts)
request.status_code = httplib.PARTIAL_CONTENT
request.status_code = http_client.PARTIAL_CONTENT
return output

5
src/calibre/srv/routes.py
View File

@ -6,13 +6,14 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, sys, inspect, re, time, numbers, json as jsonlib, textwrap
import sys, inspect, re, time, numbers, json as jsonlib, textwrap
from operator import attrgetter
from calibre.srv.errors import HTTPSimpleResponse, HTTPNotFound, RouteError
from calibre.srv.utils import http_date
from calibre.utils.serialize import msgpack_dumps, json_dumps, MSGPACK_MIME
from polyglot.builtins import iteritems, itervalues, unicode_type, range, zip
from polyglot import http_client
from polyglot.urllib import quote as urlquote
default_methods = frozenset(('HEAD', 'GET'))
@ -297,7 +298,7 @@ class Router(object):
def dispatch(self, data):
endpoint_, args = self.find_route(data.path)
if data.method not in endpoint_.methods:
raise HTTPSimpleResponse(httplib.METHOD_NOT_ALLOWED)
raise HTTPSimpleResponse(http_client.METHOD_NOT_ALLOWED)
self.read_cookies(data)

20
src/calibre/srv/tests/ajax.py
View File

@ -6,13 +6,13 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, zlib, json, base64, os
import zlib, json, base64, os
from io import BytesIO
from functools import partial
from httplib import OK, NOT_FOUND, FORBIDDEN
from calibre.ebooks.metadata.meta import get_metadata
from calibre.srv.tests.base import LibraryBaseTest
from polyglot.http_client import OK, NOT_FOUND, FORBIDDEN
from polyglot.urllib import urlencode, quote
@ -22,7 +22,7 @@ def make_request(conn, url, headers={}, prefix='/ajax', username=None, password=
conn.request(method, prefix + url, headers=headers, body=data)
r = conn.getresponse()
data = r.read()
if r.status == httplib.OK and data and data[0] in b'{[':
if r.status == OK and data and data[0] in b'{[':
data = json.loads(data)
return r, data
@ -37,10 +37,10 @@ class ContentTest(LibraryBaseTest):
request = partial(make_request, conn, prefix='/ajax/book')
r, data = request('/x')
self.ae(r.status, httplib.NOT_FOUND)
self.ae(r.status, NOT_FOUND)
r, onedata = request('/1')
self.ae(r.status, httplib.OK)
self.ae(r.status, OK)
self.ae(request('/1/' + db.server_library_id)[1], onedata)
self.ae(request('/%s?id_is_uuid=true' % db.field_for('uuid', 1))[1], onedata)
@ -63,22 +63,22 @@ class ContentTest(LibraryBaseTest):
request = partial(make_request, conn)
r, data = request('/categories')
self.ae(r.status, httplib.OK)
self.ae(r.status, OK)
r, xdata = request('/categories/' + db.server_library_id)
self.ae(r.status, httplib.OK)
self.ae(r.status, OK)
self.ae(data, xdata)
names = {x['name']:x['url'] for x in data}
for q in ('Newest', 'All books', 'Tags', 'Series', 'Authors', 'Enum', 'Composite Tags'):
self.assertIn(q, names)
r, data = request(names['Tags'], prefix='')
self.ae(r.status, httplib.OK)
self.ae(r.status, OK)
names = {x['name']:x['url'] for x in data['items']}
self.ae(set(names), set('Tag One,Tag Two,News'.split(',')))
r, data = request(names['Tag One'], prefix='')
self.ae(r.status, httplib.OK)
self.ae(r.status, OK)
self.ae(set(data['book_ids']), {1, 2})
r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"'}))
self.ae(r.status, httplib.OK)
self.ae(r.status, OK)
self.ae(set(data['book_ids']), {1, 2})
r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"', 'vl':'1'}))
self.ae(set(data['book_ids']), {2})

53
src/calibre/srv/tests/auth.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, base64, subprocess, os, cookielib, time
import base64, subprocess, os, cookielib, time
from collections import namedtuple
try:
from distutils.spawn import find_executable
@ -18,6 +18,7 @@ from calibre.srv.errors import HTTPForbidden
from calibre.srv.tests.base import BaseTest, TestServer
from calibre.srv.routes import endpoint, Router
from polyglot.builtins import iteritems, itervalues
from polyglot import http_client
from polyglot.urllib import (build_opener, HTTPBasicAuthHandler,
HTTPCookieProcessor, HTTPDigestAuthHandler, HTTPError)
@ -91,18 +92,18 @@ class TestAuth(BaseTest):
conn = server.connect()
conn.request('GET', '/open')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read(), b'open')
conn.request('GET', '/closed')
r = conn.getresponse()
self.ae(r.status, httplib.UNAUTHORIZED)
self.ae(r.status, http_client.UNAUTHORIZED)
self.ae(r.getheader('WWW-Authenticate'), b'Basic realm="%s"' % bytes(REALM))
self.assertFalse(r.read())
conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + base64.standard_b64encode(b'testuser:testpw')})
r = conn.getresponse()
self.ae(r.read(), b'closed')
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(b'closed', urlopen(server, method='basic').read())
self.ae(b'closed', urlopen(server, un='!@#$%^&*()-=_+', pw='!@#$%^&*()-=_+', method='basic').read())
@ -113,14 +114,14 @@ class TestAuth(BaseTest):
warnings = []
server.loop.log.warn = lambda *args, **kwargs: warnings.append(' '.join(args))
self.ae((httplib.OK, b'closed'), request())
self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y'))
self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('', ''))
self.ae((http_client.OK, b'closed'), request())
self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y'))
self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request('', ''))
self.ae(1, len(warnings))
self.ae((httplib.UNAUTHORIZED, b''), request('testuser', 'y'))
self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('testuser', ''))
self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request(''))
self.ae((httplib.UNAUTHORIZED, b''), request('asf', 'testpw'))
self.ae((http_client.UNAUTHORIZED, b''), request('testuser', 'y'))
self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request('testuser', ''))
self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request(''))
self.ae((http_client.UNAUTHORIZED, b''), request('asf', 'testpw'))
# }}}
def test_library_restrictions(self): # {{{
@ -169,7 +170,7 @@ class TestAuth(BaseTest):
with TestServer(r.dispatch) as server:
r.auth_controller.log = server.log
def test(conn, path, headers={}, status=httplib.OK, body=b'', request_body=b''):
def test(conn, path, headers={}, status=http_client.OK, body=b'', request_body=b''):
conn.request('GET', path, request_body, headers)
r = conn.getresponse()
self.ae(r.status, status)
@ -177,9 +178,9 @@ class TestAuth(BaseTest):
return {normalize_header_name(k):v for k, v in r.getheaders()}
conn = server.connect()
test(conn, '/open', body=b'open')
auth = parse_http_dict(test(conn, '/closed', status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
auth = parse_http_dict(test(conn, '/closed', status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
nonce = auth['nonce']
auth = parse_http_dict(test(conn, '/closed', status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
auth = parse_http_dict(test(conn, '/closed', status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
self.assertNotEqual(nonce, auth['nonce'], 'nonce was re-used')
self.ae(auth[b'realm'], bytes(REALM)), self.ae(auth[b'algorithm'], b'MD5'), self.ae(auth[b'qop'], b'auth')
self.assertNotIn('stale', auth)
@ -199,14 +200,14 @@ class TestAuth(BaseTest):
# Check stale nonces
orig, r.auth_controller.max_age_seconds = r.auth_controller.max_age_seconds, -1
auth = parse_http_dict(test(conn, '/closed', headers={
'Authorization':digest(**args)},status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
'Authorization':digest(**args)},status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
self.assertIn('stale', auth)
r.auth_controller.max_age_seconds = orig
ok_test(conn, digest(**args))
def fail_test(conn, modify, **kw):
kw['body'] = kw.get('body', b'')
kw['status'] = kw.get('status', httplib.UNAUTHORIZED)
kw['status'] = kw.get('status', http_client.UNAUTHORIZED)
args['modify'] = modify
return test(conn, '/closed', headers={'Authorization':digest(**args)}, **kw)
@ -258,13 +259,13 @@ class TestAuth(BaseTest):
warnings = []
server.loop.log.warn = lambda *args, **kwargs: warnings.append(' '.join(args))
self.ae((httplib.OK, b'closed'), request())
self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y'))
self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y'))
self.ae(httplib.FORBIDDEN, request('x', 'y')[0])
self.ae(httplib.FORBIDDEN, request()[0])
self.ae((http_client.OK, b'closed'), request())
self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y'))
self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y'))
self.ae(http_client.FORBIDDEN, request('x', 'y')[0])
self.ae(http_client.FORBIDDEN, request()[0])
time.sleep(ban_for * 60 + 0.01)
self.ae((httplib.OK, b'closed'), request())
self.ae((http_client.OK, b'closed'), request())
# }}}
def test_android_auth_workaround(self): # {{{
@ -277,7 +278,7 @@ class TestAuth(BaseTest):
# First check that unauth access fails
conn.request('GET', '/android')
r = conn.getresponse()
self.ae(r.status, httplib.UNAUTHORIZED)
self.ae(r.status, http_client.UNAUTHORIZED)
auth_handler = HTTPDigestAuthHandler()
url = 'http://localhost:%d%s' % (server.address[1], '/android')
@ -285,20 +286,20 @@ class TestAuth(BaseTest):
cj = cookielib.CookieJar()
cookie_handler = HTTPCookieProcessor(cj)
r = build_opener(auth_handler, cookie_handler).open(url)
self.ae(r.getcode(), httplib.OK)
self.ae(r.getcode(), http_client.OK)
cookies = tuple(cj)
self.ae(len(cookies), 1)
cookie = cookies[0]
self.assertIn(b':', cookie.value)
self.ae(cookie.path, b'/android')
r = build_opener(cookie_handler).open(url)
self.ae(r.getcode(), httplib.OK)
self.ae(r.getcode(), http_client.OK)
self.ae(r.read(), b'android')
# Test that a replay attack against a different URL does not work
try:
build_opener(cookie_handler).open(url+'2')
assert ('Replay attack succeeded')
except HTTPError as e:
self.ae(e.code, httplib.UNAUTHORIZED)
self.ae(e.code, http_client.UNAUTHORIZED)
# }}}

5
src/calibre/srv/tests/base.py
View File

@ -7,12 +7,13 @@ __license__ = 'GPL v3'
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import unittest, time, httplib, shutil, gc, tempfile, atexit, os
import unittest, time, shutil, gc, tempfile, atexit, os
from io import BytesIO
from functools import partial
from threading import Thread
from calibre.srv.utils import ServerLog
from polyglot import http_client
rmtree = partial(shutil.rmtree, ignore_errors=True)
@ -120,7 +121,7 @@ class TestServer(Thread):
timeout = self.loop.opts.timeout
if interface is None:
interface = self.address[0]
return httplib.HTTPConnection(interface, self.address[1], strict=True, timeout=timeout)
return http_client.HTTPConnection(interface, self.address[1], strict=True, timeout=timeout)
def change_handler(self, handler):
from calibre.srv.http_response import create_http_handler

43
src/calibre/srv/tests/content.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, zlib, json, binascii, time, os
import zlib, json, binascii, time, os
from io import BytesIO
from calibre.ebooks.metadata.epub import get_metadata
@ -14,6 +14,7 @@ from calibre.ebooks.metadata.opf2 import OPF
from calibre.srv.tests.base import LibraryBaseTest
from calibre.utils.imghdr import identify
from calibre.utils.shared_file import share_open
from polyglot import http_client
def setUpModule():
@ -32,7 +33,7 @@ class ContentTest(LibraryBaseTest):
def missing(url, body=b''):
conn.request('GET', url)
r = conn.getresponse()
self.ae(r.status, httplib.NOT_FOUND)
self.ae(r.status, http_client.NOT_FOUND)
self.ae(r.read(), body)
for prefix in ('static', 'icon'):
@ -51,7 +52,7 @@ class ContentTest(LibraryBaseTest):
raw = P(src, data=True)
conn.request('GET', url)
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
data = r.read()
if sz is None:
self.ae(data, raw)
@ -60,7 +61,7 @@ class ContentTest(LibraryBaseTest):
test_response(r)
conn.request('GET', url, headers={'If-None-Match':r.getheader('ETag')})
r = conn.getresponse()
self.ae(r.status, httplib.NOT_MODIFIED)
self.ae(r.status, http_client.NOT_MODIFIED)
self.ae(b'', r.read())
test('content-server/empty.html', '/static/empty.html')
@ -85,7 +86,7 @@ class ContentTest(LibraryBaseTest):
# Test various invalid parameters
def bad(*args):
r, data = get(*args)
self.ae(r.status, httplib.NOT_FOUND)
self.ae(r.status, http_client.NOT_FOUND)
bad('xxx', 1)
bad('fmt1', 10)
bad('fmt1', 1, 'zzzz')
@ -103,7 +104,7 @@ class ContentTest(LibraryBaseTest):
# Test fetching of format with metadata update
raw = P('quick_start/eng.epub', data=True)
r, data = get('epub', 1)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
etag = r.getheader('ETag')
self.assertIsNotNone(etag)
self.ae(r.getheader('Used-Cache'), 'no')
@ -145,39 +146,39 @@ class ContentTest(LibraryBaseTest):
os.utime(cpath, (t, t))
r, data = get('cover', 1)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(data, db.cover(1))
self.ae(r.getheader('Used-Cache'), 'no')
self.ae(r.getheader('Content-Type'), 'image/jpeg')
r, data = get('cover', 1)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(data, db.cover(1))
self.ae(r.getheader('Used-Cache'), 'yes')
r, data = get('cover', 3)
self.ae(r.status, httplib.OK) # Auto generated cover
self.ae(r.status, http_client.OK) # Auto generated cover
r, data = get('thumb', 1)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(identify(data), ('jpeg', 60, 60))
self.ae(r.getheader('Used-Cache'), 'no')
r, data = get('thumb', 1)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.getheader('Used-Cache'), 'yes')
r, data = get('thumb', 1, q='sz=100')
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(identify(data), ('jpeg', 100, 100))
self.ae(r.getheader('Used-Cache'), 'no')
r, data = get('thumb', 1, q='sz=100x100')
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.getheader('Used-Cache'), 'yes')
change_cover(1, 1)
r, data = get('thumb', 1, q='sz=100')
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(identify(data), ('jpeg', 100, 100))
self.ae(r.getheader('Used-Cache'), 'no')
# Test file sharing in cache
r, data = get('cover', 2)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no')
path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8')
@ -185,7 +186,7 @@ class ContentTest(LibraryBaseTest):
# Now force an update
change_cover(1)
r, data = get('cover', 2)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no')
path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8')
@ -193,7 +194,7 @@ class ContentTest(LibraryBaseTest):
# Do it again
change_cover(2)
r, data = get('cover', 2)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no')
self.ae(f.read(), fdata)
@ -201,7 +202,7 @@ class ContentTest(LibraryBaseTest):
# Test serving of metadata as opf
r, data = get('opf', 1)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.getheader('Content-Type'), 'application/oebps-package+xml; charset=UTF-8')
self.assertIsNotNone(r.getheader('Last-Modified'))
opf = OPF(BytesIO(data), populate_spine=False, try_to_guess_cover=False)
@ -209,17 +210,17 @@ class ContentTest(LibraryBaseTest):
self.ae(db.field_for('authors', 1), tuple(opf.authors))
conn.request('GET', '/get/opf/1', headers={'Accept-Encoding':'gzip'})
r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip')
self.ae(r.status, http_client.OK), self.ae(r.getheader('Content-Encoding'), 'gzip')
raw = r.read()
self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data)
# Test serving metadata as json
r, data = get('json', 1)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(db.field_for('title', 1), json.loads(data)['title'])
conn.request('GET', '/get/json/1', headers={'Accept-Encoding':'gzip'})
r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip')
self.ae(r.status, http_client.OK), self.ae(r.getheader('Content-Encoding'), 'gzip')
raw = r.read()
self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data)

79
src/calibre/srv/tests/http.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, hashlib, zlib, string, time, os
import hashlib, zlib, string, time, os
from io import BytesIO
from tempfile import NamedTemporaryFile
@ -15,6 +15,7 @@ from calibre.srv.tests.base import BaseTest, TestServer
from calibre.srv.utils import eintr_retry_call
from calibre.utils.monotonic import monotonic
from polyglot.builtins import iteritems, range
from polyglot import http_client
is_ci = os.environ.get('CI', '').lower() == 'true'
@ -94,7 +95,7 @@ class TestHTTP(BaseTest):
def test(al, q):
conn.request('GET', '/', headers={'Accept-Language': al})
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
q += get_translator(q)[-1].ugettext('Unknown')
self.ae(r.read(), q)
@ -136,7 +137,7 @@ class TestHTTP(BaseTest):
def raw_send(conn, raw):
conn.send(raw)
conn._HTTPConnection__state = httplib._CS_REQ_SENT
conn._HTTPConnection__state = http_client._CS_REQ_SENT
return conn.getresponse()
base_timeout = 0.5 if is_ci else 0.1
@ -144,31 +145,31 @@ class TestHTTP(BaseTest):
with TestServer(handler, timeout=base_timeout, max_header_line_size=100./1024, max_request_body_size=100./(1024*1024)) as server:
conn = server.connect()
r = raw_send(conn, b'hello\n')
self.ae(r.status, httplib.BAD_REQUEST)
self.ae(r.status, http_client.BAD_REQUEST)
self.ae(r.read(), b'HTTP requires CRLF line terminators')
r = raw_send(conn, b'\r\nGET /index.html HTTP/1.1\r\n\r\n')
self.ae(r.status, httplib.NOT_FOUND), self.ae(r.read(), b'Requested resource not found')
self.ae(r.status, http_client.NOT_FOUND), self.ae(r.read(), b'Requested resource not found')
r = raw_send(conn, b'\r\n\r\nGET /index.html HTTP/1.1\r\n\r\n')
self.ae(r.status, httplib.BAD_REQUEST)
self.ae(r.status, http_client.BAD_REQUEST)
self.ae(r.read(), b'Multiple leading empty lines not allowed')
r = raw_send(conn, b'hello world\r\n')
self.ae(r.status, httplib.BAD_REQUEST)
self.ae(r.status, http_client.BAD_REQUEST)
self.ae(r.read(), b'Malformed Request-Line')
r = raw_send(conn, b'x' * 200)
self.ae(r.status, httplib.BAD_REQUEST)
self.ae(r.status, http_client.BAD_REQUEST)
self.ae(r.read(), b'')
r = raw_send(conn, b'XXX /index.html HTTP/1.1\r\n\r\n')
self.ae(r.status, httplib.BAD_REQUEST), self.ae(r.read(), b'Unknown HTTP method')
self.ae(r.status, http_client.BAD_REQUEST), self.ae(r.read(), b'Unknown HTTP method')
# Test 404
conn.request('HEAD', '/moose')
r = conn.getresponse()
self.ae(r.status, httplib.NOT_FOUND)
self.ae(r.status, http_client.NOT_FOUND)
self.assertIsNotNone(r.getheader('Date', None))
self.ae(r.getheader('Content-Length'), str(len(body)))
self.ae(r.getheader('Content-Type'), 'text/plain; charset=UTF-8')
@ -176,7 +177,7 @@ class TestHTTP(BaseTest):
self.ae(r.read(), '')
conn.request('GET', '/choose')
r = conn.getresponse()
self.ae(r.status, httplib.NOT_FOUND)
self.ae(r.status, http_client.NOT_FOUND)
self.ae(r.read(), b'Requested resource not found')
# Test 500
@ -186,7 +187,7 @@ class TestHTTP(BaseTest):
conn = server.connect()
conn.request('GET', '/test/')
r = conn.getresponse()
self.ae(r.status, httplib.INTERNAL_SERVER_ERROR)
self.ae(r.status, http_client.INTERNAL_SERVER_ERROR)
server.loop.log.filter_level = orig
# Test 301
@ -196,7 +197,7 @@ class TestHTTP(BaseTest):
conn = server.connect()
conn.request('GET', '/')
r = conn.getresponse()
self.ae(r.status, httplib.MOVED_PERMANENTLY)
self.ae(r.status, http_client.MOVED_PERMANENTLY)
self.ae(r.getheader('Location'), '/somewhere-else')
self.ae('', r.read())
@ -206,26 +207,26 @@ class TestHTTP(BaseTest):
# Test simple GET
conn.request('GET', '/test/')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read(), b'test')
# Test TRACE
lines = ['TRACE /xxx HTTP/1.1', 'Test: value', 'Xyz: abc, def', '', '']
r = raw_send(conn, ('\r\n'.join(lines)).encode('ascii'))
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read().decode('utf-8'), '\n'.join(lines[:-2]))
# Test POST with simple body
conn.request('POST', '/test', 'body')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody')
# Test POST with chunked transfer encoding
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'4\r\nbody\r\na\r\n1234567890\r\n0\r\n\r\n')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody1234567890')
# Test various incorrect input
@ -233,39 +234,39 @@ class TestHTTP(BaseTest):
conn.request('GET', '/test' + ('a' * 200))
r = conn.getresponse()
self.ae(r.status, httplib.BAD_REQUEST)
self.ae(r.status, http_client.BAD_REQUEST)
conn = server.connect()
conn.request('GET', '/test', ('a' * 200))
r = conn.getresponse()
self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE)
self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE)
conn = server.connect()
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'x\r\nbody\r\n0\r\n\r\n')
r = conn.getresponse()
self.ae(r.status, httplib.BAD_REQUEST)
self.ae(r.status, http_client.BAD_REQUEST)
self.assertIn(b'not a valid chunk size', r.read())
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'4\r\nbody\r\n200\r\n\r\n')
r = conn.getresponse()
self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE)
self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE)
conn.request('POST', '/test', body='a'*200)
r = conn.getresponse()
self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE)
self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE)
conn = server.connect()
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'3\r\nbody\r\n0\r\n\r\n')
r = conn.getresponse()
self.ae(r.status, httplib.BAD_REQUEST), self.ae(r.read(), b'Chunk does not have trailing CRLF')
self.ae(r.status, http_client.BAD_REQUEST), self.ae(r.read(), b'Chunk does not have trailing CRLF')
conn = server.connect(timeout=base_timeout * 5)
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'30\r\nbody\r\n0\r\n\r\n')
r = conn.getresponse()
self.ae(r.status, httplib.REQUEST_TIMEOUT)
self.ae(r.status, http_client.REQUEST_TIMEOUT)
self.assertIn(b'', r.read())
server.log.filter_level = orig_level
@ -273,14 +274,14 @@ class TestHTTP(BaseTest):
# Test pipelining
responses = []
for i in range(10):
conn._HTTPConnection__state = httplib._CS_IDLE
conn._HTTPConnection__state = http_client._CS_IDLE
conn.request('GET', '/%d'%i)
responses.append(conn.response_class(conn.sock, strict=conn.strict, method=conn._method))
for i in range(10):
r = responses[i]
r.begin()
self.ae(r.read(), ('%d' % i).encode('ascii'))
conn._HTTPConnection__state = httplib._CS_IDLE
conn._HTTPConnection__state = http_client._CS_IDLE
# Test closing
server.loop.opts.timeout = 10 # ensure socket is not closed because of timeout
@ -319,12 +320,12 @@ class TestHTTP(BaseTest):
conn = server.connect()
conn.request('GET', '/an_etagged_path')
r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.read(), b'an_etagged_path')
self.ae(r.status, http_client.OK), self.ae(r.read(), b'an_etagged_path')
etag = r.getheader('ETag')
self.ae(etag, '"%s"' % hashlib.sha1('an_etagged_path').hexdigest())
conn.request('GET', '/an_etagged_path', headers={'If-None-Match':etag})
r = conn.getresponse()
self.ae(r.status, httplib.NOT_MODIFIED)
self.ae(r.status, http_client.NOT_MODIFIED)
self.ae(r.read(), b'')
# Test gzip
@ -334,7 +335,7 @@ class TestHTTP(BaseTest):
conn.request('GET', '/an_etagged_path', headers={'Accept-Encoding':'gzip'})
r = conn.getresponse()
self.ae(str(len(raw)), r.getheader('Calibre-Uncompressed-Length'))
self.ae(r.status, httplib.OK), self.ae(zlib.decompress(r.read(), 16+zlib.MAX_WBITS), raw)
self.ae(r.status, http_client.OK), self.ae(zlib.decompress(r.read(), 16+zlib.MAX_WBITS), raw)
# Test dynamic etagged content
num_calls = [0]
@ -346,13 +347,13 @@ class TestHTTP(BaseTest):
conn = server.connect()
conn.request('GET', '/an_etagged_path')
r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.read(), b'data')
self.ae(r.status, http_client.OK), self.ae(r.read(), b'data')
etag = r.getheader('ETag')
self.ae(etag, b'"xxx"')
self.ae(r.getheader('Content-Length'), '4')
conn.request('GET', '/an_etagged_path', headers={'If-None-Match':etag})
r = conn.getresponse()
self.ae(r.status, httplib.NOT_MODIFIED)
self.ae(r.status, http_client.NOT_MODIFIED)
self.ae(r.read(), b'')
self.ae(num_calls[0], 1)
@ -368,11 +369,11 @@ class TestHTTP(BaseTest):
self.ae(r.getheader('Content-Type'), guess_type(f.name)[0])
self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes')
self.ae(int(r.getheader('Content-Length')), len(fdata))
self.ae(r.status, httplib.OK), self.ae(r.read(), fdata)
self.ae(r.status, http_client.OK), self.ae(r.read(), fdata)
conn.request('GET', '/test', headers={'Range':'bytes=2-25'})
r = conn.getresponse()
self.ae(r.status, httplib.PARTIAL_CONTENT)
self.ae(r.status, http_client.PARTIAL_CONTENT)
self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes')
self.ae(type('')(r.getheader('Content-Range')), 'bytes 2-25/%d' % len(fdata))
self.ae(int(r.getheader('Content-Length')), 24)
@ -380,27 +381,27 @@ class TestHTTP(BaseTest):
conn.request('GET', '/test', headers={'Range':'bytes=100000-'})
r = conn.getresponse()
self.ae(r.status, httplib.REQUESTED_RANGE_NOT_SATISFIABLE)
self.ae(r.status, http_client.REQUESTED_RANGE_NOT_SATISFIABLE)
self.ae(type('')(r.getheader('Content-Range')), 'bytes */%d' % len(fdata))
conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':etag})
r = conn.getresponse()
self.ae(r.status, httplib.PARTIAL_CONTENT), self.ae(r.read(), fdata[25:51])
self.ae(r.status, http_client.PARTIAL_CONTENT), self.ae(r.read(), fdata[25:51])
self.ae(int(r.getheader('Content-Length')), 26)
conn.request('GET', '/test', headers={'Range':'bytes=0-1000000'})
r = conn.getresponse()
self.ae(r.status, httplib.PARTIAL_CONTENT), self.ae(r.read(), fdata)
self.ae(r.status, http_client.PARTIAL_CONTENT), self.ae(r.read(), fdata)
conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':'"nomatch"'})
r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.read(), fdata)
self.ae(r.status, http_client.OK), self.ae(r.read(), fdata)
self.assertFalse(r.getheader('Content-Range'))
self.ae(int(r.getheader('Content-Length')), len(fdata))
conn.request('GET', '/test', headers={'Range':'bytes=0-25,26-50'})
r = conn.getresponse()
self.ae(r.status, httplib.PARTIAL_CONTENT)
self.ae(r.status, http_client.PARTIAL_CONTENT)
clen = int(r.getheader('Content-Length'))
data = r.read()
self.ae(clen, len(data))
@ -415,7 +416,7 @@ class TestHTTP(BaseTest):
conn = server.connect(timeout=1)
conn.request('GET', '/test')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
rdata = r.read()
self.ae(len(data), len(rdata))
self.ae(hashlib.sha1(data).hexdigest(), hashlib.sha1(rdata).hexdigest())

13
src/calibre/srv/tests/loop.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, ssl, os, socket, time
import ssl, os, socket, time
from collections import namedtuple
from unittest import skipIf
from glob import glob
@ -18,6 +18,7 @@ from calibre.ptempfile import TemporaryDirectory
from calibre.utils.certgen import create_server_cert
from calibre.utils.monotonic import monotonic
from polyglot.builtins import range
from polyglot import http_client
is_ci = os.environ.get('CI', '').lower() == 'true'
@ -92,7 +93,7 @@ class LoopTest(BaseTest):
conn.request('GET', '/')
with self.assertRaises(socket.timeout):
res = conn.getresponse()
if str(res.status) == str(httplib.REQUEST_TIMEOUT):
if str(res.status) == str(http_client.REQUEST_TIMEOUT):
raise socket.timeout('Timeout')
raise Exception('Got unexpected response: code: %s %s headers: %r data: %r' % (
res.status, res.reason, res.getheaders(), res.read()))
@ -135,7 +136,7 @@ class LoopTest(BaseTest):
conn = server.connect(interface='127.0.0.1')
conn.request('GET', '/test', 'body')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody')
def test_ring_buffer(self):
@ -203,10 +204,10 @@ class LoopTest(BaseTest):
create_server_cert(address, ca_file, cert_file, key_file, key_size=1024)
ctx = ssl.create_default_context(cafile=ca_file)
with TestServer(lambda data:(data.path[0] + data.read()), ssl_certfile=cert_file, ssl_keyfile=key_file, listen_on=address, port=0) as server:
conn = httplib.HTTPSConnection(address, server.address[1], strict=True, context=ctx)
conn = http_client.HTTPSConnection(address, server.address[1], strict=True, context=ctx)
conn.request('GET', '/test', 'body')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody')
cert = conn.sock.getpeercert()
subject = dict(x[0] for x in cert['subject'])
@ -226,7 +227,7 @@ class LoopTest(BaseTest):
conn = server.connect()
conn.request('GET', '/test', 'body')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody')
self.ae(server.loop.bound_address[1], port)

7
src/calibre/srv/web_socket.py
View File

@ -5,7 +5,7 @@
from __future__ import (unicode_literals, division, absolute_import,
print_function)
import httplib, os, weakref, socket
import os, weakref, socket
from base64 import standard_b64encode
from collections import deque
from hashlib import sha1
@ -19,6 +19,7 @@ from calibre.srv.http_response import HTTPConnection, create_http_handler
from calibre.srv.utils import DESIRED_SEND_BUFFER_SIZE
from calibre.utils.speedups import ReadOnlyFileBuffer
from polyglot.queue import Queue, Empty
from polyglot import http_client
speedup, err = plugins['speedup']
if not speedup:
raise RuntimeError('Failed to load speedup module with error: ' + err)
@ -286,9 +287,9 @@ class WebSocketConnection(HTTPConnection):
except Exception:
ver_ok = False
if not ver_ok:
return self.simple_response(httplib.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver)
return self.simple_response(http_client.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver)
if self.method != 'GET':
return self.simple_response(httplib.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method)
return self.simple_response(http_client.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method)
response = HANDSHAKE_STR % standard_b64encode(sha1(key + GUID_STR).digest())
self.optimize_for_sending_packet()

6
src/calibre/utils/browser.py
View File

@ -5,11 +5,13 @@ __license__ = 'GPL v3'
__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import copy, httplib, ssl
import copy, ssl
from cookielib import CookieJar, Cookie
from mechanize import Browser as B, HTTPSHandler
from polyglot import http_client
class ModernHTTPSHandler(HTTPSHandler):
@ -24,7 +26,7 @@ class ModernHTTPSHandler(HTTPSHandler):
def conn_factory(hostport, **kw):
kw['context'] = self.ssl_context
return httplib.HTTPSConnection(hostport, **kw)
return http_client.HTTPSConnection(hostport, **kw)
return self.do_open(conn_factory, req)

21
src/calibre/utils/https.py
View File

@ -10,7 +10,7 @@ import ssl, socket, re
from contextlib import closing
from calibre import get_proxies
from calibre.constants import ispy3
from polyglot import http_client
from polyglot.urllib import urlsplit
has_ssl_verify = hasattr(ssl, 'create_default_context') and hasattr(ssl, '_create_unverified_context')
@ -19,19 +19,14 @@ class HTTPError(ValueError):
def __init__(self, url, code):
msg = '%s returned an unsupported http response code: %d (%s)' % (
url, code, httplib.responses.get(code, None))
url, code, http_client.responses.get(code, None))
ValueError.__init__(self, msg)
self.code = code
self.url = url
if ispy3:
import http.client as httplib
else:
import httplib
if has_ssl_verify:
class HTTPSConnection(httplib.HTTPSConnection):
class HTTPSConnection(http_client.HTTPSConnection):
def __init__(self, ssl_version, *args, **kwargs):
cafile = kwargs.pop('cert_file', None)
@ -39,7 +34,7 @@ if has_ssl_verify:
kwargs['context'] = ssl._create_unverified_context()
else:
kwargs['context'] = ssl.create_default_context(cafile=cafile)
httplib.HTTPSConnection.__init__(self, *args, **kwargs)
http_client.HTTPSConnection.__init__(self, *args, **kwargs)
else:
# Check certificate hostname {{{
# Implementation taken from python 3
@ -136,10 +131,10 @@ else:
"subjectAltName fields were found")
# }}}
class HTTPSConnection(httplib.HTTPSConnection):
class HTTPSConnection(http_client.HTTPSConnection):
def __init__(self, ssl_version, *args, **kwargs):
httplib.HTTPSConnection.__init__(self, *args, **kwargs)
http_client.HTTPSConnection.__init__(self, *args, **kwargs)
self.calibre_ssl_version = ssl_version
def connect(self):
@ -204,7 +199,7 @@ def get_https_resource_securely(
path += '?' + p.query
c.request('GET', path, headers=headers or {})
response = c.getresponse()
if response.status in (httplib.MOVED_PERMANENTLY, httplib.FOUND, httplib.SEE_OTHER):
if response.status in (http_client.MOVED_PERMANENTLY, http_client.FOUND, http_client.SEE_OTHER):
if max_redirects <= 0:
raise ValueError('Too many redirects, giving up')
newurl = response.getheader('Location', None)
@ -212,7 +207,7 @@ def get_https_resource_securely(
raise ValueError('%s returned a redirect response with no Location header' % url)
return get_https_resource_securely(
newurl, cacerts=cacerts, timeout=timeout, max_redirects=max_redirects-1, ssl_version=ssl_version, get_response=get_response)
if response.status != httplib.OK:
if response.status != http_client.OK:
raise HTTPError(url, response.status)
if get_response:
return response

2
src/calibre/web/fetch/simple.py
View File

@ -18,7 +18,6 @@ import threading
import time
import traceback
from base64 import b64decode
from httplib import responses
from calibre import browser, relpath, unicode_path
from calibre.constants import filesystem_encoding, iswindows
@ -31,6 +30,7 @@ from calibre.utils.imghdr import what
from calibre.utils.logging import Log
from calibre.web.fetch.utils import rescale_image
from polyglot.builtins import unicode_type
from polyglot.http_client import responses
from polyglot.urllib import (
URLError, quote, url2pathname, urljoin, urlparse, urlsplit, urlunparse,
urlunsplit