From bfcb301fe3da0cef93f61d3785567e27cc0597ad Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 25 Mar 2019 12:07:48 -0400 Subject: [PATCH] python3: add httplib/http.client polyglot wrapper --- src/calibre/db/cli/main.py | 8 ++-- src/calibre/gui2/icon_theme.py | 7 +-- src/calibre/srv/auth.py | 21 +++++---- src/calibre/srv/errors.py | 14 +++--- src/calibre/srv/http_request.py | 43 ++++++++--------- src/calibre/srv/http_response.py | 41 +++++++++-------- src/calibre/srv/routes.py | 5 +- src/calibre/srv/tests/ajax.py | 20 ++++---- src/calibre/srv/tests/auth.py | 53 ++++++++++----------- src/calibre/srv/tests/base.py | 5 +- src/calibre/srv/tests/content.py | 43 ++++++++--------- src/calibre/srv/tests/http.py | 79 ++++++++++++++++---------------- src/calibre/srv/tests/loop.py | 13 +++--- src/calibre/srv/web_socket.py | 7 +-- src/calibre/utils/browser.py | 6 ++- src/calibre/utils/https.py | 21 ++++----- src/calibre/web/fetch/simple.py | 2 +- 17 files changed, 198 insertions(+), 190 deletions(-) diff --git a/src/calibre/db/cli/main.py b/src/calibre/db/cli/main.py index 9833d919d4..a80da22180 100644 --- a/src/calibre/db/cli/main.py +++ b/src/calibre/db/cli/main.py @@ -4,7 +4,6 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import httplib import json import os import sys @@ -17,6 +16,7 @@ from calibre.utils.config import OptionParser, prefs from calibre.utils.localization import localize_user_manual_link from calibre.utils.lock import singleinstance from calibre.utils.serialize import MSGPACK_MIME +from polyglot import http_client from polyglot.urllib import urlencode, urlparse, urlunparse COMMANDS = ( @@ -191,13 +191,13 @@ class DBCtx(object): return m.implementation(self.db.new_api, None, *args) def interpret_http_error(self, err): - if err.code == httplib.UNAUTHORIZED: + if err.code == http_client.UNAUTHORIZED: if self.has_credentials: raise SystemExit('The username/password combination is incorrect') raise SystemExit('A username and password is required to access this server') - if err.code == httplib.FORBIDDEN: + if err.code == http_client.FORBIDDEN: raise SystemExit(err.reason) - if err.code == httplib.NOT_FOUND: + if err.code == http_client.NOT_FOUND: raise SystemExit(err.reason) def remote_run(self, name, m, *args): diff --git a/src/calibre/gui2/icon_theme.py b/src/calibre/gui2/icon_theme.py index e4c48f9562..e5d1c3a0a0 100644 --- a/src/calibre/gui2/icon_theme.py +++ b/src/calibre/gui2/icon_theme.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import os, errno, json, importlib, math, httplib, bz2, shutil, sys +import os, errno, json, importlib, math, bz2, shutil, sys from itertools import count from io import BytesIO from threading import Thread, Event @@ -35,8 +35,9 @@ from calibre.utils.img import image_from_data, Canvas, optimize_png, optimize_jp from calibre.utils.zipfile import ZipFile, ZIP_STORED from calibre.utils.filenames import atomic_rename from lzma.xz import compress, decompress -from polyglot.queue import Queue, Empty from polyglot.builtins import iteritems, map, range, reraise +from polyglot import http_client +from polyglot.queue import Queue, Empty IMAGE_EXTENSIONS = {'png', 'jpg', 'jpeg'} THEME_COVER = 'icon-theme-cover.jpg' @@ -439,7 +440,7 @@ def download_cover(cover_url, etag=None, cached=b''): etag = response.getheader('ETag', None) or None return cached, etag except HTTPError as e: - if etag and e.code == httplib.NOT_MODIFIED: + if etag and e.code == http_client.NOT_MODIFIED: return cached, etag raise diff --git a/src/calibre/srv/auth.py b/src/calibre/srv/auth.py index 51f884e28f..84fd260219 100644 --- a/src/calibre/srv/auth.py +++ b/src/calibre/srv/auth.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import binascii, os, random, struct, base64, httplib +import binascii, os, random, struct, base64 from collections import OrderedDict from hashlib import md5, sha256 from itertools import permutations @@ -16,6 +16,7 @@ from calibre.srv.errors import HTTPAuthRequired, HTTPSimpleResponse, HTTPForbidd from calibre.srv.http_request import parse_uri from calibre.srv.utils import parse_http_dict, encode_path from calibre.utils.monotonic import monotonic +from polyglot import http_client MAX_AGE_SECONDS = 3600 nonce_counter, nonce_counter_lock = 0, Lock() @@ -133,19 +134,19 @@ class DigestAuth(object): # {{{ self.nonce_count = data.get('nc') if self.algorithm not in self.valid_algorithms: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported digest algorithm') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported digest algorithm') if not (self.username and self.realm and self.nonce and self.uri and self.response): - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Digest algorithm required fields missing') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Digest algorithm required fields missing') if self.qop: if self.qop not in self.valid_qops: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported digest qop') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported digest qop') if not (self.cnonce and self.nonce_count): - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'qop present, but cnonce and nonce_count absent') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'qop present, but cnonce and nonce_count absent') else: if self.cnonce or self.nonce_count: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'qop missing') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'qop missing') def H(self, val): return md5_hex(val) @@ -201,7 +202,7 @@ class DigestAuth(object): # {{{ if log is not None: log.warn('Authorization URI mismatch: %s != %s from client: %s' % ( data.path, path, data.remote_addr)) - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The uri in the Request Line and the Authorization header do not match') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The uri in the Request Line and the Authorization header do not match') return self.response is not None and data.path == path and self.request_digest(pw, data) == self.response # }}} @@ -290,16 +291,16 @@ class AuthController(object): try: un, pw = base64_decode(rest.strip()).partition(':')[::2] except ValueError: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The username or password contained non-UTF8 encoded characters') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The username or password contained non-UTF8 encoded characters') if not un or not pw: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The username or password was empty') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The username or password was empty') if self.check(un, pw): data.username = un return log_msg = 'Failed login attempt from: %s' % data.remote_addr self.ban_list.failed(ban_key) else: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported authentication method') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported authentication method') if self.prefer_basic_auth: raise HTTPAuthRequired('Basic realm="%s"' % self.realm, log=log_msg) diff --git a/src/calibre/srv/errors.py b/src/calibre/srv/errors.py index db251b34cf..02c7c4f65e 100644 --- a/src/calibre/srv/errors.py +++ b/src/calibre/srv/errors.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib +from polyglot import http_client class JobQueueFull(Exception): @@ -30,38 +30,38 @@ class HTTPSimpleResponse(Exception): class HTTPRedirect(HTTPSimpleResponse): - def __init__(self, location, http_code=httplib.MOVED_PERMANENTLY, http_message='', close_connection=False): + def __init__(self, location, http_code=http_client.MOVED_PERMANENTLY, http_message='', close_connection=False): HTTPSimpleResponse.__init__(self, http_code, http_message, close_connection, location) class HTTPNotFound(HTTPSimpleResponse): def __init__(self, http_message='', close_connection=False): - HTTPSimpleResponse.__init__(self, httplib.NOT_FOUND, http_message, close_connection) + HTTPSimpleResponse.__init__(self, http_client.NOT_FOUND, http_message, close_connection) class HTTPAuthRequired(HTTPSimpleResponse): def __init__(self, payload, log=None): - HTTPSimpleResponse.__init__(self, httplib.UNAUTHORIZED, authenticate=payload, log=log) + HTTPSimpleResponse.__init__(self, http_client.UNAUTHORIZED, authenticate=payload, log=log) class HTTPBadRequest(HTTPSimpleResponse): def __init__(self, message, close_connection=False): - HTTPSimpleResponse.__init__(self, httplib.BAD_REQUEST, message, close_connection) + HTTPSimpleResponse.__init__(self, http_client.BAD_REQUEST, message, close_connection) class HTTPForbidden(HTTPSimpleResponse): def __init__(self, http_message='', close_connection=True, log=None): - HTTPSimpleResponse.__init__(self, httplib.FORBIDDEN, http_message, close_connection, log=log) + HTTPSimpleResponse.__init__(self, http_client.FORBIDDEN, http_message, close_connection, log=log) class HTTPInternalServerError(HTTPSimpleResponse): def __init__(self, http_message='', close_connection=True, log=None): - HTTPSimpleResponse.__init__(self, httplib.INTERNAL_SERVER_ERROR, http_message, close_connection, log=log) + HTTPSimpleResponse.__init__(self, http_client.INTERNAL_SERVER_ERROR, http_message, close_connection, log=log) class BookNotFound(HTTPNotFound): diff --git a/src/calibre/srv/http_request.py b/src/calibre/srv/http_request.py index 384fe5a04f..74ac506790 100644 --- a/src/calibre/srv/http_request.py +++ b/src/calibre/srv/http_request.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import re, httplib, repr as reprlib +import re, repr as reprlib from io import BytesIO, DEFAULT_BUFFER_SIZE from calibre import as_unicode, force_unicode @@ -14,6 +14,7 @@ from calibre.ptempfile import SpooledTemporaryFile from calibre.srv.errors import HTTPSimpleResponse from calibre.srv.loop import Connection, READ, WRITE from calibre.srv.utils import MultiDict, HTTP1, HTTP11, Accumulator +from polyglot import http_client from polyglot.urllib import unquote protocol_map = {(1, 0):HTTP1, (1, 1):HTTP11} @@ -68,29 +69,29 @@ def parse_request_uri(uri): def parse_uri(uri, parse_query=True): scheme, authority, path = parse_request_uri(uri) if path is None: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, "No path component") + raise HTTPSimpleResponse(http_client.BAD_REQUEST, "No path component") if b'#' in path: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, "Illegal #fragment in Request-URI.") + raise HTTPSimpleResponse(http_client.BAD_REQUEST, "Illegal #fragment in Request-URI.") if scheme: try: scheme = scheme.decode('ascii') except ValueError: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Un-decodeable scheme') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Un-decodeable scheme') path, qs = path.partition(b'?')[::2] if parse_query: try: query = MultiDict.create_from_query_string(qs) except Exception: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unparseable query string') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unparseable query string') else: query = None try: path = '%2F'.join(unquote(x).decode('utf-8') for x in quoted_slash.split(path)) except ValueError as e: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, as_unicode(e)) + raise HTTPSimpleResponse(http_client.BAD_REQUEST, as_unicode(e)) path = tuple(filter(None, (x.replace('%2F', '/') for x in path.split('/')))) return scheme, path, query @@ -233,7 +234,7 @@ class HTTPRequest(Connection): if line.endswith(b'\n'): line = buf.getvalue() if not line.endswith(b'\r\n'): - self.simple_response(httplib.BAD_REQUEST, 'HTTP requires CRLF line terminators') + self.simple_response(http_client.BAD_REQUEST, 'HTTP requires CRLF line terminators') return return line if not line: @@ -247,7 +248,7 @@ class HTTPRequest(Connection): self.forwarded_for = None self.path = self.query = None self.close_after_response = False - self.header_line_too_long_error_code = httplib.REQUEST_URI_TOO_LONG + self.header_line_too_long_error_code = http_client.REQUEST_URI_TOO_LONG self.response_started = False self.set_state(READ, self.parse_request_line, Accumulator(), first=True) @@ -260,28 +261,28 @@ class HTTPRequest(Connection): # Ignore a single leading empty line, as per RFC 2616 sec 4.1 if first: return self.set_state(READ, self.parse_request_line, Accumulator()) - return self.simple_response(httplib.BAD_REQUEST, 'Multiple leading empty lines not allowed') + return self.simple_response(http_client.BAD_REQUEST, 'Multiple leading empty lines not allowed') try: method, uri, req_protocol = line.strip().split(b' ', 2) rp = int(req_protocol[5]), int(req_protocol[7]) self.method = method.decode('ascii').upper() except Exception: - return self.simple_response(httplib.BAD_REQUEST, "Malformed Request-Line") + return self.simple_response(http_client.BAD_REQUEST, "Malformed Request-Line") if self.method not in HTTP_METHODS: - return self.simple_response(httplib.BAD_REQUEST, "Unknown HTTP method") + return self.simple_response(http_client.BAD_REQUEST, "Unknown HTTP method") try: self.request_protocol = protocol_map[rp] except KeyError: - return self.simple_response(httplib.HTTP_VERSION_NOT_SUPPORTED) + return self.simple_response(http_client.HTTP_VERSION_NOT_SUPPORTED) self.response_protocol = protocol_map[min((1, 1), rp)] try: self.scheme, self.path, self.query = parse_uri(uri) except HTTPSimpleResponse as e: return self.simple_response(e.http_code, e.message, close_after_response=False) - self.header_line_too_long_error_code = httplib.REQUEST_ENTITY_TOO_LARGE + self.header_line_too_long_error_code = http_client.REQUEST_ENTITY_TOO_LARGE self.set_state(READ, self.parse_header_line, HTTPHeaderParser(), Accumulator()) # }}} @@ -299,7 +300,7 @@ class HTTPRequest(Connection): try: parser(line) except ValueError: - self.simple_response(httplib.BAD_REQUEST, 'Failed to parse header line') + self.simple_response(http_client.BAD_REQUEST, 'Failed to parse header line') return if parser.finished: self.finalize_headers(parser.hdict) @@ -307,7 +308,7 @@ class HTTPRequest(Connection): def finalize_headers(self, inheaders): request_content_length = int(inheaders.get('Content-Length', 0)) if request_content_length > self.max_request_body_size: - return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE, + return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE, "The entity sent with the request exceeds the maximum " "allowed bytes (%d)." % self.max_request_body_size) # Persistent connection support @@ -334,7 +335,7 @@ class HTTPRequest(Connection): else: # Note that, even if we see "chunked", we must reject # if there is an extension we don't recognize. - return self.simple_response(httplib.NOT_IMPLEMENTED, "Unknown transfer encoding: %r" % enc) + return self.simple_response(http_client.NOT_IMPLEMENTED, "Unknown transfer encoding: %r" % enc) if inheaders.get("Expect", '').lower() == "100-continue": buf = BytesIO((HTTP11 + " 100 Continue\r\n\r\n").encode('ascii')) @@ -369,9 +370,9 @@ class HTTPRequest(Connection): try: chunk_size = int(line.strip(), 16) except Exception: - return self.simple_response(httplib.BAD_REQUEST, '%s is not a valid chunk size' % reprlib.repr(line.strip())) + return self.simple_response(http_client.BAD_REQUEST, '%s is not a valid chunk size' % reprlib.repr(line.strip())) if bytes_read[0] + chunk_size + 2 > self.max_request_body_size: - return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE, + return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE, 'Chunked request is larger than %d bytes' % self.max_request_body_size) if chunk_size == 0: self.set_state(READ, self.read_chunk_separator, inheaders, Accumulator(), buf, bytes_read, last=True) @@ -389,10 +390,10 @@ class HTTPRequest(Connection): if line is None: return if line != b'\r\n': - return self.simple_response(httplib.BAD_REQUEST, 'Chunk does not have trailing CRLF') + return self.simple_response(http_client.BAD_REQUEST, 'Chunk does not have trailing CRLF') bytes_read[0] += len(line) if bytes_read[0] > self.max_request_body_size: - return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE, + return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE, 'Chunked request is larger than %d bytes' % self.max_request_body_size) if last: self.prepare_response(inheaders, buf) @@ -402,7 +403,7 @@ class HTTPRequest(Connection): def handle_timeout(self): if self.response_started: return False - self.simple_response(httplib.REQUEST_TIMEOUT) + self.simple_response(http_client.REQUEST_TIMEOUT) return True def write(self, buf, end=None): diff --git a/src/calibre/srv/http_response.py b/src/calibre/srv/http_response.py index b64db6dab6..fc11fd0782 100644 --- a/src/calibre/srv/http_response.py +++ b/src/calibre/srv/http_response.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import os, httplib, hashlib, uuid, struct, repr as reprlib +import os, hashlib, uuid, struct, repr as reprlib from collections import namedtuple from io import BytesIO, DEFAULT_BUFFER_SIZE from itertools import chain, repeat @@ -26,6 +26,7 @@ from calibre.srv.utils import ( sort_q_values, get_translator_for_lang, Cookie, fast_now_strftime) from calibre.utils.speedups import ReadOnlyFileBuffer from calibre.utils.monotonic import monotonic +from polyglot import http_client Range = namedtuple('Range', 'start stop size') MULTIPART_SEPARATOR = uuid.uuid4().hex.decode('ascii') @@ -226,7 +227,7 @@ class RequestData(object): # {{{ self.remote_addr, self.remote_port, self.is_local_connection = remote_addr, remote_port, is_local_connection self.forwarded_for = forwarded_for self.opts = opts - self.status_code = httplib.OK + self.status_code = http_client.OK self.outcookie = Cookie() self.lang_code = self.gettext_func = self.ngettext_func = None self.set_translator(self.get_preferred_language()) @@ -402,16 +403,16 @@ class HTTPConnection(HTTPRequest): if self.response_protocol is HTTP1: # HTTP/1.0 has no 413/414/303 codes status_code = { - httplib.REQUEST_ENTITY_TOO_LARGE:httplib.BAD_REQUEST, - httplib.REQUEST_URI_TOO_LONG:httplib.BAD_REQUEST, - httplib.SEE_OTHER:httplib.FOUND + http_client.REQUEST_ENTITY_TOO_LARGE:http_client.BAD_REQUEST, + http_client.REQUEST_URI_TOO_LONG:http_client.BAD_REQUEST, + http_client.SEE_OTHER:http_client.FOUND }.get(status_code, status_code) self.close_after_response = close_after_response msg = msg.encode('utf-8') ct = 'http' if self.method == 'TRACE' else 'plain' buf = [ - '%s %d %s' % (self.response_protocol, status_code, httplib.responses[status_code]), + '%s %d %s' % (self.response_protocol, status_code, http_client.responses[status_code]), "Content-Length: %s" % len(msg), "Content-Type: text/%s; charset=UTF-8" % ct, "Date: " + http_date(), @@ -432,7 +433,7 @@ class HTTPConnection(HTTPRequest): def prepare_response(self, inheaders, request_body_file): if self.method == 'TRACE': msg = force_unicode(self.request_line, 'utf-8') + '\n' + inheaders.pretty() - return self.simple_response(httplib.OK, msg, close_after_response=False) + return self.simple_response(http_client.OK, msg, close_after_response=False) request_body_file.seek(0) outheaders = MultiDict() data = RequestData( @@ -449,28 +450,28 @@ class HTTPConnection(HTTPRequest): def send_range_not_satisfiable(self, content_length): buf = [ - '%s %d %s' % (self.response_protocol, httplib.REQUESTED_RANGE_NOT_SATISFIABLE, httplib.responses[httplib.REQUESTED_RANGE_NOT_SATISFIABLE]), + '%s %d %s' % (self.response_protocol, http_client.REQUESTED_RANGE_NOT_SATISFIABLE, http_client.responses[http_client.REQUESTED_RANGE_NOT_SATISFIABLE]), "Date: " + http_date(), "Content-Range: bytes */%d" % content_length, ] response_data = header_list_to_file(buf) - self.log_access(status_code=httplib.REQUESTED_RANGE_NOT_SATISFIABLE, response_size=response_data.sz) + self.log_access(status_code=http_client.REQUESTED_RANGE_NOT_SATISFIABLE, response_size=response_data.sz) self.response_ready(response_data) def send_not_modified(self, etag=None): buf = [ - '%s %d %s' % (self.response_protocol, httplib.NOT_MODIFIED, httplib.responses[httplib.NOT_MODIFIED]), + '%s %d %s' % (self.response_protocol, http_client.NOT_MODIFIED, http_client.responses[http_client.NOT_MODIFIED]), "Content-Length: 0", "Date: " + http_date(), ] if etag is not None: buf.append('ETag: ' + etag) response_data = header_list_to_file(buf) - self.log_access(status_code=httplib.NOT_MODIFIED, response_size=response_data.sz) + self.log_access(status_code=http_client.NOT_MODIFIED, response_size=response_data.sz) self.response_ready(response_data) def report_busy(self): - self.simple_response(httplib.SERVICE_UNAVAILABLE) + self.simple_response(http_client.SERVICE_UNAVAILABLE) def job_done(self, ok, result): if not ok: @@ -509,7 +510,7 @@ class HTTPConnection(HTTPRequest): if ct.startswith('text/') and 'charset=' not in ct: outheaders.set('Content-Type', ct + '; charset=UTF-8', replace_all=True) - buf = [HTTP11 + (' %d ' % data.status_code) + httplib.responses[data.status_code]] + buf = [HTTP11 + (' %d ' % data.status_code) + http_client.responses[data.status_code]] for header, value in sorted(iteritems(outheaders), key=itemgetter(0)): buf.append('%s: %s' % (header, value)) for morsel in itervalues(data.outcookie): @@ -530,7 +531,7 @@ class HTTPConnection(HTTPRequest): def log_access(self, status_code, response_size=None, username=None): if self.access_log is None: return - if not self.opts.log_not_found and status_code == httplib.NOT_FOUND: + if not self.opts.log_not_found and status_code == http_client.NOT_FOUND: return ff = self.forwarded_for if ff: @@ -623,7 +624,7 @@ class HTTPConnection(HTTPRequest): self.ready = ready def report_unhandled_exception(self, e, formatted_traceback): - self.simple_response(httplib.INTERNAL_SERVER_ERROR) + self.simple_response(http_client.INTERNAL_SERVER_ERROR) def finalize_output(self, output, request, is_http1): none_match = parse_if_none_match(request.inheaders.get('If-None-Match', '')) @@ -633,7 +634,7 @@ class HTTPConnection(HTTPRequest): if self.method in ('GET', 'HEAD'): self.send_not_modified(output.etag) else: - self.simple_response(httplib.PRECONDITION_FAILED) + self.simple_response(http_client.PRECONDITION_FAILED) return opts = self.opts @@ -660,10 +661,10 @@ class HTTPConnection(HTTPRequest): ct = outheaders.get('Content-Type', '').partition(';')[0] compressible = (not ct or ct.startswith('text/') or ct.startswith('image/svg') or ct.partition(';')[0] in COMPRESSIBLE_TYPES) - compressible = (compressible and request.status_code == httplib.OK and + compressible = (compressible and request.status_code == http_client.OK and (opts.compress_min_size > -1 and output.content_length >= opts.compress_min_size) and acceptable_encoding(request.inheaders.get('Accept-Encoding', '')) and not is_http1) - accept_ranges = (not compressible and output.accept_ranges is not None and request.status_code == httplib.OK and + accept_ranges = (not compressible and output.accept_ranges is not None and request.status_code == http_client.OK and not is_http1) ranges = get_ranges(request.inheaders.get('Range'), output.content_length) if output.accept_ranges and self.method in ('GET', 'HEAD') else None if_range = (request.inheaders.get('If-Range') or '').strip() @@ -680,7 +681,7 @@ class HTTPConnection(HTTPRequest): if self.method in ('GET', 'HEAD'): self.send_not_modified(output.etag) else: - self.simple_response(httplib.PRECONDITION_FAILED) + self.simple_response(http_client.PRECONDITION_FAILED) return output.ranges = None @@ -712,7 +713,7 @@ class HTTPConnection(HTTPRequest): outheaders.set('Content-Length', '%d' % size, replace_all=True) outheaders.set('Content-Type', 'multipart/byteranges; boundary=' + MULTIPART_SEPARATOR, replace_all=True) output.ranges = zip_longest(ranges, range_parts) - request.status_code = httplib.PARTIAL_CONTENT + request.status_code = http_client.PARTIAL_CONTENT return output diff --git a/src/calibre/srv/routes.py b/src/calibre/srv/routes.py index 9be1cde382..621b412c6c 100644 --- a/src/calibre/srv/routes.py +++ b/src/calibre/srv/routes.py @@ -6,13 +6,14 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, sys, inspect, re, time, numbers, json as jsonlib, textwrap +import sys, inspect, re, time, numbers, json as jsonlib, textwrap from operator import attrgetter from calibre.srv.errors import HTTPSimpleResponse, HTTPNotFound, RouteError from calibre.srv.utils import http_date from calibre.utils.serialize import msgpack_dumps, json_dumps, MSGPACK_MIME from polyglot.builtins import iteritems, itervalues, unicode_type, range, zip +from polyglot import http_client from polyglot.urllib import quote as urlquote default_methods = frozenset(('HEAD', 'GET')) @@ -297,7 +298,7 @@ class Router(object): def dispatch(self, data): endpoint_, args = self.find_route(data.path) if data.method not in endpoint_.methods: - raise HTTPSimpleResponse(httplib.METHOD_NOT_ALLOWED) + raise HTTPSimpleResponse(http_client.METHOD_NOT_ALLOWED) self.read_cookies(data) diff --git a/src/calibre/srv/tests/ajax.py b/src/calibre/srv/tests/ajax.py index 7707bd6d2b..0fade7e9d0 100644 --- a/src/calibre/srv/tests/ajax.py +++ b/src/calibre/srv/tests/ajax.py @@ -6,13 +6,13 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, zlib, json, base64, os +import zlib, json, base64, os from io import BytesIO from functools import partial -from httplib import OK, NOT_FOUND, FORBIDDEN from calibre.ebooks.metadata.meta import get_metadata from calibre.srv.tests.base import LibraryBaseTest +from polyglot.http_client import OK, NOT_FOUND, FORBIDDEN from polyglot.urllib import urlencode, quote @@ -22,7 +22,7 @@ def make_request(conn, url, headers={}, prefix='/ajax', username=None, password= conn.request(method, prefix + url, headers=headers, body=data) r = conn.getresponse() data = r.read() - if r.status == httplib.OK and data and data[0] in b'{[': + if r.status == OK and data and data[0] in b'{[': data = json.loads(data) return r, data @@ -37,10 +37,10 @@ class ContentTest(LibraryBaseTest): request = partial(make_request, conn, prefix='/ajax/book') r, data = request('/x') - self.ae(r.status, httplib.NOT_FOUND) + self.ae(r.status, NOT_FOUND) r, onedata = request('/1') - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) self.ae(request('/1/' + db.server_library_id)[1], onedata) self.ae(request('/%s?id_is_uuid=true' % db.field_for('uuid', 1))[1], onedata) @@ -63,22 +63,22 @@ class ContentTest(LibraryBaseTest): request = partial(make_request, conn) r, data = request('/categories') - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) r, xdata = request('/categories/' + db.server_library_id) - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) self.ae(data, xdata) names = {x['name']:x['url'] for x in data} for q in ('Newest', 'All books', 'Tags', 'Series', 'Authors', 'Enum', 'Composite Tags'): self.assertIn(q, names) r, data = request(names['Tags'], prefix='') - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) names = {x['name']:x['url'] for x in data['items']} self.ae(set(names), set('Tag One,Tag Two,News'.split(','))) r, data = request(names['Tag One'], prefix='') - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) self.ae(set(data['book_ids']), {1, 2}) r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"'})) - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) self.ae(set(data['book_ids']), {1, 2}) r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"', 'vl':'1'})) self.ae(set(data['book_ids']), {2}) diff --git a/src/calibre/srv/tests/auth.py b/src/calibre/srv/tests/auth.py index ea08759d04..9407516cf0 100644 --- a/src/calibre/srv/tests/auth.py +++ b/src/calibre/srv/tests/auth.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, base64, subprocess, os, cookielib, time +import base64, subprocess, os, cookielib, time from collections import namedtuple try: from distutils.spawn import find_executable @@ -18,6 +18,7 @@ from calibre.srv.errors import HTTPForbidden from calibre.srv.tests.base import BaseTest, TestServer from calibre.srv.routes import endpoint, Router from polyglot.builtins import iteritems, itervalues +from polyglot import http_client from polyglot.urllib import (build_opener, HTTPBasicAuthHandler, HTTPCookieProcessor, HTTPDigestAuthHandler, HTTPError) @@ -91,18 +92,18 @@ class TestAuth(BaseTest): conn = server.connect() conn.request('GET', '/open') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'open') conn.request('GET', '/closed') r = conn.getresponse() - self.ae(r.status, httplib.UNAUTHORIZED) + self.ae(r.status, http_client.UNAUTHORIZED) self.ae(r.getheader('WWW-Authenticate'), b'Basic realm="%s"' % bytes(REALM)) self.assertFalse(r.read()) conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + base64.standard_b64encode(b'testuser:testpw')}) r = conn.getresponse() self.ae(r.read(), b'closed') - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(b'closed', urlopen(server, method='basic').read()) self.ae(b'closed', urlopen(server, un='!@#$%^&*()-=_+', pw='!@#$%^&*()-=_+', method='basic').read()) @@ -113,14 +114,14 @@ class TestAuth(BaseTest): warnings = [] server.loop.log.warn = lambda *args, **kwargs: warnings.append(' '.join(args)) - self.ae((httplib.OK, b'closed'), request()) - self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y')) - self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('', '')) + self.ae((http_client.OK, b'closed'), request()) + self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y')) + self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request('', '')) self.ae(1, len(warnings)) - self.ae((httplib.UNAUTHORIZED, b''), request('testuser', 'y')) - self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('testuser', '')) - self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('')) - self.ae((httplib.UNAUTHORIZED, b''), request('asf', 'testpw')) + self.ae((http_client.UNAUTHORIZED, b''), request('testuser', 'y')) + self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request('testuser', '')) + self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request('')) + self.ae((http_client.UNAUTHORIZED, b''), request('asf', 'testpw')) # }}} def test_library_restrictions(self): # {{{ @@ -169,7 +170,7 @@ class TestAuth(BaseTest): with TestServer(r.dispatch) as server: r.auth_controller.log = server.log - def test(conn, path, headers={}, status=httplib.OK, body=b'', request_body=b''): + def test(conn, path, headers={}, status=http_client.OK, body=b'', request_body=b''): conn.request('GET', path, request_body, headers) r = conn.getresponse() self.ae(r.status, status) @@ -177,9 +178,9 @@ class TestAuth(BaseTest): return {normalize_header_name(k):v for k, v in r.getheaders()} conn = server.connect() test(conn, '/open', body=b'open') - auth = parse_http_dict(test(conn, '/closed', status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) + auth = parse_http_dict(test(conn, '/closed', status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) nonce = auth['nonce'] - auth = parse_http_dict(test(conn, '/closed', status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) + auth = parse_http_dict(test(conn, '/closed', status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) self.assertNotEqual(nonce, auth['nonce'], 'nonce was re-used') self.ae(auth[b'realm'], bytes(REALM)), self.ae(auth[b'algorithm'], b'MD5'), self.ae(auth[b'qop'], b'auth') self.assertNotIn('stale', auth) @@ -199,14 +200,14 @@ class TestAuth(BaseTest): # Check stale nonces orig, r.auth_controller.max_age_seconds = r.auth_controller.max_age_seconds, -1 auth = parse_http_dict(test(conn, '/closed', headers={ - 'Authorization':digest(**args)},status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) + 'Authorization':digest(**args)},status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) self.assertIn('stale', auth) r.auth_controller.max_age_seconds = orig ok_test(conn, digest(**args)) def fail_test(conn, modify, **kw): kw['body'] = kw.get('body', b'') - kw['status'] = kw.get('status', httplib.UNAUTHORIZED) + kw['status'] = kw.get('status', http_client.UNAUTHORIZED) args['modify'] = modify return test(conn, '/closed', headers={'Authorization':digest(**args)}, **kw) @@ -258,13 +259,13 @@ class TestAuth(BaseTest): warnings = [] server.loop.log.warn = lambda *args, **kwargs: warnings.append(' '.join(args)) - self.ae((httplib.OK, b'closed'), request()) - self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y')) - self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y')) - self.ae(httplib.FORBIDDEN, request('x', 'y')[0]) - self.ae(httplib.FORBIDDEN, request()[0]) + self.ae((http_client.OK, b'closed'), request()) + self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y')) + self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y')) + self.ae(http_client.FORBIDDEN, request('x', 'y')[0]) + self.ae(http_client.FORBIDDEN, request()[0]) time.sleep(ban_for * 60 + 0.01) - self.ae((httplib.OK, b'closed'), request()) + self.ae((http_client.OK, b'closed'), request()) # }}} def test_android_auth_workaround(self): # {{{ @@ -277,7 +278,7 @@ class TestAuth(BaseTest): # First check that unauth access fails conn.request('GET', '/android') r = conn.getresponse() - self.ae(r.status, httplib.UNAUTHORIZED) + self.ae(r.status, http_client.UNAUTHORIZED) auth_handler = HTTPDigestAuthHandler() url = 'http://localhost:%d%s' % (server.address[1], '/android') @@ -285,20 +286,20 @@ class TestAuth(BaseTest): cj = cookielib.CookieJar() cookie_handler = HTTPCookieProcessor(cj) r = build_opener(auth_handler, cookie_handler).open(url) - self.ae(r.getcode(), httplib.OK) + self.ae(r.getcode(), http_client.OK) cookies = tuple(cj) self.ae(len(cookies), 1) cookie = cookies[0] self.assertIn(b':', cookie.value) self.ae(cookie.path, b'/android') r = build_opener(cookie_handler).open(url) - self.ae(r.getcode(), httplib.OK) + self.ae(r.getcode(), http_client.OK) self.ae(r.read(), b'android') # Test that a replay attack against a different URL does not work try: build_opener(cookie_handler).open(url+'2') assert ('Replay attack succeeded') except HTTPError as e: - self.ae(e.code, httplib.UNAUTHORIZED) + self.ae(e.code, http_client.UNAUTHORIZED) # }}} diff --git a/src/calibre/srv/tests/base.py b/src/calibre/srv/tests/base.py index e62dcc605d..165aa2e0c5 100644 --- a/src/calibre/srv/tests/base.py +++ b/src/calibre/srv/tests/base.py @@ -7,12 +7,13 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import unittest, time, httplib, shutil, gc, tempfile, atexit, os +import unittest, time, shutil, gc, tempfile, atexit, os from io import BytesIO from functools import partial from threading import Thread from calibre.srv.utils import ServerLog +from polyglot import http_client rmtree = partial(shutil.rmtree, ignore_errors=True) @@ -120,7 +121,7 @@ class TestServer(Thread): timeout = self.loop.opts.timeout if interface is None: interface = self.address[0] - return httplib.HTTPConnection(interface, self.address[1], strict=True, timeout=timeout) + return http_client.HTTPConnection(interface, self.address[1], strict=True, timeout=timeout) def change_handler(self, handler): from calibre.srv.http_response import create_http_handler diff --git a/src/calibre/srv/tests/content.py b/src/calibre/srv/tests/content.py index e7b1950eec..fa9a12ccc7 100644 --- a/src/calibre/srv/tests/content.py +++ b/src/calibre/srv/tests/content.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, zlib, json, binascii, time, os +import zlib, json, binascii, time, os from io import BytesIO from calibre.ebooks.metadata.epub import get_metadata @@ -14,6 +14,7 @@ from calibre.ebooks.metadata.opf2 import OPF from calibre.srv.tests.base import LibraryBaseTest from calibre.utils.imghdr import identify from calibre.utils.shared_file import share_open +from polyglot import http_client def setUpModule(): @@ -32,7 +33,7 @@ class ContentTest(LibraryBaseTest): def missing(url, body=b''): conn.request('GET', url) r = conn.getresponse() - self.ae(r.status, httplib.NOT_FOUND) + self.ae(r.status, http_client.NOT_FOUND) self.ae(r.read(), body) for prefix in ('static', 'icon'): @@ -51,7 +52,7 @@ class ContentTest(LibraryBaseTest): raw = P(src, data=True) conn.request('GET', url) r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) data = r.read() if sz is None: self.ae(data, raw) @@ -60,7 +61,7 @@ class ContentTest(LibraryBaseTest): test_response(r) conn.request('GET', url, headers={'If-None-Match':r.getheader('ETag')}) r = conn.getresponse() - self.ae(r.status, httplib.NOT_MODIFIED) + self.ae(r.status, http_client.NOT_MODIFIED) self.ae(b'', r.read()) test('content-server/empty.html', '/static/empty.html') @@ -85,7 +86,7 @@ class ContentTest(LibraryBaseTest): # Test various invalid parameters def bad(*args): r, data = get(*args) - self.ae(r.status, httplib.NOT_FOUND) + self.ae(r.status, http_client.NOT_FOUND) bad('xxx', 1) bad('fmt1', 10) bad('fmt1', 1, 'zzzz') @@ -103,7 +104,7 @@ class ContentTest(LibraryBaseTest): # Test fetching of format with metadata update raw = P('quick_start/eng.epub', data=True) r, data = get('epub', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) etag = r.getheader('ETag') self.assertIsNotNone(etag) self.ae(r.getheader('Used-Cache'), 'no') @@ -145,39 +146,39 @@ class ContentTest(LibraryBaseTest): os.utime(cpath, (t, t)) r, data = get('cover', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(data, db.cover(1)) self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Content-Type'), 'image/jpeg') r, data = get('cover', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(data, db.cover(1)) self.ae(r.getheader('Used-Cache'), 'yes') r, data = get('cover', 3) - self.ae(r.status, httplib.OK) # Auto generated cover + self.ae(r.status, http_client.OK) # Auto generated cover r, data = get('thumb', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(identify(data), ('jpeg', 60, 60)) self.ae(r.getheader('Used-Cache'), 'no') r, data = get('thumb', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.getheader('Used-Cache'), 'yes') r, data = get('thumb', 1, q='sz=100') - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(identify(data), ('jpeg', 100, 100)) self.ae(r.getheader('Used-Cache'), 'no') r, data = get('thumb', 1, q='sz=100x100') - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.getheader('Used-Cache'), 'yes') change_cover(1, 1) r, data = get('thumb', 1, q='sz=100') - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(identify(data), ('jpeg', 100, 100)) self.ae(r.getheader('Used-Cache'), 'no') # Test file sharing in cache r, data = get('cover', 2) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(data, db.cover(2)) self.ae(r.getheader('Used-Cache'), 'no') path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8') @@ -185,7 +186,7 @@ class ContentTest(LibraryBaseTest): # Now force an update change_cover(1) r, data = get('cover', 2) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(data, db.cover(2)) self.ae(r.getheader('Used-Cache'), 'no') path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8') @@ -193,7 +194,7 @@ class ContentTest(LibraryBaseTest): # Do it again change_cover(2) r, data = get('cover', 2) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(data, db.cover(2)) self.ae(r.getheader('Used-Cache'), 'no') self.ae(f.read(), fdata) @@ -201,7 +202,7 @@ class ContentTest(LibraryBaseTest): # Test serving of metadata as opf r, data = get('opf', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.getheader('Content-Type'), 'application/oebps-package+xml; charset=UTF-8') self.assertIsNotNone(r.getheader('Last-Modified')) opf = OPF(BytesIO(data), populate_spine=False, try_to_guess_cover=False) @@ -209,17 +210,17 @@ class ContentTest(LibraryBaseTest): self.ae(db.field_for('authors', 1), tuple(opf.authors)) conn.request('GET', '/get/opf/1', headers={'Accept-Encoding':'gzip'}) r = conn.getresponse() - self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') + self.ae(r.status, http_client.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') raw = r.read() self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data) # Test serving metadata as json r, data = get('json', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(db.field_for('title', 1), json.loads(data)['title']) conn.request('GET', '/get/json/1', headers={'Accept-Encoding':'gzip'}) r = conn.getresponse() - self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') + self.ae(r.status, http_client.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') raw = r.read() self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data) diff --git a/src/calibre/srv/tests/http.py b/src/calibre/srv/tests/http.py index f42279795e..05b487b2a3 100644 --- a/src/calibre/srv/tests/http.py +++ b/src/calibre/srv/tests/http.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, hashlib, zlib, string, time, os +import hashlib, zlib, string, time, os from io import BytesIO from tempfile import NamedTemporaryFile @@ -15,6 +15,7 @@ from calibre.srv.tests.base import BaseTest, TestServer from calibre.srv.utils import eintr_retry_call from calibre.utils.monotonic import monotonic from polyglot.builtins import iteritems, range +from polyglot import http_client is_ci = os.environ.get('CI', '').lower() == 'true' @@ -94,7 +95,7 @@ class TestHTTP(BaseTest): def test(al, q): conn.request('GET', '/', headers={'Accept-Language': al}) r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) q += get_translator(q)[-1].ugettext('Unknown') self.ae(r.read(), q) @@ -136,7 +137,7 @@ class TestHTTP(BaseTest): def raw_send(conn, raw): conn.send(raw) - conn._HTTPConnection__state = httplib._CS_REQ_SENT + conn._HTTPConnection__state = http_client._CS_REQ_SENT return conn.getresponse() base_timeout = 0.5 if is_ci else 0.1 @@ -144,31 +145,31 @@ class TestHTTP(BaseTest): with TestServer(handler, timeout=base_timeout, max_header_line_size=100./1024, max_request_body_size=100./(1024*1024)) as server: conn = server.connect() r = raw_send(conn, b'hello\n') - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) self.ae(r.read(), b'HTTP requires CRLF line terminators') r = raw_send(conn, b'\r\nGET /index.html HTTP/1.1\r\n\r\n') - self.ae(r.status, httplib.NOT_FOUND), self.ae(r.read(), b'Requested resource not found') + self.ae(r.status, http_client.NOT_FOUND), self.ae(r.read(), b'Requested resource not found') r = raw_send(conn, b'\r\n\r\nGET /index.html HTTP/1.1\r\n\r\n') - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) self.ae(r.read(), b'Multiple leading empty lines not allowed') r = raw_send(conn, b'hello world\r\n') - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) self.ae(r.read(), b'Malformed Request-Line') r = raw_send(conn, b'x' * 200) - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) self.ae(r.read(), b'') r = raw_send(conn, b'XXX /index.html HTTP/1.1\r\n\r\n') - self.ae(r.status, httplib.BAD_REQUEST), self.ae(r.read(), b'Unknown HTTP method') + self.ae(r.status, http_client.BAD_REQUEST), self.ae(r.read(), b'Unknown HTTP method') # Test 404 conn.request('HEAD', '/moose') r = conn.getresponse() - self.ae(r.status, httplib.NOT_FOUND) + self.ae(r.status, http_client.NOT_FOUND) self.assertIsNotNone(r.getheader('Date', None)) self.ae(r.getheader('Content-Length'), str(len(body))) self.ae(r.getheader('Content-Type'), 'text/plain; charset=UTF-8') @@ -176,7 +177,7 @@ class TestHTTP(BaseTest): self.ae(r.read(), '') conn.request('GET', '/choose') r = conn.getresponse() - self.ae(r.status, httplib.NOT_FOUND) + self.ae(r.status, http_client.NOT_FOUND) self.ae(r.read(), b'Requested resource not found') # Test 500 @@ -186,7 +187,7 @@ class TestHTTP(BaseTest): conn = server.connect() conn.request('GET', '/test/') r = conn.getresponse() - self.ae(r.status, httplib.INTERNAL_SERVER_ERROR) + self.ae(r.status, http_client.INTERNAL_SERVER_ERROR) server.loop.log.filter_level = orig # Test 301 @@ -196,7 +197,7 @@ class TestHTTP(BaseTest): conn = server.connect() conn.request('GET', '/') r = conn.getresponse() - self.ae(r.status, httplib.MOVED_PERMANENTLY) + self.ae(r.status, http_client.MOVED_PERMANENTLY) self.ae(r.getheader('Location'), '/somewhere-else') self.ae('', r.read()) @@ -206,26 +207,26 @@ class TestHTTP(BaseTest): # Test simple GET conn.request('GET', '/test/') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'test') # Test TRACE lines = ['TRACE /xxx HTTP/1.1', 'Test: value', 'Xyz: abc, def', '', ''] r = raw_send(conn, ('\r\n'.join(lines)).encode('ascii')) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read().decode('utf-8'), '\n'.join(lines[:-2])) # Test POST with simple body conn.request('POST', '/test', 'body') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'testbody') # Test POST with chunked transfer encoding conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.send(b'4\r\nbody\r\na\r\n1234567890\r\n0\r\n\r\n') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'testbody1234567890') # Test various incorrect input @@ -233,39 +234,39 @@ class TestHTTP(BaseTest): conn.request('GET', '/test' + ('a' * 200)) r = conn.getresponse() - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) conn = server.connect() conn.request('GET', '/test', ('a' * 200)) r = conn.getresponse() - self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE) + self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE) conn = server.connect() conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.send(b'x\r\nbody\r\n0\r\n\r\n') r = conn.getresponse() - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) self.assertIn(b'not a valid chunk size', r.read()) conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.send(b'4\r\nbody\r\n200\r\n\r\n') r = conn.getresponse() - self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE) + self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE) conn.request('POST', '/test', body='a'*200) r = conn.getresponse() - self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE) + self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE) conn = server.connect() conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.send(b'3\r\nbody\r\n0\r\n\r\n') r = conn.getresponse() - self.ae(r.status, httplib.BAD_REQUEST), self.ae(r.read(), b'Chunk does not have trailing CRLF') + self.ae(r.status, http_client.BAD_REQUEST), self.ae(r.read(), b'Chunk does not have trailing CRLF') conn = server.connect(timeout=base_timeout * 5) conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.send(b'30\r\nbody\r\n0\r\n\r\n') r = conn.getresponse() - self.ae(r.status, httplib.REQUEST_TIMEOUT) + self.ae(r.status, http_client.REQUEST_TIMEOUT) self.assertIn(b'', r.read()) server.log.filter_level = orig_level @@ -273,14 +274,14 @@ class TestHTTP(BaseTest): # Test pipelining responses = [] for i in range(10): - conn._HTTPConnection__state = httplib._CS_IDLE + conn._HTTPConnection__state = http_client._CS_IDLE conn.request('GET', '/%d'%i) responses.append(conn.response_class(conn.sock, strict=conn.strict, method=conn._method)) for i in range(10): r = responses[i] r.begin() self.ae(r.read(), ('%d' % i).encode('ascii')) - conn._HTTPConnection__state = httplib._CS_IDLE + conn._HTTPConnection__state = http_client._CS_IDLE # Test closing server.loop.opts.timeout = 10 # ensure socket is not closed because of timeout @@ -319,12 +320,12 @@ class TestHTTP(BaseTest): conn = server.connect() conn.request('GET', '/an_etagged_path') r = conn.getresponse() - self.ae(r.status, httplib.OK), self.ae(r.read(), b'an_etagged_path') + self.ae(r.status, http_client.OK), self.ae(r.read(), b'an_etagged_path') etag = r.getheader('ETag') self.ae(etag, '"%s"' % hashlib.sha1('an_etagged_path').hexdigest()) conn.request('GET', '/an_etagged_path', headers={'If-None-Match':etag}) r = conn.getresponse() - self.ae(r.status, httplib.NOT_MODIFIED) + self.ae(r.status, http_client.NOT_MODIFIED) self.ae(r.read(), b'') # Test gzip @@ -334,7 +335,7 @@ class TestHTTP(BaseTest): conn.request('GET', '/an_etagged_path', headers={'Accept-Encoding':'gzip'}) r = conn.getresponse() self.ae(str(len(raw)), r.getheader('Calibre-Uncompressed-Length')) - self.ae(r.status, httplib.OK), self.ae(zlib.decompress(r.read(), 16+zlib.MAX_WBITS), raw) + self.ae(r.status, http_client.OK), self.ae(zlib.decompress(r.read(), 16+zlib.MAX_WBITS), raw) # Test dynamic etagged content num_calls = [0] @@ -346,13 +347,13 @@ class TestHTTP(BaseTest): conn = server.connect() conn.request('GET', '/an_etagged_path') r = conn.getresponse() - self.ae(r.status, httplib.OK), self.ae(r.read(), b'data') + self.ae(r.status, http_client.OK), self.ae(r.read(), b'data') etag = r.getheader('ETag') self.ae(etag, b'"xxx"') self.ae(r.getheader('Content-Length'), '4') conn.request('GET', '/an_etagged_path', headers={'If-None-Match':etag}) r = conn.getresponse() - self.ae(r.status, httplib.NOT_MODIFIED) + self.ae(r.status, http_client.NOT_MODIFIED) self.ae(r.read(), b'') self.ae(num_calls[0], 1) @@ -368,11 +369,11 @@ class TestHTTP(BaseTest): self.ae(r.getheader('Content-Type'), guess_type(f.name)[0]) self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes') self.ae(int(r.getheader('Content-Length')), len(fdata)) - self.ae(r.status, httplib.OK), self.ae(r.read(), fdata) + self.ae(r.status, http_client.OK), self.ae(r.read(), fdata) conn.request('GET', '/test', headers={'Range':'bytes=2-25'}) r = conn.getresponse() - self.ae(r.status, httplib.PARTIAL_CONTENT) + self.ae(r.status, http_client.PARTIAL_CONTENT) self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes') self.ae(type('')(r.getheader('Content-Range')), 'bytes 2-25/%d' % len(fdata)) self.ae(int(r.getheader('Content-Length')), 24) @@ -380,27 +381,27 @@ class TestHTTP(BaseTest): conn.request('GET', '/test', headers={'Range':'bytes=100000-'}) r = conn.getresponse() - self.ae(r.status, httplib.REQUESTED_RANGE_NOT_SATISFIABLE) + self.ae(r.status, http_client.REQUESTED_RANGE_NOT_SATISFIABLE) self.ae(type('')(r.getheader('Content-Range')), 'bytes */%d' % len(fdata)) conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':etag}) r = conn.getresponse() - self.ae(r.status, httplib.PARTIAL_CONTENT), self.ae(r.read(), fdata[25:51]) + self.ae(r.status, http_client.PARTIAL_CONTENT), self.ae(r.read(), fdata[25:51]) self.ae(int(r.getheader('Content-Length')), 26) conn.request('GET', '/test', headers={'Range':'bytes=0-1000000'}) r = conn.getresponse() - self.ae(r.status, httplib.PARTIAL_CONTENT), self.ae(r.read(), fdata) + self.ae(r.status, http_client.PARTIAL_CONTENT), self.ae(r.read(), fdata) conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':'"nomatch"'}) r = conn.getresponse() - self.ae(r.status, httplib.OK), self.ae(r.read(), fdata) + self.ae(r.status, http_client.OK), self.ae(r.read(), fdata) self.assertFalse(r.getheader('Content-Range')) self.ae(int(r.getheader('Content-Length')), len(fdata)) conn.request('GET', '/test', headers={'Range':'bytes=0-25,26-50'}) r = conn.getresponse() - self.ae(r.status, httplib.PARTIAL_CONTENT) + self.ae(r.status, http_client.PARTIAL_CONTENT) clen = int(r.getheader('Content-Length')) data = r.read() self.ae(clen, len(data)) @@ -415,7 +416,7 @@ class TestHTTP(BaseTest): conn = server.connect(timeout=1) conn.request('GET', '/test') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) rdata = r.read() self.ae(len(data), len(rdata)) self.ae(hashlib.sha1(data).hexdigest(), hashlib.sha1(rdata).hexdigest()) diff --git a/src/calibre/srv/tests/loop.py b/src/calibre/srv/tests/loop.py index af628456c7..897721cc40 100644 --- a/src/calibre/srv/tests/loop.py +++ b/src/calibre/srv/tests/loop.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, ssl, os, socket, time +import ssl, os, socket, time from collections import namedtuple from unittest import skipIf from glob import glob @@ -18,6 +18,7 @@ from calibre.ptempfile import TemporaryDirectory from calibre.utils.certgen import create_server_cert from calibre.utils.monotonic import monotonic from polyglot.builtins import range +from polyglot import http_client is_ci = os.environ.get('CI', '').lower() == 'true' @@ -92,7 +93,7 @@ class LoopTest(BaseTest): conn.request('GET', '/') with self.assertRaises(socket.timeout): res = conn.getresponse() - if str(res.status) == str(httplib.REQUEST_TIMEOUT): + if str(res.status) == str(http_client.REQUEST_TIMEOUT): raise socket.timeout('Timeout') raise Exception('Got unexpected response: code: %s %s headers: %r data: %r' % ( res.status, res.reason, res.getheaders(), res.read())) @@ -135,7 +136,7 @@ class LoopTest(BaseTest): conn = server.connect(interface='127.0.0.1') conn.request('GET', '/test', 'body') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'testbody') def test_ring_buffer(self): @@ -203,10 +204,10 @@ class LoopTest(BaseTest): create_server_cert(address, ca_file, cert_file, key_file, key_size=1024) ctx = ssl.create_default_context(cafile=ca_file) with TestServer(lambda data:(data.path[0] + data.read()), ssl_certfile=cert_file, ssl_keyfile=key_file, listen_on=address, port=0) as server: - conn = httplib.HTTPSConnection(address, server.address[1], strict=True, context=ctx) + conn = http_client.HTTPSConnection(address, server.address[1], strict=True, context=ctx) conn.request('GET', '/test', 'body') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'testbody') cert = conn.sock.getpeercert() subject = dict(x[0] for x in cert['subject']) @@ -226,7 +227,7 @@ class LoopTest(BaseTest): conn = server.connect() conn.request('GET', '/test', 'body') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'testbody') self.ae(server.loop.bound_address[1], port) diff --git a/src/calibre/srv/web_socket.py b/src/calibre/srv/web_socket.py index 5fdf7ab309..59a5cfc9f8 100644 --- a/src/calibre/srv/web_socket.py +++ b/src/calibre/srv/web_socket.py @@ -5,7 +5,7 @@ from __future__ import (unicode_literals, division, absolute_import, print_function) -import httplib, os, weakref, socket +import os, weakref, socket from base64 import standard_b64encode from collections import deque from hashlib import sha1 @@ -19,6 +19,7 @@ from calibre.srv.http_response import HTTPConnection, create_http_handler from calibre.srv.utils import DESIRED_SEND_BUFFER_SIZE from calibre.utils.speedups import ReadOnlyFileBuffer from polyglot.queue import Queue, Empty +from polyglot import http_client speedup, err = plugins['speedup'] if not speedup: raise RuntimeError('Failed to load speedup module with error: ' + err) @@ -286,9 +287,9 @@ class WebSocketConnection(HTTPConnection): except Exception: ver_ok = False if not ver_ok: - return self.simple_response(httplib.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver) + return self.simple_response(http_client.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver) if self.method != 'GET': - return self.simple_response(httplib.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method) + return self.simple_response(http_client.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method) response = HANDSHAKE_STR % standard_b64encode(sha1(key + GUID_STR).digest()) self.optimize_for_sending_packet() diff --git a/src/calibre/utils/browser.py b/src/calibre/utils/browser.py index def783eaee..47835dec1e 100644 --- a/src/calibre/utils/browser.py +++ b/src/calibre/utils/browser.py @@ -5,11 +5,13 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import copy, httplib, ssl +import copy, ssl from cookielib import CookieJar, Cookie from mechanize import Browser as B, HTTPSHandler +from polyglot import http_client + class ModernHTTPSHandler(HTTPSHandler): @@ -24,7 +26,7 @@ class ModernHTTPSHandler(HTTPSHandler): def conn_factory(hostport, **kw): kw['context'] = self.ssl_context - return httplib.HTTPSConnection(hostport, **kw) + return http_client.HTTPSConnection(hostport, **kw) return self.do_open(conn_factory, req) diff --git a/src/calibre/utils/https.py b/src/calibre/utils/https.py index 690c1f454b..9c26970d23 100644 --- a/src/calibre/utils/https.py +++ b/src/calibre/utils/https.py @@ -10,7 +10,7 @@ import ssl, socket, re from contextlib import closing from calibre import get_proxies -from calibre.constants import ispy3 +from polyglot import http_client from polyglot.urllib import urlsplit has_ssl_verify = hasattr(ssl, 'create_default_context') and hasattr(ssl, '_create_unverified_context') @@ -19,19 +19,14 @@ class HTTPError(ValueError): def __init__(self, url, code): msg = '%s returned an unsupported http response code: %d (%s)' % ( - url, code, httplib.responses.get(code, None)) + url, code, http_client.responses.get(code, None)) ValueError.__init__(self, msg) self.code = code self.url = url -if ispy3: - import http.client as httplib -else: - import httplib - if has_ssl_verify: - class HTTPSConnection(httplib.HTTPSConnection): + class HTTPSConnection(http_client.HTTPSConnection): def __init__(self, ssl_version, *args, **kwargs): cafile = kwargs.pop('cert_file', None) @@ -39,7 +34,7 @@ if has_ssl_verify: kwargs['context'] = ssl._create_unverified_context() else: kwargs['context'] = ssl.create_default_context(cafile=cafile) - httplib.HTTPSConnection.__init__(self, *args, **kwargs) + http_client.HTTPSConnection.__init__(self, *args, **kwargs) else: # Check certificate hostname {{{ # Implementation taken from python 3 @@ -136,10 +131,10 @@ else: "subjectAltName fields were found") # }}} - class HTTPSConnection(httplib.HTTPSConnection): + class HTTPSConnection(http_client.HTTPSConnection): def __init__(self, ssl_version, *args, **kwargs): - httplib.HTTPSConnection.__init__(self, *args, **kwargs) + http_client.HTTPSConnection.__init__(self, *args, **kwargs) self.calibre_ssl_version = ssl_version def connect(self): @@ -204,7 +199,7 @@ def get_https_resource_securely( path += '?' + p.query c.request('GET', path, headers=headers or {}) response = c.getresponse() - if response.status in (httplib.MOVED_PERMANENTLY, httplib.FOUND, httplib.SEE_OTHER): + if response.status in (http_client.MOVED_PERMANENTLY, http_client.FOUND, http_client.SEE_OTHER): if max_redirects <= 0: raise ValueError('Too many redirects, giving up') newurl = response.getheader('Location', None) @@ -212,7 +207,7 @@ def get_https_resource_securely( raise ValueError('%s returned a redirect response with no Location header' % url) return get_https_resource_securely( newurl, cacerts=cacerts, timeout=timeout, max_redirects=max_redirects-1, ssl_version=ssl_version, get_response=get_response) - if response.status != httplib.OK: + if response.status != http_client.OK: raise HTTPError(url, response.status) if get_response: return response diff --git a/src/calibre/web/fetch/simple.py b/src/calibre/web/fetch/simple.py index e9a5c1457f..852b00b6fa 100644 --- a/src/calibre/web/fetch/simple.py +++ b/src/calibre/web/fetch/simple.py @@ -18,7 +18,6 @@ import threading import time import traceback from base64 import b64decode -from httplib import responses from calibre import browser, relpath, unicode_path from calibre.constants import filesystem_encoding, iswindows @@ -31,6 +30,7 @@ from calibre.utils.imghdr import what from calibre.utils.logging import Log from calibre.web.fetch.utils import rescale_image from polyglot.builtins import unicode_type +from polyglot.http_client import responses from polyglot.urllib import ( URLError, quote, url2pathname, urljoin, urlparse, urlsplit, urlunparse, urlunsplit