From d7410fe7b33c3ad2fdaf761703ede0e1257dbe76 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 25 Mar 2019 10:27:57 -0400 Subject: [PATCH 01/19] python3: add polyglot wrapper for html.entities/htmlentitydefs --- src/calibre/__init__.py | 2 +- src/calibre/utils/cleantext.py | 5 +++-- src/polyglot/html_entities.py | 10 ++++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 src/polyglot/html_entities.py diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 867163fc2b..c71af8e0f4 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -614,7 +614,7 @@ def entity_to_unicode(match, exceptions=[], encoding='cp1252', return check(html5_entities[ent]) except KeyError: pass - from htmlentitydefs import name2codepoint + from polyglot.html_entities import name2codepoint try: return check(my_unichr(name2codepoint[ent])) except KeyError: diff --git a/src/calibre/utils/cleantext.py b/src/calibre/utils/cleantext.py index 9e04d5d4d6..f8ba151de3 100644 --- a/src/calibre/utils/cleantext.py +++ b/src/calibre/utils/cleantext.py @@ -2,8 +2,9 @@ __license__ = 'GPL 3' __copyright__ = '2010, sengian ' __docformat__ = 'restructuredtext en' -import re, htmlentitydefs +import re from polyglot.builtins import codepoint_to_chr, map, range +from polyglot.html_entities import name2codepoint from calibre.constants import plugins, preferred_encoding try: @@ -80,7 +81,7 @@ def unescape(text, rm=False, rchar=u''): else: # named entity try: - text = codepoint_to_chr(htmlentitydefs.name2codepoint[text[1:-1]]) + text = codepoint_to_chr(name2codepoint[text[1:-1]]) except KeyError: pass if rm: diff --git a/src/polyglot/html_entities.py b/src/polyglot/html_entities.py new file mode 100644 index 0000000000..2e20eee7f4 --- /dev/null +++ b/src/polyglot/html_entities.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +# License: GPL v3 Copyright: 2019, Eli Schwartz + +from polyglot.builtins import is_py3 + +if is_py3: + from html.entities import name2codepoint +else: + from htmlentitydefs import name2codepoint From 92c621c718c2073f124c5a816516ce4254ed6faf Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 25 Mar 2019 10:36:48 -0400 Subject: [PATCH 02/19] python3: remove deprecated use of contextlib.nested Using `with Foo() as a, Bar() as b:` is introduced in python 2.7 and deprecates the use of nested(), which is removed entirely in python3 --- src/calibre/web/feeds/news.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index fb8e75e235..f51f7e5a21 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -9,7 +9,7 @@ __docformat__ = "restructuredtext en" import os, time, traceback, re, sys, io from collections import defaultdict -from contextlib import nested, closing +from contextlib import closing from calibre import (browser, __appname__, iswindows, force_unicode, @@ -1097,7 +1097,7 @@ class BasicNewsRecipe(Recipe): if bn: img = os.path.join(imgdir, 'feed_image_%d%s'%(self.image_counter, os.path.splitext(bn))) try: - with nested(open(img, 'wb'), closing(self.browser.open(feed.image_url))) as (fi, r): + with open(img, 'wb') as fi, closing(self.browser.open(feed.image_url)) as r: fi.write(r.read()) self.image_counter += 1 feed.image_url = img @@ -1346,7 +1346,7 @@ class BasicNewsRecipe(Recipe): with open(mpath, 'wb') as mfile: mfile.write(open(mu, 'rb').read()) else: - with nested(open(mpath, 'wb'), closing(self.browser.open(mu))) as (mfile, r): + with open(mpath, 'wb') as mfile, closing(self.browser.open(mu)) as r: mfile.write(r.read()) self.report_progress(1, _('Masthead image downloaded')) self.prepare_masthead_image(mpath, outfile) @@ -1564,7 +1564,7 @@ class BasicNewsRecipe(Recipe): opf.create_spine(entries) opf.set_toc(toc) - with nested(open(opf_path, 'wb'), open(ncx_path, 'wb')) as (opf_file, ncx_file): + with open(opf_path, 'wb') as opf_file, open(ncx_path, 'wb') as ncx_file: opf.render(opf_file, ncx_file) def article_downloaded(self, request, result): From bfcb301fe3da0cef93f61d3785567e27cc0597ad Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 25 Mar 2019 12:07:48 -0400 Subject: [PATCH 03/19] python3: add httplib/http.client polyglot wrapper --- src/calibre/db/cli/main.py | 8 ++-- src/calibre/gui2/icon_theme.py | 7 +-- src/calibre/srv/auth.py | 21 +++++---- src/calibre/srv/errors.py | 14 +++--- src/calibre/srv/http_request.py | 43 ++++++++--------- src/calibre/srv/http_response.py | 41 +++++++++-------- src/calibre/srv/routes.py | 5 +- src/calibre/srv/tests/ajax.py | 20 ++++---- src/calibre/srv/tests/auth.py | 53 ++++++++++----------- src/calibre/srv/tests/base.py | 5 +- src/calibre/srv/tests/content.py | 43 ++++++++--------- src/calibre/srv/tests/http.py | 79 ++++++++++++++++---------------- src/calibre/srv/tests/loop.py | 13 +++--- src/calibre/srv/web_socket.py | 7 +-- src/calibre/utils/browser.py | 6 ++- src/calibre/utils/https.py | 21 ++++----- src/calibre/web/fetch/simple.py | 2 +- 17 files changed, 198 insertions(+), 190 deletions(-) diff --git a/src/calibre/db/cli/main.py b/src/calibre/db/cli/main.py index 9833d919d4..a80da22180 100644 --- a/src/calibre/db/cli/main.py +++ b/src/calibre/db/cli/main.py @@ -4,7 +4,6 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import httplib import json import os import sys @@ -17,6 +16,7 @@ from calibre.utils.config import OptionParser, prefs from calibre.utils.localization import localize_user_manual_link from calibre.utils.lock import singleinstance from calibre.utils.serialize import MSGPACK_MIME +from polyglot import http_client from polyglot.urllib import urlencode, urlparse, urlunparse COMMANDS = ( @@ -191,13 +191,13 @@ class DBCtx(object): return m.implementation(self.db.new_api, None, *args) def interpret_http_error(self, err): - if err.code == httplib.UNAUTHORIZED: + if err.code == http_client.UNAUTHORIZED: if self.has_credentials: raise SystemExit('The username/password combination is incorrect') raise SystemExit('A username and password is required to access this server') - if err.code == httplib.FORBIDDEN: + if err.code == http_client.FORBIDDEN: raise SystemExit(err.reason) - if err.code == httplib.NOT_FOUND: + if err.code == http_client.NOT_FOUND: raise SystemExit(err.reason) def remote_run(self, name, m, *args): diff --git a/src/calibre/gui2/icon_theme.py b/src/calibre/gui2/icon_theme.py index e4c48f9562..e5d1c3a0a0 100644 --- a/src/calibre/gui2/icon_theme.py +++ b/src/calibre/gui2/icon_theme.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import os, errno, json, importlib, math, httplib, bz2, shutil, sys +import os, errno, json, importlib, math, bz2, shutil, sys from itertools import count from io import BytesIO from threading import Thread, Event @@ -35,8 +35,9 @@ from calibre.utils.img import image_from_data, Canvas, optimize_png, optimize_jp from calibre.utils.zipfile import ZipFile, ZIP_STORED from calibre.utils.filenames import atomic_rename from lzma.xz import compress, decompress -from polyglot.queue import Queue, Empty from polyglot.builtins import iteritems, map, range, reraise +from polyglot import http_client +from polyglot.queue import Queue, Empty IMAGE_EXTENSIONS = {'png', 'jpg', 'jpeg'} THEME_COVER = 'icon-theme-cover.jpg' @@ -439,7 +440,7 @@ def download_cover(cover_url, etag=None, cached=b''): etag = response.getheader('ETag', None) or None return cached, etag except HTTPError as e: - if etag and e.code == httplib.NOT_MODIFIED: + if etag and e.code == http_client.NOT_MODIFIED: return cached, etag raise diff --git a/src/calibre/srv/auth.py b/src/calibre/srv/auth.py index 51f884e28f..84fd260219 100644 --- a/src/calibre/srv/auth.py +++ b/src/calibre/srv/auth.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import binascii, os, random, struct, base64, httplib +import binascii, os, random, struct, base64 from collections import OrderedDict from hashlib import md5, sha256 from itertools import permutations @@ -16,6 +16,7 @@ from calibre.srv.errors import HTTPAuthRequired, HTTPSimpleResponse, HTTPForbidd from calibre.srv.http_request import parse_uri from calibre.srv.utils import parse_http_dict, encode_path from calibre.utils.monotonic import monotonic +from polyglot import http_client MAX_AGE_SECONDS = 3600 nonce_counter, nonce_counter_lock = 0, Lock() @@ -133,19 +134,19 @@ class DigestAuth(object): # {{{ self.nonce_count = data.get('nc') if self.algorithm not in self.valid_algorithms: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported digest algorithm') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported digest algorithm') if not (self.username and self.realm and self.nonce and self.uri and self.response): - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Digest algorithm required fields missing') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Digest algorithm required fields missing') if self.qop: if self.qop not in self.valid_qops: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported digest qop') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported digest qop') if not (self.cnonce and self.nonce_count): - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'qop present, but cnonce and nonce_count absent') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'qop present, but cnonce and nonce_count absent') else: if self.cnonce or self.nonce_count: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'qop missing') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'qop missing') def H(self, val): return md5_hex(val) @@ -201,7 +202,7 @@ class DigestAuth(object): # {{{ if log is not None: log.warn('Authorization URI mismatch: %s != %s from client: %s' % ( data.path, path, data.remote_addr)) - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The uri in the Request Line and the Authorization header do not match') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The uri in the Request Line and the Authorization header do not match') return self.response is not None and data.path == path and self.request_digest(pw, data) == self.response # }}} @@ -290,16 +291,16 @@ class AuthController(object): try: un, pw = base64_decode(rest.strip()).partition(':')[::2] except ValueError: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The username or password contained non-UTF8 encoded characters') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The username or password contained non-UTF8 encoded characters') if not un or not pw: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The username or password was empty') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The username or password was empty') if self.check(un, pw): data.username = un return log_msg = 'Failed login attempt from: %s' % data.remote_addr self.ban_list.failed(ban_key) else: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported authentication method') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported authentication method') if self.prefer_basic_auth: raise HTTPAuthRequired('Basic realm="%s"' % self.realm, log=log_msg) diff --git a/src/calibre/srv/errors.py b/src/calibre/srv/errors.py index db251b34cf..02c7c4f65e 100644 --- a/src/calibre/srv/errors.py +++ b/src/calibre/srv/errors.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib +from polyglot import http_client class JobQueueFull(Exception): @@ -30,38 +30,38 @@ class HTTPSimpleResponse(Exception): class HTTPRedirect(HTTPSimpleResponse): - def __init__(self, location, http_code=httplib.MOVED_PERMANENTLY, http_message='', close_connection=False): + def __init__(self, location, http_code=http_client.MOVED_PERMANENTLY, http_message='', close_connection=False): HTTPSimpleResponse.__init__(self, http_code, http_message, close_connection, location) class HTTPNotFound(HTTPSimpleResponse): def __init__(self, http_message='', close_connection=False): - HTTPSimpleResponse.__init__(self, httplib.NOT_FOUND, http_message, close_connection) + HTTPSimpleResponse.__init__(self, http_client.NOT_FOUND, http_message, close_connection) class HTTPAuthRequired(HTTPSimpleResponse): def __init__(self, payload, log=None): - HTTPSimpleResponse.__init__(self, httplib.UNAUTHORIZED, authenticate=payload, log=log) + HTTPSimpleResponse.__init__(self, http_client.UNAUTHORIZED, authenticate=payload, log=log) class HTTPBadRequest(HTTPSimpleResponse): def __init__(self, message, close_connection=False): - HTTPSimpleResponse.__init__(self, httplib.BAD_REQUEST, message, close_connection) + HTTPSimpleResponse.__init__(self, http_client.BAD_REQUEST, message, close_connection) class HTTPForbidden(HTTPSimpleResponse): def __init__(self, http_message='', close_connection=True, log=None): - HTTPSimpleResponse.__init__(self, httplib.FORBIDDEN, http_message, close_connection, log=log) + HTTPSimpleResponse.__init__(self, http_client.FORBIDDEN, http_message, close_connection, log=log) class HTTPInternalServerError(HTTPSimpleResponse): def __init__(self, http_message='', close_connection=True, log=None): - HTTPSimpleResponse.__init__(self, httplib.INTERNAL_SERVER_ERROR, http_message, close_connection, log=log) + HTTPSimpleResponse.__init__(self, http_client.INTERNAL_SERVER_ERROR, http_message, close_connection, log=log) class BookNotFound(HTTPNotFound): diff --git a/src/calibre/srv/http_request.py b/src/calibre/srv/http_request.py index 384fe5a04f..74ac506790 100644 --- a/src/calibre/srv/http_request.py +++ b/src/calibre/srv/http_request.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import re, httplib, repr as reprlib +import re, repr as reprlib from io import BytesIO, DEFAULT_BUFFER_SIZE from calibre import as_unicode, force_unicode @@ -14,6 +14,7 @@ from calibre.ptempfile import SpooledTemporaryFile from calibre.srv.errors import HTTPSimpleResponse from calibre.srv.loop import Connection, READ, WRITE from calibre.srv.utils import MultiDict, HTTP1, HTTP11, Accumulator +from polyglot import http_client from polyglot.urllib import unquote protocol_map = {(1, 0):HTTP1, (1, 1):HTTP11} @@ -68,29 +69,29 @@ def parse_request_uri(uri): def parse_uri(uri, parse_query=True): scheme, authority, path = parse_request_uri(uri) if path is None: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, "No path component") + raise HTTPSimpleResponse(http_client.BAD_REQUEST, "No path component") if b'#' in path: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, "Illegal #fragment in Request-URI.") + raise HTTPSimpleResponse(http_client.BAD_REQUEST, "Illegal #fragment in Request-URI.") if scheme: try: scheme = scheme.decode('ascii') except ValueError: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Un-decodeable scheme') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Un-decodeable scheme') path, qs = path.partition(b'?')[::2] if parse_query: try: query = MultiDict.create_from_query_string(qs) except Exception: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unparseable query string') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unparseable query string') else: query = None try: path = '%2F'.join(unquote(x).decode('utf-8') for x in quoted_slash.split(path)) except ValueError as e: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, as_unicode(e)) + raise HTTPSimpleResponse(http_client.BAD_REQUEST, as_unicode(e)) path = tuple(filter(None, (x.replace('%2F', '/') for x in path.split('/')))) return scheme, path, query @@ -233,7 +234,7 @@ class HTTPRequest(Connection): if line.endswith(b'\n'): line = buf.getvalue() if not line.endswith(b'\r\n'): - self.simple_response(httplib.BAD_REQUEST, 'HTTP requires CRLF line terminators') + self.simple_response(http_client.BAD_REQUEST, 'HTTP requires CRLF line terminators') return return line if not line: @@ -247,7 +248,7 @@ class HTTPRequest(Connection): self.forwarded_for = None self.path = self.query = None self.close_after_response = False - self.header_line_too_long_error_code = httplib.REQUEST_URI_TOO_LONG + self.header_line_too_long_error_code = http_client.REQUEST_URI_TOO_LONG self.response_started = False self.set_state(READ, self.parse_request_line, Accumulator(), first=True) @@ -260,28 +261,28 @@ class HTTPRequest(Connection): # Ignore a single leading empty line, as per RFC 2616 sec 4.1 if first: return self.set_state(READ, self.parse_request_line, Accumulator()) - return self.simple_response(httplib.BAD_REQUEST, 'Multiple leading empty lines not allowed') + return self.simple_response(http_client.BAD_REQUEST, 'Multiple leading empty lines not allowed') try: method, uri, req_protocol = line.strip().split(b' ', 2) rp = int(req_protocol[5]), int(req_protocol[7]) self.method = method.decode('ascii').upper() except Exception: - return self.simple_response(httplib.BAD_REQUEST, "Malformed Request-Line") + return self.simple_response(http_client.BAD_REQUEST, "Malformed Request-Line") if self.method not in HTTP_METHODS: - return self.simple_response(httplib.BAD_REQUEST, "Unknown HTTP method") + return self.simple_response(http_client.BAD_REQUEST, "Unknown HTTP method") try: self.request_protocol = protocol_map[rp] except KeyError: - return self.simple_response(httplib.HTTP_VERSION_NOT_SUPPORTED) + return self.simple_response(http_client.HTTP_VERSION_NOT_SUPPORTED) self.response_protocol = protocol_map[min((1, 1), rp)] try: self.scheme, self.path, self.query = parse_uri(uri) except HTTPSimpleResponse as e: return self.simple_response(e.http_code, e.message, close_after_response=False) - self.header_line_too_long_error_code = httplib.REQUEST_ENTITY_TOO_LARGE + self.header_line_too_long_error_code = http_client.REQUEST_ENTITY_TOO_LARGE self.set_state(READ, self.parse_header_line, HTTPHeaderParser(), Accumulator()) # }}} @@ -299,7 +300,7 @@ class HTTPRequest(Connection): try: parser(line) except ValueError: - self.simple_response(httplib.BAD_REQUEST, 'Failed to parse header line') + self.simple_response(http_client.BAD_REQUEST, 'Failed to parse header line') return if parser.finished: self.finalize_headers(parser.hdict) @@ -307,7 +308,7 @@ class HTTPRequest(Connection): def finalize_headers(self, inheaders): request_content_length = int(inheaders.get('Content-Length', 0)) if request_content_length > self.max_request_body_size: - return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE, + return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE, "The entity sent with the request exceeds the maximum " "allowed bytes (%d)." % self.max_request_body_size) # Persistent connection support @@ -334,7 +335,7 @@ class HTTPRequest(Connection): else: # Note that, even if we see "chunked", we must reject # if there is an extension we don't recognize. - return self.simple_response(httplib.NOT_IMPLEMENTED, "Unknown transfer encoding: %r" % enc) + return self.simple_response(http_client.NOT_IMPLEMENTED, "Unknown transfer encoding: %r" % enc) if inheaders.get("Expect", '').lower() == "100-continue": buf = BytesIO((HTTP11 + " 100 Continue\r\n\r\n").encode('ascii')) @@ -369,9 +370,9 @@ class HTTPRequest(Connection): try: chunk_size = int(line.strip(), 16) except Exception: - return self.simple_response(httplib.BAD_REQUEST, '%s is not a valid chunk size' % reprlib.repr(line.strip())) + return self.simple_response(http_client.BAD_REQUEST, '%s is not a valid chunk size' % reprlib.repr(line.strip())) if bytes_read[0] + chunk_size + 2 > self.max_request_body_size: - return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE, + return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE, 'Chunked request is larger than %d bytes' % self.max_request_body_size) if chunk_size == 0: self.set_state(READ, self.read_chunk_separator, inheaders, Accumulator(), buf, bytes_read, last=True) @@ -389,10 +390,10 @@ class HTTPRequest(Connection): if line is None: return if line != b'\r\n': - return self.simple_response(httplib.BAD_REQUEST, 'Chunk does not have trailing CRLF') + return self.simple_response(http_client.BAD_REQUEST, 'Chunk does not have trailing CRLF') bytes_read[0] += len(line) if bytes_read[0] > self.max_request_body_size: - return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE, + return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE, 'Chunked request is larger than %d bytes' % self.max_request_body_size) if last: self.prepare_response(inheaders, buf) @@ -402,7 +403,7 @@ class HTTPRequest(Connection): def handle_timeout(self): if self.response_started: return False - self.simple_response(httplib.REQUEST_TIMEOUT) + self.simple_response(http_client.REQUEST_TIMEOUT) return True def write(self, buf, end=None): diff --git a/src/calibre/srv/http_response.py b/src/calibre/srv/http_response.py index b64db6dab6..fc11fd0782 100644 --- a/src/calibre/srv/http_response.py +++ b/src/calibre/srv/http_response.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import os, httplib, hashlib, uuid, struct, repr as reprlib +import os, hashlib, uuid, struct, repr as reprlib from collections import namedtuple from io import BytesIO, DEFAULT_BUFFER_SIZE from itertools import chain, repeat @@ -26,6 +26,7 @@ from calibre.srv.utils import ( sort_q_values, get_translator_for_lang, Cookie, fast_now_strftime) from calibre.utils.speedups import ReadOnlyFileBuffer from calibre.utils.monotonic import monotonic +from polyglot import http_client Range = namedtuple('Range', 'start stop size') MULTIPART_SEPARATOR = uuid.uuid4().hex.decode('ascii') @@ -226,7 +227,7 @@ class RequestData(object): # {{{ self.remote_addr, self.remote_port, self.is_local_connection = remote_addr, remote_port, is_local_connection self.forwarded_for = forwarded_for self.opts = opts - self.status_code = httplib.OK + self.status_code = http_client.OK self.outcookie = Cookie() self.lang_code = self.gettext_func = self.ngettext_func = None self.set_translator(self.get_preferred_language()) @@ -402,16 +403,16 @@ class HTTPConnection(HTTPRequest): if self.response_protocol is HTTP1: # HTTP/1.0 has no 413/414/303 codes status_code = { - httplib.REQUEST_ENTITY_TOO_LARGE:httplib.BAD_REQUEST, - httplib.REQUEST_URI_TOO_LONG:httplib.BAD_REQUEST, - httplib.SEE_OTHER:httplib.FOUND + http_client.REQUEST_ENTITY_TOO_LARGE:http_client.BAD_REQUEST, + http_client.REQUEST_URI_TOO_LONG:http_client.BAD_REQUEST, + http_client.SEE_OTHER:http_client.FOUND }.get(status_code, status_code) self.close_after_response = close_after_response msg = msg.encode('utf-8') ct = 'http' if self.method == 'TRACE' else 'plain' buf = [ - '%s %d %s' % (self.response_protocol, status_code, httplib.responses[status_code]), + '%s %d %s' % (self.response_protocol, status_code, http_client.responses[status_code]), "Content-Length: %s" % len(msg), "Content-Type: text/%s; charset=UTF-8" % ct, "Date: " + http_date(), @@ -432,7 +433,7 @@ class HTTPConnection(HTTPRequest): def prepare_response(self, inheaders, request_body_file): if self.method == 'TRACE': msg = force_unicode(self.request_line, 'utf-8') + '\n' + inheaders.pretty() - return self.simple_response(httplib.OK, msg, close_after_response=False) + return self.simple_response(http_client.OK, msg, close_after_response=False) request_body_file.seek(0) outheaders = MultiDict() data = RequestData( @@ -449,28 +450,28 @@ class HTTPConnection(HTTPRequest): def send_range_not_satisfiable(self, content_length): buf = [ - '%s %d %s' % (self.response_protocol, httplib.REQUESTED_RANGE_NOT_SATISFIABLE, httplib.responses[httplib.REQUESTED_RANGE_NOT_SATISFIABLE]), + '%s %d %s' % (self.response_protocol, http_client.REQUESTED_RANGE_NOT_SATISFIABLE, http_client.responses[http_client.REQUESTED_RANGE_NOT_SATISFIABLE]), "Date: " + http_date(), "Content-Range: bytes */%d" % content_length, ] response_data = header_list_to_file(buf) - self.log_access(status_code=httplib.REQUESTED_RANGE_NOT_SATISFIABLE, response_size=response_data.sz) + self.log_access(status_code=http_client.REQUESTED_RANGE_NOT_SATISFIABLE, response_size=response_data.sz) self.response_ready(response_data) def send_not_modified(self, etag=None): buf = [ - '%s %d %s' % (self.response_protocol, httplib.NOT_MODIFIED, httplib.responses[httplib.NOT_MODIFIED]), + '%s %d %s' % (self.response_protocol, http_client.NOT_MODIFIED, http_client.responses[http_client.NOT_MODIFIED]), "Content-Length: 0", "Date: " + http_date(), ] if etag is not None: buf.append('ETag: ' + etag) response_data = header_list_to_file(buf) - self.log_access(status_code=httplib.NOT_MODIFIED, response_size=response_data.sz) + self.log_access(status_code=http_client.NOT_MODIFIED, response_size=response_data.sz) self.response_ready(response_data) def report_busy(self): - self.simple_response(httplib.SERVICE_UNAVAILABLE) + self.simple_response(http_client.SERVICE_UNAVAILABLE) def job_done(self, ok, result): if not ok: @@ -509,7 +510,7 @@ class HTTPConnection(HTTPRequest): if ct.startswith('text/') and 'charset=' not in ct: outheaders.set('Content-Type', ct + '; charset=UTF-8', replace_all=True) - buf = [HTTP11 + (' %d ' % data.status_code) + httplib.responses[data.status_code]] + buf = [HTTP11 + (' %d ' % data.status_code) + http_client.responses[data.status_code]] for header, value in sorted(iteritems(outheaders), key=itemgetter(0)): buf.append('%s: %s' % (header, value)) for morsel in itervalues(data.outcookie): @@ -530,7 +531,7 @@ class HTTPConnection(HTTPRequest): def log_access(self, status_code, response_size=None, username=None): if self.access_log is None: return - if not self.opts.log_not_found and status_code == httplib.NOT_FOUND: + if not self.opts.log_not_found and status_code == http_client.NOT_FOUND: return ff = self.forwarded_for if ff: @@ -623,7 +624,7 @@ class HTTPConnection(HTTPRequest): self.ready = ready def report_unhandled_exception(self, e, formatted_traceback): - self.simple_response(httplib.INTERNAL_SERVER_ERROR) + self.simple_response(http_client.INTERNAL_SERVER_ERROR) def finalize_output(self, output, request, is_http1): none_match = parse_if_none_match(request.inheaders.get('If-None-Match', '')) @@ -633,7 +634,7 @@ class HTTPConnection(HTTPRequest): if self.method in ('GET', 'HEAD'): self.send_not_modified(output.etag) else: - self.simple_response(httplib.PRECONDITION_FAILED) + self.simple_response(http_client.PRECONDITION_FAILED) return opts = self.opts @@ -660,10 +661,10 @@ class HTTPConnection(HTTPRequest): ct = outheaders.get('Content-Type', '').partition(';')[0] compressible = (not ct or ct.startswith('text/') or ct.startswith('image/svg') or ct.partition(';')[0] in COMPRESSIBLE_TYPES) - compressible = (compressible and request.status_code == httplib.OK and + compressible = (compressible and request.status_code == http_client.OK and (opts.compress_min_size > -1 and output.content_length >= opts.compress_min_size) and acceptable_encoding(request.inheaders.get('Accept-Encoding', '')) and not is_http1) - accept_ranges = (not compressible and output.accept_ranges is not None and request.status_code == httplib.OK and + accept_ranges = (not compressible and output.accept_ranges is not None and request.status_code == http_client.OK and not is_http1) ranges = get_ranges(request.inheaders.get('Range'), output.content_length) if output.accept_ranges and self.method in ('GET', 'HEAD') else None if_range = (request.inheaders.get('If-Range') or '').strip() @@ -680,7 +681,7 @@ class HTTPConnection(HTTPRequest): if self.method in ('GET', 'HEAD'): self.send_not_modified(output.etag) else: - self.simple_response(httplib.PRECONDITION_FAILED) + self.simple_response(http_client.PRECONDITION_FAILED) return output.ranges = None @@ -712,7 +713,7 @@ class HTTPConnection(HTTPRequest): outheaders.set('Content-Length', '%d' % size, replace_all=True) outheaders.set('Content-Type', 'multipart/byteranges; boundary=' + MULTIPART_SEPARATOR, replace_all=True) output.ranges = zip_longest(ranges, range_parts) - request.status_code = httplib.PARTIAL_CONTENT + request.status_code = http_client.PARTIAL_CONTENT return output diff --git a/src/calibre/srv/routes.py b/src/calibre/srv/routes.py index 9be1cde382..621b412c6c 100644 --- a/src/calibre/srv/routes.py +++ b/src/calibre/srv/routes.py @@ -6,13 +6,14 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, sys, inspect, re, time, numbers, json as jsonlib, textwrap +import sys, inspect, re, time, numbers, json as jsonlib, textwrap from operator import attrgetter from calibre.srv.errors import HTTPSimpleResponse, HTTPNotFound, RouteError from calibre.srv.utils import http_date from calibre.utils.serialize import msgpack_dumps, json_dumps, MSGPACK_MIME from polyglot.builtins import iteritems, itervalues, unicode_type, range, zip +from polyglot import http_client from polyglot.urllib import quote as urlquote default_methods = frozenset(('HEAD', 'GET')) @@ -297,7 +298,7 @@ class Router(object): def dispatch(self, data): endpoint_, args = self.find_route(data.path) if data.method not in endpoint_.methods: - raise HTTPSimpleResponse(httplib.METHOD_NOT_ALLOWED) + raise HTTPSimpleResponse(http_client.METHOD_NOT_ALLOWED) self.read_cookies(data) diff --git a/src/calibre/srv/tests/ajax.py b/src/calibre/srv/tests/ajax.py index 7707bd6d2b..0fade7e9d0 100644 --- a/src/calibre/srv/tests/ajax.py +++ b/src/calibre/srv/tests/ajax.py @@ -6,13 +6,13 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, zlib, json, base64, os +import zlib, json, base64, os from io import BytesIO from functools import partial -from httplib import OK, NOT_FOUND, FORBIDDEN from calibre.ebooks.metadata.meta import get_metadata from calibre.srv.tests.base import LibraryBaseTest +from polyglot.http_client import OK, NOT_FOUND, FORBIDDEN from polyglot.urllib import urlencode, quote @@ -22,7 +22,7 @@ def make_request(conn, url, headers={}, prefix='/ajax', username=None, password= conn.request(method, prefix + url, headers=headers, body=data) r = conn.getresponse() data = r.read() - if r.status == httplib.OK and data and data[0] in b'{[': + if r.status == OK and data and data[0] in b'{[': data = json.loads(data) return r, data @@ -37,10 +37,10 @@ class ContentTest(LibraryBaseTest): request = partial(make_request, conn, prefix='/ajax/book') r, data = request('/x') - self.ae(r.status, httplib.NOT_FOUND) + self.ae(r.status, NOT_FOUND) r, onedata = request('/1') - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) self.ae(request('/1/' + db.server_library_id)[1], onedata) self.ae(request('/%s?id_is_uuid=true' % db.field_for('uuid', 1))[1], onedata) @@ -63,22 +63,22 @@ class ContentTest(LibraryBaseTest): request = partial(make_request, conn) r, data = request('/categories') - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) r, xdata = request('/categories/' + db.server_library_id) - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) self.ae(data, xdata) names = {x['name']:x['url'] for x in data} for q in ('Newest', 'All books', 'Tags', 'Series', 'Authors', 'Enum', 'Composite Tags'): self.assertIn(q, names) r, data = request(names['Tags'], prefix='') - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) names = {x['name']:x['url'] for x in data['items']} self.ae(set(names), set('Tag One,Tag Two,News'.split(','))) r, data = request(names['Tag One'], prefix='') - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) self.ae(set(data['book_ids']), {1, 2}) r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"'})) - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) self.ae(set(data['book_ids']), {1, 2}) r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"', 'vl':'1'})) self.ae(set(data['book_ids']), {2}) diff --git a/src/calibre/srv/tests/auth.py b/src/calibre/srv/tests/auth.py index ea08759d04..9407516cf0 100644 --- a/src/calibre/srv/tests/auth.py +++ b/src/calibre/srv/tests/auth.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, base64, subprocess, os, cookielib, time +import base64, subprocess, os, cookielib, time from collections import namedtuple try: from distutils.spawn import find_executable @@ -18,6 +18,7 @@ from calibre.srv.errors import HTTPForbidden from calibre.srv.tests.base import BaseTest, TestServer from calibre.srv.routes import endpoint, Router from polyglot.builtins import iteritems, itervalues +from polyglot import http_client from polyglot.urllib import (build_opener, HTTPBasicAuthHandler, HTTPCookieProcessor, HTTPDigestAuthHandler, HTTPError) @@ -91,18 +92,18 @@ class TestAuth(BaseTest): conn = server.connect() conn.request('GET', '/open') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'open') conn.request('GET', '/closed') r = conn.getresponse() - self.ae(r.status, httplib.UNAUTHORIZED) + self.ae(r.status, http_client.UNAUTHORIZED) self.ae(r.getheader('WWW-Authenticate'), b'Basic realm="%s"' % bytes(REALM)) self.assertFalse(r.read()) conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + base64.standard_b64encode(b'testuser:testpw')}) r = conn.getresponse() self.ae(r.read(), b'closed') - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(b'closed', urlopen(server, method='basic').read()) self.ae(b'closed', urlopen(server, un='!@#$%^&*()-=_+', pw='!@#$%^&*()-=_+', method='basic').read()) @@ -113,14 +114,14 @@ class TestAuth(BaseTest): warnings = [] server.loop.log.warn = lambda *args, **kwargs: warnings.append(' '.join(args)) - self.ae((httplib.OK, b'closed'), request()) - self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y')) - self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('', '')) + self.ae((http_client.OK, b'closed'), request()) + self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y')) + self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request('', '')) self.ae(1, len(warnings)) - self.ae((httplib.UNAUTHORIZED, b''), request('testuser', 'y')) - self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('testuser', '')) - self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('')) - self.ae((httplib.UNAUTHORIZED, b''), request('asf', 'testpw')) + self.ae((http_client.UNAUTHORIZED, b''), request('testuser', 'y')) + self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request('testuser', '')) + self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request('')) + self.ae((http_client.UNAUTHORIZED, b''), request('asf', 'testpw')) # }}} def test_library_restrictions(self): # {{{ @@ -169,7 +170,7 @@ class TestAuth(BaseTest): with TestServer(r.dispatch) as server: r.auth_controller.log = server.log - def test(conn, path, headers={}, status=httplib.OK, body=b'', request_body=b''): + def test(conn, path, headers={}, status=http_client.OK, body=b'', request_body=b''): conn.request('GET', path, request_body, headers) r = conn.getresponse() self.ae(r.status, status) @@ -177,9 +178,9 @@ class TestAuth(BaseTest): return {normalize_header_name(k):v for k, v in r.getheaders()} conn = server.connect() test(conn, '/open', body=b'open') - auth = parse_http_dict(test(conn, '/closed', status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) + auth = parse_http_dict(test(conn, '/closed', status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) nonce = auth['nonce'] - auth = parse_http_dict(test(conn, '/closed', status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) + auth = parse_http_dict(test(conn, '/closed', status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) self.assertNotEqual(nonce, auth['nonce'], 'nonce was re-used') self.ae(auth[b'realm'], bytes(REALM)), self.ae(auth[b'algorithm'], b'MD5'), self.ae(auth[b'qop'], b'auth') self.assertNotIn('stale', auth) @@ -199,14 +200,14 @@ class TestAuth(BaseTest): # Check stale nonces orig, r.auth_controller.max_age_seconds = r.auth_controller.max_age_seconds, -1 auth = parse_http_dict(test(conn, '/closed', headers={ - 'Authorization':digest(**args)},status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) + 'Authorization':digest(**args)},status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) self.assertIn('stale', auth) r.auth_controller.max_age_seconds = orig ok_test(conn, digest(**args)) def fail_test(conn, modify, **kw): kw['body'] = kw.get('body', b'') - kw['status'] = kw.get('status', httplib.UNAUTHORIZED) + kw['status'] = kw.get('status', http_client.UNAUTHORIZED) args['modify'] = modify return test(conn, '/closed', headers={'Authorization':digest(**args)}, **kw) @@ -258,13 +259,13 @@ class TestAuth(BaseTest): warnings = [] server.loop.log.warn = lambda *args, **kwargs: warnings.append(' '.join(args)) - self.ae((httplib.OK, b'closed'), request()) - self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y')) - self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y')) - self.ae(httplib.FORBIDDEN, request('x', 'y')[0]) - self.ae(httplib.FORBIDDEN, request()[0]) + self.ae((http_client.OK, b'closed'), request()) + self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y')) + self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y')) + self.ae(http_client.FORBIDDEN, request('x', 'y')[0]) + self.ae(http_client.FORBIDDEN, request()[0]) time.sleep(ban_for * 60 + 0.01) - self.ae((httplib.OK, b'closed'), request()) + self.ae((http_client.OK, b'closed'), request()) # }}} def test_android_auth_workaround(self): # {{{ @@ -277,7 +278,7 @@ class TestAuth(BaseTest): # First check that unauth access fails conn.request('GET', '/android') r = conn.getresponse() - self.ae(r.status, httplib.UNAUTHORIZED) + self.ae(r.status, http_client.UNAUTHORIZED) auth_handler = HTTPDigestAuthHandler() url = 'http://localhost:%d%s' % (server.address[1], '/android') @@ -285,20 +286,20 @@ class TestAuth(BaseTest): cj = cookielib.CookieJar() cookie_handler = HTTPCookieProcessor(cj) r = build_opener(auth_handler, cookie_handler).open(url) - self.ae(r.getcode(), httplib.OK) + self.ae(r.getcode(), http_client.OK) cookies = tuple(cj) self.ae(len(cookies), 1) cookie = cookies[0] self.assertIn(b':', cookie.value) self.ae(cookie.path, b'/android') r = build_opener(cookie_handler).open(url) - self.ae(r.getcode(), httplib.OK) + self.ae(r.getcode(), http_client.OK) self.ae(r.read(), b'android') # Test that a replay attack against a different URL does not work try: build_opener(cookie_handler).open(url+'2') assert ('Replay attack succeeded') except HTTPError as e: - self.ae(e.code, httplib.UNAUTHORIZED) + self.ae(e.code, http_client.UNAUTHORIZED) # }}} diff --git a/src/calibre/srv/tests/base.py b/src/calibre/srv/tests/base.py index e62dcc605d..165aa2e0c5 100644 --- a/src/calibre/srv/tests/base.py +++ b/src/calibre/srv/tests/base.py @@ -7,12 +7,13 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import unittest, time, httplib, shutil, gc, tempfile, atexit, os +import unittest, time, shutil, gc, tempfile, atexit, os from io import BytesIO from functools import partial from threading import Thread from calibre.srv.utils import ServerLog +from polyglot import http_client rmtree = partial(shutil.rmtree, ignore_errors=True) @@ -120,7 +121,7 @@ class TestServer(Thread): timeout = self.loop.opts.timeout if interface is None: interface = self.address[0] - return httplib.HTTPConnection(interface, self.address[1], strict=True, timeout=timeout) + return http_client.HTTPConnection(interface, self.address[1], strict=True, timeout=timeout) def change_handler(self, handler): from calibre.srv.http_response import create_http_handler diff --git a/src/calibre/srv/tests/content.py b/src/calibre/srv/tests/content.py index e7b1950eec..fa9a12ccc7 100644 --- a/src/calibre/srv/tests/content.py +++ b/src/calibre/srv/tests/content.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, zlib, json, binascii, time, os +import zlib, json, binascii, time, os from io import BytesIO from calibre.ebooks.metadata.epub import get_metadata @@ -14,6 +14,7 @@ from calibre.ebooks.metadata.opf2 import OPF from calibre.srv.tests.base import LibraryBaseTest from calibre.utils.imghdr import identify from calibre.utils.shared_file import share_open +from polyglot import http_client def setUpModule(): @@ -32,7 +33,7 @@ class ContentTest(LibraryBaseTest): def missing(url, body=b''): conn.request('GET', url) r = conn.getresponse() - self.ae(r.status, httplib.NOT_FOUND) + self.ae(r.status, http_client.NOT_FOUND) self.ae(r.read(), body) for prefix in ('static', 'icon'): @@ -51,7 +52,7 @@ class ContentTest(LibraryBaseTest): raw = P(src, data=True) conn.request('GET', url) r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) data = r.read() if sz is None: self.ae(data, raw) @@ -60,7 +61,7 @@ class ContentTest(LibraryBaseTest): test_response(r) conn.request('GET', url, headers={'If-None-Match':r.getheader('ETag')}) r = conn.getresponse() - self.ae(r.status, httplib.NOT_MODIFIED) + self.ae(r.status, http_client.NOT_MODIFIED) self.ae(b'', r.read()) test('content-server/empty.html', '/static/empty.html') @@ -85,7 +86,7 @@ class ContentTest(LibraryBaseTest): # Test various invalid parameters def bad(*args): r, data = get(*args) - self.ae(r.status, httplib.NOT_FOUND) + self.ae(r.status, http_client.NOT_FOUND) bad('xxx', 1) bad('fmt1', 10) bad('fmt1', 1, 'zzzz') @@ -103,7 +104,7 @@ class ContentTest(LibraryBaseTest): # Test fetching of format with metadata update raw = P('quick_start/eng.epub', data=True) r, data = get('epub', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) etag = r.getheader('ETag') self.assertIsNotNone(etag) self.ae(r.getheader('Used-Cache'), 'no') @@ -145,39 +146,39 @@ class ContentTest(LibraryBaseTest): os.utime(cpath, (t, t)) r, data = get('cover', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(data, db.cover(1)) self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Content-Type'), 'image/jpeg') r, data = get('cover', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(data, db.cover(1)) self.ae(r.getheader('Used-Cache'), 'yes') r, data = get('cover', 3) - self.ae(r.status, httplib.OK) # Auto generated cover + self.ae(r.status, http_client.OK) # Auto generated cover r, data = get('thumb', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(identify(data), ('jpeg', 60, 60)) self.ae(r.getheader('Used-Cache'), 'no') r, data = get('thumb', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.getheader('Used-Cache'), 'yes') r, data = get('thumb', 1, q='sz=100') - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(identify(data), ('jpeg', 100, 100)) self.ae(r.getheader('Used-Cache'), 'no') r, data = get('thumb', 1, q='sz=100x100') - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.getheader('Used-Cache'), 'yes') change_cover(1, 1) r, data = get('thumb', 1, q='sz=100') - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(identify(data), ('jpeg', 100, 100)) self.ae(r.getheader('Used-Cache'), 'no') # Test file sharing in cache r, data = get('cover', 2) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(data, db.cover(2)) self.ae(r.getheader('Used-Cache'), 'no') path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8') @@ -185,7 +186,7 @@ class ContentTest(LibraryBaseTest): # Now force an update change_cover(1) r, data = get('cover', 2) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(data, db.cover(2)) self.ae(r.getheader('Used-Cache'), 'no') path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8') @@ -193,7 +194,7 @@ class ContentTest(LibraryBaseTest): # Do it again change_cover(2) r, data = get('cover', 2) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(data, db.cover(2)) self.ae(r.getheader('Used-Cache'), 'no') self.ae(f.read(), fdata) @@ -201,7 +202,7 @@ class ContentTest(LibraryBaseTest): # Test serving of metadata as opf r, data = get('opf', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.getheader('Content-Type'), 'application/oebps-package+xml; charset=UTF-8') self.assertIsNotNone(r.getheader('Last-Modified')) opf = OPF(BytesIO(data), populate_spine=False, try_to_guess_cover=False) @@ -209,17 +210,17 @@ class ContentTest(LibraryBaseTest): self.ae(db.field_for('authors', 1), tuple(opf.authors)) conn.request('GET', '/get/opf/1', headers={'Accept-Encoding':'gzip'}) r = conn.getresponse() - self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') + self.ae(r.status, http_client.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') raw = r.read() self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data) # Test serving metadata as json r, data = get('json', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(db.field_for('title', 1), json.loads(data)['title']) conn.request('GET', '/get/json/1', headers={'Accept-Encoding':'gzip'}) r = conn.getresponse() - self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') + self.ae(r.status, http_client.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') raw = r.read() self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data) diff --git a/src/calibre/srv/tests/http.py b/src/calibre/srv/tests/http.py index f42279795e..05b487b2a3 100644 --- a/src/calibre/srv/tests/http.py +++ b/src/calibre/srv/tests/http.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, hashlib, zlib, string, time, os +import hashlib, zlib, string, time, os from io import BytesIO from tempfile import NamedTemporaryFile @@ -15,6 +15,7 @@ from calibre.srv.tests.base import BaseTest, TestServer from calibre.srv.utils import eintr_retry_call from calibre.utils.monotonic import monotonic from polyglot.builtins import iteritems, range +from polyglot import http_client is_ci = os.environ.get('CI', '').lower() == 'true' @@ -94,7 +95,7 @@ class TestHTTP(BaseTest): def test(al, q): conn.request('GET', '/', headers={'Accept-Language': al}) r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) q += get_translator(q)[-1].ugettext('Unknown') self.ae(r.read(), q) @@ -136,7 +137,7 @@ class TestHTTP(BaseTest): def raw_send(conn, raw): conn.send(raw) - conn._HTTPConnection__state = httplib._CS_REQ_SENT + conn._HTTPConnection__state = http_client._CS_REQ_SENT return conn.getresponse() base_timeout = 0.5 if is_ci else 0.1 @@ -144,31 +145,31 @@ class TestHTTP(BaseTest): with TestServer(handler, timeout=base_timeout, max_header_line_size=100./1024, max_request_body_size=100./(1024*1024)) as server: conn = server.connect() r = raw_send(conn, b'hello\n') - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) self.ae(r.read(), b'HTTP requires CRLF line terminators') r = raw_send(conn, b'\r\nGET /index.html HTTP/1.1\r\n\r\n') - self.ae(r.status, httplib.NOT_FOUND), self.ae(r.read(), b'Requested resource not found') + self.ae(r.status, http_client.NOT_FOUND), self.ae(r.read(), b'Requested resource not found') r = raw_send(conn, b'\r\n\r\nGET /index.html HTTP/1.1\r\n\r\n') - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) self.ae(r.read(), b'Multiple leading empty lines not allowed') r = raw_send(conn, b'hello world\r\n') - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) self.ae(r.read(), b'Malformed Request-Line') r = raw_send(conn, b'x' * 200) - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) self.ae(r.read(), b'') r = raw_send(conn, b'XXX /index.html HTTP/1.1\r\n\r\n') - self.ae(r.status, httplib.BAD_REQUEST), self.ae(r.read(), b'Unknown HTTP method') + self.ae(r.status, http_client.BAD_REQUEST), self.ae(r.read(), b'Unknown HTTP method') # Test 404 conn.request('HEAD', '/moose') r = conn.getresponse() - self.ae(r.status, httplib.NOT_FOUND) + self.ae(r.status, http_client.NOT_FOUND) self.assertIsNotNone(r.getheader('Date', None)) self.ae(r.getheader('Content-Length'), str(len(body))) self.ae(r.getheader('Content-Type'), 'text/plain; charset=UTF-8') @@ -176,7 +177,7 @@ class TestHTTP(BaseTest): self.ae(r.read(), '') conn.request('GET', '/choose') r = conn.getresponse() - self.ae(r.status, httplib.NOT_FOUND) + self.ae(r.status, http_client.NOT_FOUND) self.ae(r.read(), b'Requested resource not found') # Test 500 @@ -186,7 +187,7 @@ class TestHTTP(BaseTest): conn = server.connect() conn.request('GET', '/test/') r = conn.getresponse() - self.ae(r.status, httplib.INTERNAL_SERVER_ERROR) + self.ae(r.status, http_client.INTERNAL_SERVER_ERROR) server.loop.log.filter_level = orig # Test 301 @@ -196,7 +197,7 @@ class TestHTTP(BaseTest): conn = server.connect() conn.request('GET', '/') r = conn.getresponse() - self.ae(r.status, httplib.MOVED_PERMANENTLY) + self.ae(r.status, http_client.MOVED_PERMANENTLY) self.ae(r.getheader('Location'), '/somewhere-else') self.ae('', r.read()) @@ -206,26 +207,26 @@ class TestHTTP(BaseTest): # Test simple GET conn.request('GET', '/test/') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'test') # Test TRACE lines = ['TRACE /xxx HTTP/1.1', 'Test: value', 'Xyz: abc, def', '', ''] r = raw_send(conn, ('\r\n'.join(lines)).encode('ascii')) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read().decode('utf-8'), '\n'.join(lines[:-2])) # Test POST with simple body conn.request('POST', '/test', 'body') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'testbody') # Test POST with chunked transfer encoding conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.send(b'4\r\nbody\r\na\r\n1234567890\r\n0\r\n\r\n') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'testbody1234567890') # Test various incorrect input @@ -233,39 +234,39 @@ class TestHTTP(BaseTest): conn.request('GET', '/test' + ('a' * 200)) r = conn.getresponse() - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) conn = server.connect() conn.request('GET', '/test', ('a' * 200)) r = conn.getresponse() - self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE) + self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE) conn = server.connect() conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.send(b'x\r\nbody\r\n0\r\n\r\n') r = conn.getresponse() - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) self.assertIn(b'not a valid chunk size', r.read()) conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.send(b'4\r\nbody\r\n200\r\n\r\n') r = conn.getresponse() - self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE) + self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE) conn.request('POST', '/test', body='a'*200) r = conn.getresponse() - self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE) + self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE) conn = server.connect() conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.send(b'3\r\nbody\r\n0\r\n\r\n') r = conn.getresponse() - self.ae(r.status, httplib.BAD_REQUEST), self.ae(r.read(), b'Chunk does not have trailing CRLF') + self.ae(r.status, http_client.BAD_REQUEST), self.ae(r.read(), b'Chunk does not have trailing CRLF') conn = server.connect(timeout=base_timeout * 5) conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.send(b'30\r\nbody\r\n0\r\n\r\n') r = conn.getresponse() - self.ae(r.status, httplib.REQUEST_TIMEOUT) + self.ae(r.status, http_client.REQUEST_TIMEOUT) self.assertIn(b'', r.read()) server.log.filter_level = orig_level @@ -273,14 +274,14 @@ class TestHTTP(BaseTest): # Test pipelining responses = [] for i in range(10): - conn._HTTPConnection__state = httplib._CS_IDLE + conn._HTTPConnection__state = http_client._CS_IDLE conn.request('GET', '/%d'%i) responses.append(conn.response_class(conn.sock, strict=conn.strict, method=conn._method)) for i in range(10): r = responses[i] r.begin() self.ae(r.read(), ('%d' % i).encode('ascii')) - conn._HTTPConnection__state = httplib._CS_IDLE + conn._HTTPConnection__state = http_client._CS_IDLE # Test closing server.loop.opts.timeout = 10 # ensure socket is not closed because of timeout @@ -319,12 +320,12 @@ class TestHTTP(BaseTest): conn = server.connect() conn.request('GET', '/an_etagged_path') r = conn.getresponse() - self.ae(r.status, httplib.OK), self.ae(r.read(), b'an_etagged_path') + self.ae(r.status, http_client.OK), self.ae(r.read(), b'an_etagged_path') etag = r.getheader('ETag') self.ae(etag, '"%s"' % hashlib.sha1('an_etagged_path').hexdigest()) conn.request('GET', '/an_etagged_path', headers={'If-None-Match':etag}) r = conn.getresponse() - self.ae(r.status, httplib.NOT_MODIFIED) + self.ae(r.status, http_client.NOT_MODIFIED) self.ae(r.read(), b'') # Test gzip @@ -334,7 +335,7 @@ class TestHTTP(BaseTest): conn.request('GET', '/an_etagged_path', headers={'Accept-Encoding':'gzip'}) r = conn.getresponse() self.ae(str(len(raw)), r.getheader('Calibre-Uncompressed-Length')) - self.ae(r.status, httplib.OK), self.ae(zlib.decompress(r.read(), 16+zlib.MAX_WBITS), raw) + self.ae(r.status, http_client.OK), self.ae(zlib.decompress(r.read(), 16+zlib.MAX_WBITS), raw) # Test dynamic etagged content num_calls = [0] @@ -346,13 +347,13 @@ class TestHTTP(BaseTest): conn = server.connect() conn.request('GET', '/an_etagged_path') r = conn.getresponse() - self.ae(r.status, httplib.OK), self.ae(r.read(), b'data') + self.ae(r.status, http_client.OK), self.ae(r.read(), b'data') etag = r.getheader('ETag') self.ae(etag, b'"xxx"') self.ae(r.getheader('Content-Length'), '4') conn.request('GET', '/an_etagged_path', headers={'If-None-Match':etag}) r = conn.getresponse() - self.ae(r.status, httplib.NOT_MODIFIED) + self.ae(r.status, http_client.NOT_MODIFIED) self.ae(r.read(), b'') self.ae(num_calls[0], 1) @@ -368,11 +369,11 @@ class TestHTTP(BaseTest): self.ae(r.getheader('Content-Type'), guess_type(f.name)[0]) self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes') self.ae(int(r.getheader('Content-Length')), len(fdata)) - self.ae(r.status, httplib.OK), self.ae(r.read(), fdata) + self.ae(r.status, http_client.OK), self.ae(r.read(), fdata) conn.request('GET', '/test', headers={'Range':'bytes=2-25'}) r = conn.getresponse() - self.ae(r.status, httplib.PARTIAL_CONTENT) + self.ae(r.status, http_client.PARTIAL_CONTENT) self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes') self.ae(type('')(r.getheader('Content-Range')), 'bytes 2-25/%d' % len(fdata)) self.ae(int(r.getheader('Content-Length')), 24) @@ -380,27 +381,27 @@ class TestHTTP(BaseTest): conn.request('GET', '/test', headers={'Range':'bytes=100000-'}) r = conn.getresponse() - self.ae(r.status, httplib.REQUESTED_RANGE_NOT_SATISFIABLE) + self.ae(r.status, http_client.REQUESTED_RANGE_NOT_SATISFIABLE) self.ae(type('')(r.getheader('Content-Range')), 'bytes */%d' % len(fdata)) conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':etag}) r = conn.getresponse() - self.ae(r.status, httplib.PARTIAL_CONTENT), self.ae(r.read(), fdata[25:51]) + self.ae(r.status, http_client.PARTIAL_CONTENT), self.ae(r.read(), fdata[25:51]) self.ae(int(r.getheader('Content-Length')), 26) conn.request('GET', '/test', headers={'Range':'bytes=0-1000000'}) r = conn.getresponse() - self.ae(r.status, httplib.PARTIAL_CONTENT), self.ae(r.read(), fdata) + self.ae(r.status, http_client.PARTIAL_CONTENT), self.ae(r.read(), fdata) conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':'"nomatch"'}) r = conn.getresponse() - self.ae(r.status, httplib.OK), self.ae(r.read(), fdata) + self.ae(r.status, http_client.OK), self.ae(r.read(), fdata) self.assertFalse(r.getheader('Content-Range')) self.ae(int(r.getheader('Content-Length')), len(fdata)) conn.request('GET', '/test', headers={'Range':'bytes=0-25,26-50'}) r = conn.getresponse() - self.ae(r.status, httplib.PARTIAL_CONTENT) + self.ae(r.status, http_client.PARTIAL_CONTENT) clen = int(r.getheader('Content-Length')) data = r.read() self.ae(clen, len(data)) @@ -415,7 +416,7 @@ class TestHTTP(BaseTest): conn = server.connect(timeout=1) conn.request('GET', '/test') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) rdata = r.read() self.ae(len(data), len(rdata)) self.ae(hashlib.sha1(data).hexdigest(), hashlib.sha1(rdata).hexdigest()) diff --git a/src/calibre/srv/tests/loop.py b/src/calibre/srv/tests/loop.py index af628456c7..897721cc40 100644 --- a/src/calibre/srv/tests/loop.py +++ b/src/calibre/srv/tests/loop.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, ssl, os, socket, time +import ssl, os, socket, time from collections import namedtuple from unittest import skipIf from glob import glob @@ -18,6 +18,7 @@ from calibre.ptempfile import TemporaryDirectory from calibre.utils.certgen import create_server_cert from calibre.utils.monotonic import monotonic from polyglot.builtins import range +from polyglot import http_client is_ci = os.environ.get('CI', '').lower() == 'true' @@ -92,7 +93,7 @@ class LoopTest(BaseTest): conn.request('GET', '/') with self.assertRaises(socket.timeout): res = conn.getresponse() - if str(res.status) == str(httplib.REQUEST_TIMEOUT): + if str(res.status) == str(http_client.REQUEST_TIMEOUT): raise socket.timeout('Timeout') raise Exception('Got unexpected response: code: %s %s headers: %r data: %r' % ( res.status, res.reason, res.getheaders(), res.read())) @@ -135,7 +136,7 @@ class LoopTest(BaseTest): conn = server.connect(interface='127.0.0.1') conn.request('GET', '/test', 'body') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'testbody') def test_ring_buffer(self): @@ -203,10 +204,10 @@ class LoopTest(BaseTest): create_server_cert(address, ca_file, cert_file, key_file, key_size=1024) ctx = ssl.create_default_context(cafile=ca_file) with TestServer(lambda data:(data.path[0] + data.read()), ssl_certfile=cert_file, ssl_keyfile=key_file, listen_on=address, port=0) as server: - conn = httplib.HTTPSConnection(address, server.address[1], strict=True, context=ctx) + conn = http_client.HTTPSConnection(address, server.address[1], strict=True, context=ctx) conn.request('GET', '/test', 'body') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'testbody') cert = conn.sock.getpeercert() subject = dict(x[0] for x in cert['subject']) @@ -226,7 +227,7 @@ class LoopTest(BaseTest): conn = server.connect() conn.request('GET', '/test', 'body') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'testbody') self.ae(server.loop.bound_address[1], port) diff --git a/src/calibre/srv/web_socket.py b/src/calibre/srv/web_socket.py index 5fdf7ab309..59a5cfc9f8 100644 --- a/src/calibre/srv/web_socket.py +++ b/src/calibre/srv/web_socket.py @@ -5,7 +5,7 @@ from __future__ import (unicode_literals, division, absolute_import, print_function) -import httplib, os, weakref, socket +import os, weakref, socket from base64 import standard_b64encode from collections import deque from hashlib import sha1 @@ -19,6 +19,7 @@ from calibre.srv.http_response import HTTPConnection, create_http_handler from calibre.srv.utils import DESIRED_SEND_BUFFER_SIZE from calibre.utils.speedups import ReadOnlyFileBuffer from polyglot.queue import Queue, Empty +from polyglot import http_client speedup, err = plugins['speedup'] if not speedup: raise RuntimeError('Failed to load speedup module with error: ' + err) @@ -286,9 +287,9 @@ class WebSocketConnection(HTTPConnection): except Exception: ver_ok = False if not ver_ok: - return self.simple_response(httplib.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver) + return self.simple_response(http_client.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver) if self.method != 'GET': - return self.simple_response(httplib.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method) + return self.simple_response(http_client.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method) response = HANDSHAKE_STR % standard_b64encode(sha1(key + GUID_STR).digest()) self.optimize_for_sending_packet() diff --git a/src/calibre/utils/browser.py b/src/calibre/utils/browser.py index def783eaee..47835dec1e 100644 --- a/src/calibre/utils/browser.py +++ b/src/calibre/utils/browser.py @@ -5,11 +5,13 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import copy, httplib, ssl +import copy, ssl from cookielib import CookieJar, Cookie from mechanize import Browser as B, HTTPSHandler +from polyglot import http_client + class ModernHTTPSHandler(HTTPSHandler): @@ -24,7 +26,7 @@ class ModernHTTPSHandler(HTTPSHandler): def conn_factory(hostport, **kw): kw['context'] = self.ssl_context - return httplib.HTTPSConnection(hostport, **kw) + return http_client.HTTPSConnection(hostport, **kw) return self.do_open(conn_factory, req) diff --git a/src/calibre/utils/https.py b/src/calibre/utils/https.py index 690c1f454b..9c26970d23 100644 --- a/src/calibre/utils/https.py +++ b/src/calibre/utils/https.py @@ -10,7 +10,7 @@ import ssl, socket, re from contextlib import closing from calibre import get_proxies -from calibre.constants import ispy3 +from polyglot import http_client from polyglot.urllib import urlsplit has_ssl_verify = hasattr(ssl, 'create_default_context') and hasattr(ssl, '_create_unverified_context') @@ -19,19 +19,14 @@ class HTTPError(ValueError): def __init__(self, url, code): msg = '%s returned an unsupported http response code: %d (%s)' % ( - url, code, httplib.responses.get(code, None)) + url, code, http_client.responses.get(code, None)) ValueError.__init__(self, msg) self.code = code self.url = url -if ispy3: - import http.client as httplib -else: - import httplib - if has_ssl_verify: - class HTTPSConnection(httplib.HTTPSConnection): + class HTTPSConnection(http_client.HTTPSConnection): def __init__(self, ssl_version, *args, **kwargs): cafile = kwargs.pop('cert_file', None) @@ -39,7 +34,7 @@ if has_ssl_verify: kwargs['context'] = ssl._create_unverified_context() else: kwargs['context'] = ssl.create_default_context(cafile=cafile) - httplib.HTTPSConnection.__init__(self, *args, **kwargs) + http_client.HTTPSConnection.__init__(self, *args, **kwargs) else: # Check certificate hostname {{{ # Implementation taken from python 3 @@ -136,10 +131,10 @@ else: "subjectAltName fields were found") # }}} - class HTTPSConnection(httplib.HTTPSConnection): + class HTTPSConnection(http_client.HTTPSConnection): def __init__(self, ssl_version, *args, **kwargs): - httplib.HTTPSConnection.__init__(self, *args, **kwargs) + http_client.HTTPSConnection.__init__(self, *args, **kwargs) self.calibre_ssl_version = ssl_version def connect(self): @@ -204,7 +199,7 @@ def get_https_resource_securely( path += '?' + p.query c.request('GET', path, headers=headers or {}) response = c.getresponse() - if response.status in (httplib.MOVED_PERMANENTLY, httplib.FOUND, httplib.SEE_OTHER): + if response.status in (http_client.MOVED_PERMANENTLY, http_client.FOUND, http_client.SEE_OTHER): if max_redirects <= 0: raise ValueError('Too many redirects, giving up') newurl = response.getheader('Location', None) @@ -212,7 +207,7 @@ def get_https_resource_securely( raise ValueError('%s returned a redirect response with no Location header' % url) return get_https_resource_securely( newurl, cacerts=cacerts, timeout=timeout, max_redirects=max_redirects-1, ssl_version=ssl_version, get_response=get_response) - if response.status != httplib.OK: + if response.status != http_client.OK: raise HTTPError(url, response.status) if get_response: return response diff --git a/src/calibre/web/fetch/simple.py b/src/calibre/web/fetch/simple.py index e9a5c1457f..852b00b6fa 100644 --- a/src/calibre/web/fetch/simple.py +++ b/src/calibre/web/fetch/simple.py @@ -18,7 +18,6 @@ import threading import time import traceback from base64 import b64decode -from httplib import responses from calibre import browser, relpath, unicode_path from calibre.constants import filesystem_encoding, iswindows @@ -31,6 +30,7 @@ from calibre.utils.imghdr import what from calibre.utils.logging import Log from calibre.web.fetch.utils import rescale_image from polyglot.builtins import unicode_type +from polyglot.http_client import responses from polyglot.urllib import ( URLError, quote, url2pathname, urljoin, urlparse, urlsplit, urlunparse, urlunsplit From 727e65a203c5f4ea1aff5a64accc65e8672207cd Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 25 Mar 2019 12:07:48 -0400 Subject: [PATCH 04/19] python3: add httplib/http.client polyglot wrapper --- src/polyglot/http_client.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 src/polyglot/http_client.py diff --git a/src/polyglot/http_client.py b/src/polyglot/http_client.py new file mode 100644 index 0000000000..740d9799ff --- /dev/null +++ b/src/polyglot/http_client.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +# License: GPL v3 Copyright: 2019, Eli Schwartz + +from polyglot.builtins import is_py3 + +if is_py3: + from http.client import (responses, HTTPConnection, HTTPSConnection, + BAD_REQUEST, FOUND, FORBIDDEN, HTTP_VERSION_NOT_SUPPORTED, + INTERNAL_SERVER_ERROR, METHOD_NOT_ALLOWED, MOVED_PERMANENTLY, + NOT_FOUND, NOT_IMPLEMENTED, NOT_MODIFIED, OK, PARTIAL_CONTENT, + PRECONDITION_FAILED, REQUEST_ENTITY_TOO_LARGE, REQUEST_URI_TOO_LONG, + REQUESTED_RANGE_NOT_SATISFIABLE, REQUEST_TIMEOUT, SEE_OTHER, + SERVICE_UNAVAILABLE, UNAUTHORIZED, _CS_IDLE, _CS_REQ_SENT) +else: + from httplib import (responses, HTTPConnection, HTTPSConnection, + BAD_REQUEST, FOUND, FORBIDDEN, HTTP_VERSION_NOT_SUPPORTED, + INTERNAL_SERVER_ERROR, METHOD_NOT_ALLOWED, MOVED_PERMANENTLY, + NOT_FOUND, NOT_IMPLEMENTED, NOT_MODIFIED, OK, PARTIAL_CONTENT, + PRECONDITION_FAILED, REQUEST_ENTITY_TOO_LARGE, REQUEST_URI_TOO_LONG, + REQUESTED_RANGE_NOT_SATISFIABLE, REQUEST_TIMEOUT, SEE_OTHER, + SERVICE_UNAVAILABLE, UNAUTHORIZED, _CS_IDLE, _CS_REQ_SENT) From 6d6509df577bdd9d0aaee7a276d100baa76099a1 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 25 Mar 2019 23:57:10 -0400 Subject: [PATCH 05/19] python3: make qt image test work on python3 QByteArrays are natively bytes and ended up looking like this: u"b'xpm'" Instead use the builtin method to get the data back, and decode it to unicode. --- src/calibre/test_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 7ce4075714..17259fecd0 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -175,7 +175,7 @@ class BuildTest(unittest.TestCase): # it should just work because the hard-coded paths of the Qt # installation should work. If they do not, then it is a distro # problem. - fmts = set(map(unicode_type, QImageReader.supportedImageFormats())) + fmts = set(map(lambda x: x.data().decode('utf-8'), QImageReader.supportedImageFormats())) testf = {'jpg', 'png', 'svg', 'ico', 'gif'} self.assertEqual(testf.intersection(fmts), testf, "Qt doesn't seem to be able to load some of its image plugins. Available plugins: %s" % fmts) data = P('images/blank.png', allow_user_override=False, data=True) From 7196fa0773946a4eac62d695bfa525f050231ce8 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 01:02:45 -0400 Subject: [PATCH 06/19] python3: mark zlib plugin as only existing on python2 Fixes test failure due to checking whether all plugins work, and considering this a plugin even though it is not. --- src/calibre/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/constants.py b/src/calibre/constants.py index e5781fe5b3..05ac5bbb3e 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -168,7 +168,6 @@ class Plugins(collections.Mapping): 'icu', 'speedup', 'unicode_names', - 'zlib2', 'html', 'freetype', 'imageops', @@ -184,6 +183,7 @@ class Plugins(collections.Mapping): if not ispy3: plugins.extend([ 'monotonic', + 'zlib2', ]) if iswindows: plugins.extend(['winutil', 'wpd', 'winfonts']) From b6d4442b23b75834d802823206db68db525022a2 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 01:04:19 -0400 Subject: [PATCH 07/19] python3: mark bytestring as needed --- src/calibre/test_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 17259fecd0..45c85d1bf8 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -99,7 +99,7 @@ class BuildTest(unittest.TestCase): from calibre.utils.cleantext import test_clean_xml_chars test_clean_xml_chars() from lxml import etree - raw = '' + raw = b'' root = etree.fromstring(raw) self.assertEqual(etree.tostring(root), raw) From 2271463d2746d7cc2bea411db46bc60ebcacaadc Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 01:10:02 -0400 Subject: [PATCH 08/19] python3: don't require bytestring for unrardll comments unrardll explictly decodes to unicode Note: unrar.py already uses unicode_literals --- src/calibre/utils/unrar.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/calibre/utils/unrar.py b/src/calibre/utils/unrar.py index 882a636ba6..fc6a767843 100644 --- a/src/calibre/utils/unrar.py +++ b/src/calibre/utils/unrar.py @@ -114,20 +114,20 @@ def test_basic(): b"Rar!\x1a\x07\x00\xcf\x90s\x00\x00\r\x00\x00\x00\x00\x00\x00\x00\x14\xe7z\x00\x80#\x00\x17\x00\x00\x00\r\x00\x00\x00\x03\xc2\xb3\x96o\x00\x00\x00\x00\x1d3\x03\x00\x00\x00\x00\x00CMT\x0c\x00\x8b\xec\x8e\xef\x14\xf6\xe6h\x04\x17\xff\xcd\x0f\xffk9b\x11]^\x80\xd3dt \x90+\x00\x14\x00\x00\x00\x08\x00\x00\x00\x03\xf1\x84\x93\\\xb9]yA\x1d3\t\x00\xa4\x81\x00\x001\\sub-one\x00\xc0\x0c\x00\x8f\xec\x89\xfe.JM\x86\x82\x0c_\xfd\xfd\xd7\x11\x1a\xef@\x9eHt \x80'\x00\x0e\x00\x00\x00\x04\x00\x00\x00\x03\x9f\xa8\x17\xf8\xaf]yA\x1d3\x07\x00\xa4\x81\x00\x00one.txt\x00\x08\xbf\x08\xae\xf3\xca\x87\xfeo\xfe\xd2n\x80-Ht \x82:\x00\x18\x00\x00\x00\x10\x00\x00\x00\x03\xa86\x81\xdf\xf9fyA\x1d3\x1a\x00\xa4\x81\x00\x00\xe8\xaf\xb6\xe6\xaf\x94\xe5\xb1\x81.txt\x00\x8bh\xf6\xd4kA\\.\x00txt\x0c\x00\x8b\xec\x8e\xef\x14\xf6\xe2l\x91\x189\xff\xdf\xfe\xc2\xd3:g\x9a\x19F=cYt \x928\x00\x11\x00\x00\x00\x08\x00\x00\x00\x03\x7f\xd6\xb6\x7f\xeafyA\x1d3\x16\x00\xa4\x81\x00\x00F\xc3\xbc\xc3\x9fe.txt\x00\x01\x00F\xfc\xdfe\x00.txt\x00\xc0 Date: Tue, 26 Mar 2019 01:20:18 -0400 Subject: [PATCH 09/19] test: fix broken netifaces check This will always return True on python2 [] > 1 On python3 it is instead an error, because you cannot compare a list to an int -- instead, we want to compare the length of the returned list, to see how many interfaces are available. --- src/calibre/test_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 45c85d1bf8..2ba6492114 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -254,7 +254,7 @@ class BuildTest(unittest.TestCase): def test_netifaces(self): import netifaces - self.assertGreaterEqual(netifaces.interfaces(), 1, 'netifaces could find no network interfaces') + self.assertGreaterEqual(len(netifaces.interfaces()), 1, 'netifaces could find no network interfaces') def test_psutil(self): import psutil From 696afe85bdbfe8e231d7484b6b8504f7acf39ea9 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 01:31:34 -0400 Subject: [PATCH 10/19] python3: convert filter iterables to list in order to access as a list --- src/calibre/spell/dictionary.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/spell/dictionary.py b/src/calibre/spell/dictionary.py index 598c6e302d..5086dc08b4 100644 --- a/src/calibre/spell/dictionary.py +++ b/src/calibre/spell/dictionary.py @@ -54,7 +54,7 @@ def builtin_dictionaries(): if _builtins is None: dics = [] for lc in glob.glob(os.path.join(P('dictionaries', allow_user_override=False), '*/locales')): - locales = filter(None, open(lc, 'rb').read().decode('utf-8').splitlines()) + locales = list(filter(None, open(lc, 'rb').read().decode('utf-8').splitlines())) locale = locales[0] base = os.path.dirname(lc) dics.append(Dictionary( @@ -69,7 +69,7 @@ def custom_dictionaries(reread=False): if _custom is None or reread: dics = [] for lc in glob.glob(os.path.join(config_dir, 'dictionaries', '*/locales')): - locales = filter(None, open(lc, 'rb').read().decode('utf-8').splitlines()) + locales = list(filter(None, open(lc, 'rb').read().decode('utf-8').splitlines())) try: name, locale, locales = locales[0], locales[1], locales[1:] except IndexError: From 564f053c026fe3946d5c9c3e915ab4cbcd2694d2 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 03:15:21 -0400 Subject: [PATCH 11/19] python3: add reprlib wrapper to polyglot --- src/calibre/__init__.py | 2 +- src/calibre/db/tests/legacy.py | 4 ++-- src/calibre/library/sqlite.py | 2 +- src/calibre/srv/http_request.py | 4 ++-- src/calibre/srv/http_response.py | 4 ++-- src/calibre/srv/utils.py | 2 +- src/polyglot/reprlib.py | 10 ++++++++++ 7 files changed, 19 insertions(+), 9 deletions(-) create mode 100755 src/polyglot/reprlib.py diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index c71af8e0f4..11a2317d69 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -240,7 +240,7 @@ def prints(*args, **kwargs): file.write(arg) count += len(arg) except: - import repr as reprlib + from polyglot import reprlib arg = reprlib.repr(arg) file.write(arg) count += len(arg) diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index b0f88feb30..b7406bbca8 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -8,13 +8,13 @@ __copyright__ = '2013, Kovid Goyal ' import inspect, time, numbers from io import BytesIO -from repr import repr from functools import partial from operator import itemgetter from calibre.library.field_metadata import fm_as_dict from calibre.db.tests.base import BaseTest from polyglot.builtins import iteritems, range +from polyglot import reprlib # Utils {{{ @@ -32,7 +32,7 @@ class ET(object): oldres = getattr(old, self.func_name)(*self.args, **self.kwargs) newres = getattr(legacy, self.func_name)(*self.args, **self.kwargs) test.assertEqual(oldres, newres, 'Equivalence test for %s with args: %s and kwargs: %s failed' % ( - self.func_name, repr(self.args), repr(self.kwargs))) + self.func_name, reprlib.repr(self.args), reprlib.repr(self.kwargs))) self.retval = newres return newres diff --git a/src/calibre/library/sqlite.py b/src/calibre/library/sqlite.py index d876e96ccf..488925fe76 100644 --- a/src/calibre/library/sqlite.py +++ b/src/calibre/library/sqlite.py @@ -8,7 +8,6 @@ Wrapper for multi-threaded access to a single sqlite database connection. Serial all calls. ''' import sqlite3 as sqlite, traceback, time, uuid, sys, os -import repr as reprlib from sqlite3 import IntegrityError, OperationalError from threading import Thread from threading import RLock @@ -22,6 +21,7 @@ from calibre.constants import iswindows, DEBUG, plugins from calibre.utils.icu import sort_key from calibre import prints from polyglot.builtins import unicode_type +from polyglot import reprlib from polyglot.queue import Queue from dateutil.tz import tzoffset diff --git a/src/calibre/srv/http_request.py b/src/calibre/srv/http_request.py index 74ac506790..02afb5252f 100644 --- a/src/calibre/srv/http_request.py +++ b/src/calibre/srv/http_request.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import re, repr as reprlib +import re from io import BytesIO, DEFAULT_BUFFER_SIZE from calibre import as_unicode, force_unicode @@ -14,7 +14,7 @@ from calibre.ptempfile import SpooledTemporaryFile from calibre.srv.errors import HTTPSimpleResponse from calibre.srv.loop import Connection, READ, WRITE from calibre.srv.utils import MultiDict, HTTP1, HTTP11, Accumulator -from polyglot import http_client +from polyglot import http_client, reprlib from polyglot.urllib import unquote protocol_map = {(1, 0):HTTP1, (1, 1):HTTP11} diff --git a/src/calibre/srv/http_response.py b/src/calibre/srv/http_response.py index fc11fd0782..b5b16fbe8f 100644 --- a/src/calibre/srv/http_response.py +++ b/src/calibre/srv/http_response.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import os, hashlib, uuid, struct, repr as reprlib +import os, hashlib, uuid, struct from collections import namedtuple from io import BytesIO, DEFAULT_BUFFER_SIZE from itertools import chain, repeat @@ -26,7 +26,7 @@ from calibre.srv.utils import ( sort_q_values, get_translator_for_lang, Cookie, fast_now_strftime) from calibre.utils.speedups import ReadOnlyFileBuffer from calibre.utils.monotonic import monotonic -from polyglot import http_client +from polyglot import http_client, reprlib Range = namedtuple('Range', 'start stop size') MULTIPART_SEPARATOR = uuid.uuid4().hex.decode('ascii') diff --git a/src/calibre/srv/utils.py b/src/calibre/srv/utils.py index 7e74d2b057..0037e4e8a7 100644 --- a/src/calibre/srv/utils.py +++ b/src/calibre/srv/utils.py @@ -9,7 +9,6 @@ __copyright__ = '2015, Kovid Goyal ' import errno, socket, select, os, time from Cookie import SimpleCookie from contextlib import closing -import repr as reprlib from email.utils import formatdate from operator import itemgetter from binascii import hexlify, unhexlify @@ -23,6 +22,7 @@ from calibre.utils.socket_inheritance import set_socket_inherit from calibre.utils.logging import ThreadSafeLog from calibre.utils.shared_file import share_open, raise_winerror from polyglot.builtins import iteritems, map, unicode_type, range +from polyglot import reprlib from polyglot.urllib import parse_qs, quote as urlquote HTTP1 = 'HTTP/1.0' diff --git a/src/polyglot/reprlib.py b/src/polyglot/reprlib.py new file mode 100755 index 0000000000..2444ad1788 --- /dev/null +++ b/src/polyglot/reprlib.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +# License: GPL v3 Copyright: 2019, Eli Schwartz + +from polyglot.builtins import is_py3 + +if is_py3: + from reprlib import repr +else: + from repr import repr From 67f3ca17ddc878515daf99f7949bde77eb8f30f0 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 02:17:33 -0400 Subject: [PATCH 12/19] test: add test to try importing every file The testsuite does not currently exercise every file, but we can at least try importing it to make sure it isn't obviously broken. This additionally helps to iterate through python3 syntax-level incompatibilities. --- src/calibre/test_build.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 2ba6492114..397299264b 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -73,6 +73,27 @@ class BuildTest(unittest.TestCase): from html5_parser import parse parse('

xxx') + def test_imports(self): + import importlib + exclude = ['dbus_export.demo', 'dbus_export.gtk', 'upstream'] + if not iswindows: + exclude.extend(['iphlpapi', 'windows', 'winreg', 'winusb']) + if not isosx: + exclude.append('osx') + if not islinux: + exclude.extend(['dbus', 'linux']) + for root, dirs, files in os.walk('src/calibre'): + for dir in dirs: + if not os.path.isfile(os.path.join(root, dir, '__init__.py')): + dirs.remove(dir) + for file in files: + file, ext = os.path.splitext(file) + if ext != '.py': + continue + name = '.'.join(root.split(os.path.sep)[1:] + [file]) + if not any(x for x in exclude if x in name): + importlib.import_module(name) + def test_plugins(self): exclusions = set() if is_ci: From 06d8560ab2685f22168b502236bb64d96ed52c36 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 02:24:48 -0400 Subject: [PATCH 13/19] remove unused files which import non-existing code --- src/calibre/ebooks/lit/from_any.py | 67 ----------- src/calibre/ebooks/lrf/html/convert_to.py | 128 ---------------------- src/calibre/ebooks/pdf/from_comic.py | 21 ---- 3 files changed, 216 deletions(-) delete mode 100644 src/calibre/ebooks/lit/from_any.py delete mode 100644 src/calibre/ebooks/lrf/html/convert_to.py delete mode 100644 src/calibre/ebooks/pdf/from_comic.py diff --git a/src/calibre/ebooks/lit/from_any.py b/src/calibre/ebooks/lit/from_any.py deleted file mode 100644 index 05345b6749..0000000000 --- a/src/calibre/ebooks/lit/from_any.py +++ /dev/null @@ -1,67 +0,0 @@ -from __future__ import with_statement -from __future__ import print_function -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' -__docformat__ = 'restructuredtext en' - -''' -Convert any ebook format to LIT. -''' - -import sys, os, glob, logging - -from calibre.ebooks.epub.from_any import any2epub, formats, USAGE -from calibre.ebooks.epub import config as common_config -from calibre.ptempfile import TemporaryDirectory -from calibre.ebooks.lit.writer import oeb2lit - - -def config(defaults=None): - c = common_config(defaults=defaults, name='lit') - return c - - -def option_parser(usage=USAGE): - return config().option_parser(usage=usage%('LIT', formats())) - - -def any2lit(opts, path): - ext = os.path.splitext(path)[1] - if not ext: - raise ValueError('Unknown file type: '+path) - ext = ext.lower()[1:] - - if opts.output is None: - opts.output = os.path.splitext(os.path.basename(path))[0]+'.lit' - - opts.output = os.path.abspath(opts.output) - orig_output = opts.output - - with TemporaryDirectory('_any2lit') as tdir: - oebdir = os.path.join(tdir, 'oeb') - os.mkdir(oebdir) - opts.output = os.path.join(tdir, 'dummy.epub') - opts.profile = 'None' - orig_bfs = opts.base_font_size2 - opts.base_font_size2 = 0 - any2epub(opts, path, create_epub=False, oeb_cover=True, extract_to=oebdir) - opts.base_font_size2 = orig_bfs - opf = glob.glob(os.path.join(oebdir, '*.opf'))[0] - opts.output = orig_output - logging.getLogger('html2epub').info(_('Creating LIT file from EPUB...')) - oeb2lit(opts, opf) - - -def main(args=sys.argv): - parser = option_parser() - opts, args = parser.parse_args(args) - if len(args) < 2: - parser.print_help() - print('No input file specified.') - return 1 - any2lit(opts, args[1]) - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/lrf/html/convert_to.py b/src/calibre/ebooks/lrf/html/convert_to.py deleted file mode 100644 index 8ae9224d2c..0000000000 --- a/src/calibre/ebooks/lrf/html/convert_to.py +++ /dev/null @@ -1,128 +0,0 @@ -from __future__ import print_function -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal ' -import sys, logging, os - -from calibre import setup_cli_handlers -from calibre.utils.config import OptionParser -from calibre.ebooks import ConversionError -from calibre.ebooks.lrf.meta import get_metadata -from calibre.ebooks.lrf.lrfparser import LRFDocument -from calibre.ebooks.metadata.opf import OPFCreator - -from calibre.ebooks.lrf.objects import PageAttr, BlockAttr, TextAttr -from calibre.ebooks.lrf.pylrs.pylrs import TextStyle - - -class BlockStyle(object): - - def __init__(self, ba): - self.ba = ba - - def __str__(self): - ans = '.'+str(self.ba.id)+' {\n' - if hasattr(self.ba, 'sidemargin'): - margin = str(self.ba.sidemargin) + 'px' - ans += '\tmargin-left: %(m)s; margin-right: %(m)s;\n'%dict(m=margin) - if hasattr(self.ba, 'topskip'): - ans += '\tmargin-top: %dpx;\n'%(self.ba.topskip,) - if hasattr(self.ba, 'footskip'): - ans += '\tmargin-bottom: %dpx;\n'%(self.ba.footskip,) - if hasattr(self.ba, 'framewidth'): - ans += '\tborder-width: %dpx;\n'%(self.ba.framewidth,) - ans += '\tborder-style: solid;\n' - if hasattr(self.ba, 'framecolor'): - if self.ba.framecolor.a < 255: - ans += '\tborder-color: %s;\n'%(self.ba.framecolor.to_html()) - if hasattr(self.ba, 'bgcolor'): - if self.ba.bgcolor.a < 255: - ans += '\tbackground-color: %s;\n'%(self.ba.bgcolor.to_html()) - # TODO: Fixed size blocks - return ans + '}\n' - - -class LRFConverter(object): - - def __init__(self, document, opts, logger): - self.lrf = document - self.opts = opts - self.output_dir = opts.out - self.logger = logger - logger.info('Parsing LRF...') - self.lrf.parse() - - self.create_metadata() - self.create_styles() - - def create_metadata(self): - self.logger.info('Reading metadata...') - mi = get_metadata(self.lrf) - self.opf = OPFCreator(self.output_dir, mi) - - def create_page_styles(self): - self.page_css = '' - for obj in self.lrf.objects.values(): - if isinstance(obj, PageAttr): - selector = 'body.'+str(obj.id) - self.page_css = selector + ' {\n' - # TODO: Headers and footers - self.page_css += '}\n' - - def create_block_styles(self): - self.block_css = '' - for obj in self.lrf.objects.values(): - if isinstance(obj, BlockAttr): - self.block_css += str(BlockStyle(obj)) - - def create_text_styles(self): - self.text_css = '' - for obj in self.lrf.objects.values(): - if isinstance(obj, TextAttr): - self.text_css += str(TextStyle(obj)) - print(self.text_css) - - def create_styles(self): - self.logger.info('Creating CSS stylesheet...') - self.create_page_styles() - self.create_block_styles() - - -def option_parser(): - parser = OptionParser(usage='%prog book.lrf') - parser.add_option('--output-dir', '-o', default=None, help=( - 'Output directory in which to store created HTML files. If it does not exist, it is created. By default the current directory is used.'), dest='out') - parser.add_option('--verbose', default=False, action='store_true', dest='verbose') - return parser - - -def process_file(lrfpath, opts, logger=None): - if logger is None: - level = logging.DEBUG if opts.verbose else logging.INFO - logger = logging.getLogger('lrf2html') - setup_cli_handlers(logger, level) - if opts.out is None: - opts.out = os.getcwdu() - else: - opts.out = os.path.abspath(opts.out) - if not os.path.isdir(opts.out): - raise ConversionError(opts.out + ' is not a directory') - if not os.path.exists(opts.out): - os.makedirs(opts.out) - - document = LRFDocument(open(lrfpath, 'rb')) - LRFConverter(document, opts, logger) - - -def main(args=sys.argv): - parser = option_parser() - opts, args = parser.parse_args(args) - if len(args) != 2: - parser.print_help() - return 1 - process_file(args[1], opts) - - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/pdf/from_comic.py b/src/calibre/ebooks/pdf/from_comic.py deleted file mode 100644 index c39b660ef4..0000000000 --- a/src/calibre/ebooks/pdf/from_comic.py +++ /dev/null @@ -1,21 +0,0 @@ -from __future__ import with_statement -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' -__docformat__ = 'restructuredtext en' - -'Convert a comic in CBR/CBZ format to pdf' - -import sys -from functools import partial -from calibre.ebooks.lrf.comic.convert_from import do_convert, option_parser, config, main as _main - -convert = partial(do_convert, output_format='pdf') -main = partial(_main, output_format='pdf') - -if __name__ == '__main__': - sys.exit(main()) - -if False: - option_parser - config - From 97ab4acce5a3dd9e4f79ef217bd9db3c281e97b9 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 12:46:40 -0400 Subject: [PATCH 14/19] python3: reduce use of cmp when key can be used instead cmp does not exist on python3 --- manual/custom.py | 4 ++-- src/calibre/customize/profiles.py | 4 ++-- src/calibre/customize/ui.py | 4 ++-- src/calibre/db/cli/cmd_list_categories.py | 4 +--- src/calibre/devices/__init__.py | 3 +-- src/calibre/devices/prs505/sony_cache.py | 2 +- src/calibre/devices/usbms/device.py | 2 +- src/calibre/devices/winusb.py | 3 +-- src/calibre/ebooks/conversion/plumber.py | 2 +- src/calibre/ebooks/lit/writer.py | 3 +-- src/calibre/ebooks/metadata/lit.py | 2 +- src/calibre/ebooks/metadata/meta.py | 4 +--- src/calibre/ebooks/oeb/transforms/guide.py | 3 +-- src/calibre/ebooks/pdf/reflow.py | 10 +++++----- src/calibre/gui2/actions/choose_library.py | 3 +-- src/calibre/gui2/dialogs/catalog.py | 2 +- src/calibre/gui2/library/models.py | 2 +- src/calibre/gui2/library/views.py | 2 +- src/calibre/gui2/preferences/columns.py | 10 ++++------ src/calibre/gui2/preferences/look_feel.py | 2 +- src/calibre/gui2/preferences/main.py | 4 ++-- src/calibre/gui2/preferences/plugboard.py | 13 +------------ src/calibre/gui2/preferences/plugins.py | 2 +- src/calibre/gui2/wizard/__init__.py | 4 ++-- src/calibre/library/custom_columns.py | 4 ++-- src/calibre/library/database.py | 2 +- src/calibre/utils/smtp.py | 3 +-- src/calibre/web/feeds/news.py | 2 +- 28 files changed, 41 insertions(+), 64 deletions(-) diff --git a/manual/custom.py b/manual/custom.py index 2c9b31d514..b35ff29234 100644 --- a/manual/custom.py +++ b/manual/custom.py @@ -179,7 +179,7 @@ def generate_ebook_convert_help(preamble, app): raw += '\n\n.. contents::\n :local:' raw += '\n\n' + options - for pl in sorted(input_format_plugins(), key=lambda x:x.name): + for pl in sorted(input_format_plugins(), key=lambda x: x.name): parser, plumber = create_option_parser(['ebook-convert', 'dummyi.'+sorted(pl.file_types)[0], 'dummyo.epub', '-h'], default_log) groups = [(pl.name+ ' Options', '', g.option_list) for g in @@ -279,7 +279,7 @@ def cli_docs(app): info(bold('creating CLI documentation...')) documented_cmds, undocumented_cmds = get_cli_docs() - documented_cmds.sort(cmp=lambda x, y: cmp(x[0], y[0])) + documented_cmds.sort(key=lambda x: x[0]) undocumented_cmds.sort() documented = [' '*4 + c[0] for c in documented_cmds] diff --git a/src/calibre/customize/profiles.py b/src/calibre/customize/profiles.py index 2c7fead69f..1038d448be 100644 --- a/src/calibre/customize/profiles.py +++ b/src/calibre/customize/profiles.py @@ -233,7 +233,7 @@ input_profiles = [InputProfile, SonyReaderInput, SonyReader300Input, HanlinV5Input, CybookG3Input, CybookOpusInput, KindleInput, IlliadInput, IRexDR1000Input, IRexDR800Input, NookInput] -input_profiles.sort(cmp=lambda x,y:cmp(x.name.lower(), y.name.lower())) +input_profiles.sort(key=lambda x: x.name.lower()) # }}} @@ -870,4 +870,4 @@ output_profiles = [ KindlePaperWhite3Output, KindleOasisOutput ] -output_profiles.sort(cmp=lambda x,y:cmp(x.name.lower(), y.name.lower())) +output_profiles.sort(key=lambda x: x.name.lower()) diff --git a/src/calibre/customize/ui.py b/src/calibre/customize/ui.py index 1229cbf712..8bb9b5814f 100644 --- a/src/calibre/customize/ui.py +++ b/src/calibre/customize/ui.py @@ -727,9 +727,9 @@ def initialize_plugins(perf=False): # ipython sys.stdout, sys.stderr = ostdout, ostderr if perf: - for x in sorted(times, key=lambda x:times[x]): + for x in sorted(times, key=lambda x: times[x]): print('%50s: %.3f'%(x, times[x])) - _initialized_plugins.sort(cmp=lambda x,y:cmp(x.priority, y.priority), reverse=True) + _initialized_plugins.sort(key=lambda x: x.priority, reverse=True) reread_filetype_plugins() reread_metadata_plugins() diff --git a/src/calibre/db/cli/cmd_list_categories.py b/src/calibre/db/cli/cmd_list_categories.py index 0280455f66..6165c50877 100644 --- a/src/calibre/db/cli/cmd_list_categories.py +++ b/src/calibre/db/cli/cmd_list_categories.py @@ -149,9 +149,7 @@ def main(opts, args, dbctx): (not report_on or k in report_on) ] - categories.sort( - cmp=lambda x, y: cmp(x if x[0] != '#' else x[1:], y if y[0] != '#' else y[1:]) - ) + categories.sort(key=lambda x: x if x[0] != '#' else x[1:]) def fmtr(v): v = v or 0 diff --git a/src/calibre/devices/__init__.py b/src/calibre/devices/__init__.py index d3ee2b0f19..5ab3b38dee 100644 --- a/src/calibre/devices/__init__.py +++ b/src/calibre/devices/__init__.py @@ -82,8 +82,7 @@ def debug(ioreg_to_tmp=False, buf=None, plugins=None, out = partial(prints, file=buf) devplugins = device_plugins() if plugins is None else plugins - devplugins = list(sorted(devplugins, cmp=lambda - x,y:cmp(x.__class__.__name__, y.__class__.__name__))) + devplugins = list(sorted(devplugins, key=lambda x: x.__class__.__name__)) if plugins is None: for d in devplugins: try: diff --git a/src/calibre/devices/prs505/sony_cache.py b/src/calibre/devices/prs505/sony_cache.py index c8db25b73a..0e5a90aaee 100644 --- a/src/calibre/devices/prs505/sony_cache.py +++ b/src/calibre/devices/prs505/sony_cache.py @@ -321,7 +321,7 @@ class XMLCache(object): # Only rebase ids of nodes that are immediate children of the # record root (that way playlist/itemnodes are unaffected items = root.xpath('child::*[@id]') - items.sort(cmp=lambda x,y:cmp(int(x.get('id')), int(y.get('id')))) + items.sort(key=lambda x: int(x.get('id'))) idmap = {} for i, item in enumerate(items): old = int(item.get('id')) diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index a7eb0b3567..b025ca7abf 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -537,7 +537,7 @@ class Device(DeviceConfig, DevicePlugin): if sz > 0: nodes.append((x.split('/')[-1], sz)) - nodes.sort(cmp=lambda x, y: cmp(x[1], y[1])) + nodes.sort(key=lambda x: x[1]) if not nodes: return node return nodes[-1][0] diff --git a/src/calibre/devices/winusb.py b/src/calibre/devices/winusb.py index 0a7647cce0..dd927e5df4 100644 --- a/src/calibre/devices/winusb.py +++ b/src/calibre/devices/winusb.py @@ -995,8 +995,7 @@ def develop(): # {{{ drive_letters = set() pprint(usb_devices) print() - devplugins = list(sorted(device_plugins(), cmp=lambda - x,y:cmp(x.__class__.__name__, y.__class__.__name__))) + devplugins = list(sorted(device_plugins(), key=lambda x: x.__class__.__name__)) for dev in devplugins: dev.startup() for dev in devplugins: diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index 25ad577c8b..5d8e46fd67 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -821,7 +821,7 @@ OptionRecommendation(name='search_replace', if not html_files: raise ValueError(_('Could not find an e-book inside the archive')) html_files = [(f, os.stat(f).st_size) for f in html_files] - html_files.sort(cmp=lambda x, y: cmp(x[1], y[1])) + html_files.sort(key=lambda x: x[1]) html_files = [f[0] for f in html_files] for q in ('toc', 'index'): for f in html_files: diff --git a/src/calibre/ebooks/lit/writer.py b/src/calibre/ebooks/lit/writer.py index fd292f4a03..490c06b42f 100644 --- a/src/calibre/ebooks/lit/writer.py +++ b/src/calibre/ebooks/lit/writer.py @@ -670,8 +670,7 @@ class LitWriter(object): def _build_dchunks(self): ddata = [] directory = list(self._directory) - directory.sort(cmp=lambda x, y: - cmp(x.name.lower(), y.name.lower())) + directory.sort(key=lambda x: x.name.lower()) qrn = 1 + (1 << 2) dchunk = io.BytesIO() dcount = 0 diff --git a/src/calibre/ebooks/metadata/lit.py b/src/calibre/ebooks/metadata/lit.py index 330bf8cc86..d2f078487d 100644 --- a/src/calibre/ebooks/metadata/lit.py +++ b/src/calibre/ebooks/metadata/lit.py @@ -32,7 +32,7 @@ def get_metadata(stream): except: pass break - covers.sort(cmp=lambda x, y:cmp(len(x[0]), len(y[0])), reverse=True) + covers.sort(key=lambda x: len(x[0]), reverse=True) idx = 0 if len(covers) > 1: if covers[1][1] == covers[0][1]+'-standard': diff --git a/src/calibre/ebooks/metadata/meta.py b/src/calibre/ebooks/metadata/meta.py index de773ec8c6..7a8299dc19 100644 --- a/src/calibre/ebooks/metadata/meta.py +++ b/src/calibre/ebooks/metadata/meta.py @@ -41,8 +41,7 @@ def metadata_from_formats(formats, force_read_metadata=False, pattern=None): def _metadata_from_formats(formats, force_read_metadata=False, pattern=None): mi = MetaInformation(None, None) - formats.sort(cmp=lambda x,y: cmp(METADATA_PRIORITIES[path_to_ext(x)], - METADATA_PRIORITIES[path_to_ext(y)])) + formats.sort(key=lambda x: METADATA_PRIORITIES[path_to_ext(x)]) extensions = list(map(path_to_ext, formats)) if 'opf' in extensions: opf = formats[extensions.index('opf')] @@ -241,4 +240,3 @@ def forked_read_metadata(path, tdir): opf = metadata_to_opf(mi, default_lang='und') with lopen(os.path.join(tdir, 'metadata.opf'), 'wb') as f: f.write(opf) - diff --git a/src/calibre/ebooks/oeb/transforms/guide.py b/src/calibre/ebooks/oeb/transforms/guide.py index 29c5f2a93a..104a2fc1d5 100644 --- a/src/calibre/ebooks/oeb/transforms/guide.py +++ b/src/calibre/ebooks/oeb/transforms/guide.py @@ -28,7 +28,7 @@ class Clean(object): else: covers.append([self.oeb.guide[x], len(item.data)]) - covers.sort(cmp=lambda x,y:cmp(x[1], y[1]), reverse=True) + covers.sort(key=lambda x: x[1], reverse=True) if covers: ref = covers[0][0] if len(covers) > 1: @@ -53,4 +53,3 @@ class Clean(object): if item.title and item.title.lower() == 'start': continue self.oeb.guide.remove(x) - diff --git a/src/calibre/ebooks/pdf/reflow.py b/src/calibre/ebooks/pdf/reflow.py index a7708ea8ce..337147d8a7 100644 --- a/src/calibre/ebooks/pdf/reflow.py +++ b/src/calibre/ebooks/pdf/reflow.py @@ -169,7 +169,7 @@ class Column(object): self._post_add() def _post_add(self): - self.elements.sort(cmp=lambda x,y:cmp(x.bottom,y.bottom)) + self.elements.sort(key=lambda x: x.bottom) self.top = self.elements[0].top self.bottom = self.elements[-1].bottom self.left, self.right = sys.maxint, 0 @@ -260,7 +260,7 @@ class Region(object): def add(self, columns): if not self.columns: - for x in sorted(columns, cmp=lambda x,y: cmp(x.left, y.left)): + for x in sorted(columns, key=lambda x: x.left): self.columns.append(x) else: for i in range(len(columns)): @@ -458,7 +458,7 @@ class Page(object): self.elements = list(self.texts) for img in page.xpath('descendant::img'): self.elements.append(Image(img, self.opts, self.log, idc)) - self.elements.sort(cmp=lambda x,y:cmp(x.top, y.top)) + self.elements.sort(key=lambda x: x.top) def coalesce_fragments(self): @@ -580,7 +580,7 @@ class Page(object): def sort_into_columns(self, elem, neighbors): neighbors.add(elem) - neighbors = sorted(neighbors, cmp=lambda x,y:cmp(x.left, y.left)) + neighbors = sorted(neighbors, key=lambda x: x.left) if self.opts.verbose > 3: self.log.debug('Neighbors:', [x.to_html() for x in neighbors]) columns = [Column()] @@ -595,7 +595,7 @@ class Page(object): if not added: columns.append(Column()) columns[-1].add(x) - columns.sort(cmp=lambda x,y:cmp(x.left, y.left)) + columns.sort(key=lambda x: x.left) return columns def find_elements_in_row_of(self, x): diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py index a8adb593e4..a562958470 100644 --- a/src/calibre/gui2/actions/choose_library.py +++ b/src/calibre/gui2/actions/choose_library.py @@ -51,8 +51,7 @@ class LibraryUsageStats(object): # {{{ def write_stats(self): locs = list(self.stats.keys()) - locs.sort(cmp=lambda x, y: cmp(self.stats[x], self.stats[y]), - reverse=True) + locs.sort(key=lambda x: self.stats[x], reverse=True) for key in locs[500:]: self.stats.pop(key) gprefs.set('library_usage_stats', self.stats) diff --git a/src/calibre/gui2/dialogs/catalog.py b/src/calibre/gui2/dialogs/catalog.py index 1f32406b76..f10eab4384 100644 --- a/src/calibre/gui2/dialogs/catalog.py +++ b/src/calibre/gui2/dialogs/catalog.py @@ -93,7 +93,7 @@ class Catalog(QDialog, Ui_Dialog): else: info("No dynamic tab resources found for %s" % name) - self.widgets = sorted(self.widgets, cmp=lambda x,y:cmp(x.TITLE, y.TITLE)) + self.widgets = sorted(self.widgets, key=lambda x: x.TITLE) # Generate a sorted list of installed catalog formats/sync_enabled pairs fmts = sorted([x[0] for x in self.fmts]) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index b36f91798c..d058e396e0 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -323,7 +323,7 @@ class BooksModel(QAbstractTableModel): # {{{ return 100000 return self.db.field_metadata[name]['rec_index'] - self.column_map.sort(cmp=lambda x,y: cmp(col_idx(x), col_idx(y))) + self.column_map.sort(key=lambda x: col_idx(x)) for col in self.column_map: if col in self.orig_headers: self.headers[col] = self.orig_headers[col] diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py index a49f7c778f..a5ce76815d 100644 --- a/src/calibre/gui2/library/views.py +++ b/src/calibre/gui2/library/views.py @@ -1031,7 +1031,7 @@ class BooksView(QTableView): # {{{ h.visualIndex(x) > -1] if not pairs: pairs = [(0, 0)] - pairs.sort(cmp=lambda x,y:cmp(x[1], y[1])) + pairs.sort(key=lambda x: x[1]) i = pairs[0][0] index = self.model().index(row, i) if for_sync: diff --git a/src/calibre/gui2/preferences/columns.py b/src/calibre/gui2/preferences/columns.py index 0cd0fe5bea..3d346c41fb 100644 --- a/src/calibre/gui2/preferences/columns.py +++ b/src/calibre/gui2/preferences/columns.py @@ -69,7 +69,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): state = self.columns_state(defaults) self.hidden_cols = state['hidden_columns'] positions = state['column_positions'] - colmap.sort(cmp=lambda x,y: cmp(positions[x], positions[y])) + colmap.sort(key=lambda x: positions[x]) self.opt_columns.clear() db = model.db @@ -248,12 +248,10 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): if 'ondevice' in hidden_cols: hidden_cols.remove('ondevice') - def col_pos(x, y): - xidx = config_cols.index(x) if x in config_cols else sys.maxint - yidx = config_cols.index(y) if y in config_cols else sys.maxint - return cmp(xidx, yidx) + def col_pos(x): + return config_cols.index(x) if x in config_cols else sys.maxint positions = {} - for i, col in enumerate((sorted(model.column_map, cmp=col_pos))): + for i, col in enumerate((sorted(model.column_map, key=col_pos))): positions[col] = i state = {'hidden_columns': hidden_cols, 'column_positions':positions} self.gui.library_view.apply_state(state) diff --git a/src/calibre/gui2/preferences/look_feel.py b/src/calibre/gui2/preferences/look_feel.py index 0fb52d9a97..258ffcdc15 100644 --- a/src/calibre/gui2/preferences/look_feel.py +++ b/src/calibre/gui2/preferences/look_feel.py @@ -456,7 +456,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): if l != lang] if lang != 'en': items.append(('en', get_esc_lang('en'))) - items.sort(cmp=lambda x, y: cmp(x[1].lower(), y[1].lower())) + items.sort(key=lambda x: x[1].lower()) choices = [(y, x) for x, y in items] # Default language is the autodetected one choices = [(get_language(lang), lang)] + choices diff --git a/src/calibre/gui2/preferences/main.py b/src/calibre/gui2/preferences/main.py index 3b8cc99c45..43f4386d5b 100644 --- a/src/calibre/gui2/preferences/main.py +++ b/src/calibre/gui2/preferences/main.py @@ -171,7 +171,7 @@ class Browser(QScrollArea): # {{{ self.category_names = category_names categories = list(category_map.keys()) - categories.sort(cmp=lambda x, y: cmp(category_map[x], category_map[y])) + categories.sort(key=lambda x: category_map[x]) self.category_map = OrderedDict() for c in categories: @@ -181,7 +181,7 @@ class Browser(QScrollArea): # {{{ self.category_map[plugin.category].append(plugin) for plugins in self.category_map.values(): - plugins.sort(cmp=lambda x, y: cmp(x.name_order, y.name_order)) + plugins.sort(key=lambda x: x.name_order) self.widgets = [] self._layout = QVBoxLayout() diff --git a/src/calibre/gui2/preferences/plugboard.py b/src/calibre/gui2/preferences/plugboard.py index 5005e87735..65dea1c87d 100644 --- a/src/calibre/gui2/preferences/plugboard.py +++ b/src/calibre/gui2/preferences/plugboard.py @@ -34,17 +34,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): self.db = gui.library_view.model().db def initialize(self): - def field_cmp(x, y): - if x.startswith('#'): - if y.startswith('#'): - return cmp(x.lower(), y.lower()) - else: - return 1 - elif y.startswith('#'): - return -1 - else: - return cmp(x.lower(), y.lower()) - ConfigWidgetBase.initialize(self) self.current_plugboards = copy.deepcopy(self.db.prefs.get('plugboards',{})) @@ -73,7 +62,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): if n not in self.disabled_devices: self.disabled_devices.append(n) - self.devices.sort(cmp=lambda x, y: cmp(x.lower(), y.lower())) + self.devices.sort(key=lambda x: x.lower()) self.devices.insert(1, plugboard_save_to_disk_value) self.devices.insert(1, plugboard_content_server_value) self.device_to_formats_map[plugboard_content_server_value] = \ diff --git a/src/calibre/gui2/preferences/plugins.py b/src/calibre/gui2/preferences/plugins.py index a683eccf4d..c3c777f1e9 100644 --- a/src/calibre/gui2/preferences/plugins.py +++ b/src/calibre/gui2/preferences/plugins.py @@ -61,7 +61,7 @@ class PluginModel(QAbstractItemModel, AdaptSQP): # {{{ self.categories = sorted(self._data.keys()) for plugins in self._data.values(): - plugins.sort(cmp=lambda x, y: cmp(x.name.lower(), y.name.lower())) + plugins.sort(key=lambda x: x.name.lower()) def universal_set(self): ans = set([]) diff --git a/src/calibre/gui2/wizard/__init__.py b/src/calibre/gui2/wizard/__init__.py index ec61d6c0fe..e137babbbe 100644 --- a/src/calibre/gui2/wizard/__init__.py +++ b/src/calibre/gui2/wizard/__init__.py @@ -427,7 +427,7 @@ def get_manufacturers(): def get_devices_of(manufacturer): ans = [d for d in get_devices() if d.manufacturer == manufacturer] - return sorted(ans, cmp=lambda x,y:cmp(x.name, y.name)) + return sorted(ans, key=lambda x: x.name) class ManufacturerModel(QAbstractListModel): @@ -682,7 +682,7 @@ class LibraryPage(QWizardPage, LibraryUI): if l != lang] if lang != 'en': items.append(('en', get_esc_lang('en'))) - items.sort(cmp=lambda x, y: cmp(x[1], y[1])) + items.sort(key=lambda x: x[1]) for item in items: self.language.addItem(item[1], (item[0])) self.language.blockSignals(False) diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index 56e53ee91b..ad2ae31e39 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -216,7 +216,7 @@ class CustomColumns(object): if data['is_multiple'] and data['datatype'] == 'text': ans = ans.split(data['multiple_seps']['cache_to_list']) if ans else [] if data['display'].get('sort_alpha', False): - ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower())) + ans.sort(key=lambda x:x.lower()) return ans def get_custom_extra(self, idx, label=None, num=None, index_is_id=False): @@ -243,7 +243,7 @@ class CustomColumns(object): if data['is_multiple'] and data['datatype'] == 'text': ans = ans.split(data['multiple_seps']['cache_to_list']) if ans else [] if data['display'].get('sort_alpha', False): - ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower())) + ans.sort(key=lambda x: x.lower()) if data['datatype'] != 'series': return (ans, None) ign,lt = self.custom_table_names(data['num']) diff --git a/src/calibre/library/database.py b/src/calibre/library/database.py index b303833439..8ce65a023e 100644 --- a/src/calibre/library/database.py +++ b/src/calibre/library/database.py @@ -1018,7 +1018,7 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE; if not ans: return [] ans = [id[0] for id in ans] - ans.sort(cmp=lambda x, y: cmp(self.series_index(x, True), self.series_index(y, True))) + ans.sort(key=lambda x: self.series_index(x, True)) return ans def books_in_series_of(self, index, index_is_id=False): diff --git a/src/calibre/utils/smtp.py b/src/calibre/utils/smtp.py index d840cc4c37..8da8cc7bd2 100644 --- a/src/calibre/utils/smtp.py +++ b/src/calibre/utils/smtp.py @@ -98,8 +98,7 @@ def get_mx(host, verbose=0): if verbose: print('Find mail exchanger for', host) answers = list(dns.resolver.query(host, 'MX')) - answers.sort(cmp=lambda x, y: cmp(int(getattr(x, 'preference', sys.maxint)), - int(getattr(y, 'preference', sys.maxint)))) + answers.sort(key=lambda x: int(getattr(x, 'preference', sys.maxint))) return [str(x.exchange) for x in answers if hasattr(x, 'exchange')] diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index f51f7e5a21..101e68b510 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -760,7 +760,7 @@ class BasicNewsRecipe(Recipe): in index are not in weights, they are assumed to have a weight of 0. ''' weights = defaultdict(lambda: 0, weights) - index.sort(cmp=lambda x, y: cmp(weights[x], weights[y])) + index.sort(key=lambda x: weights[x]) return index def parse_index(self): From 9023ea8947f228ec2ef1e72f254930ff8a4d8145 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 13:00:31 -0400 Subject: [PATCH 15/19] python3: add Cookie wrapper to polyglot --- src/calibre/srv/tests/auth.py | 5 +++-- src/calibre/srv/utils.py | 2 +- src/calibre/utils/browser.py | 2 +- src/polyglot/http_cookie.py | 12 ++++++++++++ 4 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 src/polyglot/http_cookie.py diff --git a/src/calibre/srv/tests/auth.py b/src/calibre/srv/tests/auth.py index 9407516cf0..45764253e6 100644 --- a/src/calibre/srv/tests/auth.py +++ b/src/calibre/srv/tests/auth.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import base64, subprocess, os, cookielib, time +import base64, subprocess, os, time from collections import namedtuple try: from distutils.spawn import find_executable @@ -19,6 +19,7 @@ from calibre.srv.tests.base import BaseTest, TestServer from calibre.srv.routes import endpoint, Router from polyglot.builtins import iteritems, itervalues from polyglot import http_client +from polyglot.http_cookie import CookieJar from polyglot.urllib import (build_opener, HTTPBasicAuthHandler, HTTPCookieProcessor, HTTPDigestAuthHandler, HTTPError) @@ -283,7 +284,7 @@ class TestAuth(BaseTest): auth_handler = HTTPDigestAuthHandler() url = 'http://localhost:%d%s' % (server.address[1], '/android') auth_handler.add_password(realm=REALM, uri=url, user='testuser', passwd='testpw') - cj = cookielib.CookieJar() + cj = CookieJar() cookie_handler = HTTPCookieProcessor(cj) r = build_opener(auth_handler, cookie_handler).open(url) self.ae(r.getcode(), http_client.OK) diff --git a/src/calibre/srv/utils.py b/src/calibre/srv/utils.py index 0037e4e8a7..22f9d2b777 100644 --- a/src/calibre/srv/utils.py +++ b/src/calibre/srv/utils.py @@ -7,7 +7,6 @@ __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' import errno, socket, select, os, time -from Cookie import SimpleCookie from contextlib import closing from email.utils import formatdate from operator import itemgetter @@ -23,6 +22,7 @@ from calibre.utils.logging import ThreadSafeLog from calibre.utils.shared_file import share_open, raise_winerror from polyglot.builtins import iteritems, map, unicode_type, range from polyglot import reprlib +from polyglot.http_cookie import SimpleCookie from polyglot.urllib import parse_qs, quote as urlquote HTTP1 = 'HTTP/1.0' diff --git a/src/calibre/utils/browser.py b/src/calibre/utils/browser.py index 47835dec1e..1228747ee8 100644 --- a/src/calibre/utils/browser.py +++ b/src/calibre/utils/browser.py @@ -6,11 +6,11 @@ __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' import copy, ssl -from cookielib import CookieJar, Cookie from mechanize import Browser as B, HTTPSHandler from polyglot import http_client +from polyglot.http_cookie import CookieJar, Cookie class ModernHTTPSHandler(HTTPSHandler): diff --git a/src/polyglot/http_cookie.py b/src/polyglot/http_cookie.py new file mode 100644 index 0000000000..645b8b6a3a --- /dev/null +++ b/src/polyglot/http_cookie.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +# License: GPL v3 Copyright: 2019, Eli Schwartz + +from polyglot.builtins import is_py3 + +if is_py3: + from http.cookies import SimpleCookie # noqa + from http.cookiejar import CookieJar, Cookie # noqa +else: + from Cookie import SimpleCookie # noqa + from cookielib import CookieJar, Cookie # noqa From 491937360891d60b820a0d7e05aaa3bd25811e95 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 13:13:46 -0400 Subject: [PATCH 16/19] python3: do not decode uuid4 hex, as it is always a str() --- src/calibre/srv/http_response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/srv/http_response.py b/src/calibre/srv/http_response.py index b5b16fbe8f..8313d18e32 100644 --- a/src/calibre/srv/http_response.py +++ b/src/calibre/srv/http_response.py @@ -29,7 +29,7 @@ from calibre.utils.monotonic import monotonic from polyglot import http_client, reprlib Range = namedtuple('Range', 'start stop size') -MULTIPART_SEPARATOR = uuid.uuid4().hex.decode('ascii') +MULTIPART_SEPARATOR = uuid.uuid4().hex COMPRESSIBLE_TYPES = {'application/json', 'application/javascript', 'application/xml', 'application/oebps-package+xml'} if is_py3: import zlib From 24f504f16bf4320c77d058687cc529a8419d3749 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 14:26:32 -0400 Subject: [PATCH 17/19] python3: remove more cStringIO --- src/calibre/devices/__init__.py | 4 ++-- src/calibre/ebooks/conversion/plugins/tcr_input.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/devices/__init__.py b/src/calibre/devices/__init__.py index 5ab3b38dee..83ad041ff6 100644 --- a/src/calibre/devices/__init__.py +++ b/src/calibre/devices/__init__.py @@ -8,7 +8,7 @@ Device drivers. import sys, time, pprint from functools import partial -from StringIO import StringIO +from io import BytesIO DAY_MAP = dict(Sun=0, Mon=1, Tue=2, Wed=3, Thu=4, Fri=5, Sat=6) MONTH_MAP = dict(Jan=1, Feb=2, Mar=3, Apr=4, May=5, Jun=6, Jul=7, Aug=8, Sep=9, Oct=10, Nov=11, Dec=12) @@ -77,7 +77,7 @@ def debug(ioreg_to_tmp=False, buf=None, plugins=None, oldo, olde = sys.stdout, sys.stderr if buf is None: - buf = StringIO() + buf = BytesIO() sys.stdout = sys.stderr = buf out = partial(prints, file=buf) diff --git a/src/calibre/ebooks/conversion/plugins/tcr_input.py b/src/calibre/ebooks/conversion/plugins/tcr_input.py index 45903e2c56..0578eb3aac 100644 --- a/src/calibre/ebooks/conversion/plugins/tcr_input.py +++ b/src/calibre/ebooks/conversion/plugins/tcr_input.py @@ -4,7 +4,7 @@ __license__ = 'GPL 3' __copyright__ = '2009, John Schember ' __docformat__ = 'restructuredtext en' -from cStringIO import StringIO +from io import BytesIO from calibre.customize.conversion import InputFormatPlugin @@ -24,7 +24,7 @@ class TCRInput(InputFormatPlugin): raw_txt = decompress(stream) log.info('Converting text to OEB...') - stream = StringIO(raw_txt) + stream = BytesIO(raw_txt) from calibre.customize.ui import plugin_for_input_format From a75b0730a7190f5720af6a3a41ad2e3896fdd06f Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 14:37:29 -0400 Subject: [PATCH 18/19] python3: do not use types.*Type which is an alias for str Just compare to (str, unicode_type) directly, and handle the related python3 reclassification. --- src/calibre/library/catalogs/bibtex.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/calibre/library/catalogs/bibtex.py b/src/calibre/library/catalogs/bibtex.py index 4a6e8a4e27..d21d9a6fa7 100644 --- a/src/calibre/library/catalogs/bibtex.py +++ b/src/calibre/library/catalogs/bibtex.py @@ -7,7 +7,6 @@ __docformat__ = 'restructuredtext en' import re, codecs, os, numbers from collections import namedtuple -from types import StringType, UnicodeType from calibre import (strftime) from calibre.customize import CatalogPlugin @@ -15,7 +14,7 @@ from calibre.library.catalogs import FIELDS, TEMPLATE_ALLOWED_FIELDS from calibre.customize.conversion import DummyReporter from calibre.constants import preferred_encoding from calibre.ebooks.metadata import format_isbn -from polyglot.builtins import string_or_bytes +from polyglot.builtins import string_or_bytes, unicode_type class BIBTEX(CatalogPlugin): @@ -351,7 +350,7 @@ class BIBTEX(CatalogPlugin): bibtexc.ascii_bibtex = True # Check citation choice and go to default in case of bad CLI - if isinstance(opts.impcit, (StringType, UnicodeType)) : + if isinstance(opts.impcit, (str, unicode_type)) : if opts.impcit == 'False' : citation_bibtex= False elif opts.impcit == 'True' : From 02be4edee24ecc94fccfd9a6165903a60901a9f3 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 16:46:44 -0400 Subject: [PATCH 19/19] test: try to make test_import work better without assuming cwd Allows it to run properly on Windows. --- src/calibre/test_build.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 397299264b..50fb71430d 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -82,7 +82,9 @@ class BuildTest(unittest.TestCase): exclude.append('osx') if not islinux: exclude.extend(['dbus', 'linux']) - for root, dirs, files in os.walk('src/calibre'): + base = os.path.dirname(__file__) + trimpath = len(os.path.dirname(base)) + 1 + for root, dirs, files in os.walk(base): for dir in dirs: if not os.path.isfile(os.path.join(root, dir, '__init__.py')): dirs.remove(dir) @@ -90,7 +92,7 @@ class BuildTest(unittest.TestCase): file, ext = os.path.splitext(file) if ext != '.py': continue - name = '.'.join(root.split(os.path.sep)[1:] + [file]) + name = '.'.join(root[trimpath:].split(os.path.sep) + [file]) if not any(x for x in exclude if x in name): importlib.import_module(name)