diff --git a/manual/custom.py b/manual/custom.py index 2c9b31d514..b35ff29234 100644 --- a/manual/custom.py +++ b/manual/custom.py @@ -179,7 +179,7 @@ def generate_ebook_convert_help(preamble, app): raw += '\n\n.. contents::\n :local:' raw += '\n\n' + options - for pl in sorted(input_format_plugins(), key=lambda x:x.name): + for pl in sorted(input_format_plugins(), key=lambda x: x.name): parser, plumber = create_option_parser(['ebook-convert', 'dummyi.'+sorted(pl.file_types)[0], 'dummyo.epub', '-h'], default_log) groups = [(pl.name+ ' Options', '', g.option_list) for g in @@ -279,7 +279,7 @@ def cli_docs(app): info(bold('creating CLI documentation...')) documented_cmds, undocumented_cmds = get_cli_docs() - documented_cmds.sort(cmp=lambda x, y: cmp(x[0], y[0])) + documented_cmds.sort(key=lambda x: x[0]) undocumented_cmds.sort() documented = [' '*4 + c[0] for c in documented_cmds] diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 867163fc2b..11a2317d69 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -240,7 +240,7 @@ def prints(*args, **kwargs): file.write(arg) count += len(arg) except: - import repr as reprlib + from polyglot import reprlib arg = reprlib.repr(arg) file.write(arg) count += len(arg) @@ -614,7 +614,7 @@ def entity_to_unicode(match, exceptions=[], encoding='cp1252', return check(html5_entities[ent]) except KeyError: pass - from htmlentitydefs import name2codepoint + from polyglot.html_entities import name2codepoint try: return check(my_unichr(name2codepoint[ent])) except KeyError: diff --git a/src/calibre/constants.py b/src/calibre/constants.py index e5781fe5b3..05ac5bbb3e 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -168,7 +168,6 @@ class Plugins(collections.Mapping): 'icu', 'speedup', 'unicode_names', - 'zlib2', 'html', 'freetype', 'imageops', @@ -184,6 +183,7 @@ class Plugins(collections.Mapping): if not ispy3: plugins.extend([ 'monotonic', + 'zlib2', ]) if iswindows: plugins.extend(['winutil', 'wpd', 'winfonts']) diff --git a/src/calibre/customize/profiles.py b/src/calibre/customize/profiles.py index 2c7fead69f..1038d448be 100644 --- a/src/calibre/customize/profiles.py +++ b/src/calibre/customize/profiles.py @@ -233,7 +233,7 @@ input_profiles = [InputProfile, SonyReaderInput, SonyReader300Input, HanlinV5Input, CybookG3Input, CybookOpusInput, KindleInput, IlliadInput, IRexDR1000Input, IRexDR800Input, NookInput] -input_profiles.sort(cmp=lambda x,y:cmp(x.name.lower(), y.name.lower())) +input_profiles.sort(key=lambda x: x.name.lower()) # }}} @@ -870,4 +870,4 @@ output_profiles = [ KindlePaperWhite3Output, KindleOasisOutput ] -output_profiles.sort(cmp=lambda x,y:cmp(x.name.lower(), y.name.lower())) +output_profiles.sort(key=lambda x: x.name.lower()) diff --git a/src/calibre/customize/ui.py b/src/calibre/customize/ui.py index 1229cbf712..8bb9b5814f 100644 --- a/src/calibre/customize/ui.py +++ b/src/calibre/customize/ui.py @@ -727,9 +727,9 @@ def initialize_plugins(perf=False): # ipython sys.stdout, sys.stderr = ostdout, ostderr if perf: - for x in sorted(times, key=lambda x:times[x]): + for x in sorted(times, key=lambda x: times[x]): print('%50s: %.3f'%(x, times[x])) - _initialized_plugins.sort(cmp=lambda x,y:cmp(x.priority, y.priority), reverse=True) + _initialized_plugins.sort(key=lambda x: x.priority, reverse=True) reread_filetype_plugins() reread_metadata_plugins() diff --git a/src/calibre/db/cli/cmd_list_categories.py b/src/calibre/db/cli/cmd_list_categories.py index 0280455f66..6165c50877 100644 --- a/src/calibre/db/cli/cmd_list_categories.py +++ b/src/calibre/db/cli/cmd_list_categories.py @@ -149,9 +149,7 @@ def main(opts, args, dbctx): (not report_on or k in report_on) ] - categories.sort( - cmp=lambda x, y: cmp(x if x[0] != '#' else x[1:], y if y[0] != '#' else y[1:]) - ) + categories.sort(key=lambda x: x if x[0] != '#' else x[1:]) def fmtr(v): v = v or 0 diff --git a/src/calibre/db/cli/main.py b/src/calibre/db/cli/main.py index 9833d919d4..a80da22180 100644 --- a/src/calibre/db/cli/main.py +++ b/src/calibre/db/cli/main.py @@ -4,7 +4,6 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import httplib import json import os import sys @@ -17,6 +16,7 @@ from calibre.utils.config import OptionParser, prefs from calibre.utils.localization import localize_user_manual_link from calibre.utils.lock import singleinstance from calibre.utils.serialize import MSGPACK_MIME +from polyglot import http_client from polyglot.urllib import urlencode, urlparse, urlunparse COMMANDS = ( @@ -191,13 +191,13 @@ class DBCtx(object): return m.implementation(self.db.new_api, None, *args) def interpret_http_error(self, err): - if err.code == httplib.UNAUTHORIZED: + if err.code == http_client.UNAUTHORIZED: if self.has_credentials: raise SystemExit('The username/password combination is incorrect') raise SystemExit('A username and password is required to access this server') - if err.code == httplib.FORBIDDEN: + if err.code == http_client.FORBIDDEN: raise SystemExit(err.reason) - if err.code == httplib.NOT_FOUND: + if err.code == http_client.NOT_FOUND: raise SystemExit(err.reason) def remote_run(self, name, m, *args): diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index b0f88feb30..b7406bbca8 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -8,13 +8,13 @@ __copyright__ = '2013, Kovid Goyal ' import inspect, time, numbers from io import BytesIO -from repr import repr from functools import partial from operator import itemgetter from calibre.library.field_metadata import fm_as_dict from calibre.db.tests.base import BaseTest from polyglot.builtins import iteritems, range +from polyglot import reprlib # Utils {{{ @@ -32,7 +32,7 @@ class ET(object): oldres = getattr(old, self.func_name)(*self.args, **self.kwargs) newres = getattr(legacy, self.func_name)(*self.args, **self.kwargs) test.assertEqual(oldres, newres, 'Equivalence test for %s with args: %s and kwargs: %s failed' % ( - self.func_name, repr(self.args), repr(self.kwargs))) + self.func_name, reprlib.repr(self.args), reprlib.repr(self.kwargs))) self.retval = newres return newres diff --git a/src/calibre/devices/__init__.py b/src/calibre/devices/__init__.py index d3ee2b0f19..83ad041ff6 100644 --- a/src/calibre/devices/__init__.py +++ b/src/calibre/devices/__init__.py @@ -8,7 +8,7 @@ Device drivers. import sys, time, pprint from functools import partial -from StringIO import StringIO +from io import BytesIO DAY_MAP = dict(Sun=0, Mon=1, Tue=2, Wed=3, Thu=4, Fri=5, Sat=6) MONTH_MAP = dict(Jan=1, Feb=2, Mar=3, Apr=4, May=5, Jun=6, Jul=7, Aug=8, Sep=9, Oct=10, Nov=11, Dec=12) @@ -77,13 +77,12 @@ def debug(ioreg_to_tmp=False, buf=None, plugins=None, oldo, olde = sys.stdout, sys.stderr if buf is None: - buf = StringIO() + buf = BytesIO() sys.stdout = sys.stderr = buf out = partial(prints, file=buf) devplugins = device_plugins() if plugins is None else plugins - devplugins = list(sorted(devplugins, cmp=lambda - x,y:cmp(x.__class__.__name__, y.__class__.__name__))) + devplugins = list(sorted(devplugins, key=lambda x: x.__class__.__name__)) if plugins is None: for d in devplugins: try: diff --git a/src/calibre/devices/prs505/sony_cache.py b/src/calibre/devices/prs505/sony_cache.py index c8db25b73a..0e5a90aaee 100644 --- a/src/calibre/devices/prs505/sony_cache.py +++ b/src/calibre/devices/prs505/sony_cache.py @@ -321,7 +321,7 @@ class XMLCache(object): # Only rebase ids of nodes that are immediate children of the # record root (that way playlist/itemnodes are unaffected items = root.xpath('child::*[@id]') - items.sort(cmp=lambda x,y:cmp(int(x.get('id')), int(y.get('id')))) + items.sort(key=lambda x: int(x.get('id'))) idmap = {} for i, item in enumerate(items): old = int(item.get('id')) diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index a7eb0b3567..b025ca7abf 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -537,7 +537,7 @@ class Device(DeviceConfig, DevicePlugin): if sz > 0: nodes.append((x.split('/')[-1], sz)) - nodes.sort(cmp=lambda x, y: cmp(x[1], y[1])) + nodes.sort(key=lambda x: x[1]) if not nodes: return node return nodes[-1][0] diff --git a/src/calibre/devices/winusb.py b/src/calibre/devices/winusb.py index 0a7647cce0..dd927e5df4 100644 --- a/src/calibre/devices/winusb.py +++ b/src/calibre/devices/winusb.py @@ -995,8 +995,7 @@ def develop(): # {{{ drive_letters = set() pprint(usb_devices) print() - devplugins = list(sorted(device_plugins(), cmp=lambda - x,y:cmp(x.__class__.__name__, y.__class__.__name__))) + devplugins = list(sorted(device_plugins(), key=lambda x: x.__class__.__name__)) for dev in devplugins: dev.startup() for dev in devplugins: diff --git a/src/calibre/ebooks/conversion/plugins/tcr_input.py b/src/calibre/ebooks/conversion/plugins/tcr_input.py index 45903e2c56..0578eb3aac 100644 --- a/src/calibre/ebooks/conversion/plugins/tcr_input.py +++ b/src/calibre/ebooks/conversion/plugins/tcr_input.py @@ -4,7 +4,7 @@ __license__ = 'GPL 3' __copyright__ = '2009, John Schember ' __docformat__ = 'restructuredtext en' -from cStringIO import StringIO +from io import BytesIO from calibre.customize.conversion import InputFormatPlugin @@ -24,7 +24,7 @@ class TCRInput(InputFormatPlugin): raw_txt = decompress(stream) log.info('Converting text to OEB...') - stream = StringIO(raw_txt) + stream = BytesIO(raw_txt) from calibre.customize.ui import plugin_for_input_format diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index 25ad577c8b..5d8e46fd67 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -821,7 +821,7 @@ OptionRecommendation(name='search_replace', if not html_files: raise ValueError(_('Could not find an e-book inside the archive')) html_files = [(f, os.stat(f).st_size) for f in html_files] - html_files.sort(cmp=lambda x, y: cmp(x[1], y[1])) + html_files.sort(key=lambda x: x[1]) html_files = [f[0] for f in html_files] for q in ('toc', 'index'): for f in html_files: diff --git a/src/calibre/ebooks/lit/from_any.py b/src/calibre/ebooks/lit/from_any.py deleted file mode 100644 index 05345b6749..0000000000 --- a/src/calibre/ebooks/lit/from_any.py +++ /dev/null @@ -1,67 +0,0 @@ -from __future__ import with_statement -from __future__ import print_function -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' -__docformat__ = 'restructuredtext en' - -''' -Convert any ebook format to LIT. -''' - -import sys, os, glob, logging - -from calibre.ebooks.epub.from_any import any2epub, formats, USAGE -from calibre.ebooks.epub import config as common_config -from calibre.ptempfile import TemporaryDirectory -from calibre.ebooks.lit.writer import oeb2lit - - -def config(defaults=None): - c = common_config(defaults=defaults, name='lit') - return c - - -def option_parser(usage=USAGE): - return config().option_parser(usage=usage%('LIT', formats())) - - -def any2lit(opts, path): - ext = os.path.splitext(path)[1] - if not ext: - raise ValueError('Unknown file type: '+path) - ext = ext.lower()[1:] - - if opts.output is None: - opts.output = os.path.splitext(os.path.basename(path))[0]+'.lit' - - opts.output = os.path.abspath(opts.output) - orig_output = opts.output - - with TemporaryDirectory('_any2lit') as tdir: - oebdir = os.path.join(tdir, 'oeb') - os.mkdir(oebdir) - opts.output = os.path.join(tdir, 'dummy.epub') - opts.profile = 'None' - orig_bfs = opts.base_font_size2 - opts.base_font_size2 = 0 - any2epub(opts, path, create_epub=False, oeb_cover=True, extract_to=oebdir) - opts.base_font_size2 = orig_bfs - opf = glob.glob(os.path.join(oebdir, '*.opf'))[0] - opts.output = orig_output - logging.getLogger('html2epub').info(_('Creating LIT file from EPUB...')) - oeb2lit(opts, opf) - - -def main(args=sys.argv): - parser = option_parser() - opts, args = parser.parse_args(args) - if len(args) < 2: - parser.print_help() - print('No input file specified.') - return 1 - any2lit(opts, args[1]) - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/lit/writer.py b/src/calibre/ebooks/lit/writer.py index fd292f4a03..490c06b42f 100644 --- a/src/calibre/ebooks/lit/writer.py +++ b/src/calibre/ebooks/lit/writer.py @@ -670,8 +670,7 @@ class LitWriter(object): def _build_dchunks(self): ddata = [] directory = list(self._directory) - directory.sort(cmp=lambda x, y: - cmp(x.name.lower(), y.name.lower())) + directory.sort(key=lambda x: x.name.lower()) qrn = 1 + (1 << 2) dchunk = io.BytesIO() dcount = 0 diff --git a/src/calibre/ebooks/lrf/html/convert_to.py b/src/calibre/ebooks/lrf/html/convert_to.py deleted file mode 100644 index 8ae9224d2c..0000000000 --- a/src/calibre/ebooks/lrf/html/convert_to.py +++ /dev/null @@ -1,128 +0,0 @@ -from __future__ import print_function -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal ' -import sys, logging, os - -from calibre import setup_cli_handlers -from calibre.utils.config import OptionParser -from calibre.ebooks import ConversionError -from calibre.ebooks.lrf.meta import get_metadata -from calibre.ebooks.lrf.lrfparser import LRFDocument -from calibre.ebooks.metadata.opf import OPFCreator - -from calibre.ebooks.lrf.objects import PageAttr, BlockAttr, TextAttr -from calibre.ebooks.lrf.pylrs.pylrs import TextStyle - - -class BlockStyle(object): - - def __init__(self, ba): - self.ba = ba - - def __str__(self): - ans = '.'+str(self.ba.id)+' {\n' - if hasattr(self.ba, 'sidemargin'): - margin = str(self.ba.sidemargin) + 'px' - ans += '\tmargin-left: %(m)s; margin-right: %(m)s;\n'%dict(m=margin) - if hasattr(self.ba, 'topskip'): - ans += '\tmargin-top: %dpx;\n'%(self.ba.topskip,) - if hasattr(self.ba, 'footskip'): - ans += '\tmargin-bottom: %dpx;\n'%(self.ba.footskip,) - if hasattr(self.ba, 'framewidth'): - ans += '\tborder-width: %dpx;\n'%(self.ba.framewidth,) - ans += '\tborder-style: solid;\n' - if hasattr(self.ba, 'framecolor'): - if self.ba.framecolor.a < 255: - ans += '\tborder-color: %s;\n'%(self.ba.framecolor.to_html()) - if hasattr(self.ba, 'bgcolor'): - if self.ba.bgcolor.a < 255: - ans += '\tbackground-color: %s;\n'%(self.ba.bgcolor.to_html()) - # TODO: Fixed size blocks - return ans + '}\n' - - -class LRFConverter(object): - - def __init__(self, document, opts, logger): - self.lrf = document - self.opts = opts - self.output_dir = opts.out - self.logger = logger - logger.info('Parsing LRF...') - self.lrf.parse() - - self.create_metadata() - self.create_styles() - - def create_metadata(self): - self.logger.info('Reading metadata...') - mi = get_metadata(self.lrf) - self.opf = OPFCreator(self.output_dir, mi) - - def create_page_styles(self): - self.page_css = '' - for obj in self.lrf.objects.values(): - if isinstance(obj, PageAttr): - selector = 'body.'+str(obj.id) - self.page_css = selector + ' {\n' - # TODO: Headers and footers - self.page_css += '}\n' - - def create_block_styles(self): - self.block_css = '' - for obj in self.lrf.objects.values(): - if isinstance(obj, BlockAttr): - self.block_css += str(BlockStyle(obj)) - - def create_text_styles(self): - self.text_css = '' - for obj in self.lrf.objects.values(): - if isinstance(obj, TextAttr): - self.text_css += str(TextStyle(obj)) - print(self.text_css) - - def create_styles(self): - self.logger.info('Creating CSS stylesheet...') - self.create_page_styles() - self.create_block_styles() - - -def option_parser(): - parser = OptionParser(usage='%prog book.lrf') - parser.add_option('--output-dir', '-o', default=None, help=( - 'Output directory in which to store created HTML files. If it does not exist, it is created. By default the current directory is used.'), dest='out') - parser.add_option('--verbose', default=False, action='store_true', dest='verbose') - return parser - - -def process_file(lrfpath, opts, logger=None): - if logger is None: - level = logging.DEBUG if opts.verbose else logging.INFO - logger = logging.getLogger('lrf2html') - setup_cli_handlers(logger, level) - if opts.out is None: - opts.out = os.getcwdu() - else: - opts.out = os.path.abspath(opts.out) - if not os.path.isdir(opts.out): - raise ConversionError(opts.out + ' is not a directory') - if not os.path.exists(opts.out): - os.makedirs(opts.out) - - document = LRFDocument(open(lrfpath, 'rb')) - LRFConverter(document, opts, logger) - - -def main(args=sys.argv): - parser = option_parser() - opts, args = parser.parse_args(args) - if len(args) != 2: - parser.print_help() - return 1 - process_file(args[1], opts) - - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/metadata/lit.py b/src/calibre/ebooks/metadata/lit.py index 330bf8cc86..d2f078487d 100644 --- a/src/calibre/ebooks/metadata/lit.py +++ b/src/calibre/ebooks/metadata/lit.py @@ -32,7 +32,7 @@ def get_metadata(stream): except: pass break - covers.sort(cmp=lambda x, y:cmp(len(x[0]), len(y[0])), reverse=True) + covers.sort(key=lambda x: len(x[0]), reverse=True) idx = 0 if len(covers) > 1: if covers[1][1] == covers[0][1]+'-standard': diff --git a/src/calibre/ebooks/metadata/meta.py b/src/calibre/ebooks/metadata/meta.py index de773ec8c6..7a8299dc19 100644 --- a/src/calibre/ebooks/metadata/meta.py +++ b/src/calibre/ebooks/metadata/meta.py @@ -41,8 +41,7 @@ def metadata_from_formats(formats, force_read_metadata=False, pattern=None): def _metadata_from_formats(formats, force_read_metadata=False, pattern=None): mi = MetaInformation(None, None) - formats.sort(cmp=lambda x,y: cmp(METADATA_PRIORITIES[path_to_ext(x)], - METADATA_PRIORITIES[path_to_ext(y)])) + formats.sort(key=lambda x: METADATA_PRIORITIES[path_to_ext(x)]) extensions = list(map(path_to_ext, formats)) if 'opf' in extensions: opf = formats[extensions.index('opf')] @@ -241,4 +240,3 @@ def forked_read_metadata(path, tdir): opf = metadata_to_opf(mi, default_lang='und') with lopen(os.path.join(tdir, 'metadata.opf'), 'wb') as f: f.write(opf) - diff --git a/src/calibre/ebooks/oeb/transforms/guide.py b/src/calibre/ebooks/oeb/transforms/guide.py index 29c5f2a93a..104a2fc1d5 100644 --- a/src/calibre/ebooks/oeb/transforms/guide.py +++ b/src/calibre/ebooks/oeb/transforms/guide.py @@ -28,7 +28,7 @@ class Clean(object): else: covers.append([self.oeb.guide[x], len(item.data)]) - covers.sort(cmp=lambda x,y:cmp(x[1], y[1]), reverse=True) + covers.sort(key=lambda x: x[1], reverse=True) if covers: ref = covers[0][0] if len(covers) > 1: @@ -53,4 +53,3 @@ class Clean(object): if item.title and item.title.lower() == 'start': continue self.oeb.guide.remove(x) - diff --git a/src/calibre/ebooks/pdf/from_comic.py b/src/calibre/ebooks/pdf/from_comic.py deleted file mode 100644 index c39b660ef4..0000000000 --- a/src/calibre/ebooks/pdf/from_comic.py +++ /dev/null @@ -1,21 +0,0 @@ -from __future__ import with_statement -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' -__docformat__ = 'restructuredtext en' - -'Convert a comic in CBR/CBZ format to pdf' - -import sys -from functools import partial -from calibre.ebooks.lrf.comic.convert_from import do_convert, option_parser, config, main as _main - -convert = partial(do_convert, output_format='pdf') -main = partial(_main, output_format='pdf') - -if __name__ == '__main__': - sys.exit(main()) - -if False: - option_parser - config - diff --git a/src/calibre/ebooks/pdf/reflow.py b/src/calibre/ebooks/pdf/reflow.py index a7708ea8ce..337147d8a7 100644 --- a/src/calibre/ebooks/pdf/reflow.py +++ b/src/calibre/ebooks/pdf/reflow.py @@ -169,7 +169,7 @@ class Column(object): self._post_add() def _post_add(self): - self.elements.sort(cmp=lambda x,y:cmp(x.bottom,y.bottom)) + self.elements.sort(key=lambda x: x.bottom) self.top = self.elements[0].top self.bottom = self.elements[-1].bottom self.left, self.right = sys.maxint, 0 @@ -260,7 +260,7 @@ class Region(object): def add(self, columns): if not self.columns: - for x in sorted(columns, cmp=lambda x,y: cmp(x.left, y.left)): + for x in sorted(columns, key=lambda x: x.left): self.columns.append(x) else: for i in range(len(columns)): @@ -458,7 +458,7 @@ class Page(object): self.elements = list(self.texts) for img in page.xpath('descendant::img'): self.elements.append(Image(img, self.opts, self.log, idc)) - self.elements.sort(cmp=lambda x,y:cmp(x.top, y.top)) + self.elements.sort(key=lambda x: x.top) def coalesce_fragments(self): @@ -580,7 +580,7 @@ class Page(object): def sort_into_columns(self, elem, neighbors): neighbors.add(elem) - neighbors = sorted(neighbors, cmp=lambda x,y:cmp(x.left, y.left)) + neighbors = sorted(neighbors, key=lambda x: x.left) if self.opts.verbose > 3: self.log.debug('Neighbors:', [x.to_html() for x in neighbors]) columns = [Column()] @@ -595,7 +595,7 @@ class Page(object): if not added: columns.append(Column()) columns[-1].add(x) - columns.sort(cmp=lambda x,y:cmp(x.left, y.left)) + columns.sort(key=lambda x: x.left) return columns def find_elements_in_row_of(self, x): diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py index a8adb593e4..a562958470 100644 --- a/src/calibre/gui2/actions/choose_library.py +++ b/src/calibre/gui2/actions/choose_library.py @@ -51,8 +51,7 @@ class LibraryUsageStats(object): # {{{ def write_stats(self): locs = list(self.stats.keys()) - locs.sort(cmp=lambda x, y: cmp(self.stats[x], self.stats[y]), - reverse=True) + locs.sort(key=lambda x: self.stats[x], reverse=True) for key in locs[500:]: self.stats.pop(key) gprefs.set('library_usage_stats', self.stats) diff --git a/src/calibre/gui2/dialogs/catalog.py b/src/calibre/gui2/dialogs/catalog.py index 1f32406b76..f10eab4384 100644 --- a/src/calibre/gui2/dialogs/catalog.py +++ b/src/calibre/gui2/dialogs/catalog.py @@ -93,7 +93,7 @@ class Catalog(QDialog, Ui_Dialog): else: info("No dynamic tab resources found for %s" % name) - self.widgets = sorted(self.widgets, cmp=lambda x,y:cmp(x.TITLE, y.TITLE)) + self.widgets = sorted(self.widgets, key=lambda x: x.TITLE) # Generate a sorted list of installed catalog formats/sync_enabled pairs fmts = sorted([x[0] for x in self.fmts]) diff --git a/src/calibre/gui2/icon_theme.py b/src/calibre/gui2/icon_theme.py index e4c48f9562..e5d1c3a0a0 100644 --- a/src/calibre/gui2/icon_theme.py +++ b/src/calibre/gui2/icon_theme.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import os, errno, json, importlib, math, httplib, bz2, shutil, sys +import os, errno, json, importlib, math, bz2, shutil, sys from itertools import count from io import BytesIO from threading import Thread, Event @@ -35,8 +35,9 @@ from calibre.utils.img import image_from_data, Canvas, optimize_png, optimize_jp from calibre.utils.zipfile import ZipFile, ZIP_STORED from calibre.utils.filenames import atomic_rename from lzma.xz import compress, decompress -from polyglot.queue import Queue, Empty from polyglot.builtins import iteritems, map, range, reraise +from polyglot import http_client +from polyglot.queue import Queue, Empty IMAGE_EXTENSIONS = {'png', 'jpg', 'jpeg'} THEME_COVER = 'icon-theme-cover.jpg' @@ -439,7 +440,7 @@ def download_cover(cover_url, etag=None, cached=b''): etag = response.getheader('ETag', None) or None return cached, etag except HTTPError as e: - if etag and e.code == httplib.NOT_MODIFIED: + if etag and e.code == http_client.NOT_MODIFIED: return cached, etag raise diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index b36f91798c..d058e396e0 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -323,7 +323,7 @@ class BooksModel(QAbstractTableModel): # {{{ return 100000 return self.db.field_metadata[name]['rec_index'] - self.column_map.sort(cmp=lambda x,y: cmp(col_idx(x), col_idx(y))) + self.column_map.sort(key=lambda x: col_idx(x)) for col in self.column_map: if col in self.orig_headers: self.headers[col] = self.orig_headers[col] diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py index a49f7c778f..a5ce76815d 100644 --- a/src/calibre/gui2/library/views.py +++ b/src/calibre/gui2/library/views.py @@ -1031,7 +1031,7 @@ class BooksView(QTableView): # {{{ h.visualIndex(x) > -1] if not pairs: pairs = [(0, 0)] - pairs.sort(cmp=lambda x,y:cmp(x[1], y[1])) + pairs.sort(key=lambda x: x[1]) i = pairs[0][0] index = self.model().index(row, i) if for_sync: diff --git a/src/calibre/gui2/preferences/columns.py b/src/calibre/gui2/preferences/columns.py index 0cd0fe5bea..3d346c41fb 100644 --- a/src/calibre/gui2/preferences/columns.py +++ b/src/calibre/gui2/preferences/columns.py @@ -69,7 +69,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): state = self.columns_state(defaults) self.hidden_cols = state['hidden_columns'] positions = state['column_positions'] - colmap.sort(cmp=lambda x,y: cmp(positions[x], positions[y])) + colmap.sort(key=lambda x: positions[x]) self.opt_columns.clear() db = model.db @@ -248,12 +248,10 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): if 'ondevice' in hidden_cols: hidden_cols.remove('ondevice') - def col_pos(x, y): - xidx = config_cols.index(x) if x in config_cols else sys.maxint - yidx = config_cols.index(y) if y in config_cols else sys.maxint - return cmp(xidx, yidx) + def col_pos(x): + return config_cols.index(x) if x in config_cols else sys.maxint positions = {} - for i, col in enumerate((sorted(model.column_map, cmp=col_pos))): + for i, col in enumerate((sorted(model.column_map, key=col_pos))): positions[col] = i state = {'hidden_columns': hidden_cols, 'column_positions':positions} self.gui.library_view.apply_state(state) diff --git a/src/calibre/gui2/preferences/look_feel.py b/src/calibre/gui2/preferences/look_feel.py index 0fb52d9a97..258ffcdc15 100644 --- a/src/calibre/gui2/preferences/look_feel.py +++ b/src/calibre/gui2/preferences/look_feel.py @@ -456,7 +456,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): if l != lang] if lang != 'en': items.append(('en', get_esc_lang('en'))) - items.sort(cmp=lambda x, y: cmp(x[1].lower(), y[1].lower())) + items.sort(key=lambda x: x[1].lower()) choices = [(y, x) for x, y in items] # Default language is the autodetected one choices = [(get_language(lang), lang)] + choices diff --git a/src/calibre/gui2/preferences/main.py b/src/calibre/gui2/preferences/main.py index 3b8cc99c45..43f4386d5b 100644 --- a/src/calibre/gui2/preferences/main.py +++ b/src/calibre/gui2/preferences/main.py @@ -171,7 +171,7 @@ class Browser(QScrollArea): # {{{ self.category_names = category_names categories = list(category_map.keys()) - categories.sort(cmp=lambda x, y: cmp(category_map[x], category_map[y])) + categories.sort(key=lambda x: category_map[x]) self.category_map = OrderedDict() for c in categories: @@ -181,7 +181,7 @@ class Browser(QScrollArea): # {{{ self.category_map[plugin.category].append(plugin) for plugins in self.category_map.values(): - plugins.sort(cmp=lambda x, y: cmp(x.name_order, y.name_order)) + plugins.sort(key=lambda x: x.name_order) self.widgets = [] self._layout = QVBoxLayout() diff --git a/src/calibre/gui2/preferences/plugboard.py b/src/calibre/gui2/preferences/plugboard.py index 5005e87735..65dea1c87d 100644 --- a/src/calibre/gui2/preferences/plugboard.py +++ b/src/calibre/gui2/preferences/plugboard.py @@ -34,17 +34,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): self.db = gui.library_view.model().db def initialize(self): - def field_cmp(x, y): - if x.startswith('#'): - if y.startswith('#'): - return cmp(x.lower(), y.lower()) - else: - return 1 - elif y.startswith('#'): - return -1 - else: - return cmp(x.lower(), y.lower()) - ConfigWidgetBase.initialize(self) self.current_plugboards = copy.deepcopy(self.db.prefs.get('plugboards',{})) @@ -73,7 +62,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): if n not in self.disabled_devices: self.disabled_devices.append(n) - self.devices.sort(cmp=lambda x, y: cmp(x.lower(), y.lower())) + self.devices.sort(key=lambda x: x.lower()) self.devices.insert(1, plugboard_save_to_disk_value) self.devices.insert(1, plugboard_content_server_value) self.device_to_formats_map[plugboard_content_server_value] = \ diff --git a/src/calibre/gui2/preferences/plugins.py b/src/calibre/gui2/preferences/plugins.py index a683eccf4d..c3c777f1e9 100644 --- a/src/calibre/gui2/preferences/plugins.py +++ b/src/calibre/gui2/preferences/plugins.py @@ -61,7 +61,7 @@ class PluginModel(QAbstractItemModel, AdaptSQP): # {{{ self.categories = sorted(self._data.keys()) for plugins in self._data.values(): - plugins.sort(cmp=lambda x, y: cmp(x.name.lower(), y.name.lower())) + plugins.sort(key=lambda x: x.name.lower()) def universal_set(self): ans = set([]) diff --git a/src/calibre/gui2/wizard/__init__.py b/src/calibre/gui2/wizard/__init__.py index ec61d6c0fe..e137babbbe 100644 --- a/src/calibre/gui2/wizard/__init__.py +++ b/src/calibre/gui2/wizard/__init__.py @@ -427,7 +427,7 @@ def get_manufacturers(): def get_devices_of(manufacturer): ans = [d for d in get_devices() if d.manufacturer == manufacturer] - return sorted(ans, cmp=lambda x,y:cmp(x.name, y.name)) + return sorted(ans, key=lambda x: x.name) class ManufacturerModel(QAbstractListModel): @@ -682,7 +682,7 @@ class LibraryPage(QWizardPage, LibraryUI): if l != lang] if lang != 'en': items.append(('en', get_esc_lang('en'))) - items.sort(cmp=lambda x, y: cmp(x[1], y[1])) + items.sort(key=lambda x: x[1]) for item in items: self.language.addItem(item[1], (item[0])) self.language.blockSignals(False) diff --git a/src/calibre/library/catalogs/bibtex.py b/src/calibre/library/catalogs/bibtex.py index 4a6e8a4e27..d21d9a6fa7 100644 --- a/src/calibre/library/catalogs/bibtex.py +++ b/src/calibre/library/catalogs/bibtex.py @@ -7,7 +7,6 @@ __docformat__ = 'restructuredtext en' import re, codecs, os, numbers from collections import namedtuple -from types import StringType, UnicodeType from calibre import (strftime) from calibre.customize import CatalogPlugin @@ -15,7 +14,7 @@ from calibre.library.catalogs import FIELDS, TEMPLATE_ALLOWED_FIELDS from calibre.customize.conversion import DummyReporter from calibre.constants import preferred_encoding from calibre.ebooks.metadata import format_isbn -from polyglot.builtins import string_or_bytes +from polyglot.builtins import string_or_bytes, unicode_type class BIBTEX(CatalogPlugin): @@ -351,7 +350,7 @@ class BIBTEX(CatalogPlugin): bibtexc.ascii_bibtex = True # Check citation choice and go to default in case of bad CLI - if isinstance(opts.impcit, (StringType, UnicodeType)) : + if isinstance(opts.impcit, (str, unicode_type)) : if opts.impcit == 'False' : citation_bibtex= False elif opts.impcit == 'True' : diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index 56e53ee91b..ad2ae31e39 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -216,7 +216,7 @@ class CustomColumns(object): if data['is_multiple'] and data['datatype'] == 'text': ans = ans.split(data['multiple_seps']['cache_to_list']) if ans else [] if data['display'].get('sort_alpha', False): - ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower())) + ans.sort(key=lambda x:x.lower()) return ans def get_custom_extra(self, idx, label=None, num=None, index_is_id=False): @@ -243,7 +243,7 @@ class CustomColumns(object): if data['is_multiple'] and data['datatype'] == 'text': ans = ans.split(data['multiple_seps']['cache_to_list']) if ans else [] if data['display'].get('sort_alpha', False): - ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower())) + ans.sort(key=lambda x: x.lower()) if data['datatype'] != 'series': return (ans, None) ign,lt = self.custom_table_names(data['num']) diff --git a/src/calibre/library/database.py b/src/calibre/library/database.py index b303833439..8ce65a023e 100644 --- a/src/calibre/library/database.py +++ b/src/calibre/library/database.py @@ -1018,7 +1018,7 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE; if not ans: return [] ans = [id[0] for id in ans] - ans.sort(cmp=lambda x, y: cmp(self.series_index(x, True), self.series_index(y, True))) + ans.sort(key=lambda x: self.series_index(x, True)) return ans def books_in_series_of(self, index, index_is_id=False): diff --git a/src/calibre/library/sqlite.py b/src/calibre/library/sqlite.py index d876e96ccf..488925fe76 100644 --- a/src/calibre/library/sqlite.py +++ b/src/calibre/library/sqlite.py @@ -8,7 +8,6 @@ Wrapper for multi-threaded access to a single sqlite database connection. Serial all calls. ''' import sqlite3 as sqlite, traceback, time, uuid, sys, os -import repr as reprlib from sqlite3 import IntegrityError, OperationalError from threading import Thread from threading import RLock @@ -22,6 +21,7 @@ from calibre.constants import iswindows, DEBUG, plugins from calibre.utils.icu import sort_key from calibre import prints from polyglot.builtins import unicode_type +from polyglot import reprlib from polyglot.queue import Queue from dateutil.tz import tzoffset diff --git a/src/calibre/spell/dictionary.py b/src/calibre/spell/dictionary.py index 598c6e302d..5086dc08b4 100644 --- a/src/calibre/spell/dictionary.py +++ b/src/calibre/spell/dictionary.py @@ -54,7 +54,7 @@ def builtin_dictionaries(): if _builtins is None: dics = [] for lc in glob.glob(os.path.join(P('dictionaries', allow_user_override=False), '*/locales')): - locales = filter(None, open(lc, 'rb').read().decode('utf-8').splitlines()) + locales = list(filter(None, open(lc, 'rb').read().decode('utf-8').splitlines())) locale = locales[0] base = os.path.dirname(lc) dics.append(Dictionary( @@ -69,7 +69,7 @@ def custom_dictionaries(reread=False): if _custom is None or reread: dics = [] for lc in glob.glob(os.path.join(config_dir, 'dictionaries', '*/locales')): - locales = filter(None, open(lc, 'rb').read().decode('utf-8').splitlines()) + locales = list(filter(None, open(lc, 'rb').read().decode('utf-8').splitlines())) try: name, locale, locales = locales[0], locales[1], locales[1:] except IndexError: diff --git a/src/calibre/srv/auth.py b/src/calibre/srv/auth.py index 51f884e28f..84fd260219 100644 --- a/src/calibre/srv/auth.py +++ b/src/calibre/srv/auth.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import binascii, os, random, struct, base64, httplib +import binascii, os, random, struct, base64 from collections import OrderedDict from hashlib import md5, sha256 from itertools import permutations @@ -16,6 +16,7 @@ from calibre.srv.errors import HTTPAuthRequired, HTTPSimpleResponse, HTTPForbidd from calibre.srv.http_request import parse_uri from calibre.srv.utils import parse_http_dict, encode_path from calibre.utils.monotonic import monotonic +from polyglot import http_client MAX_AGE_SECONDS = 3600 nonce_counter, nonce_counter_lock = 0, Lock() @@ -133,19 +134,19 @@ class DigestAuth(object): # {{{ self.nonce_count = data.get('nc') if self.algorithm not in self.valid_algorithms: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported digest algorithm') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported digest algorithm') if not (self.username and self.realm and self.nonce and self.uri and self.response): - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Digest algorithm required fields missing') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Digest algorithm required fields missing') if self.qop: if self.qop not in self.valid_qops: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported digest qop') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported digest qop') if not (self.cnonce and self.nonce_count): - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'qop present, but cnonce and nonce_count absent') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'qop present, but cnonce and nonce_count absent') else: if self.cnonce or self.nonce_count: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'qop missing') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'qop missing') def H(self, val): return md5_hex(val) @@ -201,7 +202,7 @@ class DigestAuth(object): # {{{ if log is not None: log.warn('Authorization URI mismatch: %s != %s from client: %s' % ( data.path, path, data.remote_addr)) - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The uri in the Request Line and the Authorization header do not match') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The uri in the Request Line and the Authorization header do not match') return self.response is not None and data.path == path and self.request_digest(pw, data) == self.response # }}} @@ -290,16 +291,16 @@ class AuthController(object): try: un, pw = base64_decode(rest.strip()).partition(':')[::2] except ValueError: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The username or password contained non-UTF8 encoded characters') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The username or password contained non-UTF8 encoded characters') if not un or not pw: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The username or password was empty') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The username or password was empty') if self.check(un, pw): data.username = un return log_msg = 'Failed login attempt from: %s' % data.remote_addr self.ban_list.failed(ban_key) else: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported authentication method') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported authentication method') if self.prefer_basic_auth: raise HTTPAuthRequired('Basic realm="%s"' % self.realm, log=log_msg) diff --git a/src/calibre/srv/errors.py b/src/calibre/srv/errors.py index db251b34cf..02c7c4f65e 100644 --- a/src/calibre/srv/errors.py +++ b/src/calibre/srv/errors.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib +from polyglot import http_client class JobQueueFull(Exception): @@ -30,38 +30,38 @@ class HTTPSimpleResponse(Exception): class HTTPRedirect(HTTPSimpleResponse): - def __init__(self, location, http_code=httplib.MOVED_PERMANENTLY, http_message='', close_connection=False): + def __init__(self, location, http_code=http_client.MOVED_PERMANENTLY, http_message='', close_connection=False): HTTPSimpleResponse.__init__(self, http_code, http_message, close_connection, location) class HTTPNotFound(HTTPSimpleResponse): def __init__(self, http_message='', close_connection=False): - HTTPSimpleResponse.__init__(self, httplib.NOT_FOUND, http_message, close_connection) + HTTPSimpleResponse.__init__(self, http_client.NOT_FOUND, http_message, close_connection) class HTTPAuthRequired(HTTPSimpleResponse): def __init__(self, payload, log=None): - HTTPSimpleResponse.__init__(self, httplib.UNAUTHORIZED, authenticate=payload, log=log) + HTTPSimpleResponse.__init__(self, http_client.UNAUTHORIZED, authenticate=payload, log=log) class HTTPBadRequest(HTTPSimpleResponse): def __init__(self, message, close_connection=False): - HTTPSimpleResponse.__init__(self, httplib.BAD_REQUEST, message, close_connection) + HTTPSimpleResponse.__init__(self, http_client.BAD_REQUEST, message, close_connection) class HTTPForbidden(HTTPSimpleResponse): def __init__(self, http_message='', close_connection=True, log=None): - HTTPSimpleResponse.__init__(self, httplib.FORBIDDEN, http_message, close_connection, log=log) + HTTPSimpleResponse.__init__(self, http_client.FORBIDDEN, http_message, close_connection, log=log) class HTTPInternalServerError(HTTPSimpleResponse): def __init__(self, http_message='', close_connection=True, log=None): - HTTPSimpleResponse.__init__(self, httplib.INTERNAL_SERVER_ERROR, http_message, close_connection, log=log) + HTTPSimpleResponse.__init__(self, http_client.INTERNAL_SERVER_ERROR, http_message, close_connection, log=log) class BookNotFound(HTTPNotFound): diff --git a/src/calibre/srv/http_request.py b/src/calibre/srv/http_request.py index 384fe5a04f..02afb5252f 100644 --- a/src/calibre/srv/http_request.py +++ b/src/calibre/srv/http_request.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import re, httplib, repr as reprlib +import re from io import BytesIO, DEFAULT_BUFFER_SIZE from calibre import as_unicode, force_unicode @@ -14,6 +14,7 @@ from calibre.ptempfile import SpooledTemporaryFile from calibre.srv.errors import HTTPSimpleResponse from calibre.srv.loop import Connection, READ, WRITE from calibre.srv.utils import MultiDict, HTTP1, HTTP11, Accumulator +from polyglot import http_client, reprlib from polyglot.urllib import unquote protocol_map = {(1, 0):HTTP1, (1, 1):HTTP11} @@ -68,29 +69,29 @@ def parse_request_uri(uri): def parse_uri(uri, parse_query=True): scheme, authority, path = parse_request_uri(uri) if path is None: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, "No path component") + raise HTTPSimpleResponse(http_client.BAD_REQUEST, "No path component") if b'#' in path: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, "Illegal #fragment in Request-URI.") + raise HTTPSimpleResponse(http_client.BAD_REQUEST, "Illegal #fragment in Request-URI.") if scheme: try: scheme = scheme.decode('ascii') except ValueError: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Un-decodeable scheme') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Un-decodeable scheme') path, qs = path.partition(b'?')[::2] if parse_query: try: query = MultiDict.create_from_query_string(qs) except Exception: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unparseable query string') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unparseable query string') else: query = None try: path = '%2F'.join(unquote(x).decode('utf-8') for x in quoted_slash.split(path)) except ValueError as e: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, as_unicode(e)) + raise HTTPSimpleResponse(http_client.BAD_REQUEST, as_unicode(e)) path = tuple(filter(None, (x.replace('%2F', '/') for x in path.split('/')))) return scheme, path, query @@ -233,7 +234,7 @@ class HTTPRequest(Connection): if line.endswith(b'\n'): line = buf.getvalue() if not line.endswith(b'\r\n'): - self.simple_response(httplib.BAD_REQUEST, 'HTTP requires CRLF line terminators') + self.simple_response(http_client.BAD_REQUEST, 'HTTP requires CRLF line terminators') return return line if not line: @@ -247,7 +248,7 @@ class HTTPRequest(Connection): self.forwarded_for = None self.path = self.query = None self.close_after_response = False - self.header_line_too_long_error_code = httplib.REQUEST_URI_TOO_LONG + self.header_line_too_long_error_code = http_client.REQUEST_URI_TOO_LONG self.response_started = False self.set_state(READ, self.parse_request_line, Accumulator(), first=True) @@ -260,28 +261,28 @@ class HTTPRequest(Connection): # Ignore a single leading empty line, as per RFC 2616 sec 4.1 if first: return self.set_state(READ, self.parse_request_line, Accumulator()) - return self.simple_response(httplib.BAD_REQUEST, 'Multiple leading empty lines not allowed') + return self.simple_response(http_client.BAD_REQUEST, 'Multiple leading empty lines not allowed') try: method, uri, req_protocol = line.strip().split(b' ', 2) rp = int(req_protocol[5]), int(req_protocol[7]) self.method = method.decode('ascii').upper() except Exception: - return self.simple_response(httplib.BAD_REQUEST, "Malformed Request-Line") + return self.simple_response(http_client.BAD_REQUEST, "Malformed Request-Line") if self.method not in HTTP_METHODS: - return self.simple_response(httplib.BAD_REQUEST, "Unknown HTTP method") + return self.simple_response(http_client.BAD_REQUEST, "Unknown HTTP method") try: self.request_protocol = protocol_map[rp] except KeyError: - return self.simple_response(httplib.HTTP_VERSION_NOT_SUPPORTED) + return self.simple_response(http_client.HTTP_VERSION_NOT_SUPPORTED) self.response_protocol = protocol_map[min((1, 1), rp)] try: self.scheme, self.path, self.query = parse_uri(uri) except HTTPSimpleResponse as e: return self.simple_response(e.http_code, e.message, close_after_response=False) - self.header_line_too_long_error_code = httplib.REQUEST_ENTITY_TOO_LARGE + self.header_line_too_long_error_code = http_client.REQUEST_ENTITY_TOO_LARGE self.set_state(READ, self.parse_header_line, HTTPHeaderParser(), Accumulator()) # }}} @@ -299,7 +300,7 @@ class HTTPRequest(Connection): try: parser(line) except ValueError: - self.simple_response(httplib.BAD_REQUEST, 'Failed to parse header line') + self.simple_response(http_client.BAD_REQUEST, 'Failed to parse header line') return if parser.finished: self.finalize_headers(parser.hdict) @@ -307,7 +308,7 @@ class HTTPRequest(Connection): def finalize_headers(self, inheaders): request_content_length = int(inheaders.get('Content-Length', 0)) if request_content_length > self.max_request_body_size: - return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE, + return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE, "The entity sent with the request exceeds the maximum " "allowed bytes (%d)." % self.max_request_body_size) # Persistent connection support @@ -334,7 +335,7 @@ class HTTPRequest(Connection): else: # Note that, even if we see "chunked", we must reject # if there is an extension we don't recognize. - return self.simple_response(httplib.NOT_IMPLEMENTED, "Unknown transfer encoding: %r" % enc) + return self.simple_response(http_client.NOT_IMPLEMENTED, "Unknown transfer encoding: %r" % enc) if inheaders.get("Expect", '').lower() == "100-continue": buf = BytesIO((HTTP11 + " 100 Continue\r\n\r\n").encode('ascii')) @@ -369,9 +370,9 @@ class HTTPRequest(Connection): try: chunk_size = int(line.strip(), 16) except Exception: - return self.simple_response(httplib.BAD_REQUEST, '%s is not a valid chunk size' % reprlib.repr(line.strip())) + return self.simple_response(http_client.BAD_REQUEST, '%s is not a valid chunk size' % reprlib.repr(line.strip())) if bytes_read[0] + chunk_size + 2 > self.max_request_body_size: - return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE, + return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE, 'Chunked request is larger than %d bytes' % self.max_request_body_size) if chunk_size == 0: self.set_state(READ, self.read_chunk_separator, inheaders, Accumulator(), buf, bytes_read, last=True) @@ -389,10 +390,10 @@ class HTTPRequest(Connection): if line is None: return if line != b'\r\n': - return self.simple_response(httplib.BAD_REQUEST, 'Chunk does not have trailing CRLF') + return self.simple_response(http_client.BAD_REQUEST, 'Chunk does not have trailing CRLF') bytes_read[0] += len(line) if bytes_read[0] > self.max_request_body_size: - return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE, + return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE, 'Chunked request is larger than %d bytes' % self.max_request_body_size) if last: self.prepare_response(inheaders, buf) @@ -402,7 +403,7 @@ class HTTPRequest(Connection): def handle_timeout(self): if self.response_started: return False - self.simple_response(httplib.REQUEST_TIMEOUT) + self.simple_response(http_client.REQUEST_TIMEOUT) return True def write(self, buf, end=None): diff --git a/src/calibre/srv/http_response.py b/src/calibre/srv/http_response.py index b64db6dab6..8313d18e32 100644 --- a/src/calibre/srv/http_response.py +++ b/src/calibre/srv/http_response.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import os, httplib, hashlib, uuid, struct, repr as reprlib +import os, hashlib, uuid, struct from collections import namedtuple from io import BytesIO, DEFAULT_BUFFER_SIZE from itertools import chain, repeat @@ -26,9 +26,10 @@ from calibre.srv.utils import ( sort_q_values, get_translator_for_lang, Cookie, fast_now_strftime) from calibre.utils.speedups import ReadOnlyFileBuffer from calibre.utils.monotonic import monotonic +from polyglot import http_client, reprlib Range = namedtuple('Range', 'start stop size') -MULTIPART_SEPARATOR = uuid.uuid4().hex.decode('ascii') +MULTIPART_SEPARATOR = uuid.uuid4().hex COMPRESSIBLE_TYPES = {'application/json', 'application/javascript', 'application/xml', 'application/oebps-package+xml'} if is_py3: import zlib @@ -226,7 +227,7 @@ class RequestData(object): # {{{ self.remote_addr, self.remote_port, self.is_local_connection = remote_addr, remote_port, is_local_connection self.forwarded_for = forwarded_for self.opts = opts - self.status_code = httplib.OK + self.status_code = http_client.OK self.outcookie = Cookie() self.lang_code = self.gettext_func = self.ngettext_func = None self.set_translator(self.get_preferred_language()) @@ -402,16 +403,16 @@ class HTTPConnection(HTTPRequest): if self.response_protocol is HTTP1: # HTTP/1.0 has no 413/414/303 codes status_code = { - httplib.REQUEST_ENTITY_TOO_LARGE:httplib.BAD_REQUEST, - httplib.REQUEST_URI_TOO_LONG:httplib.BAD_REQUEST, - httplib.SEE_OTHER:httplib.FOUND + http_client.REQUEST_ENTITY_TOO_LARGE:http_client.BAD_REQUEST, + http_client.REQUEST_URI_TOO_LONG:http_client.BAD_REQUEST, + http_client.SEE_OTHER:http_client.FOUND }.get(status_code, status_code) self.close_after_response = close_after_response msg = msg.encode('utf-8') ct = 'http' if self.method == 'TRACE' else 'plain' buf = [ - '%s %d %s' % (self.response_protocol, status_code, httplib.responses[status_code]), + '%s %d %s' % (self.response_protocol, status_code, http_client.responses[status_code]), "Content-Length: %s" % len(msg), "Content-Type: text/%s; charset=UTF-8" % ct, "Date: " + http_date(), @@ -432,7 +433,7 @@ class HTTPConnection(HTTPRequest): def prepare_response(self, inheaders, request_body_file): if self.method == 'TRACE': msg = force_unicode(self.request_line, 'utf-8') + '\n' + inheaders.pretty() - return self.simple_response(httplib.OK, msg, close_after_response=False) + return self.simple_response(http_client.OK, msg, close_after_response=False) request_body_file.seek(0) outheaders = MultiDict() data = RequestData( @@ -449,28 +450,28 @@ class HTTPConnection(HTTPRequest): def send_range_not_satisfiable(self, content_length): buf = [ - '%s %d %s' % (self.response_protocol, httplib.REQUESTED_RANGE_NOT_SATISFIABLE, httplib.responses[httplib.REQUESTED_RANGE_NOT_SATISFIABLE]), + '%s %d %s' % (self.response_protocol, http_client.REQUESTED_RANGE_NOT_SATISFIABLE, http_client.responses[http_client.REQUESTED_RANGE_NOT_SATISFIABLE]), "Date: " + http_date(), "Content-Range: bytes */%d" % content_length, ] response_data = header_list_to_file(buf) - self.log_access(status_code=httplib.REQUESTED_RANGE_NOT_SATISFIABLE, response_size=response_data.sz) + self.log_access(status_code=http_client.REQUESTED_RANGE_NOT_SATISFIABLE, response_size=response_data.sz) self.response_ready(response_data) def send_not_modified(self, etag=None): buf = [ - '%s %d %s' % (self.response_protocol, httplib.NOT_MODIFIED, httplib.responses[httplib.NOT_MODIFIED]), + '%s %d %s' % (self.response_protocol, http_client.NOT_MODIFIED, http_client.responses[http_client.NOT_MODIFIED]), "Content-Length: 0", "Date: " + http_date(), ] if etag is not None: buf.append('ETag: ' + etag) response_data = header_list_to_file(buf) - self.log_access(status_code=httplib.NOT_MODIFIED, response_size=response_data.sz) + self.log_access(status_code=http_client.NOT_MODIFIED, response_size=response_data.sz) self.response_ready(response_data) def report_busy(self): - self.simple_response(httplib.SERVICE_UNAVAILABLE) + self.simple_response(http_client.SERVICE_UNAVAILABLE) def job_done(self, ok, result): if not ok: @@ -509,7 +510,7 @@ class HTTPConnection(HTTPRequest): if ct.startswith('text/') and 'charset=' not in ct: outheaders.set('Content-Type', ct + '; charset=UTF-8', replace_all=True) - buf = [HTTP11 + (' %d ' % data.status_code) + httplib.responses[data.status_code]] + buf = [HTTP11 + (' %d ' % data.status_code) + http_client.responses[data.status_code]] for header, value in sorted(iteritems(outheaders), key=itemgetter(0)): buf.append('%s: %s' % (header, value)) for morsel in itervalues(data.outcookie): @@ -530,7 +531,7 @@ class HTTPConnection(HTTPRequest): def log_access(self, status_code, response_size=None, username=None): if self.access_log is None: return - if not self.opts.log_not_found and status_code == httplib.NOT_FOUND: + if not self.opts.log_not_found and status_code == http_client.NOT_FOUND: return ff = self.forwarded_for if ff: @@ -623,7 +624,7 @@ class HTTPConnection(HTTPRequest): self.ready = ready def report_unhandled_exception(self, e, formatted_traceback): - self.simple_response(httplib.INTERNAL_SERVER_ERROR) + self.simple_response(http_client.INTERNAL_SERVER_ERROR) def finalize_output(self, output, request, is_http1): none_match = parse_if_none_match(request.inheaders.get('If-None-Match', '')) @@ -633,7 +634,7 @@ class HTTPConnection(HTTPRequest): if self.method in ('GET', 'HEAD'): self.send_not_modified(output.etag) else: - self.simple_response(httplib.PRECONDITION_FAILED) + self.simple_response(http_client.PRECONDITION_FAILED) return opts = self.opts @@ -660,10 +661,10 @@ class HTTPConnection(HTTPRequest): ct = outheaders.get('Content-Type', '').partition(';')[0] compressible = (not ct or ct.startswith('text/') or ct.startswith('image/svg') or ct.partition(';')[0] in COMPRESSIBLE_TYPES) - compressible = (compressible and request.status_code == httplib.OK and + compressible = (compressible and request.status_code == http_client.OK and (opts.compress_min_size > -1 and output.content_length >= opts.compress_min_size) and acceptable_encoding(request.inheaders.get('Accept-Encoding', '')) and not is_http1) - accept_ranges = (not compressible and output.accept_ranges is not None and request.status_code == httplib.OK and + accept_ranges = (not compressible and output.accept_ranges is not None and request.status_code == http_client.OK and not is_http1) ranges = get_ranges(request.inheaders.get('Range'), output.content_length) if output.accept_ranges and self.method in ('GET', 'HEAD') else None if_range = (request.inheaders.get('If-Range') or '').strip() @@ -680,7 +681,7 @@ class HTTPConnection(HTTPRequest): if self.method in ('GET', 'HEAD'): self.send_not_modified(output.etag) else: - self.simple_response(httplib.PRECONDITION_FAILED) + self.simple_response(http_client.PRECONDITION_FAILED) return output.ranges = None @@ -712,7 +713,7 @@ class HTTPConnection(HTTPRequest): outheaders.set('Content-Length', '%d' % size, replace_all=True) outheaders.set('Content-Type', 'multipart/byteranges; boundary=' + MULTIPART_SEPARATOR, replace_all=True) output.ranges = zip_longest(ranges, range_parts) - request.status_code = httplib.PARTIAL_CONTENT + request.status_code = http_client.PARTIAL_CONTENT return output diff --git a/src/calibre/srv/routes.py b/src/calibre/srv/routes.py index 9be1cde382..621b412c6c 100644 --- a/src/calibre/srv/routes.py +++ b/src/calibre/srv/routes.py @@ -6,13 +6,14 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, sys, inspect, re, time, numbers, json as jsonlib, textwrap +import sys, inspect, re, time, numbers, json as jsonlib, textwrap from operator import attrgetter from calibre.srv.errors import HTTPSimpleResponse, HTTPNotFound, RouteError from calibre.srv.utils import http_date from calibre.utils.serialize import msgpack_dumps, json_dumps, MSGPACK_MIME from polyglot.builtins import iteritems, itervalues, unicode_type, range, zip +from polyglot import http_client from polyglot.urllib import quote as urlquote default_methods = frozenset(('HEAD', 'GET')) @@ -297,7 +298,7 @@ class Router(object): def dispatch(self, data): endpoint_, args = self.find_route(data.path) if data.method not in endpoint_.methods: - raise HTTPSimpleResponse(httplib.METHOD_NOT_ALLOWED) + raise HTTPSimpleResponse(http_client.METHOD_NOT_ALLOWED) self.read_cookies(data) diff --git a/src/calibre/srv/tests/ajax.py b/src/calibre/srv/tests/ajax.py index 7707bd6d2b..0fade7e9d0 100644 --- a/src/calibre/srv/tests/ajax.py +++ b/src/calibre/srv/tests/ajax.py @@ -6,13 +6,13 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, zlib, json, base64, os +import zlib, json, base64, os from io import BytesIO from functools import partial -from httplib import OK, NOT_FOUND, FORBIDDEN from calibre.ebooks.metadata.meta import get_metadata from calibre.srv.tests.base import LibraryBaseTest +from polyglot.http_client import OK, NOT_FOUND, FORBIDDEN from polyglot.urllib import urlencode, quote @@ -22,7 +22,7 @@ def make_request(conn, url, headers={}, prefix='/ajax', username=None, password= conn.request(method, prefix + url, headers=headers, body=data) r = conn.getresponse() data = r.read() - if r.status == httplib.OK and data and data[0] in b'{[': + if r.status == OK and data and data[0] in b'{[': data = json.loads(data) return r, data @@ -37,10 +37,10 @@ class ContentTest(LibraryBaseTest): request = partial(make_request, conn, prefix='/ajax/book') r, data = request('/x') - self.ae(r.status, httplib.NOT_FOUND) + self.ae(r.status, NOT_FOUND) r, onedata = request('/1') - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) self.ae(request('/1/' + db.server_library_id)[1], onedata) self.ae(request('/%s?id_is_uuid=true' % db.field_for('uuid', 1))[1], onedata) @@ -63,22 +63,22 @@ class ContentTest(LibraryBaseTest): request = partial(make_request, conn) r, data = request('/categories') - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) r, xdata = request('/categories/' + db.server_library_id) - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) self.ae(data, xdata) names = {x['name']:x['url'] for x in data} for q in ('Newest', 'All books', 'Tags', 'Series', 'Authors', 'Enum', 'Composite Tags'): self.assertIn(q, names) r, data = request(names['Tags'], prefix='') - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) names = {x['name']:x['url'] for x in data['items']} self.ae(set(names), set('Tag One,Tag Two,News'.split(','))) r, data = request(names['Tag One'], prefix='') - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) self.ae(set(data['book_ids']), {1, 2}) r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"'})) - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) self.ae(set(data['book_ids']), {1, 2}) r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"', 'vl':'1'})) self.ae(set(data['book_ids']), {2}) diff --git a/src/calibre/srv/tests/auth.py b/src/calibre/srv/tests/auth.py index ea08759d04..45764253e6 100644 --- a/src/calibre/srv/tests/auth.py +++ b/src/calibre/srv/tests/auth.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, base64, subprocess, os, cookielib, time +import base64, subprocess, os, time from collections import namedtuple try: from distutils.spawn import find_executable @@ -18,6 +18,8 @@ from calibre.srv.errors import HTTPForbidden from calibre.srv.tests.base import BaseTest, TestServer from calibre.srv.routes import endpoint, Router from polyglot.builtins import iteritems, itervalues +from polyglot import http_client +from polyglot.http_cookie import CookieJar from polyglot.urllib import (build_opener, HTTPBasicAuthHandler, HTTPCookieProcessor, HTTPDigestAuthHandler, HTTPError) @@ -91,18 +93,18 @@ class TestAuth(BaseTest): conn = server.connect() conn.request('GET', '/open') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'open') conn.request('GET', '/closed') r = conn.getresponse() - self.ae(r.status, httplib.UNAUTHORIZED) + self.ae(r.status, http_client.UNAUTHORIZED) self.ae(r.getheader('WWW-Authenticate'), b'Basic realm="%s"' % bytes(REALM)) self.assertFalse(r.read()) conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + base64.standard_b64encode(b'testuser:testpw')}) r = conn.getresponse() self.ae(r.read(), b'closed') - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(b'closed', urlopen(server, method='basic').read()) self.ae(b'closed', urlopen(server, un='!@#$%^&*()-=_+', pw='!@#$%^&*()-=_+', method='basic').read()) @@ -113,14 +115,14 @@ class TestAuth(BaseTest): warnings = [] server.loop.log.warn = lambda *args, **kwargs: warnings.append(' '.join(args)) - self.ae((httplib.OK, b'closed'), request()) - self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y')) - self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('', '')) + self.ae((http_client.OK, b'closed'), request()) + self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y')) + self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request('', '')) self.ae(1, len(warnings)) - self.ae((httplib.UNAUTHORIZED, b''), request('testuser', 'y')) - self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('testuser', '')) - self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('')) - self.ae((httplib.UNAUTHORIZED, b''), request('asf', 'testpw')) + self.ae((http_client.UNAUTHORIZED, b''), request('testuser', 'y')) + self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request('testuser', '')) + self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request('')) + self.ae((http_client.UNAUTHORIZED, b''), request('asf', 'testpw')) # }}} def test_library_restrictions(self): # {{{ @@ -169,7 +171,7 @@ class TestAuth(BaseTest): with TestServer(r.dispatch) as server: r.auth_controller.log = server.log - def test(conn, path, headers={}, status=httplib.OK, body=b'', request_body=b''): + def test(conn, path, headers={}, status=http_client.OK, body=b'', request_body=b''): conn.request('GET', path, request_body, headers) r = conn.getresponse() self.ae(r.status, status) @@ -177,9 +179,9 @@ class TestAuth(BaseTest): return {normalize_header_name(k):v for k, v in r.getheaders()} conn = server.connect() test(conn, '/open', body=b'open') - auth = parse_http_dict(test(conn, '/closed', status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) + auth = parse_http_dict(test(conn, '/closed', status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) nonce = auth['nonce'] - auth = parse_http_dict(test(conn, '/closed', status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) + auth = parse_http_dict(test(conn, '/closed', status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) self.assertNotEqual(nonce, auth['nonce'], 'nonce was re-used') self.ae(auth[b'realm'], bytes(REALM)), self.ae(auth[b'algorithm'], b'MD5'), self.ae(auth[b'qop'], b'auth') self.assertNotIn('stale', auth) @@ -199,14 +201,14 @@ class TestAuth(BaseTest): # Check stale nonces orig, r.auth_controller.max_age_seconds = r.auth_controller.max_age_seconds, -1 auth = parse_http_dict(test(conn, '/closed', headers={ - 'Authorization':digest(**args)},status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) + 'Authorization':digest(**args)},status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) self.assertIn('stale', auth) r.auth_controller.max_age_seconds = orig ok_test(conn, digest(**args)) def fail_test(conn, modify, **kw): kw['body'] = kw.get('body', b'') - kw['status'] = kw.get('status', httplib.UNAUTHORIZED) + kw['status'] = kw.get('status', http_client.UNAUTHORIZED) args['modify'] = modify return test(conn, '/closed', headers={'Authorization':digest(**args)}, **kw) @@ -258,13 +260,13 @@ class TestAuth(BaseTest): warnings = [] server.loop.log.warn = lambda *args, **kwargs: warnings.append(' '.join(args)) - self.ae((httplib.OK, b'closed'), request()) - self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y')) - self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y')) - self.ae(httplib.FORBIDDEN, request('x', 'y')[0]) - self.ae(httplib.FORBIDDEN, request()[0]) + self.ae((http_client.OK, b'closed'), request()) + self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y')) + self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y')) + self.ae(http_client.FORBIDDEN, request('x', 'y')[0]) + self.ae(http_client.FORBIDDEN, request()[0]) time.sleep(ban_for * 60 + 0.01) - self.ae((httplib.OK, b'closed'), request()) + self.ae((http_client.OK, b'closed'), request()) # }}} def test_android_auth_workaround(self): # {{{ @@ -277,28 +279,28 @@ class TestAuth(BaseTest): # First check that unauth access fails conn.request('GET', '/android') r = conn.getresponse() - self.ae(r.status, httplib.UNAUTHORIZED) + self.ae(r.status, http_client.UNAUTHORIZED) auth_handler = HTTPDigestAuthHandler() url = 'http://localhost:%d%s' % (server.address[1], '/android') auth_handler.add_password(realm=REALM, uri=url, user='testuser', passwd='testpw') - cj = cookielib.CookieJar() + cj = CookieJar() cookie_handler = HTTPCookieProcessor(cj) r = build_opener(auth_handler, cookie_handler).open(url) - self.ae(r.getcode(), httplib.OK) + self.ae(r.getcode(), http_client.OK) cookies = tuple(cj) self.ae(len(cookies), 1) cookie = cookies[0] self.assertIn(b':', cookie.value) self.ae(cookie.path, b'/android') r = build_opener(cookie_handler).open(url) - self.ae(r.getcode(), httplib.OK) + self.ae(r.getcode(), http_client.OK) self.ae(r.read(), b'android') # Test that a replay attack against a different URL does not work try: build_opener(cookie_handler).open(url+'2') assert ('Replay attack succeeded') except HTTPError as e: - self.ae(e.code, httplib.UNAUTHORIZED) + self.ae(e.code, http_client.UNAUTHORIZED) # }}} diff --git a/src/calibre/srv/tests/base.py b/src/calibre/srv/tests/base.py index e62dcc605d..165aa2e0c5 100644 --- a/src/calibre/srv/tests/base.py +++ b/src/calibre/srv/tests/base.py @@ -7,12 +7,13 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import unittest, time, httplib, shutil, gc, tempfile, atexit, os +import unittest, time, shutil, gc, tempfile, atexit, os from io import BytesIO from functools import partial from threading import Thread from calibre.srv.utils import ServerLog +from polyglot import http_client rmtree = partial(shutil.rmtree, ignore_errors=True) @@ -120,7 +121,7 @@ class TestServer(Thread): timeout = self.loop.opts.timeout if interface is None: interface = self.address[0] - return httplib.HTTPConnection(interface, self.address[1], strict=True, timeout=timeout) + return http_client.HTTPConnection(interface, self.address[1], strict=True, timeout=timeout) def change_handler(self, handler): from calibre.srv.http_response import create_http_handler diff --git a/src/calibre/srv/tests/content.py b/src/calibre/srv/tests/content.py index e7b1950eec..fa9a12ccc7 100644 --- a/src/calibre/srv/tests/content.py +++ b/src/calibre/srv/tests/content.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, zlib, json, binascii, time, os +import zlib, json, binascii, time, os from io import BytesIO from calibre.ebooks.metadata.epub import get_metadata @@ -14,6 +14,7 @@ from calibre.ebooks.metadata.opf2 import OPF from calibre.srv.tests.base import LibraryBaseTest from calibre.utils.imghdr import identify from calibre.utils.shared_file import share_open +from polyglot import http_client def setUpModule(): @@ -32,7 +33,7 @@ class ContentTest(LibraryBaseTest): def missing(url, body=b''): conn.request('GET', url) r = conn.getresponse() - self.ae(r.status, httplib.NOT_FOUND) + self.ae(r.status, http_client.NOT_FOUND) self.ae(r.read(), body) for prefix in ('static', 'icon'): @@ -51,7 +52,7 @@ class ContentTest(LibraryBaseTest): raw = P(src, data=True) conn.request('GET', url) r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) data = r.read() if sz is None: self.ae(data, raw) @@ -60,7 +61,7 @@ class ContentTest(LibraryBaseTest): test_response(r) conn.request('GET', url, headers={'If-None-Match':r.getheader('ETag')}) r = conn.getresponse() - self.ae(r.status, httplib.NOT_MODIFIED) + self.ae(r.status, http_client.NOT_MODIFIED) self.ae(b'', r.read()) test('content-server/empty.html', '/static/empty.html') @@ -85,7 +86,7 @@ class ContentTest(LibraryBaseTest): # Test various invalid parameters def bad(*args): r, data = get(*args) - self.ae(r.status, httplib.NOT_FOUND) + self.ae(r.status, http_client.NOT_FOUND) bad('xxx', 1) bad('fmt1', 10) bad('fmt1', 1, 'zzzz') @@ -103,7 +104,7 @@ class ContentTest(LibraryBaseTest): # Test fetching of format with metadata update raw = P('quick_start/eng.epub', data=True) r, data = get('epub', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) etag = r.getheader('ETag') self.assertIsNotNone(etag) self.ae(r.getheader('Used-Cache'), 'no') @@ -145,39 +146,39 @@ class ContentTest(LibraryBaseTest): os.utime(cpath, (t, t)) r, data = get('cover', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(data, db.cover(1)) self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Content-Type'), 'image/jpeg') r, data = get('cover', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(data, db.cover(1)) self.ae(r.getheader('Used-Cache'), 'yes') r, data = get('cover', 3) - self.ae(r.status, httplib.OK) # Auto generated cover + self.ae(r.status, http_client.OK) # Auto generated cover r, data = get('thumb', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(identify(data), ('jpeg', 60, 60)) self.ae(r.getheader('Used-Cache'), 'no') r, data = get('thumb', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.getheader('Used-Cache'), 'yes') r, data = get('thumb', 1, q='sz=100') - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(identify(data), ('jpeg', 100, 100)) self.ae(r.getheader('Used-Cache'), 'no') r, data = get('thumb', 1, q='sz=100x100') - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.getheader('Used-Cache'), 'yes') change_cover(1, 1) r, data = get('thumb', 1, q='sz=100') - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(identify(data), ('jpeg', 100, 100)) self.ae(r.getheader('Used-Cache'), 'no') # Test file sharing in cache r, data = get('cover', 2) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(data, db.cover(2)) self.ae(r.getheader('Used-Cache'), 'no') path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8') @@ -185,7 +186,7 @@ class ContentTest(LibraryBaseTest): # Now force an update change_cover(1) r, data = get('cover', 2) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(data, db.cover(2)) self.ae(r.getheader('Used-Cache'), 'no') path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8') @@ -193,7 +194,7 @@ class ContentTest(LibraryBaseTest): # Do it again change_cover(2) r, data = get('cover', 2) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(data, db.cover(2)) self.ae(r.getheader('Used-Cache'), 'no') self.ae(f.read(), fdata) @@ -201,7 +202,7 @@ class ContentTest(LibraryBaseTest): # Test serving of metadata as opf r, data = get('opf', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.getheader('Content-Type'), 'application/oebps-package+xml; charset=UTF-8') self.assertIsNotNone(r.getheader('Last-Modified')) opf = OPF(BytesIO(data), populate_spine=False, try_to_guess_cover=False) @@ -209,17 +210,17 @@ class ContentTest(LibraryBaseTest): self.ae(db.field_for('authors', 1), tuple(opf.authors)) conn.request('GET', '/get/opf/1', headers={'Accept-Encoding':'gzip'}) r = conn.getresponse() - self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') + self.ae(r.status, http_client.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') raw = r.read() self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data) # Test serving metadata as json r, data = get('json', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(db.field_for('title', 1), json.loads(data)['title']) conn.request('GET', '/get/json/1', headers={'Accept-Encoding':'gzip'}) r = conn.getresponse() - self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') + self.ae(r.status, http_client.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') raw = r.read() self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data) diff --git a/src/calibre/srv/tests/http.py b/src/calibre/srv/tests/http.py index f42279795e..05b487b2a3 100644 --- a/src/calibre/srv/tests/http.py +++ b/src/calibre/srv/tests/http.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, hashlib, zlib, string, time, os +import hashlib, zlib, string, time, os from io import BytesIO from tempfile import NamedTemporaryFile @@ -15,6 +15,7 @@ from calibre.srv.tests.base import BaseTest, TestServer from calibre.srv.utils import eintr_retry_call from calibre.utils.monotonic import monotonic from polyglot.builtins import iteritems, range +from polyglot import http_client is_ci = os.environ.get('CI', '').lower() == 'true' @@ -94,7 +95,7 @@ class TestHTTP(BaseTest): def test(al, q): conn.request('GET', '/', headers={'Accept-Language': al}) r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) q += get_translator(q)[-1].ugettext('Unknown') self.ae(r.read(), q) @@ -136,7 +137,7 @@ class TestHTTP(BaseTest): def raw_send(conn, raw): conn.send(raw) - conn._HTTPConnection__state = httplib._CS_REQ_SENT + conn._HTTPConnection__state = http_client._CS_REQ_SENT return conn.getresponse() base_timeout = 0.5 if is_ci else 0.1 @@ -144,31 +145,31 @@ class TestHTTP(BaseTest): with TestServer(handler, timeout=base_timeout, max_header_line_size=100./1024, max_request_body_size=100./(1024*1024)) as server: conn = server.connect() r = raw_send(conn, b'hello\n') - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) self.ae(r.read(), b'HTTP requires CRLF line terminators') r = raw_send(conn, b'\r\nGET /index.html HTTP/1.1\r\n\r\n') - self.ae(r.status, httplib.NOT_FOUND), self.ae(r.read(), b'Requested resource not found') + self.ae(r.status, http_client.NOT_FOUND), self.ae(r.read(), b'Requested resource not found') r = raw_send(conn, b'\r\n\r\nGET /index.html HTTP/1.1\r\n\r\n') - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) self.ae(r.read(), b'Multiple leading empty lines not allowed') r = raw_send(conn, b'hello world\r\n') - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) self.ae(r.read(), b'Malformed Request-Line') r = raw_send(conn, b'x' * 200) - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) self.ae(r.read(), b'') r = raw_send(conn, b'XXX /index.html HTTP/1.1\r\n\r\n') - self.ae(r.status, httplib.BAD_REQUEST), self.ae(r.read(), b'Unknown HTTP method') + self.ae(r.status, http_client.BAD_REQUEST), self.ae(r.read(), b'Unknown HTTP method') # Test 404 conn.request('HEAD', '/moose') r = conn.getresponse() - self.ae(r.status, httplib.NOT_FOUND) + self.ae(r.status, http_client.NOT_FOUND) self.assertIsNotNone(r.getheader('Date', None)) self.ae(r.getheader('Content-Length'), str(len(body))) self.ae(r.getheader('Content-Type'), 'text/plain; charset=UTF-8') @@ -176,7 +177,7 @@ class TestHTTP(BaseTest): self.ae(r.read(), '') conn.request('GET', '/choose') r = conn.getresponse() - self.ae(r.status, httplib.NOT_FOUND) + self.ae(r.status, http_client.NOT_FOUND) self.ae(r.read(), b'Requested resource not found') # Test 500 @@ -186,7 +187,7 @@ class TestHTTP(BaseTest): conn = server.connect() conn.request('GET', '/test/') r = conn.getresponse() - self.ae(r.status, httplib.INTERNAL_SERVER_ERROR) + self.ae(r.status, http_client.INTERNAL_SERVER_ERROR) server.loop.log.filter_level = orig # Test 301 @@ -196,7 +197,7 @@ class TestHTTP(BaseTest): conn = server.connect() conn.request('GET', '/') r = conn.getresponse() - self.ae(r.status, httplib.MOVED_PERMANENTLY) + self.ae(r.status, http_client.MOVED_PERMANENTLY) self.ae(r.getheader('Location'), '/somewhere-else') self.ae('', r.read()) @@ -206,26 +207,26 @@ class TestHTTP(BaseTest): # Test simple GET conn.request('GET', '/test/') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'test') # Test TRACE lines = ['TRACE /xxx HTTP/1.1', 'Test: value', 'Xyz: abc, def', '', ''] r = raw_send(conn, ('\r\n'.join(lines)).encode('ascii')) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read().decode('utf-8'), '\n'.join(lines[:-2])) # Test POST with simple body conn.request('POST', '/test', 'body') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'testbody') # Test POST with chunked transfer encoding conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.send(b'4\r\nbody\r\na\r\n1234567890\r\n0\r\n\r\n') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'testbody1234567890') # Test various incorrect input @@ -233,39 +234,39 @@ class TestHTTP(BaseTest): conn.request('GET', '/test' + ('a' * 200)) r = conn.getresponse() - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) conn = server.connect() conn.request('GET', '/test', ('a' * 200)) r = conn.getresponse() - self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE) + self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE) conn = server.connect() conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.send(b'x\r\nbody\r\n0\r\n\r\n') r = conn.getresponse() - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) self.assertIn(b'not a valid chunk size', r.read()) conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.send(b'4\r\nbody\r\n200\r\n\r\n') r = conn.getresponse() - self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE) + self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE) conn.request('POST', '/test', body='a'*200) r = conn.getresponse() - self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE) + self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE) conn = server.connect() conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.send(b'3\r\nbody\r\n0\r\n\r\n') r = conn.getresponse() - self.ae(r.status, httplib.BAD_REQUEST), self.ae(r.read(), b'Chunk does not have trailing CRLF') + self.ae(r.status, http_client.BAD_REQUEST), self.ae(r.read(), b'Chunk does not have trailing CRLF') conn = server.connect(timeout=base_timeout * 5) conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.send(b'30\r\nbody\r\n0\r\n\r\n') r = conn.getresponse() - self.ae(r.status, httplib.REQUEST_TIMEOUT) + self.ae(r.status, http_client.REQUEST_TIMEOUT) self.assertIn(b'', r.read()) server.log.filter_level = orig_level @@ -273,14 +274,14 @@ class TestHTTP(BaseTest): # Test pipelining responses = [] for i in range(10): - conn._HTTPConnection__state = httplib._CS_IDLE + conn._HTTPConnection__state = http_client._CS_IDLE conn.request('GET', '/%d'%i) responses.append(conn.response_class(conn.sock, strict=conn.strict, method=conn._method)) for i in range(10): r = responses[i] r.begin() self.ae(r.read(), ('%d' % i).encode('ascii')) - conn._HTTPConnection__state = httplib._CS_IDLE + conn._HTTPConnection__state = http_client._CS_IDLE # Test closing server.loop.opts.timeout = 10 # ensure socket is not closed because of timeout @@ -319,12 +320,12 @@ class TestHTTP(BaseTest): conn = server.connect() conn.request('GET', '/an_etagged_path') r = conn.getresponse() - self.ae(r.status, httplib.OK), self.ae(r.read(), b'an_etagged_path') + self.ae(r.status, http_client.OK), self.ae(r.read(), b'an_etagged_path') etag = r.getheader('ETag') self.ae(etag, '"%s"' % hashlib.sha1('an_etagged_path').hexdigest()) conn.request('GET', '/an_etagged_path', headers={'If-None-Match':etag}) r = conn.getresponse() - self.ae(r.status, httplib.NOT_MODIFIED) + self.ae(r.status, http_client.NOT_MODIFIED) self.ae(r.read(), b'') # Test gzip @@ -334,7 +335,7 @@ class TestHTTP(BaseTest): conn.request('GET', '/an_etagged_path', headers={'Accept-Encoding':'gzip'}) r = conn.getresponse() self.ae(str(len(raw)), r.getheader('Calibre-Uncompressed-Length')) - self.ae(r.status, httplib.OK), self.ae(zlib.decompress(r.read(), 16+zlib.MAX_WBITS), raw) + self.ae(r.status, http_client.OK), self.ae(zlib.decompress(r.read(), 16+zlib.MAX_WBITS), raw) # Test dynamic etagged content num_calls = [0] @@ -346,13 +347,13 @@ class TestHTTP(BaseTest): conn = server.connect() conn.request('GET', '/an_etagged_path') r = conn.getresponse() - self.ae(r.status, httplib.OK), self.ae(r.read(), b'data') + self.ae(r.status, http_client.OK), self.ae(r.read(), b'data') etag = r.getheader('ETag') self.ae(etag, b'"xxx"') self.ae(r.getheader('Content-Length'), '4') conn.request('GET', '/an_etagged_path', headers={'If-None-Match':etag}) r = conn.getresponse() - self.ae(r.status, httplib.NOT_MODIFIED) + self.ae(r.status, http_client.NOT_MODIFIED) self.ae(r.read(), b'') self.ae(num_calls[0], 1) @@ -368,11 +369,11 @@ class TestHTTP(BaseTest): self.ae(r.getheader('Content-Type'), guess_type(f.name)[0]) self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes') self.ae(int(r.getheader('Content-Length')), len(fdata)) - self.ae(r.status, httplib.OK), self.ae(r.read(), fdata) + self.ae(r.status, http_client.OK), self.ae(r.read(), fdata) conn.request('GET', '/test', headers={'Range':'bytes=2-25'}) r = conn.getresponse() - self.ae(r.status, httplib.PARTIAL_CONTENT) + self.ae(r.status, http_client.PARTIAL_CONTENT) self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes') self.ae(type('')(r.getheader('Content-Range')), 'bytes 2-25/%d' % len(fdata)) self.ae(int(r.getheader('Content-Length')), 24) @@ -380,27 +381,27 @@ class TestHTTP(BaseTest): conn.request('GET', '/test', headers={'Range':'bytes=100000-'}) r = conn.getresponse() - self.ae(r.status, httplib.REQUESTED_RANGE_NOT_SATISFIABLE) + self.ae(r.status, http_client.REQUESTED_RANGE_NOT_SATISFIABLE) self.ae(type('')(r.getheader('Content-Range')), 'bytes */%d' % len(fdata)) conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':etag}) r = conn.getresponse() - self.ae(r.status, httplib.PARTIAL_CONTENT), self.ae(r.read(), fdata[25:51]) + self.ae(r.status, http_client.PARTIAL_CONTENT), self.ae(r.read(), fdata[25:51]) self.ae(int(r.getheader('Content-Length')), 26) conn.request('GET', '/test', headers={'Range':'bytes=0-1000000'}) r = conn.getresponse() - self.ae(r.status, httplib.PARTIAL_CONTENT), self.ae(r.read(), fdata) + self.ae(r.status, http_client.PARTIAL_CONTENT), self.ae(r.read(), fdata) conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':'"nomatch"'}) r = conn.getresponse() - self.ae(r.status, httplib.OK), self.ae(r.read(), fdata) + self.ae(r.status, http_client.OK), self.ae(r.read(), fdata) self.assertFalse(r.getheader('Content-Range')) self.ae(int(r.getheader('Content-Length')), len(fdata)) conn.request('GET', '/test', headers={'Range':'bytes=0-25,26-50'}) r = conn.getresponse() - self.ae(r.status, httplib.PARTIAL_CONTENT) + self.ae(r.status, http_client.PARTIAL_CONTENT) clen = int(r.getheader('Content-Length')) data = r.read() self.ae(clen, len(data)) @@ -415,7 +416,7 @@ class TestHTTP(BaseTest): conn = server.connect(timeout=1) conn.request('GET', '/test') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) rdata = r.read() self.ae(len(data), len(rdata)) self.ae(hashlib.sha1(data).hexdigest(), hashlib.sha1(rdata).hexdigest()) diff --git a/src/calibre/srv/tests/loop.py b/src/calibre/srv/tests/loop.py index af628456c7..897721cc40 100644 --- a/src/calibre/srv/tests/loop.py +++ b/src/calibre/srv/tests/loop.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, ssl, os, socket, time +import ssl, os, socket, time from collections import namedtuple from unittest import skipIf from glob import glob @@ -18,6 +18,7 @@ from calibre.ptempfile import TemporaryDirectory from calibre.utils.certgen import create_server_cert from calibre.utils.monotonic import monotonic from polyglot.builtins import range +from polyglot import http_client is_ci = os.environ.get('CI', '').lower() == 'true' @@ -92,7 +93,7 @@ class LoopTest(BaseTest): conn.request('GET', '/') with self.assertRaises(socket.timeout): res = conn.getresponse() - if str(res.status) == str(httplib.REQUEST_TIMEOUT): + if str(res.status) == str(http_client.REQUEST_TIMEOUT): raise socket.timeout('Timeout') raise Exception('Got unexpected response: code: %s %s headers: %r data: %r' % ( res.status, res.reason, res.getheaders(), res.read())) @@ -135,7 +136,7 @@ class LoopTest(BaseTest): conn = server.connect(interface='127.0.0.1') conn.request('GET', '/test', 'body') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'testbody') def test_ring_buffer(self): @@ -203,10 +204,10 @@ class LoopTest(BaseTest): create_server_cert(address, ca_file, cert_file, key_file, key_size=1024) ctx = ssl.create_default_context(cafile=ca_file) with TestServer(lambda data:(data.path[0] + data.read()), ssl_certfile=cert_file, ssl_keyfile=key_file, listen_on=address, port=0) as server: - conn = httplib.HTTPSConnection(address, server.address[1], strict=True, context=ctx) + conn = http_client.HTTPSConnection(address, server.address[1], strict=True, context=ctx) conn.request('GET', '/test', 'body') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'testbody') cert = conn.sock.getpeercert() subject = dict(x[0] for x in cert['subject']) @@ -226,7 +227,7 @@ class LoopTest(BaseTest): conn = server.connect() conn.request('GET', '/test', 'body') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'testbody') self.ae(server.loop.bound_address[1], port) diff --git a/src/calibre/srv/utils.py b/src/calibre/srv/utils.py index 7e74d2b057..22f9d2b777 100644 --- a/src/calibre/srv/utils.py +++ b/src/calibre/srv/utils.py @@ -7,9 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' import errno, socket, select, os, time -from Cookie import SimpleCookie from contextlib import closing -import repr as reprlib from email.utils import formatdate from operator import itemgetter from binascii import hexlify, unhexlify @@ -23,6 +21,8 @@ from calibre.utils.socket_inheritance import set_socket_inherit from calibre.utils.logging import ThreadSafeLog from calibre.utils.shared_file import share_open, raise_winerror from polyglot.builtins import iteritems, map, unicode_type, range +from polyglot import reprlib +from polyglot.http_cookie import SimpleCookie from polyglot.urllib import parse_qs, quote as urlquote HTTP1 = 'HTTP/1.0' diff --git a/src/calibre/srv/web_socket.py b/src/calibre/srv/web_socket.py index 5fdf7ab309..59a5cfc9f8 100644 --- a/src/calibre/srv/web_socket.py +++ b/src/calibre/srv/web_socket.py @@ -5,7 +5,7 @@ from __future__ import (unicode_literals, division, absolute_import, print_function) -import httplib, os, weakref, socket +import os, weakref, socket from base64 import standard_b64encode from collections import deque from hashlib import sha1 @@ -19,6 +19,7 @@ from calibre.srv.http_response import HTTPConnection, create_http_handler from calibre.srv.utils import DESIRED_SEND_BUFFER_SIZE from calibre.utils.speedups import ReadOnlyFileBuffer from polyglot.queue import Queue, Empty +from polyglot import http_client speedup, err = plugins['speedup'] if not speedup: raise RuntimeError('Failed to load speedup module with error: ' + err) @@ -286,9 +287,9 @@ class WebSocketConnection(HTTPConnection): except Exception: ver_ok = False if not ver_ok: - return self.simple_response(httplib.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver) + return self.simple_response(http_client.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver) if self.method != 'GET': - return self.simple_response(httplib.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method) + return self.simple_response(http_client.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method) response = HANDSHAKE_STR % standard_b64encode(sha1(key + GUID_STR).digest()) self.optimize_for_sending_packet() diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 7ce4075714..50fb71430d 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -73,6 +73,29 @@ class BuildTest(unittest.TestCase): from html5_parser import parse parse('

xxx') + def test_imports(self): + import importlib + exclude = ['dbus_export.demo', 'dbus_export.gtk', 'upstream'] + if not iswindows: + exclude.extend(['iphlpapi', 'windows', 'winreg', 'winusb']) + if not isosx: + exclude.append('osx') + if not islinux: + exclude.extend(['dbus', 'linux']) + base = os.path.dirname(__file__) + trimpath = len(os.path.dirname(base)) + 1 + for root, dirs, files in os.walk(base): + for dir in dirs: + if not os.path.isfile(os.path.join(root, dir, '__init__.py')): + dirs.remove(dir) + for file in files: + file, ext = os.path.splitext(file) + if ext != '.py': + continue + name = '.'.join(root[trimpath:].split(os.path.sep) + [file]) + if not any(x for x in exclude if x in name): + importlib.import_module(name) + def test_plugins(self): exclusions = set() if is_ci: @@ -99,7 +122,7 @@ class BuildTest(unittest.TestCase): from calibre.utils.cleantext import test_clean_xml_chars test_clean_xml_chars() from lxml import etree - raw = '' + raw = b'' root = etree.fromstring(raw) self.assertEqual(etree.tostring(root), raw) @@ -175,7 +198,7 @@ class BuildTest(unittest.TestCase): # it should just work because the hard-coded paths of the Qt # installation should work. If they do not, then it is a distro # problem. - fmts = set(map(unicode_type, QImageReader.supportedImageFormats())) + fmts = set(map(lambda x: x.data().decode('utf-8'), QImageReader.supportedImageFormats())) testf = {'jpg', 'png', 'svg', 'ico', 'gif'} self.assertEqual(testf.intersection(fmts), testf, "Qt doesn't seem to be able to load some of its image plugins. Available plugins: %s" % fmts) data = P('images/blank.png', allow_user_override=False, data=True) @@ -254,7 +277,7 @@ class BuildTest(unittest.TestCase): def test_netifaces(self): import netifaces - self.assertGreaterEqual(netifaces.interfaces(), 1, 'netifaces could find no network interfaces') + self.assertGreaterEqual(len(netifaces.interfaces()), 1, 'netifaces could find no network interfaces') def test_psutil(self): import psutil diff --git a/src/calibre/utils/browser.py b/src/calibre/utils/browser.py index def783eaee..1228747ee8 100644 --- a/src/calibre/utils/browser.py +++ b/src/calibre/utils/browser.py @@ -5,11 +5,13 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import copy, httplib, ssl -from cookielib import CookieJar, Cookie +import copy, ssl from mechanize import Browser as B, HTTPSHandler +from polyglot import http_client +from polyglot.http_cookie import CookieJar, Cookie + class ModernHTTPSHandler(HTTPSHandler): @@ -24,7 +26,7 @@ class ModernHTTPSHandler(HTTPSHandler): def conn_factory(hostport, **kw): kw['context'] = self.ssl_context - return httplib.HTTPSConnection(hostport, **kw) + return http_client.HTTPSConnection(hostport, **kw) return self.do_open(conn_factory, req) diff --git a/src/calibre/utils/cleantext.py b/src/calibre/utils/cleantext.py index 9e04d5d4d6..f8ba151de3 100644 --- a/src/calibre/utils/cleantext.py +++ b/src/calibre/utils/cleantext.py @@ -2,8 +2,9 @@ __license__ = 'GPL 3' __copyright__ = '2010, sengian ' __docformat__ = 'restructuredtext en' -import re, htmlentitydefs +import re from polyglot.builtins import codepoint_to_chr, map, range +from polyglot.html_entities import name2codepoint from calibre.constants import plugins, preferred_encoding try: @@ -80,7 +81,7 @@ def unescape(text, rm=False, rchar=u''): else: # named entity try: - text = codepoint_to_chr(htmlentitydefs.name2codepoint[text[1:-1]]) + text = codepoint_to_chr(name2codepoint[text[1:-1]]) except KeyError: pass if rm: diff --git a/src/calibre/utils/https.py b/src/calibre/utils/https.py index 690c1f454b..9c26970d23 100644 --- a/src/calibre/utils/https.py +++ b/src/calibre/utils/https.py @@ -10,7 +10,7 @@ import ssl, socket, re from contextlib import closing from calibre import get_proxies -from calibre.constants import ispy3 +from polyglot import http_client from polyglot.urllib import urlsplit has_ssl_verify = hasattr(ssl, 'create_default_context') and hasattr(ssl, '_create_unverified_context') @@ -19,19 +19,14 @@ class HTTPError(ValueError): def __init__(self, url, code): msg = '%s returned an unsupported http response code: %d (%s)' % ( - url, code, httplib.responses.get(code, None)) + url, code, http_client.responses.get(code, None)) ValueError.__init__(self, msg) self.code = code self.url = url -if ispy3: - import http.client as httplib -else: - import httplib - if has_ssl_verify: - class HTTPSConnection(httplib.HTTPSConnection): + class HTTPSConnection(http_client.HTTPSConnection): def __init__(self, ssl_version, *args, **kwargs): cafile = kwargs.pop('cert_file', None) @@ -39,7 +34,7 @@ if has_ssl_verify: kwargs['context'] = ssl._create_unverified_context() else: kwargs['context'] = ssl.create_default_context(cafile=cafile) - httplib.HTTPSConnection.__init__(self, *args, **kwargs) + http_client.HTTPSConnection.__init__(self, *args, **kwargs) else: # Check certificate hostname {{{ # Implementation taken from python 3 @@ -136,10 +131,10 @@ else: "subjectAltName fields were found") # }}} - class HTTPSConnection(httplib.HTTPSConnection): + class HTTPSConnection(http_client.HTTPSConnection): def __init__(self, ssl_version, *args, **kwargs): - httplib.HTTPSConnection.__init__(self, *args, **kwargs) + http_client.HTTPSConnection.__init__(self, *args, **kwargs) self.calibre_ssl_version = ssl_version def connect(self): @@ -204,7 +199,7 @@ def get_https_resource_securely( path += '?' + p.query c.request('GET', path, headers=headers or {}) response = c.getresponse() - if response.status in (httplib.MOVED_PERMANENTLY, httplib.FOUND, httplib.SEE_OTHER): + if response.status in (http_client.MOVED_PERMANENTLY, http_client.FOUND, http_client.SEE_OTHER): if max_redirects <= 0: raise ValueError('Too many redirects, giving up') newurl = response.getheader('Location', None) @@ -212,7 +207,7 @@ def get_https_resource_securely( raise ValueError('%s returned a redirect response with no Location header' % url) return get_https_resource_securely( newurl, cacerts=cacerts, timeout=timeout, max_redirects=max_redirects-1, ssl_version=ssl_version, get_response=get_response) - if response.status != httplib.OK: + if response.status != http_client.OK: raise HTTPError(url, response.status) if get_response: return response diff --git a/src/calibre/utils/smtp.py b/src/calibre/utils/smtp.py index d840cc4c37..8da8cc7bd2 100644 --- a/src/calibre/utils/smtp.py +++ b/src/calibre/utils/smtp.py @@ -98,8 +98,7 @@ def get_mx(host, verbose=0): if verbose: print('Find mail exchanger for', host) answers = list(dns.resolver.query(host, 'MX')) - answers.sort(cmp=lambda x, y: cmp(int(getattr(x, 'preference', sys.maxint)), - int(getattr(y, 'preference', sys.maxint)))) + answers.sort(key=lambda x: int(getattr(x, 'preference', sys.maxint))) return [str(x.exchange) for x in answers if hasattr(x, 'exchange')] diff --git a/src/calibre/utils/unrar.py b/src/calibre/utils/unrar.py index 882a636ba6..fc6a767843 100644 --- a/src/calibre/utils/unrar.py +++ b/src/calibre/utils/unrar.py @@ -114,20 +114,20 @@ def test_basic(): b"Rar!\x1a\x07\x00\xcf\x90s\x00\x00\r\x00\x00\x00\x00\x00\x00\x00\x14\xe7z\x00\x80#\x00\x17\x00\x00\x00\r\x00\x00\x00\x03\xc2\xb3\x96o\x00\x00\x00\x00\x1d3\x03\x00\x00\x00\x00\x00CMT\x0c\x00\x8b\xec\x8e\xef\x14\xf6\xe6h\x04\x17\xff\xcd\x0f\xffk9b\x11]^\x80\xd3dt \x90+\x00\x14\x00\x00\x00\x08\x00\x00\x00\x03\xf1\x84\x93\\\xb9]yA\x1d3\t\x00\xa4\x81\x00\x001\\sub-one\x00\xc0\x0c\x00\x8f\xec\x89\xfe.JM\x86\x82\x0c_\xfd\xfd\xd7\x11\x1a\xef@\x9eHt \x80'\x00\x0e\x00\x00\x00\x04\x00\x00\x00\x03\x9f\xa8\x17\xf8\xaf]yA\x1d3\x07\x00\xa4\x81\x00\x00one.txt\x00\x08\xbf\x08\xae\xf3\xca\x87\xfeo\xfe\xd2n\x80-Ht \x82:\x00\x18\x00\x00\x00\x10\x00\x00\x00\x03\xa86\x81\xdf\xf9fyA\x1d3\x1a\x00\xa4\x81\x00\x00\xe8\xaf\xb6\xe6\xaf\x94\xe5\xb1\x81.txt\x00\x8bh\xf6\xd4kA\\.\x00txt\x0c\x00\x8b\xec\x8e\xef\x14\xf6\xe2l\x91\x189\xff\xdf\xfe\xc2\xd3:g\x9a\x19F=cYt \x928\x00\x11\x00\x00\x00\x08\x00\x00\x00\x03\x7f\xd6\xb6\x7f\xeafyA\x1d3\x16\x00\xa4\x81\x00\x00F\xc3\xbc\xc3\x9fe.txt\x00\x01\x00F\xfc\xdfe\x00.txt\x00\xc0 + +from polyglot.builtins import is_py3 + +if is_py3: + from html.entities import name2codepoint +else: + from htmlentitydefs import name2codepoint diff --git a/src/polyglot/http_client.py b/src/polyglot/http_client.py new file mode 100644 index 0000000000..740d9799ff --- /dev/null +++ b/src/polyglot/http_client.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +# License: GPL v3 Copyright: 2019, Eli Schwartz + +from polyglot.builtins import is_py3 + +if is_py3: + from http.client import (responses, HTTPConnection, HTTPSConnection, + BAD_REQUEST, FOUND, FORBIDDEN, HTTP_VERSION_NOT_SUPPORTED, + INTERNAL_SERVER_ERROR, METHOD_NOT_ALLOWED, MOVED_PERMANENTLY, + NOT_FOUND, NOT_IMPLEMENTED, NOT_MODIFIED, OK, PARTIAL_CONTENT, + PRECONDITION_FAILED, REQUEST_ENTITY_TOO_LARGE, REQUEST_URI_TOO_LONG, + REQUESTED_RANGE_NOT_SATISFIABLE, REQUEST_TIMEOUT, SEE_OTHER, + SERVICE_UNAVAILABLE, UNAUTHORIZED, _CS_IDLE, _CS_REQ_SENT) +else: + from httplib import (responses, HTTPConnection, HTTPSConnection, + BAD_REQUEST, FOUND, FORBIDDEN, HTTP_VERSION_NOT_SUPPORTED, + INTERNAL_SERVER_ERROR, METHOD_NOT_ALLOWED, MOVED_PERMANENTLY, + NOT_FOUND, NOT_IMPLEMENTED, NOT_MODIFIED, OK, PARTIAL_CONTENT, + PRECONDITION_FAILED, REQUEST_ENTITY_TOO_LARGE, REQUEST_URI_TOO_LONG, + REQUESTED_RANGE_NOT_SATISFIABLE, REQUEST_TIMEOUT, SEE_OTHER, + SERVICE_UNAVAILABLE, UNAUTHORIZED, _CS_IDLE, _CS_REQ_SENT) diff --git a/src/polyglot/http_cookie.py b/src/polyglot/http_cookie.py new file mode 100644 index 0000000000..645b8b6a3a --- /dev/null +++ b/src/polyglot/http_cookie.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +# License: GPL v3 Copyright: 2019, Eli Schwartz + +from polyglot.builtins import is_py3 + +if is_py3: + from http.cookies import SimpleCookie # noqa + from http.cookiejar import CookieJar, Cookie # noqa +else: + from Cookie import SimpleCookie # noqa + from cookielib import CookieJar, Cookie # noqa diff --git a/src/polyglot/reprlib.py b/src/polyglot/reprlib.py new file mode 100755 index 0000000000..2444ad1788 --- /dev/null +++ b/src/polyglot/reprlib.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +# License: GPL v3 Copyright: 2019, Eli Schwartz + +from polyglot.builtins import is_py3 + +if is_py3: + from reprlib import repr +else: + from repr import repr