Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'py3' of https://github.com/eli-schwartz/calibre

Browse Source
This commit is contained in:
Kovid Goyal 2019-03-27 05:55:10 +05:30
commit 55b39e16bb
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
63 changed files with 357 additions and 510 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
manual/custom.py
View File

@ -179,7 +179,7 @@ def generate_ebook_convert_help(preamble, app):
raw += '\n\n.. contents::\n :local:' raw += '\n\n.. contents::\n :local:'
raw += '\n\n' + options raw += '\n\n' + options
for pl in sorted(input_format_plugins(), key=lambda x:x.name): for pl in sorted(input_format_plugins(), key=lambda x: x.name):
parser, plumber = create_option_parser(['ebook-convert', parser, plumber = create_option_parser(['ebook-convert',
'dummyi.'+sorted(pl.file_types)[0], 'dummyo.epub', '-h'], default_log) 'dummyi.'+sorted(pl.file_types)[0], 'dummyo.epub', '-h'], default_log)
groups = [(pl.name+ ' Options', '', g.option_list) for g in groups = [(pl.name+ ' Options', '', g.option_list) for g in
@ -279,7 +279,7 @@ def cli_docs(app):
info(bold('creating CLI documentation...')) info(bold('creating CLI documentation...'))
documented_cmds, undocumented_cmds = get_cli_docs() documented_cmds, undocumented_cmds = get_cli_docs()
documented_cmds.sort(cmp=lambda x, y: cmp(x[0], y[0])) documented_cmds.sort(key=lambda x: x[0])
undocumented_cmds.sort() undocumented_cmds.sort()
documented = [' '*4 + c[0] for c in documented_cmds] documented = [' '*4 + c[0] for c in documented_cmds]

4
src/calibre/__init__.py
View File

@ -240,7 +240,7 @@ def prints(*args, **kwargs):
file.write(arg) file.write(arg)
count += len(arg) count += len(arg)
except: except:
import repr as reprlib from polyglot import reprlib
arg = reprlib.repr(arg) arg = reprlib.repr(arg)
file.write(arg) file.write(arg)
count += len(arg) count += len(arg)
@ -614,7 +614,7 @@ def entity_to_unicode(match, exceptions=[], encoding='cp1252',
return check(html5_entities[ent]) return check(html5_entities[ent])
except KeyError: except KeyError:
pass pass
from htmlentitydefs import name2codepoint from polyglot.html_entities import name2codepoint
try: try:
return check(my_unichr(name2codepoint[ent])) return check(my_unichr(name2codepoint[ent]))
except KeyError: except KeyError:

2
src/calibre/constants.py
View File

@ -168,7 +168,6 @@ class Plugins(collections.Mapping):
'icu', 'icu',
'speedup', 'speedup',
'unicode_names', 'unicode_names',
'zlib2',
'html', 'html',
'freetype', 'freetype',
'imageops', 'imageops',
@ -184,6 +183,7 @@ class Plugins(collections.Mapping):
if not ispy3: if not ispy3:
plugins.extend([ plugins.extend([
'monotonic', 'monotonic',
'zlib2',
]) ])
if iswindows: if iswindows:
plugins.extend(['winutil', 'wpd', 'winfonts']) plugins.extend(['winutil', 'wpd', 'winfonts'])

4
src/calibre/customize/profiles.py
View File

@ -233,7 +233,7 @@ input_profiles = [InputProfile, SonyReaderInput, SonyReader300Input,
HanlinV5Input, CybookG3Input, CybookOpusInput, KindleInput, IlliadInput, HanlinV5Input, CybookG3Input, CybookOpusInput, KindleInput, IlliadInput,
IRexDR1000Input, IRexDR800Input, NookInput] IRexDR1000Input, IRexDR800Input, NookInput]
input_profiles.sort(cmp=lambda x,y:cmp(x.name.lower(), y.name.lower())) input_profiles.sort(key=lambda x: x.name.lower())
# }}} # }}}
@ -870,4 +870,4 @@ output_profiles = [
KindlePaperWhite3Output, KindleOasisOutput KindlePaperWhite3Output, KindleOasisOutput
] ]
output_profiles.sort(cmp=lambda x,y:cmp(x.name.lower(), y.name.lower())) output_profiles.sort(key=lambda x: x.name.lower())

4
src/calibre/customize/ui.py
View File

@ -727,9 +727,9 @@ def initialize_plugins(perf=False):
# ipython # ipython
sys.stdout, sys.stderr = ostdout, ostderr sys.stdout, sys.stderr = ostdout, ostderr
if perf: if perf:
for x in sorted(times, key=lambda x:times[x]): for x in sorted(times, key=lambda x: times[x]):
print('%50s: %.3f'%(x, times[x])) print('%50s: %.3f'%(x, times[x]))
_initialized_plugins.sort(cmp=lambda x,y:cmp(x.priority, y.priority), reverse=True) _initialized_plugins.sort(key=lambda x: x.priority, reverse=True)
reread_filetype_plugins() reread_filetype_plugins()
reread_metadata_plugins() reread_metadata_plugins()

4
src/calibre/db/cli/cmd_list_categories.py
View File

@ -149,9 +149,7 @@ def main(opts, args, dbctx):
(not report_on or k in report_on) (not report_on or k in report_on)
] ]
categories.sort( categories.sort(key=lambda x: x if x[0] != '#' else x[1:])
cmp=lambda x, y: cmp(x if x[0] != '#' else x[1:], y if y[0] != '#' else y[1:])
)
def fmtr(v): def fmtr(v):
v = v or 0 v = v or 0

8
src/calibre/db/cli/main.py
View File

@ -4,7 +4,6 @@
from __future__ import absolute_import, division, print_function, unicode_literals from __future__ import absolute_import, division, print_function, unicode_literals
import httplib
import json import json
import os import os
import sys import sys
@ -17,6 +16,7 @@ from calibre.utils.config import OptionParser, prefs
from calibre.utils.localization import localize_user_manual_link from calibre.utils.localization import localize_user_manual_link
from calibre.utils.lock import singleinstance from calibre.utils.lock import singleinstance
from calibre.utils.serialize import MSGPACK_MIME from calibre.utils.serialize import MSGPACK_MIME
from polyglot import http_client
from polyglot.urllib import urlencode, urlparse, urlunparse from polyglot.urllib import urlencode, urlparse, urlunparse
COMMANDS = ( COMMANDS = (
@ -191,13 +191,13 @@ class DBCtx(object):
return m.implementation(self.db.new_api, None, *args) return m.implementation(self.db.new_api, None, *args)
def interpret_http_error(self, err): def interpret_http_error(self, err):
if err.code == httplib.UNAUTHORIZED: if err.code == http_client.UNAUTHORIZED:
if self.has_credentials: if self.has_credentials:
raise SystemExit('The username/password combination is incorrect') raise SystemExit('The username/password combination is incorrect')
raise SystemExit('A username and password is required to access this server') raise SystemExit('A username and password is required to access this server')
if err.code == httplib.FORBIDDEN: if err.code == http_client.FORBIDDEN:
raise SystemExit(err.reason) raise SystemExit(err.reason)
if err.code == httplib.NOT_FOUND: if err.code == http_client.NOT_FOUND:
raise SystemExit(err.reason) raise SystemExit(err.reason)
def remote_run(self, name, m, *args): def remote_run(self, name, m, *args):

4
src/calibre/db/tests/legacy.py
View File

@ -8,13 +8,13 @@ __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
import inspect, time, numbers import inspect, time, numbers
from io import BytesIO from io import BytesIO
from repr import repr
from functools import partial from functools import partial
from operator import itemgetter from operator import itemgetter
from calibre.library.field_metadata import fm_as_dict from calibre.library.field_metadata import fm_as_dict
from calibre.db.tests.base import BaseTest from calibre.db.tests.base import BaseTest
from polyglot.builtins import iteritems, range from polyglot.builtins import iteritems, range
from polyglot import reprlib
# Utils {{{ # Utils {{{
@ -32,7 +32,7 @@ class ET(object):
oldres = getattr(old, self.func_name)(*self.args, **self.kwargs) oldres = getattr(old, self.func_name)(*self.args, **self.kwargs)
newres = getattr(legacy, self.func_name)(*self.args, **self.kwargs) newres = getattr(legacy, self.func_name)(*self.args, **self.kwargs)
test.assertEqual(oldres, newres, 'Equivalence test for %s with args: %s and kwargs: %s failed' % ( test.assertEqual(oldres, newres, 'Equivalence test for %s with args: %s and kwargs: %s failed' % (
self.func_name, repr(self.args), repr(self.kwargs))) self.func_name, reprlib.repr(self.args), reprlib.repr(self.kwargs)))
self.retval = newres self.retval = newres
return newres return newres

7
src/calibre/devices/__init__.py
View File

@ -8,7 +8,7 @@ Device drivers.
import sys, time, pprint import sys, time, pprint
from functools import partial from functools import partial
from StringIO import StringIO from io import BytesIO
DAY_MAP = dict(Sun=0, Mon=1, Tue=2, Wed=3, Thu=4, Fri=5, Sat=6) DAY_MAP = dict(Sun=0, Mon=1, Tue=2, Wed=3, Thu=4, Fri=5, Sat=6)
MONTH_MAP = dict(Jan=1, Feb=2, Mar=3, Apr=4, May=5, Jun=6, Jul=7, Aug=8, Sep=9, Oct=10, Nov=11, Dec=12) MONTH_MAP = dict(Jan=1, Feb=2, Mar=3, Apr=4, May=5, Jun=6, Jul=7, Aug=8, Sep=9, Oct=10, Nov=11, Dec=12)
@ -77,13 +77,12 @@ def debug(ioreg_to_tmp=False, buf=None, plugins=None,
oldo, olde = sys.stdout, sys.stderr oldo, olde = sys.stdout, sys.stderr
if buf is None: if buf is None:
buf = StringIO() buf = BytesIO()
sys.stdout = sys.stderr = buf sys.stdout = sys.stderr = buf
out = partial(prints, file=buf) out = partial(prints, file=buf)
devplugins = device_plugins() if plugins is None else plugins devplugins = device_plugins() if plugins is None else plugins
devplugins = list(sorted(devplugins, cmp=lambda devplugins = list(sorted(devplugins, key=lambda x: x.__class__.__name__))
x,y:cmp(x.__class__.__name__, y.__class__.__name__)))
if plugins is None: if plugins is None:
for d in devplugins: for d in devplugins:
try: try:

2
src/calibre/devices/prs505/sony_cache.py
View File

@ -321,7 +321,7 @@ class XMLCache(object):
# Only rebase ids of nodes that are immediate children of the # Only rebase ids of nodes that are immediate children of the
# record root (that way playlist/itemnodes are unaffected # record root (that way playlist/itemnodes are unaffected
items = root.xpath('child::*[@id]') items = root.xpath('child::*[@id]')
items.sort(cmp=lambda x,y:cmp(int(x.get('id')), int(y.get('id')))) items.sort(key=lambda x: int(x.get('id')))
idmap = {} idmap = {}
for i, item in enumerate(items): for i, item in enumerate(items):
old = int(item.get('id')) old = int(item.get('id'))

2
src/calibre/devices/usbms/device.py
View File

@ -537,7 +537,7 @@ class Device(DeviceConfig, DevicePlugin):
if sz > 0: if sz > 0:
nodes.append((x.split('/')[-1], sz)) nodes.append((x.split('/')[-1], sz))
nodes.sort(cmp=lambda x, y: cmp(x[1], y[1])) nodes.sort(key=lambda x: x[1])
if not nodes: if not nodes:
return node return node
return nodes[-1][0] return nodes[-1][0]

3
src/calibre/devices/winusb.py
View File

@ -995,8 +995,7 @@ def develop(): # {{{
drive_letters = set() drive_letters = set()
pprint(usb_devices) pprint(usb_devices)
print() print()
devplugins = list(sorted(device_plugins(), cmp=lambda devplugins = list(sorted(device_plugins(), key=lambda x: x.__class__.__name__))
x,y:cmp(x.__class__.__name__, y.__class__.__name__)))
for dev in devplugins: for dev in devplugins:
dev.startup() dev.startup()
for dev in devplugins: for dev in devplugins:

4
src/calibre/ebooks/conversion/plugins/tcr_input.py
View File

@ -4,7 +4,7 @@ __license__ = 'GPL 3'
__copyright__ = '2009, John Schember <john@nachtimwald.com>' __copyright__ = '2009, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
from cStringIO import StringIO from io import BytesIO
from calibre.customize.conversion import InputFormatPlugin from calibre.customize.conversion import InputFormatPlugin
@ -24,7 +24,7 @@ class TCRInput(InputFormatPlugin):
raw_txt = decompress(stream) raw_txt = decompress(stream)
log.info('Converting text to OEB...') log.info('Converting text to OEB...')
stream = StringIO(raw_txt) stream = BytesIO(raw_txt)
from calibre.customize.ui import plugin_for_input_format from calibre.customize.ui import plugin_for_input_format

2
src/calibre/ebooks/conversion/plumber.py
View File

@ -821,7 +821,7 @@ OptionRecommendation(name='search_replace',
if not html_files: if not html_files:
raise ValueError(_('Could not find an e-book inside the archive')) raise ValueError(_('Could not find an e-book inside the archive'))
html_files = [(f, os.stat(f).st_size) for f in html_files] html_files = [(f, os.stat(f).st_size) for f in html_files]
html_files.sort(cmp=lambda x, y: cmp(x[1], y[1])) html_files.sort(key=lambda x: x[1])
html_files = [f[0] for f in html_files] html_files = [f[0] for f in html_files]
for q in ('toc', 'index'): for q in ('toc', 'index'):
for f in html_files: for f in html_files:

67
src/calibre/ebooks/lit/from_any.py
View File

@ -1,67 +0,0 @@
from __future__ import with_statement
from __future__ import print_function
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
__docformat__ = 'restructuredtext en'
'''
Convert any ebook format to LIT.
'''
import sys, os, glob, logging
from calibre.ebooks.epub.from_any import any2epub, formats, USAGE
from calibre.ebooks.epub import config as common_config
from calibre.ptempfile import TemporaryDirectory
from calibre.ebooks.lit.writer import oeb2lit
def config(defaults=None):
c = common_config(defaults=defaults, name='lit')
return c
def option_parser(usage=USAGE):
return config().option_parser(usage=usage%('LIT', formats()))
def any2lit(opts, path):
ext = os.path.splitext(path)[1]
if not ext:
raise ValueError('Unknown file type: '+path)
ext = ext.lower()[1:]
if opts.output is None:
opts.output = os.path.splitext(os.path.basename(path))[0]+'.lit'
opts.output = os.path.abspath(opts.output)
orig_output = opts.output
with TemporaryDirectory('_any2lit') as tdir:
oebdir = os.path.join(tdir, 'oeb')
os.mkdir(oebdir)
opts.output = os.path.join(tdir, 'dummy.epub')
opts.profile = 'None'
orig_bfs = opts.base_font_size2
opts.base_font_size2 = 0
any2epub(opts, path, create_epub=False, oeb_cover=True, extract_to=oebdir)
opts.base_font_size2 = orig_bfs
opf = glob.glob(os.path.join(oebdir, '*.opf'))[0]
opts.output = orig_output
logging.getLogger('html2epub').info(_('Creating LIT file from EPUB...'))
oeb2lit(opts, opf)
def main(args=sys.argv):
parser = option_parser()
opts, args = parser.parse_args(args)
if len(args) < 2:
parser.print_help()
print('No input file specified.')
return 1
any2lit(opts, args[1])
return 0
if __name__ == '__main__':
sys.exit(main())

3
src/calibre/ebooks/lit/writer.py
View File

@ -670,8 +670,7 @@ class LitWriter(object):
def _build_dchunks(self): def _build_dchunks(self):
ddata = [] ddata = []
directory = list(self._directory) directory = list(self._directory)
directory.sort(cmp=lambda x, y: directory.sort(key=lambda x: x.name.lower())
cmp(x.name.lower(), y.name.lower()))
qrn = 1 + (1 << 2) qrn = 1 + (1 << 2)
dchunk = io.BytesIO() dchunk = io.BytesIO()
dcount = 0 dcount = 0

128
src/calibre/ebooks/lrf/html/convert_to.py
View File

@ -1,128 +0,0 @@
from __future__ import print_function
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
import sys, logging, os
from calibre import setup_cli_handlers
from calibre.utils.config import OptionParser
from calibre.ebooks import ConversionError
from calibre.ebooks.lrf.meta import get_metadata
from calibre.ebooks.lrf.lrfparser import LRFDocument
from calibre.ebooks.metadata.opf import OPFCreator
from calibre.ebooks.lrf.objects import PageAttr, BlockAttr, TextAttr
from calibre.ebooks.lrf.pylrs.pylrs import TextStyle
class BlockStyle(object):
def __init__(self, ba):
self.ba = ba
def __str__(self):
ans = '.'+str(self.ba.id)+' {\n'
if hasattr(self.ba, 'sidemargin'):
margin = str(self.ba.sidemargin) + 'px'
ans += '\tmargin-left: %(m)s; margin-right: %(m)s;\n'%dict(m=margin)
if hasattr(self.ba, 'topskip'):
ans += '\tmargin-top: %dpx;\n'%(self.ba.topskip,)
if hasattr(self.ba, 'footskip'):
ans += '\tmargin-bottom: %dpx;\n'%(self.ba.footskip,)
if hasattr(self.ba, 'framewidth'):
ans += '\tborder-width: %dpx;\n'%(self.ba.framewidth,)
ans += '\tborder-style: solid;\n'
if hasattr(self.ba, 'framecolor'):
if self.ba.framecolor.a < 255:
ans += '\tborder-color: %s;\n'%(self.ba.framecolor.to_html())
if hasattr(self.ba, 'bgcolor'):
if self.ba.bgcolor.a < 255:
ans += '\tbackground-color: %s;\n'%(self.ba.bgcolor.to_html())
# TODO: Fixed size blocks
return ans + '}\n'
class LRFConverter(object):
def __init__(self, document, opts, logger):
self.lrf = document
self.opts = opts
self.output_dir = opts.out
self.logger = logger
logger.info('Parsing LRF...')
self.lrf.parse()
self.create_metadata()
self.create_styles()
def create_metadata(self):
self.logger.info('Reading metadata...')
mi = get_metadata(self.lrf)
self.opf = OPFCreator(self.output_dir, mi)
def create_page_styles(self):
self.page_css = ''
for obj in self.lrf.objects.values():
if isinstance(obj, PageAttr):
selector = 'body.'+str(obj.id)
self.page_css = selector + ' {\n'
# TODO: Headers and footers
self.page_css += '}\n'
def create_block_styles(self):
self.block_css = ''
for obj in self.lrf.objects.values():
if isinstance(obj, BlockAttr):
self.block_css += str(BlockStyle(obj))
def create_text_styles(self):
self.text_css = ''
for obj in self.lrf.objects.values():
if isinstance(obj, TextAttr):
self.text_css += str(TextStyle(obj))
print(self.text_css)
def create_styles(self):
self.logger.info('Creating CSS stylesheet...')
self.create_page_styles()
self.create_block_styles()
def option_parser():
parser = OptionParser(usage='%prog book.lrf')
parser.add_option('--output-dir', '-o', default=None, help=(
'Output directory in which to store created HTML files. If it does not exist, it is created. By default the current directory is used.'), dest='out')
parser.add_option('--verbose', default=False, action='store_true', dest='verbose')
return parser
def process_file(lrfpath, opts, logger=None):
if logger is None:
level = logging.DEBUG if opts.verbose else logging.INFO
logger = logging.getLogger('lrf2html')
setup_cli_handlers(logger, level)
if opts.out is None:
opts.out = os.getcwdu()
else:
opts.out = os.path.abspath(opts.out)
if not os.path.isdir(opts.out):
raise ConversionError(opts.out + ' is not a directory')
if not os.path.exists(opts.out):
os.makedirs(opts.out)
document = LRFDocument(open(lrfpath, 'rb'))
LRFConverter(document, opts, logger)
def main(args=sys.argv):
parser = option_parser()
opts, args = parser.parse_args(args)
if len(args) != 2:
parser.print_help()
return 1
process_file(args[1], opts)
return 0
if __name__ == '__main__':
sys.exit(main())

2
src/calibre/ebooks/metadata/lit.py
View File

@ -32,7 +32,7 @@ def get_metadata(stream):
except: except:
pass pass
break break
covers.sort(cmp=lambda x, y:cmp(len(x[0]), len(y[0])), reverse=True) covers.sort(key=lambda x: len(x[0]), reverse=True)
idx = 0 idx = 0
if len(covers) > 1: if len(covers) > 1:
if covers[1][1] == covers[0][1]+'-standard': if covers[1][1] == covers[0][1]+'-standard':

4
src/calibre/ebooks/metadata/meta.py
View File

@ -41,8 +41,7 @@ def metadata_from_formats(formats, force_read_metadata=False, pattern=None):
def _metadata_from_formats(formats, force_read_metadata=False, pattern=None): def _metadata_from_formats(formats, force_read_metadata=False, pattern=None):
mi = MetaInformation(None, None) mi = MetaInformation(None, None)
formats.sort(cmp=lambda x,y: cmp(METADATA_PRIORITIES[path_to_ext(x)], formats.sort(key=lambda x: METADATA_PRIORITIES[path_to_ext(x)])
METADATA_PRIORITIES[path_to_ext(y)]))
extensions = list(map(path_to_ext, formats)) extensions = list(map(path_to_ext, formats))
if 'opf' in extensions: if 'opf' in extensions:
opf = formats[extensions.index('opf')] opf = formats[extensions.index('opf')]
@ -241,4 +240,3 @@ def forked_read_metadata(path, tdir):
opf = metadata_to_opf(mi, default_lang='und') opf = metadata_to_opf(mi, default_lang='und')
with lopen(os.path.join(tdir, 'metadata.opf'), 'wb') as f: with lopen(os.path.join(tdir, 'metadata.opf'), 'wb') as f:
f.write(opf) f.write(opf)

3
src/calibre/ebooks/oeb/transforms/guide.py
View File

@ -28,7 +28,7 @@ class Clean(object):
else: else:
covers.append([self.oeb.guide[x], len(item.data)]) covers.append([self.oeb.guide[x], len(item.data)])
covers.sort(cmp=lambda x,y:cmp(x[1], y[1]), reverse=True) covers.sort(key=lambda x: x[1], reverse=True)
if covers: if covers:
ref = covers[0][0] ref = covers[0][0]
if len(covers) > 1: if len(covers) > 1:
@ -53,4 +53,3 @@ class Clean(object):
if item.title and item.title.lower() == 'start': if item.title and item.title.lower() == 'start':
continue continue
self.oeb.guide.remove(x) self.oeb.guide.remove(x)

21
src/calibre/ebooks/pdf/from_comic.py
View File

@ -1,21 +0,0 @@
from __future__ import with_statement
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
__docformat__ = 'restructuredtext en'
'Convert a comic in CBR/CBZ format to pdf'
import sys
from functools import partial
from calibre.ebooks.lrf.comic.convert_from import do_convert, option_parser, config, main as _main
convert = partial(do_convert, output_format='pdf')
main = partial(_main, output_format='pdf')
if __name__ == '__main__':
sys.exit(main())
if False:
option_parser
config

10
src/calibre/ebooks/pdf/reflow.py
View File

@ -169,7 +169,7 @@ class Column(object):
self._post_add() self._post_add()
def _post_add(self): def _post_add(self):
self.elements.sort(cmp=lambda x,y:cmp(x.bottom,y.bottom)) self.elements.sort(key=lambda x: x.bottom)
self.top = self.elements[0].top self.top = self.elements[0].top
self.bottom = self.elements[-1].bottom self.bottom = self.elements[-1].bottom
self.left, self.right = sys.maxint, 0 self.left, self.right = sys.maxint, 0
@ -260,7 +260,7 @@ class Region(object):
def add(self, columns): def add(self, columns):
if not self.columns: if not self.columns:
for x in sorted(columns, cmp=lambda x,y: cmp(x.left, y.left)): for x in sorted(columns, key=lambda x: x.left):
self.columns.append(x) self.columns.append(x)
else: else:
for i in range(len(columns)): for i in range(len(columns)):
@ -458,7 +458,7 @@ class Page(object):
self.elements = list(self.texts) self.elements = list(self.texts)
for img in page.xpath('descendant::img'): for img in page.xpath('descendant::img'):
self.elements.append(Image(img, self.opts, self.log, idc)) self.elements.append(Image(img, self.opts, self.log, idc))
self.elements.sort(cmp=lambda x,y:cmp(x.top, y.top)) self.elements.sort(key=lambda x: x.top)
def coalesce_fragments(self): def coalesce_fragments(self):
@ -580,7 +580,7 @@ class Page(object):
def sort_into_columns(self, elem, neighbors): def sort_into_columns(self, elem, neighbors):
neighbors.add(elem) neighbors.add(elem)
neighbors = sorted(neighbors, cmp=lambda x,y:cmp(x.left, y.left)) neighbors = sorted(neighbors, key=lambda x: x.left)
if self.opts.verbose > 3: if self.opts.verbose > 3:
self.log.debug('Neighbors:', [x.to_html() for x in neighbors]) self.log.debug('Neighbors:', [x.to_html() for x in neighbors])
columns = [Column()] columns = [Column()]
@ -595,7 +595,7 @@ class Page(object):
if not added: if not added:
columns.append(Column()) columns.append(Column())
columns[-1].add(x) columns[-1].add(x)
columns.sort(cmp=lambda x,y:cmp(x.left, y.left)) columns.sort(key=lambda x: x.left)
return columns return columns
def find_elements_in_row_of(self, x): def find_elements_in_row_of(self, x):

3
src/calibre/gui2/actions/choose_library.py
View File

@ -51,8 +51,7 @@ class LibraryUsageStats(object): # {{{
def write_stats(self): def write_stats(self):
locs = list(self.stats.keys()) locs = list(self.stats.keys())
locs.sort(cmp=lambda x, y: cmp(self.stats[x], self.stats[y]), locs.sort(key=lambda x: self.stats[x], reverse=True)
reverse=True)
for key in locs[500:]: for key in locs[500:]:
self.stats.pop(key) self.stats.pop(key)
gprefs.set('library_usage_stats', self.stats) gprefs.set('library_usage_stats', self.stats)

2
src/calibre/gui2/dialogs/catalog.py
View File

@ -93,7 +93,7 @@ class Catalog(QDialog, Ui_Dialog):
else: else:
info("No dynamic tab resources found for %s" % name) info("No dynamic tab resources found for %s" % name)
self.widgets = sorted(self.widgets, cmp=lambda x,y:cmp(x.TITLE, y.TITLE)) self.widgets = sorted(self.widgets, key=lambda x: x.TITLE)
# Generate a sorted list of installed catalog formats/sync_enabled pairs # Generate a sorted list of installed catalog formats/sync_enabled pairs
fmts = sorted([x[0] for x in self.fmts]) fmts = sorted([x[0] for x in self.fmts])

7
src/calibre/gui2/icon_theme.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import os, errno, json, importlib, math, httplib, bz2, shutil, sys import os, errno, json, importlib, math, bz2, shutil, sys
from itertools import count from itertools import count
from io import BytesIO from io import BytesIO
from threading import Thread, Event from threading import Thread, Event
@ -35,8 +35,9 @@ from calibre.utils.img import image_from_data, Canvas, optimize_png, optimize_jp
from calibre.utils.zipfile import ZipFile, ZIP_STORED from calibre.utils.zipfile import ZipFile, ZIP_STORED
from calibre.utils.filenames import atomic_rename from calibre.utils.filenames import atomic_rename
from lzma.xz import compress, decompress from lzma.xz import compress, decompress
from polyglot.queue import Queue, Empty
from polyglot.builtins import iteritems, map, range, reraise from polyglot.builtins import iteritems, map, range, reraise
from polyglot import http_client
from polyglot.queue import Queue, Empty
IMAGE_EXTENSIONS = {'png', 'jpg', 'jpeg'} IMAGE_EXTENSIONS = {'png', 'jpg', 'jpeg'}
THEME_COVER = 'icon-theme-cover.jpg' THEME_COVER = 'icon-theme-cover.jpg'
@ -439,7 +440,7 @@ def download_cover(cover_url, etag=None, cached=b''):
etag = response.getheader('ETag', None) or None etag = response.getheader('ETag', None) or None
return cached, etag return cached, etag
except HTTPError as e: except HTTPError as e:
if etag and e.code == httplib.NOT_MODIFIED: if etag and e.code == http_client.NOT_MODIFIED:
return cached, etag return cached, etag
raise raise

2
src/calibre/gui2/library/models.py
View File

@ -323,7 +323,7 @@ class BooksModel(QAbstractTableModel): # {{{
return 100000 return 100000
return self.db.field_metadata[name]['rec_index'] return self.db.field_metadata[name]['rec_index']
self.column_map.sort(cmp=lambda x,y: cmp(col_idx(x), col_idx(y))) self.column_map.sort(key=lambda x: col_idx(x))
for col in self.column_map: for col in self.column_map:
if col in self.orig_headers: if col in self.orig_headers:
self.headers[col] = self.orig_headers[col] self.headers[col] = self.orig_headers[col]

2
src/calibre/gui2/library/views.py
View File

@ -1031,7 +1031,7 @@ class BooksView(QTableView): # {{{
h.visualIndex(x) > -1] h.visualIndex(x) > -1]
if not pairs: if not pairs:
pairs = [(0, 0)] pairs = [(0, 0)]
pairs.sort(cmp=lambda x,y:cmp(x[1], y[1])) pairs.sort(key=lambda x: x[1])
i = pairs[0][0] i = pairs[0][0]
index = self.model().index(row, i) index = self.model().index(row, i)
if for_sync: if for_sync:

10
src/calibre/gui2/preferences/columns.py
View File

@ -69,7 +69,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
state = self.columns_state(defaults) state = self.columns_state(defaults)
self.hidden_cols = state['hidden_columns'] self.hidden_cols = state['hidden_columns']
positions = state['column_positions'] positions = state['column_positions']
colmap.sort(cmp=lambda x,y: cmp(positions[x], positions[y])) colmap.sort(key=lambda x: positions[x])
self.opt_columns.clear() self.opt_columns.clear()
db = model.db db = model.db
@ -248,12 +248,10 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
if 'ondevice' in hidden_cols: if 'ondevice' in hidden_cols:
hidden_cols.remove('ondevice') hidden_cols.remove('ondevice')
def col_pos(x, y): def col_pos(x):
xidx = config_cols.index(x) if x in config_cols else sys.maxint return config_cols.index(x) if x in config_cols else sys.maxint
yidx = config_cols.index(y) if y in config_cols else sys.maxint
return cmp(xidx, yidx)
positions = {} positions = {}
for i, col in enumerate((sorted(model.column_map, cmp=col_pos))): for i, col in enumerate((sorted(model.column_map, key=col_pos))):
positions[col] = i positions[col] = i
state = {'hidden_columns': hidden_cols, 'column_positions':positions} state = {'hidden_columns': hidden_cols, 'column_positions':positions}
self.gui.library_view.apply_state(state) self.gui.library_view.apply_state(state)

2
src/calibre/gui2/preferences/look_feel.py
View File

@ -456,7 +456,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
if l != lang] if l != lang]
if lang != 'en': if lang != 'en':
items.append(('en', get_esc_lang('en'))) items.append(('en', get_esc_lang('en')))
items.sort(cmp=lambda x, y: cmp(x[1].lower(), y[1].lower())) items.sort(key=lambda x: x[1].lower())
choices = [(y, x) for x, y in items] choices = [(y, x) for x, y in items]
# Default language is the autodetected one # Default language is the autodetected one
choices = [(get_language(lang), lang)] + choices choices = [(get_language(lang), lang)] + choices

4
src/calibre/gui2/preferences/main.py
View File

@ -171,7 +171,7 @@ class Browser(QScrollArea): # {{{
self.category_names = category_names self.category_names = category_names
categories = list(category_map.keys()) categories = list(category_map.keys())
categories.sort(cmp=lambda x, y: cmp(category_map[x], category_map[y])) categories.sort(key=lambda x: category_map[x])
self.category_map = OrderedDict() self.category_map = OrderedDict()
for c in categories: for c in categories:
@ -181,7 +181,7 @@ class Browser(QScrollArea): # {{{
self.category_map[plugin.category].append(plugin) self.category_map[plugin.category].append(plugin)
for plugins in self.category_map.values(): for plugins in self.category_map.values():
plugins.sort(cmp=lambda x, y: cmp(x.name_order, y.name_order)) plugins.sort(key=lambda x: x.name_order)
self.widgets = [] self.widgets = []
self._layout = QVBoxLayout() self._layout = QVBoxLayout()

13
src/calibre/gui2/preferences/plugboard.py
View File

@ -34,17 +34,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
self.db = gui.library_view.model().db self.db = gui.library_view.model().db
def initialize(self): def initialize(self):
def field_cmp(x, y):
if x.startswith('#'):
if y.startswith('#'):
return cmp(x.lower(), y.lower())
else:
return 1
elif y.startswith('#'):
return -1
else:
return cmp(x.lower(), y.lower())
ConfigWidgetBase.initialize(self) ConfigWidgetBase.initialize(self)
self.current_plugboards = copy.deepcopy(self.db.prefs.get('plugboards',{})) self.current_plugboards = copy.deepcopy(self.db.prefs.get('plugboards',{}))
@ -73,7 +62,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
if n not in self.disabled_devices: if n not in self.disabled_devices:
self.disabled_devices.append(n) self.disabled_devices.append(n)
self.devices.sort(cmp=lambda x, y: cmp(x.lower(), y.lower())) self.devices.sort(key=lambda x: x.lower())
self.devices.insert(1, plugboard_save_to_disk_value) self.devices.insert(1, plugboard_save_to_disk_value)
self.devices.insert(1, plugboard_content_server_value) self.devices.insert(1, plugboard_content_server_value)
self.device_to_formats_map[plugboard_content_server_value] = \ self.device_to_formats_map[plugboard_content_server_value] = \

2
src/calibre/gui2/preferences/plugins.py
View File

@ -61,7 +61,7 @@ class PluginModel(QAbstractItemModel, AdaptSQP): # {{{
self.categories = sorted(self._data.keys()) self.categories = sorted(self._data.keys())
for plugins in self._data.values(): for plugins in self._data.values():
plugins.sort(cmp=lambda x, y: cmp(x.name.lower(), y.name.lower())) plugins.sort(key=lambda x: x.name.lower())
def universal_set(self): def universal_set(self):
ans = set([]) ans = set([])

4
src/calibre/gui2/wizard/__init__.py
View File

@ -427,7 +427,7 @@ def get_manufacturers():
def get_devices_of(manufacturer): def get_devices_of(manufacturer):
ans = [d for d in get_devices() if d.manufacturer == manufacturer] ans = [d for d in get_devices() if d.manufacturer == manufacturer]
return sorted(ans, cmp=lambda x,y:cmp(x.name, y.name)) return sorted(ans, key=lambda x: x.name)
class ManufacturerModel(QAbstractListModel): class ManufacturerModel(QAbstractListModel):
@ -682,7 +682,7 @@ class LibraryPage(QWizardPage, LibraryUI):
if l != lang] if l != lang]
if lang != 'en': if lang != 'en':
items.append(('en', get_esc_lang('en'))) items.append(('en', get_esc_lang('en')))
items.sort(cmp=lambda x, y: cmp(x[1], y[1])) items.sort(key=lambda x: x[1])
for item in items: for item in items:
self.language.addItem(item[1], (item[0])) self.language.addItem(item[1], (item[0]))
self.language.blockSignals(False) self.language.blockSignals(False)

5
src/calibre/library/catalogs/bibtex.py
View File

@ -7,7 +7,6 @@ __docformat__ = 'restructuredtext en'
import re, codecs, os, numbers import re, codecs, os, numbers
from collections import namedtuple from collections import namedtuple
from types import StringType, UnicodeType
from calibre import (strftime) from calibre import (strftime)
from calibre.customize import CatalogPlugin from calibre.customize import CatalogPlugin
@ -15,7 +14,7 @@ from calibre.library.catalogs import FIELDS, TEMPLATE_ALLOWED_FIELDS
from calibre.customize.conversion import DummyReporter from calibre.customize.conversion import DummyReporter
from calibre.constants import preferred_encoding from calibre.constants import preferred_encoding
from calibre.ebooks.metadata import format_isbn from calibre.ebooks.metadata import format_isbn
from polyglot.builtins import string_or_bytes from polyglot.builtins import string_or_bytes, unicode_type
class BIBTEX(CatalogPlugin): class BIBTEX(CatalogPlugin):
@ -351,7 +350,7 @@ class BIBTEX(CatalogPlugin):
bibtexc.ascii_bibtex = True bibtexc.ascii_bibtex = True
# Check citation choice and go to default in case of bad CLI # Check citation choice and go to default in case of bad CLI
if isinstance(opts.impcit, (StringType, UnicodeType)) : if isinstance(opts.impcit, (str, unicode_type)) :
if opts.impcit == 'False' : if opts.impcit == 'False' :
citation_bibtex= False citation_bibtex= False
elif opts.impcit == 'True' : elif opts.impcit == 'True' :

4
src/calibre/library/custom_columns.py
View File

@ -216,7 +216,7 @@ class CustomColumns(object):
if data['is_multiple'] and data['datatype'] == 'text': if data['is_multiple'] and data['datatype'] == 'text':
ans = ans.split(data['multiple_seps']['cache_to_list']) if ans else [] ans = ans.split(data['multiple_seps']['cache_to_list']) if ans else []
if data['display'].get('sort_alpha', False): if data['display'].get('sort_alpha', False):
ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower())) ans.sort(key=lambda x:x.lower())
return ans return ans
def get_custom_extra(self, idx, label=None, num=None, index_is_id=False): def get_custom_extra(self, idx, label=None, num=None, index_is_id=False):
@ -243,7 +243,7 @@ class CustomColumns(object):
if data['is_multiple'] and data['datatype'] == 'text': if data['is_multiple'] and data['datatype'] == 'text':
ans = ans.split(data['multiple_seps']['cache_to_list']) if ans else [] ans = ans.split(data['multiple_seps']['cache_to_list']) if ans else []
if data['display'].get('sort_alpha', False): if data['display'].get('sort_alpha', False):
ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower())) ans.sort(key=lambda x: x.lower())
if data['datatype'] != 'series': if data['datatype'] != 'series':
return (ans, None) return (ans, None)
ign,lt = self.custom_table_names(data['num']) ign,lt = self.custom_table_names(data['num'])

2
src/calibre/library/database.py
View File

@ -1018,7 +1018,7 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE;
if not ans: if not ans:
return [] return []
ans = [id[0] for id in ans] ans = [id[0] for id in ans]
ans.sort(cmp=lambda x, y: cmp(self.series_index(x, True), self.series_index(y, True))) ans.sort(key=lambda x: self.series_index(x, True))
return ans return ans
def books_in_series_of(self, index, index_is_id=False): def books_in_series_of(self, index, index_is_id=False):

2
src/calibre/library/sqlite.py
View File

@ -8,7 +8,6 @@ Wrapper for multi-threaded access to a single sqlite database connection. Serial
all calls. all calls.
''' '''
import sqlite3 as sqlite, traceback, time, uuid, sys, os import sqlite3 as sqlite, traceback, time, uuid, sys, os
import repr as reprlib
from sqlite3 import IntegrityError, OperationalError from sqlite3 import IntegrityError, OperationalError
from threading import Thread from threading import Thread
from threading import RLock from threading import RLock
@ -22,6 +21,7 @@ from calibre.constants import iswindows, DEBUG, plugins
from calibre.utils.icu import sort_key from calibre.utils.icu import sort_key
from calibre import prints from calibre import prints
from polyglot.builtins import unicode_type from polyglot.builtins import unicode_type
from polyglot import reprlib
from polyglot.queue import Queue from polyglot.queue import Queue
from dateutil.tz import tzoffset from dateutil.tz import tzoffset

4
src/calibre/spell/dictionary.py
View File

@ -54,7 +54,7 @@ def builtin_dictionaries():
if _builtins is None: if _builtins is None:
dics = [] dics = []
for lc in glob.glob(os.path.join(P('dictionaries', allow_user_override=False), '*/locales')): for lc in glob.glob(os.path.join(P('dictionaries', allow_user_override=False), '*/locales')):
locales = filter(None, open(lc, 'rb').read().decode('utf-8').splitlines()) locales = list(filter(None, open(lc, 'rb').read().decode('utf-8').splitlines()))
locale = locales[0] locale = locales[0]
base = os.path.dirname(lc) base = os.path.dirname(lc)
dics.append(Dictionary( dics.append(Dictionary(
@ -69,7 +69,7 @@ def custom_dictionaries(reread=False):
if _custom is None or reread: if _custom is None or reread:
dics = [] dics = []
for lc in glob.glob(os.path.join(config_dir, 'dictionaries', '*/locales')): for lc in glob.glob(os.path.join(config_dir, 'dictionaries', '*/locales')):
locales = filter(None, open(lc, 'rb').read().decode('utf-8').splitlines()) locales = list(filter(None, open(lc, 'rb').read().decode('utf-8').splitlines()))
try: try:
name, locale, locales = locales[0], locales[1], locales[1:] name, locale, locales = locales[0], locales[1], locales[1:]
except IndexError: except IndexError:

21
src/calibre/srv/auth.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import binascii, os, random, struct, base64, httplib import binascii, os, random, struct, base64
from collections import OrderedDict from collections import OrderedDict
from hashlib import md5, sha256 from hashlib import md5, sha256
from itertools import permutations from itertools import permutations
@ -16,6 +16,7 @@ from calibre.srv.errors import HTTPAuthRequired, HTTPSimpleResponse, HTTPForbidd
from calibre.srv.http_request import parse_uri from calibre.srv.http_request import parse_uri
from calibre.srv.utils import parse_http_dict, encode_path from calibre.srv.utils import parse_http_dict, encode_path
from calibre.utils.monotonic import monotonic from calibre.utils.monotonic import monotonic
from polyglot import http_client
MAX_AGE_SECONDS = 3600 MAX_AGE_SECONDS = 3600
nonce_counter, nonce_counter_lock = 0, Lock() nonce_counter, nonce_counter_lock = 0, Lock()
@ -133,19 +134,19 @@ class DigestAuth(object): # {{{
self.nonce_count = data.get('nc') self.nonce_count = data.get('nc')
if self.algorithm not in self.valid_algorithms: if self.algorithm not in self.valid_algorithms:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported digest algorithm') raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported digest algorithm')
if not (self.username and self.realm and self.nonce and self.uri and self.response): if not (self.username and self.realm and self.nonce and self.uri and self.response):
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Digest algorithm required fields missing') raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Digest algorithm required fields missing')
if self.qop: if self.qop:
if self.qop not in self.valid_qops: if self.qop not in self.valid_qops:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported digest qop') raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported digest qop')
if not (self.cnonce and self.nonce_count): if not (self.cnonce and self.nonce_count):
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'qop present, but cnonce and nonce_count absent') raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'qop present, but cnonce and nonce_count absent')
else: else:
if self.cnonce or self.nonce_count: if self.cnonce or self.nonce_count:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'qop missing') raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'qop missing')
def H(self, val): def H(self, val):
return md5_hex(val) return md5_hex(val)
@ -201,7 +202,7 @@ class DigestAuth(object): # {{{
if log is not None: if log is not None:
log.warn('Authorization URI mismatch: %s != %s from client: %s' % ( log.warn('Authorization URI mismatch: %s != %s from client: %s' % (
data.path, path, data.remote_addr)) data.path, path, data.remote_addr))
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The uri in the Request Line and the Authorization header do not match') raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The uri in the Request Line and the Authorization header do not match')
return self.response is not None and data.path == path and self.request_digest(pw, data) == self.response return self.response is not None and data.path == path and self.request_digest(pw, data) == self.response
# }}} # }}}
@ -290,16 +291,16 @@ class AuthController(object):
try: try:
un, pw = base64_decode(rest.strip()).partition(':')[::2] un, pw = base64_decode(rest.strip()).partition(':')[::2]
except ValueError: except ValueError:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The username or password contained non-UTF8 encoded characters') raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The username or password contained non-UTF8 encoded characters')
if not un or not pw: if not un or not pw:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The username or password was empty') raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The username or password was empty')
if self.check(un, pw): if self.check(un, pw):
data.username = un data.username = un
return return
log_msg = 'Failed login attempt from: %s' % data.remote_addr log_msg = 'Failed login attempt from: %s' % data.remote_addr
self.ban_list.failed(ban_key) self.ban_list.failed(ban_key)
else: else:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported authentication method') raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported authentication method')
if self.prefer_basic_auth: if self.prefer_basic_auth:
raise HTTPAuthRequired('Basic realm="%s"' % self.realm, log=log_msg) raise HTTPAuthRequired('Basic realm="%s"' % self.realm, log=log_msg)

14
src/calibre/srv/errors.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib from polyglot import http_client
class JobQueueFull(Exception): class JobQueueFull(Exception):
@ -30,38 +30,38 @@ class HTTPSimpleResponse(Exception):
class HTTPRedirect(HTTPSimpleResponse): class HTTPRedirect(HTTPSimpleResponse):
def __init__(self, location, http_code=httplib.MOVED_PERMANENTLY, http_message='', close_connection=False): def __init__(self, location, http_code=http_client.MOVED_PERMANENTLY, http_message='', close_connection=False):
HTTPSimpleResponse.__init__(self, http_code, http_message, close_connection, location) HTTPSimpleResponse.__init__(self, http_code, http_message, close_connection, location)
class HTTPNotFound(HTTPSimpleResponse): class HTTPNotFound(HTTPSimpleResponse):
def __init__(self, http_message='', close_connection=False): def __init__(self, http_message='', close_connection=False):
HTTPSimpleResponse.__init__(self, httplib.NOT_FOUND, http_message, close_connection) HTTPSimpleResponse.__init__(self, http_client.NOT_FOUND, http_message, close_connection)
class HTTPAuthRequired(HTTPSimpleResponse): class HTTPAuthRequired(HTTPSimpleResponse):
def __init__(self, payload, log=None): def __init__(self, payload, log=None):
HTTPSimpleResponse.__init__(self, httplib.UNAUTHORIZED, authenticate=payload, log=log) HTTPSimpleResponse.__init__(self, http_client.UNAUTHORIZED, authenticate=payload, log=log)
class HTTPBadRequest(HTTPSimpleResponse): class HTTPBadRequest(HTTPSimpleResponse):
def __init__(self, message, close_connection=False): def __init__(self, message, close_connection=False):
HTTPSimpleResponse.__init__(self, httplib.BAD_REQUEST, message, close_connection) HTTPSimpleResponse.__init__(self, http_client.BAD_REQUEST, message, close_connection)
class HTTPForbidden(HTTPSimpleResponse): class HTTPForbidden(HTTPSimpleResponse):
def __init__(self, http_message='', close_connection=True, log=None): def __init__(self, http_message='', close_connection=True, log=None):
HTTPSimpleResponse.__init__(self, httplib.FORBIDDEN, http_message, close_connection, log=log) HTTPSimpleResponse.__init__(self, http_client.FORBIDDEN, http_message, close_connection, log=log)
class HTTPInternalServerError(HTTPSimpleResponse): class HTTPInternalServerError(HTTPSimpleResponse):
def __init__(self, http_message='', close_connection=True, log=None): def __init__(self, http_message='', close_connection=True, log=None):
HTTPSimpleResponse.__init__(self, httplib.INTERNAL_SERVER_ERROR, http_message, close_connection, log=log) HTTPSimpleResponse.__init__(self, http_client.INTERNAL_SERVER_ERROR, http_message, close_connection, log=log)
class BookNotFound(HTTPNotFound): class BookNotFound(HTTPNotFound):

43
src/calibre/srv/http_request.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import re, httplib, repr as reprlib import re
from io import BytesIO, DEFAULT_BUFFER_SIZE from io import BytesIO, DEFAULT_BUFFER_SIZE
from calibre import as_unicode, force_unicode from calibre import as_unicode, force_unicode
@ -14,6 +14,7 @@ from calibre.ptempfile import SpooledTemporaryFile
from calibre.srv.errors import HTTPSimpleResponse from calibre.srv.errors import HTTPSimpleResponse
from calibre.srv.loop import Connection, READ, WRITE from calibre.srv.loop import Connection, READ, WRITE
from calibre.srv.utils import MultiDict, HTTP1, HTTP11, Accumulator from calibre.srv.utils import MultiDict, HTTP1, HTTP11, Accumulator
from polyglot import http_client, reprlib
from polyglot.urllib import unquote from polyglot.urllib import unquote
protocol_map = {(1, 0):HTTP1, (1, 1):HTTP11} protocol_map = {(1, 0):HTTP1, (1, 1):HTTP11}
@ -68,29 +69,29 @@ def parse_request_uri(uri):
def parse_uri(uri, parse_query=True): def parse_uri(uri, parse_query=True):
scheme, authority, path = parse_request_uri(uri) scheme, authority, path = parse_request_uri(uri)
if path is None: if path is None:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, "No path component") raise HTTPSimpleResponse(http_client.BAD_REQUEST, "No path component")
if b'#' in path: if b'#' in path:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, "Illegal #fragment in Request-URI.") raise HTTPSimpleResponse(http_client.BAD_REQUEST, "Illegal #fragment in Request-URI.")
if scheme: if scheme:
try: try:
scheme = scheme.decode('ascii') scheme = scheme.decode('ascii')
except ValueError: except ValueError:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Un-decodeable scheme') raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Un-decodeable scheme')
path, qs = path.partition(b'?')[::2] path, qs = path.partition(b'?')[::2]
if parse_query: if parse_query:
try: try:
query = MultiDict.create_from_query_string(qs) query = MultiDict.create_from_query_string(qs)
except Exception: except Exception:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unparseable query string') raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unparseable query string')
else: else:
query = None query = None
try: try:
path = '%2F'.join(unquote(x).decode('utf-8') for x in quoted_slash.split(path)) path = '%2F'.join(unquote(x).decode('utf-8') for x in quoted_slash.split(path))
except ValueError as e: except ValueError as e:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, as_unicode(e)) raise HTTPSimpleResponse(http_client.BAD_REQUEST, as_unicode(e))
path = tuple(filter(None, (x.replace('%2F', '/') for x in path.split('/')))) path = tuple(filter(None, (x.replace('%2F', '/') for x in path.split('/'))))
return scheme, path, query return scheme, path, query
@ -233,7 +234,7 @@ class HTTPRequest(Connection):
if line.endswith(b'\n'): if line.endswith(b'\n'):
line = buf.getvalue() line = buf.getvalue()
if not line.endswith(b'\r\n'): if not line.endswith(b'\r\n'):
self.simple_response(httplib.BAD_REQUEST, 'HTTP requires CRLF line terminators') self.simple_response(http_client.BAD_REQUEST, 'HTTP requires CRLF line terminators')
return return
return line return line
if not line: if not line:
@ -247,7 +248,7 @@ class HTTPRequest(Connection):
self.forwarded_for = None self.forwarded_for = None
self.path = self.query = None self.path = self.query = None
self.close_after_response = False self.close_after_response = False
self.header_line_too_long_error_code = httplib.REQUEST_URI_TOO_LONG self.header_line_too_long_error_code = http_client.REQUEST_URI_TOO_LONG
self.response_started = False self.response_started = False
self.set_state(READ, self.parse_request_line, Accumulator(), first=True) self.set_state(READ, self.parse_request_line, Accumulator(), first=True)
@ -260,28 +261,28 @@ class HTTPRequest(Connection):
# Ignore a single leading empty line, as per RFC 2616 sec 4.1 # Ignore a single leading empty line, as per RFC 2616 sec 4.1
if first: if first:
return self.set_state(READ, self.parse_request_line, Accumulator()) return self.set_state(READ, self.parse_request_line, Accumulator())
return self.simple_response(httplib.BAD_REQUEST, 'Multiple leading empty lines not allowed') return self.simple_response(http_client.BAD_REQUEST, 'Multiple leading empty lines not allowed')
try: try:
method, uri, req_protocol = line.strip().split(b' ', 2) method, uri, req_protocol = line.strip().split(b' ', 2)
rp = int(req_protocol[5]), int(req_protocol[7]) rp = int(req_protocol[5]), int(req_protocol[7])
self.method = method.decode('ascii').upper() self.method = method.decode('ascii').upper()
except Exception: except Exception:
return self.simple_response(httplib.BAD_REQUEST, "Malformed Request-Line") return self.simple_response(http_client.BAD_REQUEST, "Malformed Request-Line")
if self.method not in HTTP_METHODS: if self.method not in HTTP_METHODS:
return self.simple_response(httplib.BAD_REQUEST, "Unknown HTTP method") return self.simple_response(http_client.BAD_REQUEST, "Unknown HTTP method")
try: try:
self.request_protocol = protocol_map[rp] self.request_protocol = protocol_map[rp]
except KeyError: except KeyError:
return self.simple_response(httplib.HTTP_VERSION_NOT_SUPPORTED) return self.simple_response(http_client.HTTP_VERSION_NOT_SUPPORTED)
self.response_protocol = protocol_map[min((1, 1), rp)] self.response_protocol = protocol_map[min((1, 1), rp)]
try: try:
self.scheme, self.path, self.query = parse_uri(uri) self.scheme, self.path, self.query = parse_uri(uri)
except HTTPSimpleResponse as e: except HTTPSimpleResponse as e:
return self.simple_response(e.http_code, e.message, close_after_response=False) return self.simple_response(e.http_code, e.message, close_after_response=False)
self.header_line_too_long_error_code = httplib.REQUEST_ENTITY_TOO_LARGE self.header_line_too_long_error_code = http_client.REQUEST_ENTITY_TOO_LARGE
self.set_state(READ, self.parse_header_line, HTTPHeaderParser(), Accumulator()) self.set_state(READ, self.parse_header_line, HTTPHeaderParser(), Accumulator())
# }}} # }}}
@ -299,7 +300,7 @@ class HTTPRequest(Connection):
try: try:
parser(line) parser(line)
except ValueError: except ValueError:
self.simple_response(httplib.BAD_REQUEST, 'Failed to parse header line') self.simple_response(http_client.BAD_REQUEST, 'Failed to parse header line')
return return
if parser.finished: if parser.finished:
self.finalize_headers(parser.hdict) self.finalize_headers(parser.hdict)
@ -307,7 +308,7 @@ class HTTPRequest(Connection):
def finalize_headers(self, inheaders): def finalize_headers(self, inheaders):
request_content_length = int(inheaders.get('Content-Length', 0)) request_content_length = int(inheaders.get('Content-Length', 0))
if request_content_length > self.max_request_body_size: if request_content_length > self.max_request_body_size:
return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE, return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE,
"The entity sent with the request exceeds the maximum " "The entity sent with the request exceeds the maximum "
"allowed bytes (%d)." % self.max_request_body_size) "allowed bytes (%d)." % self.max_request_body_size)
# Persistent connection support # Persistent connection support
@ -334,7 +335,7 @@ class HTTPRequest(Connection):
else: else:
# Note that, even if we see "chunked", we must reject # Note that, even if we see "chunked", we must reject
# if there is an extension we don't recognize. # if there is an extension we don't recognize.
return self.simple_response(httplib.NOT_IMPLEMENTED, "Unknown transfer encoding: %r" % enc) return self.simple_response(http_client.NOT_IMPLEMENTED, "Unknown transfer encoding: %r" % enc)
if inheaders.get("Expect", '').lower() == "100-continue": if inheaders.get("Expect", '').lower() == "100-continue":
buf = BytesIO((HTTP11 + " 100 Continue\r\n\r\n").encode('ascii')) buf = BytesIO((HTTP11 + " 100 Continue\r\n\r\n").encode('ascii'))
@ -369,9 +370,9 @@ class HTTPRequest(Connection):
try: try:
chunk_size = int(line.strip(), 16) chunk_size = int(line.strip(), 16)
except Exception: except Exception:
return self.simple_response(httplib.BAD_REQUEST, '%s is not a valid chunk size' % reprlib.repr(line.strip())) return self.simple_response(http_client.BAD_REQUEST, '%s is not a valid chunk size' % reprlib.repr(line.strip()))
if bytes_read[0] + chunk_size + 2 > self.max_request_body_size: if bytes_read[0] + chunk_size + 2 > self.max_request_body_size:
return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE, return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE,
'Chunked request is larger than %d bytes' % self.max_request_body_size) 'Chunked request is larger than %d bytes' % self.max_request_body_size)
if chunk_size == 0: if chunk_size == 0:
self.set_state(READ, self.read_chunk_separator, inheaders, Accumulator(), buf, bytes_read, last=True) self.set_state(READ, self.read_chunk_separator, inheaders, Accumulator(), buf, bytes_read, last=True)
@ -389,10 +390,10 @@ class HTTPRequest(Connection):
if line is None: if line is None:
return return
if line != b'\r\n': if line != b'\r\n':
return self.simple_response(httplib.BAD_REQUEST, 'Chunk does not have trailing CRLF') return self.simple_response(http_client.BAD_REQUEST, 'Chunk does not have trailing CRLF')
bytes_read[0] += len(line) bytes_read[0] += len(line)
if bytes_read[0] > self.max_request_body_size: if bytes_read[0] > self.max_request_body_size:
return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE, return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE,
'Chunked request is larger than %d bytes' % self.max_request_body_size) 'Chunked request is larger than %d bytes' % self.max_request_body_size)
if last: if last:
self.prepare_response(inheaders, buf) self.prepare_response(inheaders, buf)
@ -402,7 +403,7 @@ class HTTPRequest(Connection):
def handle_timeout(self): def handle_timeout(self):
if self.response_started: if self.response_started:
return False return False
self.simple_response(httplib.REQUEST_TIMEOUT) self.simple_response(http_client.REQUEST_TIMEOUT)
return True return True
def write(self, buf, end=None): def write(self, buf, end=None):

43
src/calibre/srv/http_response.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import os, httplib, hashlib, uuid, struct, repr as reprlib import os, hashlib, uuid, struct
from collections import namedtuple from collections import namedtuple
from io import BytesIO, DEFAULT_BUFFER_SIZE from io import BytesIO, DEFAULT_BUFFER_SIZE
from itertools import chain, repeat from itertools import chain, repeat
@ -26,9 +26,10 @@ from calibre.srv.utils import (
sort_q_values, get_translator_for_lang, Cookie, fast_now_strftime) sort_q_values, get_translator_for_lang, Cookie, fast_now_strftime)
from calibre.utils.speedups import ReadOnlyFileBuffer from calibre.utils.speedups import ReadOnlyFileBuffer
from calibre.utils.monotonic import monotonic from calibre.utils.monotonic import monotonic
from polyglot import http_client, reprlib
Range = namedtuple('Range', 'start stop size') Range = namedtuple('Range', 'start stop size')
MULTIPART_SEPARATOR = uuid.uuid4().hex.decode('ascii') MULTIPART_SEPARATOR = uuid.uuid4().hex
COMPRESSIBLE_TYPES = {'application/json', 'application/javascript', 'application/xml', 'application/oebps-package+xml'} COMPRESSIBLE_TYPES = {'application/json', 'application/javascript', 'application/xml', 'application/oebps-package+xml'}
if is_py3: if is_py3:
import zlib import zlib
@ -226,7 +227,7 @@ class RequestData(object): # {{{
self.remote_addr, self.remote_port, self.is_local_connection = remote_addr, remote_port, is_local_connection self.remote_addr, self.remote_port, self.is_local_connection = remote_addr, remote_port, is_local_connection
self.forwarded_for = forwarded_for self.forwarded_for = forwarded_for
self.opts = opts self.opts = opts
self.status_code = httplib.OK self.status_code = http_client.OK
self.outcookie = Cookie() self.outcookie = Cookie()
self.lang_code = self.gettext_func = self.ngettext_func = None self.lang_code = self.gettext_func = self.ngettext_func = None
self.set_translator(self.get_preferred_language()) self.set_translator(self.get_preferred_language())
@ -402,16 +403,16 @@ class HTTPConnection(HTTPRequest):
if self.response_protocol is HTTP1: if self.response_protocol is HTTP1:
# HTTP/1.0 has no 413/414/303 codes # HTTP/1.0 has no 413/414/303 codes
status_code = { status_code = {
httplib.REQUEST_ENTITY_TOO_LARGE:httplib.BAD_REQUEST, http_client.REQUEST_ENTITY_TOO_LARGE:http_client.BAD_REQUEST,
httplib.REQUEST_URI_TOO_LONG:httplib.BAD_REQUEST, http_client.REQUEST_URI_TOO_LONG:http_client.BAD_REQUEST,
httplib.SEE_OTHER:httplib.FOUND http_client.SEE_OTHER:http_client.FOUND
}.get(status_code, status_code) }.get(status_code, status_code)
self.close_after_response = close_after_response self.close_after_response = close_after_response
msg = msg.encode('utf-8') msg = msg.encode('utf-8')
ct = 'http' if self.method == 'TRACE' else 'plain' ct = 'http' if self.method == 'TRACE' else 'plain'
buf = [ buf = [
'%s %d %s' % (self.response_protocol, status_code, httplib.responses[status_code]), '%s %d %s' % (self.response_protocol, status_code, http_client.responses[status_code]),
"Content-Length: %s" % len(msg), "Content-Length: %s" % len(msg),
"Content-Type: text/%s; charset=UTF-8" % ct, "Content-Type: text/%s; charset=UTF-8" % ct,
"Date: " + http_date(), "Date: " + http_date(),
@ -432,7 +433,7 @@ class HTTPConnection(HTTPRequest):
def prepare_response(self, inheaders, request_body_file): def prepare_response(self, inheaders, request_body_file):
if self.method == 'TRACE': if self.method == 'TRACE':
msg = force_unicode(self.request_line, 'utf-8') + '\n' + inheaders.pretty() msg = force_unicode(self.request_line, 'utf-8') + '\n' + inheaders.pretty()
return self.simple_response(httplib.OK, msg, close_after_response=False) return self.simple_response(http_client.OK, msg, close_after_response=False)
request_body_file.seek(0) request_body_file.seek(0)
outheaders = MultiDict() outheaders = MultiDict()
data = RequestData( data = RequestData(
@ -449,28 +450,28 @@ class HTTPConnection(HTTPRequest):
def send_range_not_satisfiable(self, content_length): def send_range_not_satisfiable(self, content_length):
buf = [ buf = [
'%s %d %s' % (self.response_protocol, httplib.REQUESTED_RANGE_NOT_SATISFIABLE, httplib.responses[httplib.REQUESTED_RANGE_NOT_SATISFIABLE]), '%s %d %s' % (self.response_protocol, http_client.REQUESTED_RANGE_NOT_SATISFIABLE, http_client.responses[http_client.REQUESTED_RANGE_NOT_SATISFIABLE]),
"Date: " + http_date(), "Date: " + http_date(),
"Content-Range: bytes */%d" % content_length, "Content-Range: bytes */%d" % content_length,
] ]
response_data = header_list_to_file(buf) response_data = header_list_to_file(buf)
self.log_access(status_code=httplib.REQUESTED_RANGE_NOT_SATISFIABLE, response_size=response_data.sz) self.log_access(status_code=http_client.REQUESTED_RANGE_NOT_SATISFIABLE, response_size=response_data.sz)
self.response_ready(response_data) self.response_ready(response_data)
def send_not_modified(self, etag=None): def send_not_modified(self, etag=None):
buf = [ buf = [
'%s %d %s' % (self.response_protocol, httplib.NOT_MODIFIED, httplib.responses[httplib.NOT_MODIFIED]), '%s %d %s' % (self.response_protocol, http_client.NOT_MODIFIED, http_client.responses[http_client.NOT_MODIFIED]),
"Content-Length: 0", "Content-Length: 0",
"Date: " + http_date(), "Date: " + http_date(),
] ]
if etag is not None: if etag is not None:
buf.append('ETag: ' + etag) buf.append('ETag: ' + etag)
response_data = header_list_to_file(buf) response_data = header_list_to_file(buf)
self.log_access(status_code=httplib.NOT_MODIFIED, response_size=response_data.sz) self.log_access(status_code=http_client.NOT_MODIFIED, response_size=response_data.sz)
self.response_ready(response_data) self.response_ready(response_data)
def report_busy(self): def report_busy(self):
self.simple_response(httplib.SERVICE_UNAVAILABLE) self.simple_response(http_client.SERVICE_UNAVAILABLE)
def job_done(self, ok, result): def job_done(self, ok, result):
if not ok: if not ok:
@ -509,7 +510,7 @@ class HTTPConnection(HTTPRequest):
if ct.startswith('text/') and 'charset=' not in ct: if ct.startswith('text/') and 'charset=' not in ct:
outheaders.set('Content-Type', ct + '; charset=UTF-8', replace_all=True) outheaders.set('Content-Type', ct + '; charset=UTF-8', replace_all=True)
buf = [HTTP11 + (' %d ' % data.status_code) + httplib.responses[data.status_code]] buf = [HTTP11 + (' %d ' % data.status_code) + http_client.responses[data.status_code]]
for header, value in sorted(iteritems(outheaders), key=itemgetter(0)): for header, value in sorted(iteritems(outheaders), key=itemgetter(0)):
buf.append('%s: %s' % (header, value)) buf.append('%s: %s' % (header, value))
for morsel in itervalues(data.outcookie): for morsel in itervalues(data.outcookie):
@ -530,7 +531,7 @@ class HTTPConnection(HTTPRequest):
def log_access(self, status_code, response_size=None, username=None): def log_access(self, status_code, response_size=None, username=None):
if self.access_log is None: if self.access_log is None:
return return
if not self.opts.log_not_found and status_code == httplib.NOT_FOUND: if not self.opts.log_not_found and status_code == http_client.NOT_FOUND:
return return
ff = self.forwarded_for ff = self.forwarded_for
if ff: if ff:
@ -623,7 +624,7 @@ class HTTPConnection(HTTPRequest):
self.ready = ready self.ready = ready
def report_unhandled_exception(self, e, formatted_traceback): def report_unhandled_exception(self, e, formatted_traceback):
self.simple_response(httplib.INTERNAL_SERVER_ERROR) self.simple_response(http_client.INTERNAL_SERVER_ERROR)
def finalize_output(self, output, request, is_http1): def finalize_output(self, output, request, is_http1):
none_match = parse_if_none_match(request.inheaders.get('If-None-Match', '')) none_match = parse_if_none_match(request.inheaders.get('If-None-Match', ''))
@ -633,7 +634,7 @@ class HTTPConnection(HTTPRequest):
if self.method in ('GET', 'HEAD'): if self.method in ('GET', 'HEAD'):
self.send_not_modified(output.etag) self.send_not_modified(output.etag)
else: else:
self.simple_response(httplib.PRECONDITION_FAILED) self.simple_response(http_client.PRECONDITION_FAILED)
return return
opts = self.opts opts = self.opts
@ -660,10 +661,10 @@ class HTTPConnection(HTTPRequest):
ct = outheaders.get('Content-Type', '').partition(';')[0] ct = outheaders.get('Content-Type', '').partition(';')[0]
compressible = (not ct or ct.startswith('text/') or ct.startswith('image/svg') or compressible = (not ct or ct.startswith('text/') or ct.startswith('image/svg') or
ct.partition(';')[0] in COMPRESSIBLE_TYPES) ct.partition(';')[0] in COMPRESSIBLE_TYPES)
compressible = (compressible and request.status_code == httplib.OK and compressible = (compressible and request.status_code == http_client.OK and
(opts.compress_min_size > -1 and output.content_length >= opts.compress_min_size) and (opts.compress_min_size > -1 and output.content_length >= opts.compress_min_size) and
acceptable_encoding(request.inheaders.get('Accept-Encoding', '')) and not is_http1) acceptable_encoding(request.inheaders.get('Accept-Encoding', '')) and not is_http1)
accept_ranges = (not compressible and output.accept_ranges is not None and request.status_code == httplib.OK and accept_ranges = (not compressible and output.accept_ranges is not None and request.status_code == http_client.OK and
not is_http1) not is_http1)
ranges = get_ranges(request.inheaders.get('Range'), output.content_length) if output.accept_ranges and self.method in ('GET', 'HEAD') else None ranges = get_ranges(request.inheaders.get('Range'), output.content_length) if output.accept_ranges and self.method in ('GET', 'HEAD') else None
if_range = (request.inheaders.get('If-Range') or '').strip() if_range = (request.inheaders.get('If-Range') or '').strip()
@ -680,7 +681,7 @@ class HTTPConnection(HTTPRequest):
if self.method in ('GET', 'HEAD'): if self.method in ('GET', 'HEAD'):
self.send_not_modified(output.etag) self.send_not_modified(output.etag)
else: else:
self.simple_response(httplib.PRECONDITION_FAILED) self.simple_response(http_client.PRECONDITION_FAILED)
return return
output.ranges = None output.ranges = None
@ -712,7 +713,7 @@ class HTTPConnection(HTTPRequest):
outheaders.set('Content-Length', '%d' % size, replace_all=True) outheaders.set('Content-Length', '%d' % size, replace_all=True)
outheaders.set('Content-Type', 'multipart/byteranges; boundary=' + MULTIPART_SEPARATOR, replace_all=True) outheaders.set('Content-Type', 'multipart/byteranges; boundary=' + MULTIPART_SEPARATOR, replace_all=True)
output.ranges = zip_longest(ranges, range_parts) output.ranges = zip_longest(ranges, range_parts)
request.status_code = httplib.PARTIAL_CONTENT request.status_code = http_client.PARTIAL_CONTENT
return output return output

5
src/calibre/srv/routes.py
View File

@ -6,13 +6,14 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, sys, inspect, re, time, numbers, json as jsonlib, textwrap import sys, inspect, re, time, numbers, json as jsonlib, textwrap
from operator import attrgetter from operator import attrgetter
from calibre.srv.errors import HTTPSimpleResponse, HTTPNotFound, RouteError from calibre.srv.errors import HTTPSimpleResponse, HTTPNotFound, RouteError
from calibre.srv.utils import http_date from calibre.srv.utils import http_date
from calibre.utils.serialize import msgpack_dumps, json_dumps, MSGPACK_MIME from calibre.utils.serialize import msgpack_dumps, json_dumps, MSGPACK_MIME
from polyglot.builtins import iteritems, itervalues, unicode_type, range, zip from polyglot.builtins import iteritems, itervalues, unicode_type, range, zip
from polyglot import http_client
from polyglot.urllib import quote as urlquote from polyglot.urllib import quote as urlquote
default_methods = frozenset(('HEAD', 'GET')) default_methods = frozenset(('HEAD', 'GET'))
@ -297,7 +298,7 @@ class Router(object):
def dispatch(self, data): def dispatch(self, data):
endpoint_, args = self.find_route(data.path) endpoint_, args = self.find_route(data.path)
if data.method not in endpoint_.methods: if data.method not in endpoint_.methods:
raise HTTPSimpleResponse(httplib.METHOD_NOT_ALLOWED) raise HTTPSimpleResponse(http_client.METHOD_NOT_ALLOWED)
self.read_cookies(data) self.read_cookies(data)

20
src/calibre/srv/tests/ajax.py
View File

@ -6,13 +6,13 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, zlib, json, base64, os import zlib, json, base64, os
from io import BytesIO from io import BytesIO
from functools import partial from functools import partial
from httplib import OK, NOT_FOUND, FORBIDDEN
from calibre.ebooks.metadata.meta import get_metadata from calibre.ebooks.metadata.meta import get_metadata
from calibre.srv.tests.base import LibraryBaseTest from calibre.srv.tests.base import LibraryBaseTest
from polyglot.http_client import OK, NOT_FOUND, FORBIDDEN
from polyglot.urllib import urlencode, quote from polyglot.urllib import urlencode, quote
@ -22,7 +22,7 @@ def make_request(conn, url, headers={}, prefix='/ajax', username=None, password=
conn.request(method, prefix + url, headers=headers, body=data) conn.request(method, prefix + url, headers=headers, body=data)
r = conn.getresponse() r = conn.getresponse()
data = r.read() data = r.read()
if r.status == httplib.OK and data and data[0] in b'{[': if r.status == OK and data and data[0] in b'{[':
data = json.loads(data) data = json.loads(data)
return r, data return r, data
@ -37,10 +37,10 @@ class ContentTest(LibraryBaseTest):
request = partial(make_request, conn, prefix='/ajax/book') request = partial(make_request, conn, prefix='/ajax/book')
r, data = request('/x') r, data = request('/x')
self.ae(r.status, httplib.NOT_FOUND) self.ae(r.status, NOT_FOUND)
r, onedata = request('/1') r, onedata = request('/1')
self.ae(r.status, httplib.OK) self.ae(r.status, OK)
self.ae(request('/1/' + db.server_library_id)[1], onedata) self.ae(request('/1/' + db.server_library_id)[1], onedata)
self.ae(request('/%s?id_is_uuid=true' % db.field_for('uuid', 1))[1], onedata) self.ae(request('/%s?id_is_uuid=true' % db.field_for('uuid', 1))[1], onedata)
@ -63,22 +63,22 @@ class ContentTest(LibraryBaseTest):
request = partial(make_request, conn) request = partial(make_request, conn)
r, data = request('/categories') r, data = request('/categories')
self.ae(r.status, httplib.OK) self.ae(r.status, OK)
r, xdata = request('/categories/' + db.server_library_id) r, xdata = request('/categories/' + db.server_library_id)
self.ae(r.status, httplib.OK) self.ae(r.status, OK)
self.ae(data, xdata) self.ae(data, xdata)
names = {x['name']:x['url'] for x in data} names = {x['name']:x['url'] for x in data}
for q in ('Newest', 'All books', 'Tags', 'Series', 'Authors', 'Enum', 'Composite Tags'): for q in ('Newest', 'All books', 'Tags', 'Series', 'Authors', 'Enum', 'Composite Tags'):
self.assertIn(q, names) self.assertIn(q, names)
r, data = request(names['Tags'], prefix='') r, data = request(names['Tags'], prefix='')
self.ae(r.status, httplib.OK) self.ae(r.status, OK)
names = {x['name']:x['url'] for x in data['items']} names = {x['name']:x['url'] for x in data['items']}
self.ae(set(names), set('Tag One,Tag Two,News'.split(','))) self.ae(set(names), set('Tag One,Tag Two,News'.split(',')))
r, data = request(names['Tag One'], prefix='') r, data = request(names['Tag One'], prefix='')
self.ae(r.status, httplib.OK) self.ae(r.status, OK)
self.ae(set(data['book_ids']), {1, 2}) self.ae(set(data['book_ids']), {1, 2})
r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"'})) r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"'}))
self.ae(r.status, httplib.OK) self.ae(r.status, OK)
self.ae(set(data['book_ids']), {1, 2}) self.ae(set(data['book_ids']), {1, 2})
r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"', 'vl':'1'})) r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"', 'vl':'1'}))
self.ae(set(data['book_ids']), {2}) self.ae(set(data['book_ids']), {2})

56
src/calibre/srv/tests/auth.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, base64, subprocess, os, cookielib, time import base64, subprocess, os, time
from collections import namedtuple from collections import namedtuple
try: try:
from distutils.spawn import find_executable from distutils.spawn import find_executable
@ -18,6 +18,8 @@ from calibre.srv.errors import HTTPForbidden
from calibre.srv.tests.base import BaseTest, TestServer from calibre.srv.tests.base import BaseTest, TestServer
from calibre.srv.routes import endpoint, Router from calibre.srv.routes import endpoint, Router
from polyglot.builtins import iteritems, itervalues from polyglot.builtins import iteritems, itervalues
from polyglot import http_client
from polyglot.http_cookie import CookieJar
from polyglot.urllib import (build_opener, HTTPBasicAuthHandler, from polyglot.urllib import (build_opener, HTTPBasicAuthHandler,
HTTPCookieProcessor, HTTPDigestAuthHandler, HTTPError) HTTPCookieProcessor, HTTPDigestAuthHandler, HTTPError)
@ -91,18 +93,18 @@ class TestAuth(BaseTest):
conn = server.connect() conn = server.connect()
conn.request('GET', '/open') conn.request('GET', '/open')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(r.read(), b'open') self.ae(r.read(), b'open')
conn.request('GET', '/closed') conn.request('GET', '/closed')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.UNAUTHORIZED) self.ae(r.status, http_client.UNAUTHORIZED)
self.ae(r.getheader('WWW-Authenticate'), b'Basic realm="%s"' % bytes(REALM)) self.ae(r.getheader('WWW-Authenticate'), b'Basic realm="%s"' % bytes(REALM))
self.assertFalse(r.read()) self.assertFalse(r.read())
conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + base64.standard_b64encode(b'testuser:testpw')}) conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + base64.standard_b64encode(b'testuser:testpw')})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.read(), b'closed') self.ae(r.read(), b'closed')
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(b'closed', urlopen(server, method='basic').read()) self.ae(b'closed', urlopen(server, method='basic').read())
self.ae(b'closed', urlopen(server, un='!@#$%^&*()-=_+', pw='!@#$%^&*()-=_+', method='basic').read()) self.ae(b'closed', urlopen(server, un='!@#$%^&*()-=_+', pw='!@#$%^&*()-=_+', method='basic').read())
@ -113,14 +115,14 @@ class TestAuth(BaseTest):
warnings = [] warnings = []
server.loop.log.warn = lambda *args, **kwargs: warnings.append(' '.join(args)) server.loop.log.warn = lambda *args, **kwargs: warnings.append(' '.join(args))
self.ae((httplib.OK, b'closed'), request()) self.ae((http_client.OK, b'closed'), request())
self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y')) self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y'))
self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('', '')) self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request('', ''))
self.ae(1, len(warnings)) self.ae(1, len(warnings))
self.ae((httplib.UNAUTHORIZED, b''), request('testuser', 'y')) self.ae((http_client.UNAUTHORIZED, b''), request('testuser', 'y'))
self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('testuser', '')) self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request('testuser', ''))
self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('')) self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request(''))
self.ae((httplib.UNAUTHORIZED, b''), request('asf', 'testpw')) self.ae((http_client.UNAUTHORIZED, b''), request('asf', 'testpw'))
# }}} # }}}
def test_library_restrictions(self): # {{{ def test_library_restrictions(self): # {{{
@ -169,7 +171,7 @@ class TestAuth(BaseTest):
with TestServer(r.dispatch) as server: with TestServer(r.dispatch) as server:
r.auth_controller.log = server.log r.auth_controller.log = server.log
def test(conn, path, headers={}, status=httplib.OK, body=b'', request_body=b''): def test(conn, path, headers={}, status=http_client.OK, body=b'', request_body=b''):
conn.request('GET', path, request_body, headers) conn.request('GET', path, request_body, headers)
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, status) self.ae(r.status, status)
@ -177,9 +179,9 @@ class TestAuth(BaseTest):
return {normalize_header_name(k):v for k, v in r.getheaders()} return {normalize_header_name(k):v for k, v in r.getheaders()}
conn = server.connect() conn = server.connect()
test(conn, '/open', body=b'open') test(conn, '/open', body=b'open')
auth = parse_http_dict(test(conn, '/closed', status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) auth = parse_http_dict(test(conn, '/closed', status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
nonce = auth['nonce'] nonce = auth['nonce']
auth = parse_http_dict(test(conn, '/closed', status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) auth = parse_http_dict(test(conn, '/closed', status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
self.assertNotEqual(nonce, auth['nonce'], 'nonce was re-used') self.assertNotEqual(nonce, auth['nonce'], 'nonce was re-used')
self.ae(auth[b'realm'], bytes(REALM)), self.ae(auth[b'algorithm'], b'MD5'), self.ae(auth[b'qop'], b'auth') self.ae(auth[b'realm'], bytes(REALM)), self.ae(auth[b'algorithm'], b'MD5'), self.ae(auth[b'qop'], b'auth')
self.assertNotIn('stale', auth) self.assertNotIn('stale', auth)
@ -199,14 +201,14 @@ class TestAuth(BaseTest):
# Check stale nonces # Check stale nonces
orig, r.auth_controller.max_age_seconds = r.auth_controller.max_age_seconds, -1 orig, r.auth_controller.max_age_seconds = r.auth_controller.max_age_seconds, -1
auth = parse_http_dict(test(conn, '/closed', headers={ auth = parse_http_dict(test(conn, '/closed', headers={
'Authorization':digest(**args)},status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) 'Authorization':digest(**args)},status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
self.assertIn('stale', auth) self.assertIn('stale', auth)
r.auth_controller.max_age_seconds = orig r.auth_controller.max_age_seconds = orig
ok_test(conn, digest(**args)) ok_test(conn, digest(**args))
def fail_test(conn, modify, **kw): def fail_test(conn, modify, **kw):
kw['body'] = kw.get('body', b'') kw['body'] = kw.get('body', b'')
kw['status'] = kw.get('status', httplib.UNAUTHORIZED) kw['status'] = kw.get('status', http_client.UNAUTHORIZED)
args['modify'] = modify args['modify'] = modify
return test(conn, '/closed', headers={'Authorization':digest(**args)}, **kw) return test(conn, '/closed', headers={'Authorization':digest(**args)}, **kw)
@ -258,13 +260,13 @@ class TestAuth(BaseTest):
warnings = [] warnings = []
server.loop.log.warn = lambda *args, **kwargs: warnings.append(' '.join(args)) server.loop.log.warn = lambda *args, **kwargs: warnings.append(' '.join(args))
self.ae((httplib.OK, b'closed'), request()) self.ae((http_client.OK, b'closed'), request())
self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y')) self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y'))
self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y')) self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y'))
self.ae(httplib.FORBIDDEN, request('x', 'y')[0]) self.ae(http_client.FORBIDDEN, request('x', 'y')[0])
self.ae(httplib.FORBIDDEN, request()[0]) self.ae(http_client.FORBIDDEN, request()[0])
time.sleep(ban_for * 60 + 0.01) time.sleep(ban_for * 60 + 0.01)
self.ae((httplib.OK, b'closed'), request()) self.ae((http_client.OK, b'closed'), request())
# }}} # }}}
def test_android_auth_workaround(self): # {{{ def test_android_auth_workaround(self): # {{{
@ -277,28 +279,28 @@ class TestAuth(BaseTest):
# First check that unauth access fails # First check that unauth access fails
conn.request('GET', '/android') conn.request('GET', '/android')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.UNAUTHORIZED) self.ae(r.status, http_client.UNAUTHORIZED)
auth_handler = HTTPDigestAuthHandler() auth_handler = HTTPDigestAuthHandler()
url = 'http://localhost:%d%s' % (server.address[1], '/android') url = 'http://localhost:%d%s' % (server.address[1], '/android')
auth_handler.add_password(realm=REALM, uri=url, user='testuser', passwd='testpw') auth_handler.add_password(realm=REALM, uri=url, user='testuser', passwd='testpw')
cj = cookielib.CookieJar() cj = CookieJar()
cookie_handler = HTTPCookieProcessor(cj) cookie_handler = HTTPCookieProcessor(cj)
r = build_opener(auth_handler, cookie_handler).open(url) r = build_opener(auth_handler, cookie_handler).open(url)
self.ae(r.getcode(), httplib.OK) self.ae(r.getcode(), http_client.OK)
cookies = tuple(cj) cookies = tuple(cj)
self.ae(len(cookies), 1) self.ae(len(cookies), 1)
cookie = cookies[0] cookie = cookies[0]
self.assertIn(b':', cookie.value) self.assertIn(b':', cookie.value)
self.ae(cookie.path, b'/android') self.ae(cookie.path, b'/android')
r = build_opener(cookie_handler).open(url) r = build_opener(cookie_handler).open(url)
self.ae(r.getcode(), httplib.OK) self.ae(r.getcode(), http_client.OK)
self.ae(r.read(), b'android') self.ae(r.read(), b'android')
# Test that a replay attack against a different URL does not work # Test that a replay attack against a different URL does not work
try: try:
build_opener(cookie_handler).open(url+'2') build_opener(cookie_handler).open(url+'2')
assert ('Replay attack succeeded') assert ('Replay attack succeeded')
except HTTPError as e: except HTTPError as e:
self.ae(e.code, httplib.UNAUTHORIZED) self.ae(e.code, http_client.UNAUTHORIZED)
# }}} # }}}

5
src/calibre/srv/tests/base.py
View File

@ -7,12 +7,13 @@ __license__ = 'GPL v3'
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>' __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
import unittest, time, httplib, shutil, gc, tempfile, atexit, os import unittest, time, shutil, gc, tempfile, atexit, os
from io import BytesIO from io import BytesIO
from functools import partial from functools import partial
from threading import Thread from threading import Thread
from calibre.srv.utils import ServerLog from calibre.srv.utils import ServerLog
from polyglot import http_client
rmtree = partial(shutil.rmtree, ignore_errors=True) rmtree = partial(shutil.rmtree, ignore_errors=True)
@ -120,7 +121,7 @@ class TestServer(Thread):
timeout = self.loop.opts.timeout timeout = self.loop.opts.timeout
if interface is None: if interface is None:
interface = self.address[0] interface = self.address[0]
return httplib.HTTPConnection(interface, self.address[1], strict=True, timeout=timeout) return http_client.HTTPConnection(interface, self.address[1], strict=True, timeout=timeout)
def change_handler(self, handler): def change_handler(self, handler):
from calibre.srv.http_response import create_http_handler from calibre.srv.http_response import create_http_handler

43
src/calibre/srv/tests/content.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, zlib, json, binascii, time, os import zlib, json, binascii, time, os
from io import BytesIO from io import BytesIO
from calibre.ebooks.metadata.epub import get_metadata from calibre.ebooks.metadata.epub import get_metadata
@ -14,6 +14,7 @@ from calibre.ebooks.metadata.opf2 import OPF
from calibre.srv.tests.base import LibraryBaseTest from calibre.srv.tests.base import LibraryBaseTest
from calibre.utils.imghdr import identify from calibre.utils.imghdr import identify
from calibre.utils.shared_file import share_open from calibre.utils.shared_file import share_open
from polyglot import http_client
def setUpModule(): def setUpModule():
@ -32,7 +33,7 @@ class ContentTest(LibraryBaseTest):
def missing(url, body=b''): def missing(url, body=b''):
conn.request('GET', url) conn.request('GET', url)
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.NOT_FOUND) self.ae(r.status, http_client.NOT_FOUND)
self.ae(r.read(), body) self.ae(r.read(), body)
for prefix in ('static', 'icon'): for prefix in ('static', 'icon'):
@ -51,7 +52,7 @@ class ContentTest(LibraryBaseTest):
raw = P(src, data=True) raw = P(src, data=True)
conn.request('GET', url) conn.request('GET', url)
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
data = r.read() data = r.read()
if sz is None: if sz is None:
self.ae(data, raw) self.ae(data, raw)
@ -60,7 +61,7 @@ class ContentTest(LibraryBaseTest):
test_response(r) test_response(r)
conn.request('GET', url, headers={'If-None-Match':r.getheader('ETag')}) conn.request('GET', url, headers={'If-None-Match':r.getheader('ETag')})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.NOT_MODIFIED) self.ae(r.status, http_client.NOT_MODIFIED)
self.ae(b'', r.read()) self.ae(b'', r.read())
test('content-server/empty.html', '/static/empty.html') test('content-server/empty.html', '/static/empty.html')
@ -85,7 +86,7 @@ class ContentTest(LibraryBaseTest):
# Test various invalid parameters # Test various invalid parameters
def bad(*args): def bad(*args):
r, data = get(*args) r, data = get(*args)
self.ae(r.status, httplib.NOT_FOUND) self.ae(r.status, http_client.NOT_FOUND)
bad('xxx', 1) bad('xxx', 1)
bad('fmt1', 10) bad('fmt1', 10)
bad('fmt1', 1, 'zzzz') bad('fmt1', 1, 'zzzz')
@ -103,7 +104,7 @@ class ContentTest(LibraryBaseTest):
# Test fetching of format with metadata update # Test fetching of format with metadata update
raw = P('quick_start/eng.epub', data=True) raw = P('quick_start/eng.epub', data=True)
r, data = get('epub', 1) r, data = get('epub', 1)
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
etag = r.getheader('ETag') etag = r.getheader('ETag')
self.assertIsNotNone(etag) self.assertIsNotNone(etag)
self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Used-Cache'), 'no')
@ -145,39 +146,39 @@ class ContentTest(LibraryBaseTest):
os.utime(cpath, (t, t)) os.utime(cpath, (t, t))
r, data = get('cover', 1) r, data = get('cover', 1)
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(data, db.cover(1)) self.ae(data, db.cover(1))
self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Used-Cache'), 'no')
self.ae(r.getheader('Content-Type'), 'image/jpeg') self.ae(r.getheader('Content-Type'), 'image/jpeg')
r, data = get('cover', 1) r, data = get('cover', 1)
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(data, db.cover(1)) self.ae(data, db.cover(1))
self.ae(r.getheader('Used-Cache'), 'yes') self.ae(r.getheader('Used-Cache'), 'yes')
r, data = get('cover', 3) r, data = get('cover', 3)
self.ae(r.status, httplib.OK) # Auto generated cover self.ae(r.status, http_client.OK) # Auto generated cover
r, data = get('thumb', 1) r, data = get('thumb', 1)
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(identify(data), ('jpeg', 60, 60)) self.ae(identify(data), ('jpeg', 60, 60))
self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Used-Cache'), 'no')
r, data = get('thumb', 1) r, data = get('thumb', 1)
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(r.getheader('Used-Cache'), 'yes') self.ae(r.getheader('Used-Cache'), 'yes')
r, data = get('thumb', 1, q='sz=100') r, data = get('thumb', 1, q='sz=100')
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(identify(data), ('jpeg', 100, 100)) self.ae(identify(data), ('jpeg', 100, 100))
self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Used-Cache'), 'no')
r, data = get('thumb', 1, q='sz=100x100') r, data = get('thumb', 1, q='sz=100x100')
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(r.getheader('Used-Cache'), 'yes') self.ae(r.getheader('Used-Cache'), 'yes')
change_cover(1, 1) change_cover(1, 1)
r, data = get('thumb', 1, q='sz=100') r, data = get('thumb', 1, q='sz=100')
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(identify(data), ('jpeg', 100, 100)) self.ae(identify(data), ('jpeg', 100, 100))
self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Used-Cache'), 'no')
# Test file sharing in cache # Test file sharing in cache
r, data = get('cover', 2) r, data = get('cover', 2)
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(data, db.cover(2)) self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Used-Cache'), 'no')
path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8') path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8')
@ -185,7 +186,7 @@ class ContentTest(LibraryBaseTest):
# Now force an update # Now force an update
change_cover(1) change_cover(1)
r, data = get('cover', 2) r, data = get('cover', 2)
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(data, db.cover(2)) self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Used-Cache'), 'no')
path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8') path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8')
@ -193,7 +194,7 @@ class ContentTest(LibraryBaseTest):
# Do it again # Do it again
change_cover(2) change_cover(2)
r, data = get('cover', 2) r, data = get('cover', 2)
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(data, db.cover(2)) self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Used-Cache'), 'no')
self.ae(f.read(), fdata) self.ae(f.read(), fdata)
@ -201,7 +202,7 @@ class ContentTest(LibraryBaseTest):
# Test serving of metadata as opf # Test serving of metadata as opf
r, data = get('opf', 1) r, data = get('opf', 1)
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(r.getheader('Content-Type'), 'application/oebps-package+xml; charset=UTF-8') self.ae(r.getheader('Content-Type'), 'application/oebps-package+xml; charset=UTF-8')
self.assertIsNotNone(r.getheader('Last-Modified')) self.assertIsNotNone(r.getheader('Last-Modified'))
opf = OPF(BytesIO(data), populate_spine=False, try_to_guess_cover=False) opf = OPF(BytesIO(data), populate_spine=False, try_to_guess_cover=False)
@ -209,17 +210,17 @@ class ContentTest(LibraryBaseTest):
self.ae(db.field_for('authors', 1), tuple(opf.authors)) self.ae(db.field_for('authors', 1), tuple(opf.authors))
conn.request('GET', '/get/opf/1', headers={'Accept-Encoding':'gzip'}) conn.request('GET', '/get/opf/1', headers={'Accept-Encoding':'gzip'})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') self.ae(r.status, http_client.OK), self.ae(r.getheader('Content-Encoding'), 'gzip')
raw = r.read() raw = r.read()
self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data) self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data)
# Test serving metadata as json # Test serving metadata as json
r, data = get('json', 1) r, data = get('json', 1)
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(db.field_for('title', 1), json.loads(data)['title']) self.ae(db.field_for('title', 1), json.loads(data)['title'])
conn.request('GET', '/get/json/1', headers={'Accept-Encoding':'gzip'}) conn.request('GET', '/get/json/1', headers={'Accept-Encoding':'gzip'})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') self.ae(r.status, http_client.OK), self.ae(r.getheader('Content-Encoding'), 'gzip')
raw = r.read() raw = r.read()
self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data) self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data)

79
src/calibre/srv/tests/http.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, hashlib, zlib, string, time, os import hashlib, zlib, string, time, os
from io import BytesIO from io import BytesIO
from tempfile import NamedTemporaryFile from tempfile import NamedTemporaryFile
@ -15,6 +15,7 @@ from calibre.srv.tests.base import BaseTest, TestServer
from calibre.srv.utils import eintr_retry_call from calibre.srv.utils import eintr_retry_call
from calibre.utils.monotonic import monotonic from calibre.utils.monotonic import monotonic
from polyglot.builtins import iteritems, range from polyglot.builtins import iteritems, range
from polyglot import http_client
is_ci = os.environ.get('CI', '').lower() == 'true' is_ci = os.environ.get('CI', '').lower() == 'true'
@ -94,7 +95,7 @@ class TestHTTP(BaseTest):
def test(al, q): def test(al, q):
conn.request('GET', '/', headers={'Accept-Language': al}) conn.request('GET', '/', headers={'Accept-Language': al})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
q += get_translator(q)[-1].ugettext('Unknown') q += get_translator(q)[-1].ugettext('Unknown')
self.ae(r.read(), q) self.ae(r.read(), q)
@ -136,7 +137,7 @@ class TestHTTP(BaseTest):
def raw_send(conn, raw): def raw_send(conn, raw):
conn.send(raw) conn.send(raw)
conn._HTTPConnection__state = httplib._CS_REQ_SENT conn._HTTPConnection__state = http_client._CS_REQ_SENT
return conn.getresponse() return conn.getresponse()
base_timeout = 0.5 if is_ci else 0.1 base_timeout = 0.5 if is_ci else 0.1
@ -144,31 +145,31 @@ class TestHTTP(BaseTest):
with TestServer(handler, timeout=base_timeout, max_header_line_size=100./1024, max_request_body_size=100./(1024*1024)) as server: with TestServer(handler, timeout=base_timeout, max_header_line_size=100./1024, max_request_body_size=100./(1024*1024)) as server:
conn = server.connect() conn = server.connect()
r = raw_send(conn, b'hello\n') r = raw_send(conn, b'hello\n')
self.ae(r.status, httplib.BAD_REQUEST) self.ae(r.status, http_client.BAD_REQUEST)
self.ae(r.read(), b'HTTP requires CRLF line terminators') self.ae(r.read(), b'HTTP requires CRLF line terminators')
r = raw_send(conn, b'\r\nGET /index.html HTTP/1.1\r\n\r\n') r = raw_send(conn, b'\r\nGET /index.html HTTP/1.1\r\n\r\n')
self.ae(r.status, httplib.NOT_FOUND), self.ae(r.read(), b'Requested resource not found') self.ae(r.status, http_client.NOT_FOUND), self.ae(r.read(), b'Requested resource not found')
r = raw_send(conn, b'\r\n\r\nGET /index.html HTTP/1.1\r\n\r\n') r = raw_send(conn, b'\r\n\r\nGET /index.html HTTP/1.1\r\n\r\n')
self.ae(r.status, httplib.BAD_REQUEST) self.ae(r.status, http_client.BAD_REQUEST)
self.ae(r.read(), b'Multiple leading empty lines not allowed') self.ae(r.read(), b'Multiple leading empty lines not allowed')
r = raw_send(conn, b'hello world\r\n') r = raw_send(conn, b'hello world\r\n')
self.ae(r.status, httplib.BAD_REQUEST) self.ae(r.status, http_client.BAD_REQUEST)
self.ae(r.read(), b'Malformed Request-Line') self.ae(r.read(), b'Malformed Request-Line')
r = raw_send(conn, b'x' * 200) r = raw_send(conn, b'x' * 200)
self.ae(r.status, httplib.BAD_REQUEST) self.ae(r.status, http_client.BAD_REQUEST)
self.ae(r.read(), b'') self.ae(r.read(), b'')
r = raw_send(conn, b'XXX /index.html HTTP/1.1\r\n\r\n') r = raw_send(conn, b'XXX /index.html HTTP/1.1\r\n\r\n')
self.ae(r.status, httplib.BAD_REQUEST), self.ae(r.read(), b'Unknown HTTP method') self.ae(r.status, http_client.BAD_REQUEST), self.ae(r.read(), b'Unknown HTTP method')
# Test 404 # Test 404
conn.request('HEAD', '/moose') conn.request('HEAD', '/moose')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.NOT_FOUND) self.ae(r.status, http_client.NOT_FOUND)
self.assertIsNotNone(r.getheader('Date', None)) self.assertIsNotNone(r.getheader('Date', None))
self.ae(r.getheader('Content-Length'), str(len(body))) self.ae(r.getheader('Content-Length'), str(len(body)))
self.ae(r.getheader('Content-Type'), 'text/plain; charset=UTF-8') self.ae(r.getheader('Content-Type'), 'text/plain; charset=UTF-8')
@ -176,7 +177,7 @@ class TestHTTP(BaseTest):
self.ae(r.read(), '') self.ae(r.read(), '')
conn.request('GET', '/choose') conn.request('GET', '/choose')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.NOT_FOUND) self.ae(r.status, http_client.NOT_FOUND)
self.ae(r.read(), b'Requested resource not found') self.ae(r.read(), b'Requested resource not found')
# Test 500 # Test 500
@ -186,7 +187,7 @@ class TestHTTP(BaseTest):
conn = server.connect() conn = server.connect()
conn.request('GET', '/test/') conn.request('GET', '/test/')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.INTERNAL_SERVER_ERROR) self.ae(r.status, http_client.INTERNAL_SERVER_ERROR)
server.loop.log.filter_level = orig server.loop.log.filter_level = orig
# Test 301 # Test 301
@ -196,7 +197,7 @@ class TestHTTP(BaseTest):
conn = server.connect() conn = server.connect()
conn.request('GET', '/') conn.request('GET', '/')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.MOVED_PERMANENTLY) self.ae(r.status, http_client.MOVED_PERMANENTLY)
self.ae(r.getheader('Location'), '/somewhere-else') self.ae(r.getheader('Location'), '/somewhere-else')
self.ae('', r.read()) self.ae('', r.read())
@ -206,26 +207,26 @@ class TestHTTP(BaseTest):
# Test simple GET # Test simple GET
conn.request('GET', '/test/') conn.request('GET', '/test/')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(r.read(), b'test') self.ae(r.read(), b'test')
# Test TRACE # Test TRACE
lines = ['TRACE /xxx HTTP/1.1', 'Test: value', 'Xyz: abc, def', '', ''] lines = ['TRACE /xxx HTTP/1.1', 'Test: value', 'Xyz: abc, def', '', '']
r = raw_send(conn, ('\r\n'.join(lines)).encode('ascii')) r = raw_send(conn, ('\r\n'.join(lines)).encode('ascii'))
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(r.read().decode('utf-8'), '\n'.join(lines[:-2])) self.ae(r.read().decode('utf-8'), '\n'.join(lines[:-2]))
# Test POST with simple body # Test POST with simple body
conn.request('POST', '/test', 'body') conn.request('POST', '/test', 'body')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody') self.ae(r.read(), b'testbody')
# Test POST with chunked transfer encoding # Test POST with chunked transfer encoding
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'4\r\nbody\r\na\r\n1234567890\r\n0\r\n\r\n') conn.send(b'4\r\nbody\r\na\r\n1234567890\r\n0\r\n\r\n')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody1234567890') self.ae(r.read(), b'testbody1234567890')
# Test various incorrect input # Test various incorrect input
@ -233,39 +234,39 @@ class TestHTTP(BaseTest):
conn.request('GET', '/test' + ('a' * 200)) conn.request('GET', '/test' + ('a' * 200))
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.BAD_REQUEST) self.ae(r.status, http_client.BAD_REQUEST)
conn = server.connect() conn = server.connect()
conn.request('GET', '/test', ('a' * 200)) conn.request('GET', '/test', ('a' * 200))
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE) self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE)
conn = server.connect() conn = server.connect()
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'x\r\nbody\r\n0\r\n\r\n') conn.send(b'x\r\nbody\r\n0\r\n\r\n')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.BAD_REQUEST) self.ae(r.status, http_client.BAD_REQUEST)
self.assertIn(b'not a valid chunk size', r.read()) self.assertIn(b'not a valid chunk size', r.read())
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'4\r\nbody\r\n200\r\n\r\n') conn.send(b'4\r\nbody\r\n200\r\n\r\n')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE) self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE)
conn.request('POST', '/test', body='a'*200) conn.request('POST', '/test', body='a'*200)
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE) self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE)
conn = server.connect() conn = server.connect()
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'3\r\nbody\r\n0\r\n\r\n') conn.send(b'3\r\nbody\r\n0\r\n\r\n')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.BAD_REQUEST), self.ae(r.read(), b'Chunk does not have trailing CRLF') self.ae(r.status, http_client.BAD_REQUEST), self.ae(r.read(), b'Chunk does not have trailing CRLF')
conn = server.connect(timeout=base_timeout * 5) conn = server.connect(timeout=base_timeout * 5)
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'30\r\nbody\r\n0\r\n\r\n') conn.send(b'30\r\nbody\r\n0\r\n\r\n')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.REQUEST_TIMEOUT) self.ae(r.status, http_client.REQUEST_TIMEOUT)
self.assertIn(b'', r.read()) self.assertIn(b'', r.read())
server.log.filter_level = orig_level server.log.filter_level = orig_level
@ -273,14 +274,14 @@ class TestHTTP(BaseTest):
# Test pipelining # Test pipelining
responses = [] responses = []
for i in range(10): for i in range(10):
conn._HTTPConnection__state = httplib._CS_IDLE conn._HTTPConnection__state = http_client._CS_IDLE
conn.request('GET', '/%d'%i) conn.request('GET', '/%d'%i)
responses.append(conn.response_class(conn.sock, strict=conn.strict, method=conn._method)) responses.append(conn.response_class(conn.sock, strict=conn.strict, method=conn._method))
for i in range(10): for i in range(10):
r = responses[i] r = responses[i]
r.begin() r.begin()
self.ae(r.read(), ('%d' % i).encode('ascii')) self.ae(r.read(), ('%d' % i).encode('ascii'))
conn._HTTPConnection__state = httplib._CS_IDLE conn._HTTPConnection__state = http_client._CS_IDLE
# Test closing # Test closing
server.loop.opts.timeout = 10 # ensure socket is not closed because of timeout server.loop.opts.timeout = 10 # ensure socket is not closed because of timeout
@ -319,12 +320,12 @@ class TestHTTP(BaseTest):
conn = server.connect() conn = server.connect()
conn.request('GET', '/an_etagged_path') conn.request('GET', '/an_etagged_path')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.read(), b'an_etagged_path') self.ae(r.status, http_client.OK), self.ae(r.read(), b'an_etagged_path')
etag = r.getheader('ETag') etag = r.getheader('ETag')
self.ae(etag, '"%s"' % hashlib.sha1('an_etagged_path').hexdigest()) self.ae(etag, '"%s"' % hashlib.sha1('an_etagged_path').hexdigest())
conn.request('GET', '/an_etagged_path', headers={'If-None-Match':etag}) conn.request('GET', '/an_etagged_path', headers={'If-None-Match':etag})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.NOT_MODIFIED) self.ae(r.status, http_client.NOT_MODIFIED)
self.ae(r.read(), b'') self.ae(r.read(), b'')
# Test gzip # Test gzip
@ -334,7 +335,7 @@ class TestHTTP(BaseTest):
conn.request('GET', '/an_etagged_path', headers={'Accept-Encoding':'gzip'}) conn.request('GET', '/an_etagged_path', headers={'Accept-Encoding':'gzip'})
r = conn.getresponse() r = conn.getresponse()
self.ae(str(len(raw)), r.getheader('Calibre-Uncompressed-Length')) self.ae(str(len(raw)), r.getheader('Calibre-Uncompressed-Length'))
self.ae(r.status, httplib.OK), self.ae(zlib.decompress(r.read(), 16+zlib.MAX_WBITS), raw) self.ae(r.status, http_client.OK), self.ae(zlib.decompress(r.read(), 16+zlib.MAX_WBITS), raw)
# Test dynamic etagged content # Test dynamic etagged content
num_calls = [0] num_calls = [0]
@ -346,13 +347,13 @@ class TestHTTP(BaseTest):
conn = server.connect() conn = server.connect()
conn.request('GET', '/an_etagged_path') conn.request('GET', '/an_etagged_path')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.read(), b'data') self.ae(r.status, http_client.OK), self.ae(r.read(), b'data')
etag = r.getheader('ETag') etag = r.getheader('ETag')
self.ae(etag, b'"xxx"') self.ae(etag, b'"xxx"')
self.ae(r.getheader('Content-Length'), '4') self.ae(r.getheader('Content-Length'), '4')
conn.request('GET', '/an_etagged_path', headers={'If-None-Match':etag}) conn.request('GET', '/an_etagged_path', headers={'If-None-Match':etag})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.NOT_MODIFIED) self.ae(r.status, http_client.NOT_MODIFIED)
self.ae(r.read(), b'') self.ae(r.read(), b'')
self.ae(num_calls[0], 1) self.ae(num_calls[0], 1)
@ -368,11 +369,11 @@ class TestHTTP(BaseTest):
self.ae(r.getheader('Content-Type'), guess_type(f.name)[0]) self.ae(r.getheader('Content-Type'), guess_type(f.name)[0])
self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes') self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes')
self.ae(int(r.getheader('Content-Length')), len(fdata)) self.ae(int(r.getheader('Content-Length')), len(fdata))
self.ae(r.status, httplib.OK), self.ae(r.read(), fdata) self.ae(r.status, http_client.OK), self.ae(r.read(), fdata)
conn.request('GET', '/test', headers={'Range':'bytes=2-25'}) conn.request('GET', '/test', headers={'Range':'bytes=2-25'})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.PARTIAL_CONTENT) self.ae(r.status, http_client.PARTIAL_CONTENT)
self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes') self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes')
self.ae(type('')(r.getheader('Content-Range')), 'bytes 2-25/%d' % len(fdata)) self.ae(type('')(r.getheader('Content-Range')), 'bytes 2-25/%d' % len(fdata))
self.ae(int(r.getheader('Content-Length')), 24) self.ae(int(r.getheader('Content-Length')), 24)
@ -380,27 +381,27 @@ class TestHTTP(BaseTest):
conn.request('GET', '/test', headers={'Range':'bytes=100000-'}) conn.request('GET', '/test', headers={'Range':'bytes=100000-'})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.REQUESTED_RANGE_NOT_SATISFIABLE) self.ae(r.status, http_client.REQUESTED_RANGE_NOT_SATISFIABLE)
self.ae(type('')(r.getheader('Content-Range')), 'bytes */%d' % len(fdata)) self.ae(type('')(r.getheader('Content-Range')), 'bytes */%d' % len(fdata))
conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':etag}) conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':etag})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.PARTIAL_CONTENT), self.ae(r.read(), fdata[25:51]) self.ae(r.status, http_client.PARTIAL_CONTENT), self.ae(r.read(), fdata[25:51])
self.ae(int(r.getheader('Content-Length')), 26) self.ae(int(r.getheader('Content-Length')), 26)
conn.request('GET', '/test', headers={'Range':'bytes=0-1000000'}) conn.request('GET', '/test', headers={'Range':'bytes=0-1000000'})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.PARTIAL_CONTENT), self.ae(r.read(), fdata) self.ae(r.status, http_client.PARTIAL_CONTENT), self.ae(r.read(), fdata)
conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':'"nomatch"'}) conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':'"nomatch"'})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.read(), fdata) self.ae(r.status, http_client.OK), self.ae(r.read(), fdata)
self.assertFalse(r.getheader('Content-Range')) self.assertFalse(r.getheader('Content-Range'))
self.ae(int(r.getheader('Content-Length')), len(fdata)) self.ae(int(r.getheader('Content-Length')), len(fdata))
conn.request('GET', '/test', headers={'Range':'bytes=0-25,26-50'}) conn.request('GET', '/test', headers={'Range':'bytes=0-25,26-50'})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.PARTIAL_CONTENT) self.ae(r.status, http_client.PARTIAL_CONTENT)
clen = int(r.getheader('Content-Length')) clen = int(r.getheader('Content-Length'))
data = r.read() data = r.read()
self.ae(clen, len(data)) self.ae(clen, len(data))
@ -415,7 +416,7 @@ class TestHTTP(BaseTest):
conn = server.connect(timeout=1) conn = server.connect(timeout=1)
conn.request('GET', '/test') conn.request('GET', '/test')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
rdata = r.read() rdata = r.read()
self.ae(len(data), len(rdata)) self.ae(len(data), len(rdata))
self.ae(hashlib.sha1(data).hexdigest(), hashlib.sha1(rdata).hexdigest()) self.ae(hashlib.sha1(data).hexdigest(), hashlib.sha1(rdata).hexdigest())

13
src/calibre/srv/tests/loop.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, ssl, os, socket, time import ssl, os, socket, time
from collections import namedtuple from collections import namedtuple
from unittest import skipIf from unittest import skipIf
from glob import glob from glob import glob
@ -18,6 +18,7 @@ from calibre.ptempfile import TemporaryDirectory
from calibre.utils.certgen import create_server_cert from calibre.utils.certgen import create_server_cert
from calibre.utils.monotonic import monotonic from calibre.utils.monotonic import monotonic
from polyglot.builtins import range from polyglot.builtins import range
from polyglot import http_client
is_ci = os.environ.get('CI', '').lower() == 'true' is_ci = os.environ.get('CI', '').lower() == 'true'
@ -92,7 +93,7 @@ class LoopTest(BaseTest):
conn.request('GET', '/') conn.request('GET', '/')
with self.assertRaises(socket.timeout): with self.assertRaises(socket.timeout):
res = conn.getresponse() res = conn.getresponse()
if str(res.status) == str(httplib.REQUEST_TIMEOUT): if str(res.status) == str(http_client.REQUEST_TIMEOUT):
raise socket.timeout('Timeout') raise socket.timeout('Timeout')
raise Exception('Got unexpected response: code: %s %s headers: %r data: %r' % ( raise Exception('Got unexpected response: code: %s %s headers: %r data: %r' % (
res.status, res.reason, res.getheaders(), res.read())) res.status, res.reason, res.getheaders(), res.read()))
@ -135,7 +136,7 @@ class LoopTest(BaseTest):
conn = server.connect(interface='127.0.0.1') conn = server.connect(interface='127.0.0.1')
conn.request('GET', '/test', 'body') conn.request('GET', '/test', 'body')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody') self.ae(r.read(), b'testbody')
def test_ring_buffer(self): def test_ring_buffer(self):
@ -203,10 +204,10 @@ class LoopTest(BaseTest):
create_server_cert(address, ca_file, cert_file, key_file, key_size=1024) create_server_cert(address, ca_file, cert_file, key_file, key_size=1024)
ctx = ssl.create_default_context(cafile=ca_file) ctx = ssl.create_default_context(cafile=ca_file)
with TestServer(lambda data:(data.path[0] + data.read()), ssl_certfile=cert_file, ssl_keyfile=key_file, listen_on=address, port=0) as server: with TestServer(lambda data:(data.path[0] + data.read()), ssl_certfile=cert_file, ssl_keyfile=key_file, listen_on=address, port=0) as server:
conn = httplib.HTTPSConnection(address, server.address[1], strict=True, context=ctx) conn = http_client.HTTPSConnection(address, server.address[1], strict=True, context=ctx)
conn.request('GET', '/test', 'body') conn.request('GET', '/test', 'body')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody') self.ae(r.read(), b'testbody')
cert = conn.sock.getpeercert() cert = conn.sock.getpeercert()
subject = dict(x[0] for x in cert['subject']) subject = dict(x[0] for x in cert['subject'])
@ -226,7 +227,7 @@ class LoopTest(BaseTest):
conn = server.connect() conn = server.connect()
conn.request('GET', '/test', 'body') conn.request('GET', '/test', 'body')
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK) self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody') self.ae(r.read(), b'testbody')
self.ae(server.loop.bound_address[1], port) self.ae(server.loop.bound_address[1], port)

4
src/calibre/srv/utils.py
View File

@ -7,9 +7,7 @@ __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import errno, socket, select, os, time import errno, socket, select, os, time
from Cookie import SimpleCookie
from contextlib import closing from contextlib import closing
import repr as reprlib
from email.utils import formatdate from email.utils import formatdate
from operator import itemgetter from operator import itemgetter
from binascii import hexlify, unhexlify from binascii import hexlify, unhexlify
@ -23,6 +21,8 @@ from calibre.utils.socket_inheritance import set_socket_inherit
from calibre.utils.logging import ThreadSafeLog from calibre.utils.logging import ThreadSafeLog
from calibre.utils.shared_file import share_open, raise_winerror from calibre.utils.shared_file import share_open, raise_winerror
from polyglot.builtins import iteritems, map, unicode_type, range from polyglot.builtins import iteritems, map, unicode_type, range
from polyglot import reprlib
from polyglot.http_cookie import SimpleCookie
from polyglot.urllib import parse_qs, quote as urlquote from polyglot.urllib import parse_qs, quote as urlquote
HTTP1 = 'HTTP/1.0' HTTP1 = 'HTTP/1.0'

7
src/calibre/srv/web_socket.py
View File

@ -5,7 +5,7 @@
from __future__ import (unicode_literals, division, absolute_import, from __future__ import (unicode_literals, division, absolute_import,
print_function) print_function)
import httplib, os, weakref, socket import os, weakref, socket
from base64 import standard_b64encode from base64 import standard_b64encode
from collections import deque from collections import deque
from hashlib import sha1 from hashlib import sha1
@ -19,6 +19,7 @@ from calibre.srv.http_response import HTTPConnection, create_http_handler
from calibre.srv.utils import DESIRED_SEND_BUFFER_SIZE from calibre.srv.utils import DESIRED_SEND_BUFFER_SIZE
from calibre.utils.speedups import ReadOnlyFileBuffer from calibre.utils.speedups import ReadOnlyFileBuffer
from polyglot.queue import Queue, Empty from polyglot.queue import Queue, Empty
from polyglot import http_client
speedup, err = plugins['speedup'] speedup, err = plugins['speedup']
if not speedup: if not speedup:
raise RuntimeError('Failed to load speedup module with error: ' + err) raise RuntimeError('Failed to load speedup module with error: ' + err)
@ -286,9 +287,9 @@ class WebSocketConnection(HTTPConnection):
except Exception: except Exception:
ver_ok = False ver_ok = False
if not ver_ok: if not ver_ok:
return self.simple_response(httplib.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver) return self.simple_response(http_client.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver)
if self.method != 'GET': if self.method != 'GET':
return self.simple_response(httplib.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method) return self.simple_response(http_client.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method)
response = HANDSHAKE_STR % standard_b64encode(sha1(key + GUID_STR).digest()) response = HANDSHAKE_STR % standard_b64encode(sha1(key + GUID_STR).digest())
self.optimize_for_sending_packet() self.optimize_for_sending_packet()

29
src/calibre/test_build.py
View File

@ -73,6 +73,29 @@ class BuildTest(unittest.TestCase):
from html5_parser import parse from html5_parser import parse
parse('<p>xxx') parse('<p>xxx')
def test_imports(self):
import importlib
exclude = ['dbus_export.demo', 'dbus_export.gtk', 'upstream']
if not iswindows:
exclude.extend(['iphlpapi', 'windows', 'winreg', 'winusb'])
if not isosx:
exclude.append('osx')
if not islinux:
exclude.extend(['dbus', 'linux'])
base = os.path.dirname(__file__)
trimpath = len(os.path.dirname(base)) + 1
for root, dirs, files in os.walk(base):
for dir in dirs:
if not os.path.isfile(os.path.join(root, dir, '__init__.py')):
dirs.remove(dir)
for file in files:
file, ext = os.path.splitext(file)
if ext != '.py':
continue
name = '.'.join(root[trimpath:].split(os.path.sep) + [file])
if not any(x for x in exclude if x in name):
importlib.import_module(name)
def test_plugins(self): def test_plugins(self):
exclusions = set() exclusions = set()
if is_ci: if is_ci:
@ -99,7 +122,7 @@ class BuildTest(unittest.TestCase):
from calibre.utils.cleantext import test_clean_xml_chars from calibre.utils.cleantext import test_clean_xml_chars
test_clean_xml_chars() test_clean_xml_chars()
from lxml import etree from lxml import etree
raw = '<a/>' raw = b'<a/>'
root = etree.fromstring(raw) root = etree.fromstring(raw)
self.assertEqual(etree.tostring(root), raw) self.assertEqual(etree.tostring(root), raw)
@ -175,7 +198,7 @@ class BuildTest(unittest.TestCase):
# it should just work because the hard-coded paths of the Qt # it should just work because the hard-coded paths of the Qt
# installation should work. If they do not, then it is a distro # installation should work. If they do not, then it is a distro
# problem. # problem.
fmts = set(map(unicode_type, QImageReader.supportedImageFormats())) fmts = set(map(lambda x: x.data().decode('utf-8'), QImageReader.supportedImageFormats()))
testf = {'jpg', 'png', 'svg', 'ico', 'gif'} testf = {'jpg', 'png', 'svg', 'ico', 'gif'}
self.assertEqual(testf.intersection(fmts), testf, "Qt doesn't seem to be able to load some of its image plugins. Available plugins: %s" % fmts) self.assertEqual(testf.intersection(fmts), testf, "Qt doesn't seem to be able to load some of its image plugins. Available plugins: %s" % fmts)
data = P('images/blank.png', allow_user_override=False, data=True) data = P('images/blank.png', allow_user_override=False, data=True)
@ -254,7 +277,7 @@ class BuildTest(unittest.TestCase):
def test_netifaces(self): def test_netifaces(self):
import netifaces import netifaces
self.assertGreaterEqual(netifaces.interfaces(), 1, 'netifaces could find no network interfaces') self.assertGreaterEqual(len(netifaces.interfaces()), 1, 'netifaces could find no network interfaces')
def test_psutil(self): def test_psutil(self):
import psutil import psutil

8
src/calibre/utils/browser.py
View File

@ -5,11 +5,13 @@ __license__ = 'GPL v3'
__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>' __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
import copy, httplib, ssl import copy, ssl
from cookielib import CookieJar, Cookie
from mechanize import Browser as B, HTTPSHandler from mechanize import Browser as B, HTTPSHandler
from polyglot import http_client
from polyglot.http_cookie import CookieJar, Cookie
class ModernHTTPSHandler(HTTPSHandler): class ModernHTTPSHandler(HTTPSHandler):
@ -24,7 +26,7 @@ class ModernHTTPSHandler(HTTPSHandler):
def conn_factory(hostport, **kw): def conn_factory(hostport, **kw):
kw['context'] = self.ssl_context kw['context'] = self.ssl_context
return httplib.HTTPSConnection(hostport, **kw) return http_client.HTTPSConnection(hostport, **kw)
return self.do_open(conn_factory, req) return self.do_open(conn_factory, req)

5
src/calibre/utils/cleantext.py
View File

@ -2,8 +2,9 @@ __license__ = 'GPL 3'
__copyright__ = '2010, sengian <sengian1@gmail.com>' __copyright__ = '2010, sengian <sengian1@gmail.com>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
import re, htmlentitydefs import re
from polyglot.builtins import codepoint_to_chr, map, range from polyglot.builtins import codepoint_to_chr, map, range
from polyglot.html_entities import name2codepoint
from calibre.constants import plugins, preferred_encoding from calibre.constants import plugins, preferred_encoding
try: try:
@ -80,7 +81,7 @@ def unescape(text, rm=False, rchar=u''):
else: else:
# named entity # named entity
try: try:
text = codepoint_to_chr(htmlentitydefs.name2codepoint[text[1:-1]]) text = codepoint_to_chr(name2codepoint[text[1:-1]])
except KeyError: except KeyError:
pass pass
if rm: if rm:

21
src/calibre/utils/https.py
View File

@ -10,7 +10,7 @@ import ssl, socket, re
from contextlib import closing from contextlib import closing
from calibre import get_proxies from calibre import get_proxies
from calibre.constants import ispy3 from polyglot import http_client
from polyglot.urllib import urlsplit from polyglot.urllib import urlsplit
has_ssl_verify = hasattr(ssl, 'create_default_context') and hasattr(ssl, '_create_unverified_context') has_ssl_verify = hasattr(ssl, 'create_default_context') and hasattr(ssl, '_create_unverified_context')
@ -19,19 +19,14 @@ class HTTPError(ValueError):
def __init__(self, url, code): def __init__(self, url, code):
msg = '%s returned an unsupported http response code: %d (%s)' % ( msg = '%s returned an unsupported http response code: %d (%s)' % (
url, code, httplib.responses.get(code, None)) url, code, http_client.responses.get(code, None))
ValueError.__init__(self, msg) ValueError.__init__(self, msg)
self.code = code self.code = code
self.url = url self.url = url
if ispy3:
import http.client as httplib
else:
import httplib
if has_ssl_verify: if has_ssl_verify:
class HTTPSConnection(httplib.HTTPSConnection): class HTTPSConnection(http_client.HTTPSConnection):
def __init__(self, ssl_version, *args, **kwargs): def __init__(self, ssl_version, *args, **kwargs):
cafile = kwargs.pop('cert_file', None) cafile = kwargs.pop('cert_file', None)
@ -39,7 +34,7 @@ if has_ssl_verify:
kwargs['context'] = ssl._create_unverified_context() kwargs['context'] = ssl._create_unverified_context()
else: else:
kwargs['context'] = ssl.create_default_context(cafile=cafile) kwargs['context'] = ssl.create_default_context(cafile=cafile)
httplib.HTTPSConnection.__init__(self, *args, **kwargs) http_client.HTTPSConnection.__init__(self, *args, **kwargs)
else: else:
# Check certificate hostname {{{ # Check certificate hostname {{{
# Implementation taken from python 3 # Implementation taken from python 3
@ -136,10 +131,10 @@ else:
"subjectAltName fields were found") "subjectAltName fields were found")
# }}} # }}}
class HTTPSConnection(httplib.HTTPSConnection): class HTTPSConnection(http_client.HTTPSConnection):
def __init__(self, ssl_version, *args, **kwargs): def __init__(self, ssl_version, *args, **kwargs):
httplib.HTTPSConnection.__init__(self, *args, **kwargs) http_client.HTTPSConnection.__init__(self, *args, **kwargs)
self.calibre_ssl_version = ssl_version self.calibre_ssl_version = ssl_version
def connect(self): def connect(self):
@ -204,7 +199,7 @@ def get_https_resource_securely(
path += '?' + p.query path += '?' + p.query
c.request('GET', path, headers=headers or {}) c.request('GET', path, headers=headers or {})
response = c.getresponse() response = c.getresponse()
if response.status in (httplib.MOVED_PERMANENTLY, httplib.FOUND, httplib.SEE_OTHER): if response.status in (http_client.MOVED_PERMANENTLY, http_client.FOUND, http_client.SEE_OTHER):
if max_redirects <= 0: if max_redirects <= 0:
raise ValueError('Too many redirects, giving up') raise ValueError('Too many redirects, giving up')
newurl = response.getheader('Location', None) newurl = response.getheader('Location', None)
@ -212,7 +207,7 @@ def get_https_resource_securely(
raise ValueError('%s returned a redirect response with no Location header' % url) raise ValueError('%s returned a redirect response with no Location header' % url)
return get_https_resource_securely( return get_https_resource_securely(
newurl, cacerts=cacerts, timeout=timeout, max_redirects=max_redirects-1, ssl_version=ssl_version, get_response=get_response) newurl, cacerts=cacerts, timeout=timeout, max_redirects=max_redirects-1, ssl_version=ssl_version, get_response=get_response)
if response.status != httplib.OK: if response.status != http_client.OK:
raise HTTPError(url, response.status) raise HTTPError(url, response.status)
if get_response: if get_response:
return response return response

3
src/calibre/utils/smtp.py
View File

@ -98,8 +98,7 @@ def get_mx(host, verbose=0):
if verbose: if verbose:
print('Find mail exchanger for', host) print('Find mail exchanger for', host)
answers = list(dns.resolver.query(host, 'MX')) answers = list(dns.resolver.query(host, 'MX'))
answers.sort(cmp=lambda x, y: cmp(int(getattr(x, 'preference', sys.maxint)), answers.sort(key=lambda x: int(getattr(x, 'preference', sys.maxint)))
int(getattr(y, 'preference', sys.maxint))))
return [str(x.exchange) for x in answers if hasattr(x, 'exchange')] return [str(x.exchange) for x in answers if hasattr(x, 'exchange')]

22
src/calibre/utils/unrar.py
View File

@ -114,20 +114,20 @@ def test_basic():
b"Rar!\x1a\x07\x00\xcf\x90s\x00\x00\r\x00\x00\x00\x00\x00\x00\x00\x14\xe7z\x00\x80#\x00\x17\x00\x00\x00\r\x00\x00\x00\x03\xc2\xb3\x96o\x00\x00\x00\x00\x1d3\x03\x00\x00\x00\x00\x00CMT\x0c\x00\x8b\xec\x8e\xef\x14\xf6\xe6h\x04\x17\xff\xcd\x0f\xffk9b\x11]^\x80\xd3dt \x90+\x00\x14\x00\x00\x00\x08\x00\x00\x00\x03\xf1\x84\x93\\\xb9]yA\x1d3\t\x00\xa4\x81\x00\x001\\sub-one\x00\xc0\x0c\x00\x8f\xec\x89\xfe.JM\x86\x82\x0c_\xfd\xfd\xd7\x11\x1a\xef@\x9eHt \x80'\x00\x0e\x00\x00\x00\x04\x00\x00\x00\x03\x9f\xa8\x17\xf8\xaf]yA\x1d3\x07\x00\xa4\x81\x00\x00one.txt\x00\x08\xbf\x08\xae\xf3\xca\x87\xfeo\xfe\xd2n\x80-Ht \x82:\x00\x18\x00\x00\x00\x10\x00\x00\x00\x03\xa86\x81\xdf\xf9fyA\x1d3\x1a\x00\xa4\x81\x00\x00\xe8\xaf\xb6\xe6\xaf\x94\xe5\xb1\x81.txt\x00\x8bh\xf6\xd4kA\\.\x00txt\x0c\x00\x8b\xec\x8e\xef\x14\xf6\xe2l\x91\x189\xff\xdf\xfe\xc2\xd3:g\x9a\x19F=cYt \x928\x00\x11\x00\x00\x00\x08\x00\x00\x00\x03\x7f\xd6\xb6\x7f\xeafyA\x1d3\x16\x00\xa4\x81\x00\x00F\xc3\xbc\xc3\x9fe.txt\x00\x01\x00F\xfc\xdfe\x00.txt\x00\xc0<D\xfe\xc8\xef\xbc\xd1\x04I?\xfd\xff\xdbF)]\xe8\xb9\xe1t \x90/\x00\x13\x00\x00\x00\x08\x00\x00\x00\x03\x1a$\x932\xc2]yA\x1d3\r\x00\xa4\x81\x00\x002\\sub-two.txt\x00\xc0\x10\x00S\xec\xcb\x7f\x8b\xa5(\x0b\x01\xcb\xef\xdf\xf6t\x89\x97z\x0eft \x90)\x00\r\x00\x00\x00\r\x00\x00\x00\x03c\x89K\xd3\xc8fyA\x140\x07\x00\xff\xa1\x00\x00symlink\x00\xc02/sub-two.txt\xeb\x86t\xe0\x90#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\xb9]yA\x140\x01\x00\xedA\x00\x001\x00\xc0\xe0Dt\xe0\x90#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\xc2]yA\x140\x01\x00\xedA\x00\x002\x00\xc0u\xa1t \x80,\x00\r\x00\x00\x00\r\x00\x00\x00\x03T\xea\x04\xca\xe6\x84yA\x140\x0c\x00\xa4\x81\x00\x00uncompresseduncompressed\n\xda\x10t \x900\x00\x0e\x00\x00\x00\x04\x00\x00\x00\x035K.\xa6\x18\x85yA\x1d5\x0e\x00\xa4\x81\x00\x00max-compressed\x00\xc0\x00\x08\xbf\x08\xae\xf2\xcc\x01s\xf8\xff\xec\x96\xe8\xc4={\x00@\x07\x00") # noqa }}} b"Rar!\x1a\x07\x00\xcf\x90s\x00\x00\r\x00\x00\x00\x00\x00\x00\x00\x14\xe7z\x00\x80#\x00\x17\x00\x00\x00\r\x00\x00\x00\x03\xc2\xb3\x96o\x00\x00\x00\x00\x1d3\x03\x00\x00\x00\x00\x00CMT\x0c\x00\x8b\xec\x8e\xef\x14\xf6\xe6h\x04\x17\xff\xcd\x0f\xffk9b\x11]^\x80\xd3dt \x90+\x00\x14\x00\x00\x00\x08\x00\x00\x00\x03\xf1\x84\x93\\\xb9]yA\x1d3\t\x00\xa4\x81\x00\x001\\sub-one\x00\xc0\x0c\x00\x8f\xec\x89\xfe.JM\x86\x82\x0c_\xfd\xfd\xd7\x11\x1a\xef@\x9eHt \x80'\x00\x0e\x00\x00\x00\x04\x00\x00\x00\x03\x9f\xa8\x17\xf8\xaf]yA\x1d3\x07\x00\xa4\x81\x00\x00one.txt\x00\x08\xbf\x08\xae\xf3\xca\x87\xfeo\xfe\xd2n\x80-Ht \x82:\x00\x18\x00\x00\x00\x10\x00\x00\x00\x03\xa86\x81\xdf\xf9fyA\x1d3\x1a\x00\xa4\x81\x00\x00\xe8\xaf\xb6\xe6\xaf\x94\xe5\xb1\x81.txt\x00\x8bh\xf6\xd4kA\\.\x00txt\x0c\x00\x8b\xec\x8e\xef\x14\xf6\xe2l\x91\x189\xff\xdf\xfe\xc2\xd3:g\x9a\x19F=cYt \x928\x00\x11\x00\x00\x00\x08\x00\x00\x00\x03\x7f\xd6\xb6\x7f\xeafyA\x1d3\x16\x00\xa4\x81\x00\x00F\xc3\xbc\xc3\x9fe.txt\x00\x01\x00F\xfc\xdfe\x00.txt\x00\xc0<D\xfe\xc8\xef\xbc\xd1\x04I?\xfd\xff\xdbF)]\xe8\xb9\xe1t \x90/\x00\x13\x00\x00\x00\x08\x00\x00\x00\x03\x1a$\x932\xc2]yA\x1d3\r\x00\xa4\x81\x00\x002\\sub-two.txt\x00\xc0\x10\x00S\xec\xcb\x7f\x8b\xa5(\x0b\x01\xcb\xef\xdf\xf6t\x89\x97z\x0eft \x90)\x00\r\x00\x00\x00\r\x00\x00\x00\x03c\x89K\xd3\xc8fyA\x140\x07\x00\xff\xa1\x00\x00symlink\x00\xc02/sub-two.txt\xeb\x86t\xe0\x90#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\xb9]yA\x140\x01\x00\xedA\x00\x001\x00\xc0\xe0Dt\xe0\x90#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\xc2]yA\x140\x01\x00\xedA\x00\x002\x00\xc0u\xa1t \x80,\x00\r\x00\x00\x00\r\x00\x00\x00\x03T\xea\x04\xca\xe6\x84yA\x140\x0c\x00\xa4\x81\x00\x00uncompresseduncompressed\n\xda\x10t \x900\x00\x0e\x00\x00\x00\x04\x00\x00\x00\x035K.\xa6\x18\x85yA\x1d5\x0e\x00\xa4\x81\x00\x00max-compressed\x00\xc0\x00\x08\xbf\x08\xae\xf2\xcc\x01s\xf8\xff\xec\x96\xe8\xc4={\x00@\x07\x00") # noqa }}}
tdata = { tdata = {
u'1': b'', '1': b'',
u'1/sub-one': b'sub-one\n', '1/sub-one': b'sub-one\n',
u'2': b'', '2': b'',
u'2/sub-two.txt': b'sub-two\n', '2/sub-two.txt': b'sub-two\n',
u'F\xfc\xdfe.txt': b'unicode\n', 'F\xfc\xdfe.txt': b'unicode\n',
u'max-compressed': b'max\n', 'max-compressed': b'max\n',
u'one.txt': b'one\n', 'one.txt': b'one\n',
u'symlink': b'2/sub-two.txt', 'symlink': b'2/sub-two.txt',
u'uncompressed': b'uncompressed\n', 'uncompressed': b'uncompressed\n',
u'\u8bf6\u6bd4\u5c41.txt': b'chinese unicode\n'} '\u8bf6\u6bd4\u5c41.txt': b'chinese unicode\n'}
def do_test(stream): def do_test(stream):
c = comment(stream) c = comment(stream)
if c != b'some comment\n': if c != 'some comment\n':
raise ValueError('Comment not read: %r != %r' % (c, b'some comment\n')) raise ValueError('Comment not read: %r != %r' % (c, b'some comment\n'))
if set(names(stream)) != { if set(names(stream)) != {
'1/sub-one', 'one.txt', '2/sub-two.txt', '诶比屁.txt', 'Füße.txt', '1/sub-one', 'one.txt', '2/sub-two.txt', '诶比屁.txt', 'Füße.txt',

10
src/calibre/web/feeds/news.py
View File

@ -9,7 +9,7 @@ __docformat__ = "restructuredtext en"
import os, time, traceback, re, sys, io import os, time, traceback, re, sys, io
from collections import defaultdict from collections import defaultdict
from contextlib import nested, closing from contextlib import closing
from calibre import (browser, __appname__, iswindows, force_unicode, from calibre import (browser, __appname__, iswindows, force_unicode,
@ -760,7 +760,7 @@ class BasicNewsRecipe(Recipe):
in index are not in weights, they are assumed to have a weight of 0. in index are not in weights, they are assumed to have a weight of 0.
''' '''
weights = defaultdict(lambda: 0, weights) weights = defaultdict(lambda: 0, weights)
index.sort(cmp=lambda x, y: cmp(weights[x], weights[y])) index.sort(key=lambda x: weights[x])
return index return index
def parse_index(self): def parse_index(self):
@ -1097,7 +1097,7 @@ class BasicNewsRecipe(Recipe):
if bn: if bn:
img = os.path.join(imgdir, 'feed_image_%d%s'%(self.image_counter, os.path.splitext(bn))) img = os.path.join(imgdir, 'feed_image_%d%s'%(self.image_counter, os.path.splitext(bn)))
try: try:
with nested(open(img, 'wb'), closing(self.browser.open(feed.image_url))) as (fi, r): with open(img, 'wb') as fi, closing(self.browser.open(feed.image_url)) as r:
fi.write(r.read()) fi.write(r.read())
self.image_counter += 1 self.image_counter += 1
feed.image_url = img feed.image_url = img
@ -1346,7 +1346,7 @@ class BasicNewsRecipe(Recipe):
with open(mpath, 'wb') as mfile: with open(mpath, 'wb') as mfile:
mfile.write(open(mu, 'rb').read()) mfile.write(open(mu, 'rb').read())
else: else:
with nested(open(mpath, 'wb'), closing(self.browser.open(mu))) as (mfile, r): with open(mpath, 'wb') as mfile, closing(self.browser.open(mu)) as r:
mfile.write(r.read()) mfile.write(r.read())
self.report_progress(1, _('Masthead image downloaded')) self.report_progress(1, _('Masthead image downloaded'))
self.prepare_masthead_image(mpath, outfile) self.prepare_masthead_image(mpath, outfile)
@ -1564,7 +1564,7 @@ class BasicNewsRecipe(Recipe):
opf.create_spine(entries) opf.create_spine(entries)
opf.set_toc(toc) opf.set_toc(toc)
with nested(open(opf_path, 'wb'), open(ncx_path, 'wb')) as (opf_file, ncx_file): with open(opf_path, 'wb') as opf_file, open(ncx_path, 'wb') as ncx_file:
opf.render(opf_file, ncx_file) opf.render(opf_file, ncx_file)
def article_downloaded(self, request, result): def article_downloaded(self, request, result):

2
src/calibre/web/fetch/simple.py
View File

@ -18,7 +18,6 @@ import threading
import time import time
import traceback import traceback
from base64 import b64decode from base64 import b64decode
from httplib import responses
from calibre import browser, relpath, unicode_path from calibre import browser, relpath, unicode_path
from calibre.constants import filesystem_encoding, iswindows from calibre.constants import filesystem_encoding, iswindows
@ -31,6 +30,7 @@ from calibre.utils.imghdr import what
from calibre.utils.logging import Log from calibre.utils.logging import Log
from calibre.web.fetch.utils import rescale_image from calibre.web.fetch.utils import rescale_image
from polyglot.builtins import unicode_type from polyglot.builtins import unicode_type
from polyglot.http_client import responses
from polyglot.urllib import ( from polyglot.urllib import (
URLError, quote, url2pathname, urljoin, urlparse, urlsplit, urlunparse, URLError, quote, url2pathname, urljoin, urlparse, urlsplit, urlunparse,
urlunsplit urlunsplit

10
src/polyglot/html_entities.py Normal file
View File

@ -0,0 +1,10 @@
#!/usr/bin/env python2
# vim:fileencoding=utf-8
# License: GPL v3 Copyright: 2019, Eli Schwartz <eschwartz@archlinux.org>
from polyglot.builtins import is_py3
if is_py3:
from html.entities import name2codepoint
else:
from htmlentitydefs import name2codepoint

22
src/polyglot/http_client.py Normal file
View File

@ -0,0 +1,22 @@
#!/usr/bin/env python2
# vim:fileencoding=utf-8
# License: GPL v3 Copyright: 2019, Eli Schwartz <eschwartz@archlinux.org>
from polyglot.builtins import is_py3
if is_py3:
from http.client import (responses, HTTPConnection, HTTPSConnection,
BAD_REQUEST, FOUND, FORBIDDEN, HTTP_VERSION_NOT_SUPPORTED,
INTERNAL_SERVER_ERROR, METHOD_NOT_ALLOWED, MOVED_PERMANENTLY,
NOT_FOUND, NOT_IMPLEMENTED, NOT_MODIFIED, OK, PARTIAL_CONTENT,
PRECONDITION_FAILED, REQUEST_ENTITY_TOO_LARGE, REQUEST_URI_TOO_LONG,
REQUESTED_RANGE_NOT_SATISFIABLE, REQUEST_TIMEOUT, SEE_OTHER,
SERVICE_UNAVAILABLE, UNAUTHORIZED, _CS_IDLE, _CS_REQ_SENT)
else:
from httplib import (responses, HTTPConnection, HTTPSConnection,
BAD_REQUEST, FOUND, FORBIDDEN, HTTP_VERSION_NOT_SUPPORTED,
INTERNAL_SERVER_ERROR, METHOD_NOT_ALLOWED, MOVED_PERMANENTLY,
NOT_FOUND, NOT_IMPLEMENTED, NOT_MODIFIED, OK, PARTIAL_CONTENT,
PRECONDITION_FAILED, REQUEST_ENTITY_TOO_LARGE, REQUEST_URI_TOO_LONG,
REQUESTED_RANGE_NOT_SATISFIABLE, REQUEST_TIMEOUT, SEE_OTHER,
SERVICE_UNAVAILABLE, UNAUTHORIZED, _CS_IDLE, _CS_REQ_SENT)

12
src/polyglot/http_cookie.py Normal file
View File

@ -0,0 +1,12 @@
#!/usr/bin/env python2
# vim:fileencoding=utf-8
# License: GPL v3 Copyright: 2019, Eli Schwartz <eschwartz@archlinux.org>
from polyglot.builtins import is_py3
if is_py3:
from http.cookies import SimpleCookie # noqa
from http.cookiejar import CookieJar, Cookie # noqa
else:
from Cookie import SimpleCookie # noqa
from cookielib import CookieJar, Cookie # noqa

10
src/polyglot/reprlib.py Executable file
View File

@ -0,0 +1,10 @@
#!/usr/bin/env python2
# vim:fileencoding=utf-8
# License: GPL v3 Copyright: 2019, Eli Schwartz <eschwartz@archlinux.org>
from polyglot.builtins import is_py3
if is_py3:
from reprlib import repr
else:
from repr import repr