Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'py3' of https://github.com/eli-schwartz/calibre

Browse Source
This commit is contained in:
Kovid Goyal 2019-03-27 05:55:10 +05:30
commit 55b39e16bb
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
63 changed files with 357 additions and 510 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
manual/custom.py
View File

@ -179,7 +179,7 @@ def generate_ebook_convert_help(preamble, app):
raw += '\n\n.. contents::\n :local:'
raw += '\n\n' + options
for pl in sorted(input_format_plugins(), key=lambda x:x.name):
for pl in sorted(input_format_plugins(), key=lambda x: x.name):
parser, plumber = create_option_parser(['ebook-convert',
'dummyi.'+sorted(pl.file_types)[0], 'dummyo.epub', '-h'], default_log)
groups = [(pl.name+ ' Options', '', g.option_list) for g in
@ -279,7 +279,7 @@ def cli_docs(app):
info(bold('creating CLI documentation...'))
documented_cmds, undocumented_cmds = get_cli_docs()
documented_cmds.sort(cmp=lambda x, y: cmp(x[0], y[0]))
documented_cmds.sort(key=lambda x: x[0])
undocumented_cmds.sort()
documented = [' '*4 + c[0] for c in documented_cmds]

4
src/calibre/__init__.py
View File

@ -240,7 +240,7 @@ def prints(*args, **kwargs):
file.write(arg)
count += len(arg)
except:
import repr as reprlib
from polyglot import reprlib
arg = reprlib.repr(arg)
file.write(arg)
count += len(arg)
@ -614,7 +614,7 @@ def entity_to_unicode(match, exceptions=[], encoding='cp1252',
return check(html5_entities[ent])
except KeyError:
pass
from htmlentitydefs import name2codepoint
from polyglot.html_entities import name2codepoint
try:
return check(my_unichr(name2codepoint[ent]))
except KeyError:

2
src/calibre/constants.py
View File

@ -168,7 +168,6 @@ class Plugins(collections.Mapping):
'icu',
'speedup',
'unicode_names',
'zlib2',
'html',
'freetype',
'imageops',
@ -184,6 +183,7 @@ class Plugins(collections.Mapping):
if not ispy3:
plugins.extend([
'monotonic',
'zlib2',
])
if iswindows:
plugins.extend(['winutil', 'wpd', 'winfonts'])

4
src/calibre/customize/profiles.py
View File

@ -233,7 +233,7 @@ input_profiles = [InputProfile, SonyReaderInput, SonyReader300Input,
HanlinV5Input, CybookG3Input, CybookOpusInput, KindleInput, IlliadInput,
IRexDR1000Input, IRexDR800Input, NookInput]
input_profiles.sort(cmp=lambda x,y:cmp(x.name.lower(), y.name.lower()))
input_profiles.sort(key=lambda x: x.name.lower())
# }}}
@ -870,4 +870,4 @@ output_profiles = [
KindlePaperWhite3Output, KindleOasisOutput
]
output_profiles.sort(cmp=lambda x,y:cmp(x.name.lower(), y.name.lower()))
output_profiles.sort(key=lambda x: x.name.lower())

4
src/calibre/customize/ui.py
View File

@ -727,9 +727,9 @@ def initialize_plugins(perf=False):
# ipython
sys.stdout, sys.stderr = ostdout, ostderr
if perf:
for x in sorted(times, key=lambda x:times[x]):
for x in sorted(times, key=lambda x: times[x]):
print('%50s: %.3f'%(x, times[x]))
_initialized_plugins.sort(cmp=lambda x,y:cmp(x.priority, y.priority), reverse=True)
_initialized_plugins.sort(key=lambda x: x.priority, reverse=True)
reread_filetype_plugins()
reread_metadata_plugins()

4
src/calibre/db/cli/cmd_list_categories.py
View File

@ -149,9 +149,7 @@ def main(opts, args, dbctx):
(not report_on or k in report_on)
]
categories.sort(
cmp=lambda x, y: cmp(x if x[0] != '#' else x[1:], y if y[0] != '#' else y[1:])
)
categories.sort(key=lambda x: x if x[0] != '#' else x[1:])
def fmtr(v):
v = v or 0

8
src/calibre/db/cli/main.py
View File

@ -4,7 +4,6 @@
from __future__ import absolute_import, division, print_function, unicode_literals
import httplib
import json
import os
import sys
@ -17,6 +16,7 @@ from calibre.utils.config import OptionParser, prefs
from calibre.utils.localization import localize_user_manual_link
from calibre.utils.lock import singleinstance
from calibre.utils.serialize import MSGPACK_MIME
from polyglot import http_client
from polyglot.urllib import urlencode, urlparse, urlunparse
COMMANDS = (
@ -191,13 +191,13 @@ class DBCtx(object):
return m.implementation(self.db.new_api, None, *args)
def interpret_http_error(self, err):
if err.code == httplib.UNAUTHORIZED:
if err.code == http_client.UNAUTHORIZED:
if self.has_credentials:
raise SystemExit('The username/password combination is incorrect')
raise SystemExit('A username and password is required to access this server')
if err.code == httplib.FORBIDDEN:
if err.code == http_client.FORBIDDEN:
raise SystemExit(err.reason)
if err.code == httplib.NOT_FOUND:
if err.code == http_client.NOT_FOUND:
raise SystemExit(err.reason)
def remote_run(self, name, m, *args):

4
src/calibre/db/tests/legacy.py
View File

@ -8,13 +8,13 @@ __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
import inspect, time, numbers
from io import BytesIO
from repr import repr
from functools import partial
from operator import itemgetter
from calibre.library.field_metadata import fm_as_dict
from calibre.db.tests.base import BaseTest
from polyglot.builtins import iteritems, range
from polyglot import reprlib
# Utils {{{
@ -32,7 +32,7 @@ class ET(object):
oldres = getattr(old, self.func_name)(*self.args, **self.kwargs)
newres = getattr(legacy, self.func_name)(*self.args, **self.kwargs)
test.assertEqual(oldres, newres, 'Equivalence test for %s with args: %s and kwargs: %s failed' % (
self.func_name, repr(self.args), repr(self.kwargs)))
self.func_name, reprlib.repr(self.args), reprlib.repr(self.kwargs)))
self.retval = newres
return newres

7
src/calibre/devices/__init__.py
View File

@ -8,7 +8,7 @@ Device drivers.
import sys, time, pprint
from functools import partial
from StringIO import StringIO
from io import BytesIO
DAY_MAP = dict(Sun=0, Mon=1, Tue=2, Wed=3, Thu=4, Fri=5, Sat=6)
MONTH_MAP = dict(Jan=1, Feb=2, Mar=3, Apr=4, May=5, Jun=6, Jul=7, Aug=8, Sep=9, Oct=10, Nov=11, Dec=12)
@ -77,13 +77,12 @@ def debug(ioreg_to_tmp=False, buf=None, plugins=None,
oldo, olde = sys.stdout, sys.stderr
if buf is None:
buf = StringIO()
buf = BytesIO()
sys.stdout = sys.stderr = buf
out = partial(prints, file=buf)
devplugins = device_plugins() if plugins is None else plugins
devplugins = list(sorted(devplugins, cmp=lambda
x,y:cmp(x.__class__.__name__, y.__class__.__name__)))
devplugins = list(sorted(devplugins, key=lambda x: x.__class__.__name__))
if plugins is None:
for d in devplugins:
try:

2
src/calibre/devices/prs505/sony_cache.py
View File

@ -321,7 +321,7 @@ class XMLCache(object):
# Only rebase ids of nodes that are immediate children of the
# record root (that way playlist/itemnodes are unaffected
items = root.xpath('child::*[@id]')
items.sort(cmp=lambda x,y:cmp(int(x.get('id')), int(y.get('id'))))
items.sort(key=lambda x: int(x.get('id')))
idmap = {}
for i, item in enumerate(items):
old = int(item.get('id'))

2
src/calibre/devices/usbms/device.py
View File

@ -537,7 +537,7 @@ class Device(DeviceConfig, DevicePlugin):
if sz > 0:
nodes.append((x.split('/')[-1], sz))
nodes.sort(cmp=lambda x, y: cmp(x[1], y[1]))
nodes.sort(key=lambda x: x[1])
if not nodes:
return node
return nodes[-1][0]

3
src/calibre/devices/winusb.py
View File

@ -995,8 +995,7 @@ def develop(): # {{{
drive_letters = set()
pprint(usb_devices)
print()
devplugins = list(sorted(device_plugins(), cmp=lambda
x,y:cmp(x.__class__.__name__, y.__class__.__name__)))
devplugins = list(sorted(device_plugins(), key=lambda x: x.__class__.__name__))
for dev in devplugins:
dev.startup()
for dev in devplugins:

4
src/calibre/ebooks/conversion/plugins/tcr_input.py
View File

@ -4,7 +4,7 @@ __license__ = 'GPL 3'
__copyright__ = '2009, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en'
from cStringIO import StringIO
from io import BytesIO
from calibre.customize.conversion import InputFormatPlugin
@ -24,7 +24,7 @@ class TCRInput(InputFormatPlugin):
raw_txt = decompress(stream)
log.info('Converting text to OEB...')
stream = StringIO(raw_txt)
stream = BytesIO(raw_txt)
from calibre.customize.ui import plugin_for_input_format

2
src/calibre/ebooks/conversion/plumber.py
View File

@ -821,7 +821,7 @@ OptionRecommendation(name='search_replace',
if not html_files:
raise ValueError(_('Could not find an e-book inside the archive'))
html_files = [(f, os.stat(f).st_size) for f in html_files]
html_files.sort(cmp=lambda x, y: cmp(x[1], y[1]))
html_files.sort(key=lambda x: x[1])
html_files = [f[0] for f in html_files]
for q in ('toc', 'index'):
for f in html_files:

67
src/calibre/ebooks/lit/from_any.py
View File

@ -1,67 +0,0 @@
from __future__ import with_statement
from __future__ import print_function
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
__docformat__ = 'restructuredtext en'
'''
Convert any ebook format to LIT.
'''
import sys, os, glob, logging
from calibre.ebooks.epub.from_any import any2epub, formats, USAGE
from calibre.ebooks.epub import config as common_config
from calibre.ptempfile import TemporaryDirectory
from calibre.ebooks.lit.writer import oeb2lit
def config(defaults=None):
c = common_config(defaults=defaults, name='lit')
return c
def option_parser(usage=USAGE):
return config().option_parser(usage=usage%('LIT', formats()))
def any2lit(opts, path):
ext = os.path.splitext(path)[1]
if not ext:
raise ValueError('Unknown file type: '+path)
ext = ext.lower()[1:]
if opts.output is None:
opts.output = os.path.splitext(os.path.basename(path))[0]+'.lit'
opts.output = os.path.abspath(opts.output)
orig_output = opts.output
with TemporaryDirectory('_any2lit') as tdir:
oebdir = os.path.join(tdir, 'oeb')
os.mkdir(oebdir)
opts.output = os.path.join(tdir, 'dummy.epub')
opts.profile = 'None'
orig_bfs = opts.base_font_size2
opts.base_font_size2 = 0
any2epub(opts, path, create_epub=False, oeb_cover=True, extract_to=oebdir)
opts.base_font_size2 = orig_bfs
opf = glob.glob(os.path.join(oebdir, '*.opf'))[0]
opts.output = orig_output
logging.getLogger('html2epub').info(_('Creating LIT file from EPUB...'))
oeb2lit(opts, opf)
def main(args=sys.argv):
parser = option_parser()
opts, args = parser.parse_args(args)
if len(args) < 2:
parser.print_help()
print('No input file specified.')
return 1
any2lit(opts, args[1])
return 0
if __name__ == '__main__':
sys.exit(main())

3
src/calibre/ebooks/lit/writer.py
View File

@ -670,8 +670,7 @@ class LitWriter(object):
def _build_dchunks(self):
ddata = []
directory = list(self._directory)
directory.sort(cmp=lambda x, y:
cmp(x.name.lower(), y.name.lower()))
directory.sort(key=lambda x: x.name.lower())
qrn = 1 + (1 << 2)
dchunk = io.BytesIO()
dcount = 0

128
src/calibre/ebooks/lrf/html/convert_to.py
View File

@ -1,128 +0,0 @@
from __future__ import print_function
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
import sys, logging, os
from calibre import setup_cli_handlers
from calibre.utils.config import OptionParser
from calibre.ebooks import ConversionError
from calibre.ebooks.lrf.meta import get_metadata
from calibre.ebooks.lrf.lrfparser import LRFDocument
from calibre.ebooks.metadata.opf import OPFCreator
from calibre.ebooks.lrf.objects import PageAttr, BlockAttr, TextAttr
from calibre.ebooks.lrf.pylrs.pylrs import TextStyle
class BlockStyle(object):
def __init__(self, ba):
self.ba = ba
def __str__(self):
ans = '.'+str(self.ba.id)+' {\n'
if hasattr(self.ba, 'sidemargin'):
margin = str(self.ba.sidemargin) + 'px'
ans += '\tmargin-left: %(m)s; margin-right: %(m)s;\n'%dict(m=margin)
if hasattr(self.ba, 'topskip'):
ans += '\tmargin-top: %dpx;\n'%(self.ba.topskip,)
if hasattr(self.ba, 'footskip'):
ans += '\tmargin-bottom: %dpx;\n'%(self.ba.footskip,)
if hasattr(self.ba, 'framewidth'):
ans += '\tborder-width: %dpx;\n'%(self.ba.framewidth,)
ans += '\tborder-style: solid;\n'
if hasattr(self.ba, 'framecolor'):
if self.ba.framecolor.a < 255:
ans += '\tborder-color: %s;\n'%(self.ba.framecolor.to_html())
if hasattr(self.ba, 'bgcolor'):
if self.ba.bgcolor.a < 255:
ans += '\tbackground-color: %s;\n'%(self.ba.bgcolor.to_html())
# TODO: Fixed size blocks
return ans + '}\n'
class LRFConverter(object):
def __init__(self, document, opts, logger):
self.lrf = document
self.opts = opts
self.output_dir = opts.out
self.logger = logger
logger.info('Parsing LRF...')
self.lrf.parse()
self.create_metadata()
self.create_styles()
def create_metadata(self):
self.logger.info('Reading metadata...')
mi = get_metadata(self.lrf)
self.opf = OPFCreator(self.output_dir, mi)
def create_page_styles(self):
self.page_css = ''
for obj in self.lrf.objects.values():
if isinstance(obj, PageAttr):
selector = 'body.'+str(obj.id)
self.page_css = selector + ' {\n'
# TODO: Headers and footers
self.page_css += '}\n'
def create_block_styles(self):
self.block_css = ''
for obj in self.lrf.objects.values():
if isinstance(obj, BlockAttr):
self.block_css += str(BlockStyle(obj))
def create_text_styles(self):
self.text_css = ''
for obj in self.lrf.objects.values():
if isinstance(obj, TextAttr):
self.text_css += str(TextStyle(obj))
print(self.text_css)
def create_styles(self):
self.logger.info('Creating CSS stylesheet...')
self.create_page_styles()
self.create_block_styles()
def option_parser():
parser = OptionParser(usage='%prog book.lrf')
parser.add_option('--output-dir', '-o', default=None, help=(
'Output directory in which to store created HTML files. If it does not exist, it is created. By default the current directory is used.'), dest='out')
parser.add_option('--verbose', default=False, action='store_true', dest='verbose')
return parser
def process_file(lrfpath, opts, logger=None):
if logger is None:
level = logging.DEBUG if opts.verbose else logging.INFO
logger = logging.getLogger('lrf2html')
setup_cli_handlers(logger, level)
if opts.out is None:
opts.out = os.getcwdu()
else:
opts.out = os.path.abspath(opts.out)
if not os.path.isdir(opts.out):
raise ConversionError(opts.out + ' is not a directory')
if not os.path.exists(opts.out):
os.makedirs(opts.out)
document = LRFDocument(open(lrfpath, 'rb'))
LRFConverter(document, opts, logger)
def main(args=sys.argv):
parser = option_parser()
opts, args = parser.parse_args(args)
if len(args) != 2:
parser.print_help()
return 1
process_file(args[1], opts)
return 0
if __name__ == '__main__':
sys.exit(main())

2
src/calibre/ebooks/metadata/lit.py
View File

@ -32,7 +32,7 @@ def get_metadata(stream):
except:
pass
break
covers.sort(cmp=lambda x, y:cmp(len(x[0]), len(y[0])), reverse=True)
covers.sort(key=lambda x: len(x[0]), reverse=True)
idx = 0
if len(covers) > 1:
if covers[1][1] == covers[0][1]+'-standard':

4
src/calibre/ebooks/metadata/meta.py
View File

@ -41,8 +41,7 @@ def metadata_from_formats(formats, force_read_metadata=False, pattern=None):
def _metadata_from_formats(formats, force_read_metadata=False, pattern=None):
mi = MetaInformation(None, None)
formats.sort(cmp=lambda x,y: cmp(METADATA_PRIORITIES[path_to_ext(x)],
METADATA_PRIORITIES[path_to_ext(y)]))
formats.sort(key=lambda x: METADATA_PRIORITIES[path_to_ext(x)])
extensions = list(map(path_to_ext, formats))
if 'opf' in extensions:
opf = formats[extensions.index('opf')]
@ -241,4 +240,3 @@ def forked_read_metadata(path, tdir):
opf = metadata_to_opf(mi, default_lang='und')
with lopen(os.path.join(tdir, 'metadata.opf'), 'wb') as f:
f.write(opf)

3
src/calibre/ebooks/oeb/transforms/guide.py
View File

@ -28,7 +28,7 @@ class Clean(object):
else:
covers.append([self.oeb.guide[x], len(item.data)])
covers.sort(cmp=lambda x,y:cmp(x[1], y[1]), reverse=True)
covers.sort(key=lambda x: x[1], reverse=True)
if covers:
ref = covers[0][0]
if len(covers) > 1:
@ -53,4 +53,3 @@ class Clean(object):
if item.title and item.title.lower() == 'start':
continue
self.oeb.guide.remove(x)

21
src/calibre/ebooks/pdf/from_comic.py
View File

@ -1,21 +0,0 @@
from __future__ import with_statement
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
__docformat__ = 'restructuredtext en'
'Convert a comic in CBR/CBZ format to pdf'
import sys
from functools import partial
from calibre.ebooks.lrf.comic.convert_from import do_convert, option_parser, config, main as _main
convert = partial(do_convert, output_format='pdf')
main = partial(_main, output_format='pdf')
if __name__ == '__main__':
sys.exit(main())
if False:
option_parser
config

10
src/calibre/ebooks/pdf/reflow.py
View File

@ -169,7 +169,7 @@ class Column(object):
self._post_add()
def _post_add(self):
self.elements.sort(cmp=lambda x,y:cmp(x.bottom,y.bottom))
self.elements.sort(key=lambda x: x.bottom)
self.top = self.elements[0].top
self.bottom = self.elements[-1].bottom
self.left, self.right = sys.maxint, 0
@ -260,7 +260,7 @@ class Region(object):
def add(self, columns):
if not self.columns:
for x in sorted(columns, cmp=lambda x,y: cmp(x.left, y.left)):
for x in sorted(columns, key=lambda x: x.left):
self.columns.append(x)
else:
for i in range(len(columns)):
@ -458,7 +458,7 @@ class Page(object):
self.elements = list(self.texts)
for img in page.xpath('descendant::img'):
self.elements.append(Image(img, self.opts, self.log, idc))
self.elements.sort(cmp=lambda x,y:cmp(x.top, y.top))
self.elements.sort(key=lambda x: x.top)
def coalesce_fragments(self):
@ -580,7 +580,7 @@ class Page(object):
def sort_into_columns(self, elem, neighbors):
neighbors.add(elem)
neighbors = sorted(neighbors, cmp=lambda x,y:cmp(x.left, y.left))
neighbors = sorted(neighbors, key=lambda x: x.left)
if self.opts.verbose > 3:
self.log.debug('Neighbors:', [x.to_html() for x in neighbors])
columns = [Column()]
@ -595,7 +595,7 @@ class Page(object):
if not added:
columns.append(Column())
columns[-1].add(x)
columns.sort(cmp=lambda x,y:cmp(x.left, y.left))
columns.sort(key=lambda x: x.left)
return columns
def find_elements_in_row_of(self, x):

3
src/calibre/gui2/actions/choose_library.py
View File

@ -51,8 +51,7 @@ class LibraryUsageStats(object): # {{{
def write_stats(self):
locs = list(self.stats.keys())
locs.sort(cmp=lambda x, y: cmp(self.stats[x], self.stats[y]),
reverse=True)
locs.sort(key=lambda x: self.stats[x], reverse=True)
for key in locs[500:]:
self.stats.pop(key)
gprefs.set('library_usage_stats', self.stats)

2
src/calibre/gui2/dialogs/catalog.py
View File

@ -93,7 +93,7 @@ class Catalog(QDialog, Ui_Dialog):
else:
info("No dynamic tab resources found for %s" % name)
self.widgets = sorted(self.widgets, cmp=lambda x,y:cmp(x.TITLE, y.TITLE))
self.widgets = sorted(self.widgets, key=lambda x: x.TITLE)
# Generate a sorted list of installed catalog formats/sync_enabled pairs
fmts = sorted([x[0] for x in self.fmts])

7
src/calibre/gui2/icon_theme.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import os, errno, json, importlib, math, httplib, bz2, shutil, sys
import os, errno, json, importlib, math, bz2, shutil, sys
from itertools import count
from io import BytesIO
from threading import Thread, Event
@ -35,8 +35,9 @@ from calibre.utils.img import image_from_data, Canvas, optimize_png, optimize_jp
from calibre.utils.zipfile import ZipFile, ZIP_STORED
from calibre.utils.filenames import atomic_rename
from lzma.xz import compress, decompress
from polyglot.queue import Queue, Empty
from polyglot.builtins import iteritems, map, range, reraise
from polyglot import http_client
from polyglot.queue import Queue, Empty
IMAGE_EXTENSIONS = {'png', 'jpg', 'jpeg'}
THEME_COVER = 'icon-theme-cover.jpg'
@ -439,7 +440,7 @@ def download_cover(cover_url, etag=None, cached=b''):
etag = response.getheader('ETag', None) or None
return cached, etag
except HTTPError as e:
if etag and e.code == httplib.NOT_MODIFIED:
if etag and e.code == http_client.NOT_MODIFIED:
return cached, etag
raise

2
src/calibre/gui2/library/models.py
View File

@ -323,7 +323,7 @@ class BooksModel(QAbstractTableModel): # {{{
return 100000
return self.db.field_metadata[name]['rec_index']
self.column_map.sort(cmp=lambda x,y: cmp(col_idx(x), col_idx(y)))
self.column_map.sort(key=lambda x: col_idx(x))
for col in self.column_map:
if col in self.orig_headers:
self.headers[col] = self.orig_headers[col]

2
src/calibre/gui2/library/views.py
View File

@ -1031,7 +1031,7 @@ class BooksView(QTableView): # {{{
h.visualIndex(x) > -1]
if not pairs:
pairs = [(0, 0)]
pairs.sort(cmp=lambda x,y:cmp(x[1], y[1]))
pairs.sort(key=lambda x: x[1])
i = pairs[0][0]
index = self.model().index(row, i)
if for_sync:

10
src/calibre/gui2/preferences/columns.py
View File

@ -69,7 +69,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
state = self.columns_state(defaults)
self.hidden_cols = state['hidden_columns']
positions = state['column_positions']
colmap.sort(cmp=lambda x,y: cmp(positions[x], positions[y]))
colmap.sort(key=lambda x: positions[x])
self.opt_columns.clear()
db = model.db
@ -248,12 +248,10 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
if 'ondevice' in hidden_cols:
hidden_cols.remove('ondevice')
def col_pos(x, y):
xidx = config_cols.index(x) if x in config_cols else sys.maxint
yidx = config_cols.index(y) if y in config_cols else sys.maxint
return cmp(xidx, yidx)
def col_pos(x):
return config_cols.index(x) if x in config_cols else sys.maxint
positions = {}
for i, col in enumerate((sorted(model.column_map, cmp=col_pos))):
for i, col in enumerate((sorted(model.column_map, key=col_pos))):
positions[col] = i
state = {'hidden_columns': hidden_cols, 'column_positions':positions}
self.gui.library_view.apply_state(state)

2
src/calibre/gui2/preferences/look_feel.py
View File

@ -456,7 +456,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
if l != lang]
if lang != 'en':
items.append(('en', get_esc_lang('en')))
items.sort(cmp=lambda x, y: cmp(x[1].lower(), y[1].lower()))
items.sort(key=lambda x: x[1].lower())
choices = [(y, x) for x, y in items]
# Default language is the autodetected one
choices = [(get_language(lang), lang)] + choices

4
src/calibre/gui2/preferences/main.py
View File

@ -171,7 +171,7 @@ class Browser(QScrollArea): # {{{
self.category_names = category_names
categories = list(category_map.keys())
categories.sort(cmp=lambda x, y: cmp(category_map[x], category_map[y]))
categories.sort(key=lambda x: category_map[x])
self.category_map = OrderedDict()
for c in categories:
@ -181,7 +181,7 @@ class Browser(QScrollArea): # {{{
self.category_map[plugin.category].append(plugin)
for plugins in self.category_map.values():
plugins.sort(cmp=lambda x, y: cmp(x.name_order, y.name_order))
plugins.sort(key=lambda x: x.name_order)
self.widgets = []
self._layout = QVBoxLayout()

13
src/calibre/gui2/preferences/plugboard.py
View File

@ -34,17 +34,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
self.db = gui.library_view.model().db
def initialize(self):
def field_cmp(x, y):
if x.startswith('#'):
if y.startswith('#'):
return cmp(x.lower(), y.lower())
else:
return 1
elif y.startswith('#'):
return -1
else:
return cmp(x.lower(), y.lower())
ConfigWidgetBase.initialize(self)
self.current_plugboards = copy.deepcopy(self.db.prefs.get('plugboards',{}))
@ -73,7 +62,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
if n not in self.disabled_devices:
self.disabled_devices.append(n)
self.devices.sort(cmp=lambda x, y: cmp(x.lower(), y.lower()))
self.devices.sort(key=lambda x: x.lower())
self.devices.insert(1, plugboard_save_to_disk_value)
self.devices.insert(1, plugboard_content_server_value)
self.device_to_formats_map[plugboard_content_server_value] = \

2
src/calibre/gui2/preferences/plugins.py
View File

@ -61,7 +61,7 @@ class PluginModel(QAbstractItemModel, AdaptSQP): # {{{
self.categories = sorted(self._data.keys())
for plugins in self._data.values():
plugins.sort(cmp=lambda x, y: cmp(x.name.lower(), y.name.lower()))
plugins.sort(key=lambda x: x.name.lower())
def universal_set(self):
ans = set([])

4
src/calibre/gui2/wizard/__init__.py
View File

@ -427,7 +427,7 @@ def get_manufacturers():
def get_devices_of(manufacturer):
ans = [d for d in get_devices() if d.manufacturer == manufacturer]
return sorted(ans, cmp=lambda x,y:cmp(x.name, y.name))
return sorted(ans, key=lambda x: x.name)
class ManufacturerModel(QAbstractListModel):
@ -682,7 +682,7 @@ class LibraryPage(QWizardPage, LibraryUI):
if l != lang]
if lang != 'en':
items.append(('en', get_esc_lang('en')))
items.sort(cmp=lambda x, y: cmp(x[1], y[1]))
items.sort(key=lambda x: x[1])
for item in items:
self.language.addItem(item[1], (item[0]))
self.language.blockSignals(False)

5
src/calibre/library/catalogs/bibtex.py
View File

@ -7,7 +7,6 @@ __docformat__ = 'restructuredtext en'
import re, codecs, os, numbers
from collections import namedtuple
from types import StringType, UnicodeType
from calibre import (strftime)
from calibre.customize import CatalogPlugin
@ -15,7 +14,7 @@ from calibre.library.catalogs import FIELDS, TEMPLATE_ALLOWED_FIELDS
from calibre.customize.conversion import DummyReporter
from calibre.constants import preferred_encoding
from calibre.ebooks.metadata import format_isbn
from polyglot.builtins import string_or_bytes
from polyglot.builtins import string_or_bytes, unicode_type
class BIBTEX(CatalogPlugin):
@ -351,7 +350,7 @@ class BIBTEX(CatalogPlugin):
bibtexc.ascii_bibtex = True
# Check citation choice and go to default in case of bad CLI
if isinstance(opts.impcit, (StringType, UnicodeType)) :
if isinstance(opts.impcit, (str, unicode_type)) :
if opts.impcit == 'False' :
citation_bibtex= False
elif opts.impcit == 'True' :

4
src/calibre/library/custom_columns.py
View File

@ -216,7 +216,7 @@ class CustomColumns(object):
if data['is_multiple'] and data['datatype'] == 'text':
ans = ans.split(data['multiple_seps']['cache_to_list']) if ans else []
if data['display'].get('sort_alpha', False):
ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower()))
ans.sort(key=lambda x:x.lower())
return ans
def get_custom_extra(self, idx, label=None, num=None, index_is_id=False):
@ -243,7 +243,7 @@ class CustomColumns(object):
if data['is_multiple'] and data['datatype'] == 'text':
ans = ans.split(data['multiple_seps']['cache_to_list']) if ans else []
if data['display'].get('sort_alpha', False):
ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower()))
ans.sort(key=lambda x: x.lower())
if data['datatype'] != 'series':
return (ans, None)
ign,lt = self.custom_table_names(data['num'])

2
src/calibre/library/database.py
View File

@ -1018,7 +1018,7 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE;
if not ans:
return []
ans = [id[0] for id in ans]
ans.sort(cmp=lambda x, y: cmp(self.series_index(x, True), self.series_index(y, True)))
ans.sort(key=lambda x: self.series_index(x, True))
return ans
def books_in_series_of(self, index, index_is_id=False):

2
src/calibre/library/sqlite.py
View File

@ -8,7 +8,6 @@ Wrapper for multi-threaded access to a single sqlite database connection. Serial
all calls.
'''
import sqlite3 as sqlite, traceback, time, uuid, sys, os
import repr as reprlib
from sqlite3 import IntegrityError, OperationalError
from threading import Thread
from threading import RLock
@ -22,6 +21,7 @@ from calibre.constants import iswindows, DEBUG, plugins
from calibre.utils.icu import sort_key
from calibre import prints
from polyglot.builtins import unicode_type
from polyglot import reprlib
from polyglot.queue import Queue
from dateutil.tz import tzoffset

4
src/calibre/spell/dictionary.py
View File

@ -54,7 +54,7 @@ def builtin_dictionaries():
if _builtins is None:
dics = []
for lc in glob.glob(os.path.join(P('dictionaries', allow_user_override=False), '*/locales')):
locales = filter(None, open(lc, 'rb').read().decode('utf-8').splitlines())
locales = list(filter(None, open(lc, 'rb').read().decode('utf-8').splitlines()))
locale = locales[0]
base = os.path.dirname(lc)
dics.append(Dictionary(
@ -69,7 +69,7 @@ def custom_dictionaries(reread=False):
if _custom is None or reread:
dics = []
for lc in glob.glob(os.path.join(config_dir, 'dictionaries', '*/locales')):
locales = filter(None, open(lc, 'rb').read().decode('utf-8').splitlines())
locales = list(filter(None, open(lc, 'rb').read().decode('utf-8').splitlines()))
try:
name, locale, locales = locales[0], locales[1], locales[1:]
except IndexError:

21
src/calibre/srv/auth.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import binascii, os, random, struct, base64, httplib
import binascii, os, random, struct, base64
from collections import OrderedDict
from hashlib import md5, sha256
from itertools import permutations
@ -16,6 +16,7 @@ from calibre.srv.errors import HTTPAuthRequired, HTTPSimpleResponse, HTTPForbidd
from calibre.srv.http_request import parse_uri
from calibre.srv.utils import parse_http_dict, encode_path
from calibre.utils.monotonic import monotonic
from polyglot import http_client
MAX_AGE_SECONDS = 3600
nonce_counter, nonce_counter_lock = 0, Lock()
@ -133,19 +134,19 @@ class DigestAuth(object): # {{{
self.nonce_count = data.get('nc')
if self.algorithm not in self.valid_algorithms:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported digest algorithm')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported digest algorithm')
if not (self.username and self.realm and self.nonce and self.uri and self.response):
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Digest algorithm required fields missing')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Digest algorithm required fields missing')
if self.qop:
if self.qop not in self.valid_qops:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported digest qop')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported digest qop')
if not (self.cnonce and self.nonce_count):
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'qop present, but cnonce and nonce_count absent')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'qop present, but cnonce and nonce_count absent')
else:
if self.cnonce or self.nonce_count:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'qop missing')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'qop missing')
def H(self, val):
return md5_hex(val)
@ -201,7 +202,7 @@ class DigestAuth(object): # {{{
if log is not None:
log.warn('Authorization URI mismatch: %s != %s from client: %s' % (
data.path, path, data.remote_addr))
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The uri in the Request Line and the Authorization header do not match')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The uri in the Request Line and the Authorization header do not match')
return self.response is not None and data.path == path and self.request_digest(pw, data) == self.response
# }}}
@ -290,16 +291,16 @@ class AuthController(object):
try:
un, pw = base64_decode(rest.strip()).partition(':')[::2]
except ValueError:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The username or password contained non-UTF8 encoded characters')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The username or password contained non-UTF8 encoded characters')
if not un or not pw:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The username or password was empty')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The username or password was empty')
if self.check(un, pw):
data.username = un
return
log_msg = 'Failed login attempt from: %s' % data.remote_addr
self.ban_list.failed(ban_key)
else:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported authentication method')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported authentication method')
if self.prefer_basic_auth:
raise HTTPAuthRequired('Basic realm="%s"' % self.realm, log=log_msg)

14
src/calibre/srv/errors.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib
from polyglot import http_client
class JobQueueFull(Exception):
@ -30,38 +30,38 @@ class HTTPSimpleResponse(Exception):
class HTTPRedirect(HTTPSimpleResponse):
def __init__(self, location, http_code=httplib.MOVED_PERMANENTLY, http_message='', close_connection=False):
def __init__(self, location, http_code=http_client.MOVED_PERMANENTLY, http_message='', close_connection=False):
HTTPSimpleResponse.__init__(self, http_code, http_message, close_connection, location)
class HTTPNotFound(HTTPSimpleResponse):
def __init__(self, http_message='', close_connection=False):
HTTPSimpleResponse.__init__(self, httplib.NOT_FOUND, http_message, close_connection)
HTTPSimpleResponse.__init__(self, http_client.NOT_FOUND, http_message, close_connection)
class HTTPAuthRequired(HTTPSimpleResponse):
def __init__(self, payload, log=None):
HTTPSimpleResponse.__init__(self, httplib.UNAUTHORIZED, authenticate=payload, log=log)
HTTPSimpleResponse.__init__(self, http_client.UNAUTHORIZED, authenticate=payload, log=log)
class HTTPBadRequest(HTTPSimpleResponse):
def __init__(self, message, close_connection=False):
HTTPSimpleResponse.__init__(self, httplib.BAD_REQUEST, message, close_connection)
HTTPSimpleResponse.__init__(self, http_client.BAD_REQUEST, message, close_connection)
class HTTPForbidden(HTTPSimpleResponse):
def __init__(self, http_message='', close_connection=True, log=None):
HTTPSimpleResponse.__init__(self, httplib.FORBIDDEN, http_message, close_connection, log=log)
HTTPSimpleResponse.__init__(self, http_client.FORBIDDEN, http_message, close_connection, log=log)
class HTTPInternalServerError(HTTPSimpleResponse):
def __init__(self, http_message='', close_connection=True, log=None):
HTTPSimpleResponse.__init__(self, httplib.INTERNAL_SERVER_ERROR, http_message, close_connection, log=log)
HTTPSimpleResponse.__init__(self, http_client.INTERNAL_SERVER_ERROR, http_message, close_connection, log=log)
class BookNotFound(HTTPNotFound):

43
src/calibre/srv/http_request.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import re, httplib, repr as reprlib
import re
from io import BytesIO, DEFAULT_BUFFER_SIZE
from calibre import as_unicode, force_unicode
@ -14,6 +14,7 @@ from calibre.ptempfile import SpooledTemporaryFile
from calibre.srv.errors import HTTPSimpleResponse
from calibre.srv.loop import Connection, READ, WRITE
from calibre.srv.utils import MultiDict, HTTP1, HTTP11, Accumulator
from polyglot import http_client, reprlib
from polyglot.urllib import unquote
protocol_map = {(1, 0):HTTP1, (1, 1):HTTP11}
@ -68,29 +69,29 @@ def parse_request_uri(uri):
def parse_uri(uri, parse_query=True):
scheme, authority, path = parse_request_uri(uri)
if path is None:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, "No path component")
raise HTTPSimpleResponse(http_client.BAD_REQUEST, "No path component")
if b'#' in path:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, "Illegal #fragment in Request-URI.")
raise HTTPSimpleResponse(http_client.BAD_REQUEST, "Illegal #fragment in Request-URI.")
if scheme:
try:
scheme = scheme.decode('ascii')
except ValueError:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Un-decodeable scheme')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Un-decodeable scheme')
path, qs = path.partition(b'?')[::2]
if parse_query:
try:
query = MultiDict.create_from_query_string(qs)
except Exception:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unparseable query string')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unparseable query string')
else:
query = None
try:
path = '%2F'.join(unquote(x).decode('utf-8') for x in quoted_slash.split(path))
except ValueError as e:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, as_unicode(e))
raise HTTPSimpleResponse(http_client.BAD_REQUEST, as_unicode(e))
path = tuple(filter(None, (x.replace('%2F', '/') for x in path.split('/'))))
return scheme, path, query
@ -233,7 +234,7 @@ class HTTPRequest(Connection):
if line.endswith(b'\n'):
line = buf.getvalue()
if not line.endswith(b'\r\n'):
self.simple_response(httplib.BAD_REQUEST, 'HTTP requires CRLF line terminators')
self.simple_response(http_client.BAD_REQUEST, 'HTTP requires CRLF line terminators')
return
return line
if not line:
@ -247,7 +248,7 @@ class HTTPRequest(Connection):
self.forwarded_for = None
self.path = self.query = None
self.close_after_response = False
self.header_line_too_long_error_code = httplib.REQUEST_URI_TOO_LONG
self.header_line_too_long_error_code = http_client.REQUEST_URI_TOO_LONG
self.response_started = False
self.set_state(READ, self.parse_request_line, Accumulator(), first=True)
@ -260,28 +261,28 @@ class HTTPRequest(Connection):
# Ignore a single leading empty line, as per RFC 2616 sec 4.1
if first:
return self.set_state(READ, self.parse_request_line, Accumulator())
return self.simple_response(httplib.BAD_REQUEST, 'Multiple leading empty lines not allowed')
return self.simple_response(http_client.BAD_REQUEST, 'Multiple leading empty lines not allowed')
try:
method, uri, req_protocol = line.strip().split(b' ', 2)
rp = int(req_protocol[5]), int(req_protocol[7])
self.method = method.decode('ascii').upper()
except Exception:
return self.simple_response(httplib.BAD_REQUEST, "Malformed Request-Line")
return self.simple_response(http_client.BAD_REQUEST, "Malformed Request-Line")
if self.method not in HTTP_METHODS:
return self.simple_response(httplib.BAD_REQUEST, "Unknown HTTP method")
return self.simple_response(http_client.BAD_REQUEST, "Unknown HTTP method")
try:
self.request_protocol = protocol_map[rp]
except KeyError:
return self.simple_response(httplib.HTTP_VERSION_NOT_SUPPORTED)
return self.simple_response(http_client.HTTP_VERSION_NOT_SUPPORTED)
self.response_protocol = protocol_map[min((1, 1), rp)]
try:
self.scheme, self.path, self.query = parse_uri(uri)
except HTTPSimpleResponse as e:
return self.simple_response(e.http_code, e.message, close_after_response=False)
self.header_line_too_long_error_code = httplib.REQUEST_ENTITY_TOO_LARGE
self.header_line_too_long_error_code = http_client.REQUEST_ENTITY_TOO_LARGE
self.set_state(READ, self.parse_header_line, HTTPHeaderParser(), Accumulator())
# }}}
@ -299,7 +300,7 @@ class HTTPRequest(Connection):
try:
parser(line)
except ValueError:
self.simple_response(httplib.BAD_REQUEST, 'Failed to parse header line')
self.simple_response(http_client.BAD_REQUEST, 'Failed to parse header line')
return
if parser.finished:
self.finalize_headers(parser.hdict)
@ -307,7 +308,7 @@ class HTTPRequest(Connection):
def finalize_headers(self, inheaders):
request_content_length = int(inheaders.get('Content-Length', 0))
if request_content_length > self.max_request_body_size:
return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE,
return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE,
"The entity sent with the request exceeds the maximum "
"allowed bytes (%d)." % self.max_request_body_size)
# Persistent connection support
@ -334,7 +335,7 @@ class HTTPRequest(Connection):
else:
# Note that, even if we see "chunked", we must reject
# if there is an extension we don't recognize.
return self.simple_response(httplib.NOT_IMPLEMENTED, "Unknown transfer encoding: %r" % enc)
return self.simple_response(http_client.NOT_IMPLEMENTED, "Unknown transfer encoding: %r" % enc)
if inheaders.get("Expect", '').lower() == "100-continue":
buf = BytesIO((HTTP11 + " 100 Continue\r\n\r\n").encode('ascii'))
@ -369,9 +370,9 @@ class HTTPRequest(Connection):
try:
chunk_size = int(line.strip(), 16)
except Exception:
return self.simple_response(httplib.BAD_REQUEST, '%s is not a valid chunk size' % reprlib.repr(line.strip()))
return self.simple_response(http_client.BAD_REQUEST, '%s is not a valid chunk size' % reprlib.repr(line.strip()))
if bytes_read[0] + chunk_size + 2 > self.max_request_body_size:
return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE,
return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE,
'Chunked request is larger than %d bytes' % self.max_request_body_size)
if chunk_size == 0:
self.set_state(READ, self.read_chunk_separator, inheaders, Accumulator(), buf, bytes_read, last=True)
@ -389,10 +390,10 @@ class HTTPRequest(Connection):
if line is None:
return
if line != b'\r\n':
return self.simple_response(httplib.BAD_REQUEST, 'Chunk does not have trailing CRLF')
return self.simple_response(http_client.BAD_REQUEST, 'Chunk does not have trailing CRLF')
bytes_read[0] += len(line)
if bytes_read[0] > self.max_request_body_size:
return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE,
return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE,
'Chunked request is larger than %d bytes' % self.max_request_body_size)
if last:
self.prepare_response(inheaders, buf)
@ -402,7 +403,7 @@ class HTTPRequest(Connection):
def handle_timeout(self):
if self.response_started:
return False
self.simple_response(httplib.REQUEST_TIMEOUT)
self.simple_response(http_client.REQUEST_TIMEOUT)
return True
def write(self, buf, end=None):

43
src/calibre/srv/http_response.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import os, httplib, hashlib, uuid, struct, repr as reprlib
import os, hashlib, uuid, struct
from collections import namedtuple
from io import BytesIO, DEFAULT_BUFFER_SIZE
from itertools import chain, repeat
@ -26,9 +26,10 @@ from calibre.srv.utils import (
sort_q_values, get_translator_for_lang, Cookie, fast_now_strftime)
from calibre.utils.speedups import ReadOnlyFileBuffer
from calibre.utils.monotonic import monotonic
from polyglot import http_client, reprlib
Range = namedtuple('Range', 'start stop size')
MULTIPART_SEPARATOR = uuid.uuid4().hex.decode('ascii')
MULTIPART_SEPARATOR = uuid.uuid4().hex
COMPRESSIBLE_TYPES = {'application/json', 'application/javascript', 'application/xml', 'application/oebps-package+xml'}
if is_py3:
import zlib
@ -226,7 +227,7 @@ class RequestData(object): # {{{
self.remote_addr, self.remote_port, self.is_local_connection = remote_addr, remote_port, is_local_connection
self.forwarded_for = forwarded_for
self.opts = opts
self.status_code = httplib.OK
self.status_code = http_client.OK
self.outcookie = Cookie()
self.lang_code = self.gettext_func = self.ngettext_func = None
self.set_translator(self.get_preferred_language())
@ -402,16 +403,16 @@ class HTTPConnection(HTTPRequest):
if self.response_protocol is HTTP1:
# HTTP/1.0 has no 413/414/303 codes
status_code = {
httplib.REQUEST_ENTITY_TOO_LARGE:httplib.BAD_REQUEST,
httplib.REQUEST_URI_TOO_LONG:httplib.BAD_REQUEST,
httplib.SEE_OTHER:httplib.FOUND
http_client.REQUEST_ENTITY_TOO_LARGE:http_client.BAD_REQUEST,
http_client.REQUEST_URI_TOO_LONG:http_client.BAD_REQUEST,
http_client.SEE_OTHER:http_client.FOUND
}.get(status_code, status_code)
self.close_after_response = close_after_response
msg = msg.encode('utf-8')
ct = 'http' if self.method == 'TRACE' else 'plain'
buf = [
'%s %d %s' % (self.response_protocol, status_code, httplib.responses[status_code]),
'%s %d %s' % (self.response_protocol, status_code, http_client.responses[status_code]),
"Content-Length: %s" % len(msg),
"Content-Type: text/%s; charset=UTF-8" % ct,
"Date: " + http_date(),
@ -432,7 +433,7 @@ class HTTPConnection(HTTPRequest):
def prepare_response(self, inheaders, request_body_file):
if self.method == 'TRACE':
msg = force_unicode(self.request_line, 'utf-8') + '\n' + inheaders.pretty()
return self.simple_response(httplib.OK, msg, close_after_response=False)
return self.simple_response(http_client.OK, msg, close_after_response=False)
request_body_file.seek(0)
outheaders = MultiDict()
data = RequestData(
@ -449,28 +450,28 @@ class HTTPConnection(HTTPRequest):
def send_range_not_satisfiable(self, content_length):
buf = [
'%s %d %s' % (self.response_protocol, httplib.REQUESTED_RANGE_NOT_SATISFIABLE, httplib.responses[httplib.REQUESTED_RANGE_NOT_SATISFIABLE]),
'%s %d %s' % (self.response_protocol, http_client.REQUESTED_RANGE_NOT_SATISFIABLE, http_client.responses[http_client.REQUESTED_RANGE_NOT_SATISFIABLE]),
"Date: " + http_date(),
"Content-Range: bytes */%d" % content_length,
]
response_data = header_list_to_file(buf)
self.log_access(status_code=httplib.REQUESTED_RANGE_NOT_SATISFIABLE, response_size=response_data.sz)
self.log_access(status_code=http_client.REQUESTED_RANGE_NOT_SATISFIABLE, response_size=response_data.sz)
self.response_ready(response_data)
def send_not_modified(self, etag=None):
buf = [
'%s %d %s' % (self.response_protocol, httplib.NOT_MODIFIED, httplib.responses[httplib.NOT_MODIFIED]),
'%s %d %s' % (self.response_protocol, http_client.NOT_MODIFIED, http_client.responses[http_client.NOT_MODIFIED]),
"Content-Length: 0",
"Date: " + http_date(),
]
if etag is not None:
buf.append('ETag: ' + etag)
response_data = header_list_to_file(buf)
self.log_access(status_code=httplib.NOT_MODIFIED, response_size=response_data.sz)
self.log_access(status_code=http_client.NOT_MODIFIED, response_size=response_data.sz)
self.response_ready(response_data)
def report_busy(self):
self.simple_response(httplib.SERVICE_UNAVAILABLE)
self.simple_response(http_client.SERVICE_UNAVAILABLE)
def job_done(self, ok, result):
if not ok:
@ -509,7 +510,7 @@ class HTTPConnection(HTTPRequest):
if ct.startswith('text/') and 'charset=' not in ct:
outheaders.set('Content-Type', ct + '; charset=UTF-8', replace_all=True)
buf = [HTTP11 + (' %d ' % data.status_code) + httplib.responses[data.status_code]]
buf = [HTTP11 + (' %d ' % data.status_code) + http_client.responses[data.status_code]]
for header, value in sorted(iteritems(outheaders), key=itemgetter(0)):
buf.append('%s: %s' % (header, value))
for morsel in itervalues(data.outcookie):
@ -530,7 +531,7 @@ class HTTPConnection(HTTPRequest):
def log_access(self, status_code, response_size=None, username=None):
if self.access_log is None:
return
if not self.opts.log_not_found and status_code == httplib.NOT_FOUND:
if not self.opts.log_not_found and status_code == http_client.NOT_FOUND:
return
ff = self.forwarded_for
if ff:
@ -623,7 +624,7 @@ class HTTPConnection(HTTPRequest):
self.ready = ready
def report_unhandled_exception(self, e, formatted_traceback):
self.simple_response(httplib.INTERNAL_SERVER_ERROR)
self.simple_response(http_client.INTERNAL_SERVER_ERROR)
def finalize_output(self, output, request, is_http1):
none_match = parse_if_none_match(request.inheaders.get('If-None-Match', ''))
@ -633,7 +634,7 @@ class HTTPConnection(HTTPRequest):
if self.method in ('GET', 'HEAD'):
self.send_not_modified(output.etag)
else:
self.simple_response(httplib.PRECONDITION_FAILED)
self.simple_response(http_client.PRECONDITION_FAILED)
return
opts = self.opts
@ -660,10 +661,10 @@ class HTTPConnection(HTTPRequest):
ct = outheaders.get('Content-Type', '').partition(';')[0]
compressible = (not ct or ct.startswith('text/') or ct.startswith('image/svg') or
ct.partition(';')[0] in COMPRESSIBLE_TYPES)
compressible = (compressible and request.status_code == httplib.OK and
compressible = (compressible and request.status_code == http_client.OK and
(opts.compress_min_size > -1 and output.content_length >= opts.compress_min_size) and
acceptable_encoding(request.inheaders.get('Accept-Encoding', '')) and not is_http1)
accept_ranges = (not compressible and output.accept_ranges is not None and request.status_code == httplib.OK and
accept_ranges = (not compressible and output.accept_ranges is not None and request.status_code == http_client.OK and
not is_http1)
ranges = get_ranges(request.inheaders.get('Range'), output.content_length) if output.accept_ranges and self.method in ('GET', 'HEAD') else None
if_range = (request.inheaders.get('If-Range') or '').strip()
@ -680,7 +681,7 @@ class HTTPConnection(HTTPRequest):
if self.method in ('GET', 'HEAD'):
self.send_not_modified(output.etag)
else:
self.simple_response(httplib.PRECONDITION_FAILED)
self.simple_response(http_client.PRECONDITION_FAILED)
return
output.ranges = None
@ -712,7 +713,7 @@ class HTTPConnection(HTTPRequest):
outheaders.set('Content-Length', '%d' % size, replace_all=True)
outheaders.set('Content-Type', 'multipart/byteranges; boundary=' + MULTIPART_SEPARATOR, replace_all=True)
output.ranges = zip_longest(ranges, range_parts)
request.status_code = httplib.PARTIAL_CONTENT
request.status_code = http_client.PARTIAL_CONTENT
return output

5
src/calibre/srv/routes.py
View File

@ -6,13 +6,14 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, sys, inspect, re, time, numbers, json as jsonlib, textwrap
import sys, inspect, re, time, numbers, json as jsonlib, textwrap
from operator import attrgetter
from calibre.srv.errors import HTTPSimpleResponse, HTTPNotFound, RouteError
from calibre.srv.utils import http_date
from calibre.utils.serialize import msgpack_dumps, json_dumps, MSGPACK_MIME
from polyglot.builtins import iteritems, itervalues, unicode_type, range, zip
from polyglot import http_client
from polyglot.urllib import quote as urlquote
default_methods = frozenset(('HEAD', 'GET'))
@ -297,7 +298,7 @@ class Router(object):
def dispatch(self, data):
endpoint_, args = self.find_route(data.path)
if data.method not in endpoint_.methods:
raise HTTPSimpleResponse(httplib.METHOD_NOT_ALLOWED)
raise HTTPSimpleResponse(http_client.METHOD_NOT_ALLOWED)
self.read_cookies(data)

20
src/calibre/srv/tests/ajax.py
View File

@ -6,13 +6,13 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, zlib, json, base64, os
import zlib, json, base64, os
from io import BytesIO
from functools import partial
from httplib import OK, NOT_FOUND, FORBIDDEN
from calibre.ebooks.metadata.meta import get_metadata
from calibre.srv.tests.base import LibraryBaseTest
from polyglot.http_client import OK, NOT_FOUND, FORBIDDEN
from polyglot.urllib import urlencode, quote
@ -22,7 +22,7 @@ def make_request(conn, url, headers={}, prefix='/ajax', username=None, password=
conn.request(method, prefix + url, headers=headers, body=data)
r = conn.getresponse()
data = r.read()
if r.status == httplib.OK and data and data[0] in b'{[':
if r.status == OK and data and data[0] in b'{[':
data = json.loads(data)
return r, data
@ -37,10 +37,10 @@ class ContentTest(LibraryBaseTest):
request = partial(make_request, conn, prefix='/ajax/book')
r, data = request('/x')
self.ae(r.status, httplib.NOT_FOUND)
self.ae(r.status, NOT_FOUND)
r, onedata = request('/1')
self.ae(r.status, httplib.OK)
self.ae(r.status, OK)
self.ae(request('/1/' + db.server_library_id)[1], onedata)
self.ae(request('/%s?id_is_uuid=true' % db.field_for('uuid', 1))[1], onedata)
@ -63,22 +63,22 @@ class ContentTest(LibraryBaseTest):
request = partial(make_request, conn)
r, data = request('/categories')
self.ae(r.status, httplib.OK)
self.ae(r.status, OK)
r, xdata = request('/categories/' + db.server_library_id)
self.ae(r.status, httplib.OK)
self.ae(r.status, OK)
self.ae(data, xdata)
names = {x['name']:x['url'] for x in data}
for q in ('Newest', 'All books', 'Tags', 'Series', 'Authors', 'Enum', 'Composite Tags'):
self.assertIn(q, names)
r, data = request(names['Tags'], prefix='')
self.ae(r.status, httplib.OK)
self.ae(r.status, OK)
names = {x['name']:x['url'] for x in data['items']}
self.ae(set(names), set('Tag One,Tag Two,News'.split(',')))
r, data = request(names['Tag One'], prefix='')
self.ae(r.status, httplib.OK)
self.ae(r.status, OK)
self.ae(set(data['book_ids']), {1, 2})
r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"'}))
self.ae(r.status, httplib.OK)
self.ae(r.status, OK)
self.ae(set(data['book_ids']), {1, 2})
r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"', 'vl':'1'}))
self.ae(set(data['book_ids']), {2})

56
src/calibre/srv/tests/auth.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, base64, subprocess, os, cookielib, time
import base64, subprocess, os, time
from collections import namedtuple
try:
from distutils.spawn import find_executable
@ -18,6 +18,8 @@ from calibre.srv.errors import HTTPForbidden
from calibre.srv.tests.base import BaseTest, TestServer
from calibre.srv.routes import endpoint, Router
from polyglot.builtins import iteritems, itervalues
from polyglot import http_client
from polyglot.http_cookie import CookieJar
from polyglot.urllib import (build_opener, HTTPBasicAuthHandler,
HTTPCookieProcessor, HTTPDigestAuthHandler, HTTPError)
@ -91,18 +93,18 @@ class TestAuth(BaseTest):
conn = server.connect()
conn.request('GET', '/open')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read(), b'open')
conn.request('GET', '/closed')
r = conn.getresponse()
self.ae(r.status, httplib.UNAUTHORIZED)
self.ae(r.status, http_client.UNAUTHORIZED)
self.ae(r.getheader('WWW-Authenticate'), b'Basic realm="%s"' % bytes(REALM))
self.assertFalse(r.read())
conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + base64.standard_b64encode(b'testuser:testpw')})
r = conn.getresponse()
self.ae(r.read(), b'closed')
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(b'closed', urlopen(server, method='basic').read())
self.ae(b'closed', urlopen(server, un='!@#$%^&*()-=_+', pw='!@#$%^&*()-=_+', method='basic').read())
@ -113,14 +115,14 @@ class TestAuth(BaseTest):
warnings = []
server.loop.log.warn = lambda *args, **kwargs: warnings.append(' '.join(args))
self.ae((httplib.OK, b'closed'), request())
self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y'))
self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('', ''))
self.ae((http_client.OK, b'closed'), request())
self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y'))
self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request('', ''))
self.ae(1, len(warnings))
self.ae((httplib.UNAUTHORIZED, b''), request('testuser', 'y'))
self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('testuser', ''))
self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request(''))
self.ae((httplib.UNAUTHORIZED, b''), request('asf', 'testpw'))
self.ae((http_client.UNAUTHORIZED, b''), request('testuser', 'y'))
self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request('testuser', ''))
self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request(''))
self.ae((http_client.UNAUTHORIZED, b''), request('asf', 'testpw'))
# }}}
def test_library_restrictions(self): # {{{
@ -169,7 +171,7 @@ class TestAuth(BaseTest):
with TestServer(r.dispatch) as server:
r.auth_controller.log = server.log
def test(conn, path, headers={}, status=httplib.OK, body=b'', request_body=b''):
def test(conn, path, headers={}, status=http_client.OK, body=b'', request_body=b''):
conn.request('GET', path, request_body, headers)
r = conn.getresponse()
self.ae(r.status, status)
@ -177,9 +179,9 @@ class TestAuth(BaseTest):
return {normalize_header_name(k):v for k, v in r.getheaders()}
conn = server.connect()
test(conn, '/open', body=b'open')
auth = parse_http_dict(test(conn, '/closed', status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
auth = parse_http_dict(test(conn, '/closed', status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
nonce = auth['nonce']
auth = parse_http_dict(test(conn, '/closed', status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
auth = parse_http_dict(test(conn, '/closed', status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
self.assertNotEqual(nonce, auth['nonce'], 'nonce was re-used')
self.ae(auth[b'realm'], bytes(REALM)), self.ae(auth[b'algorithm'], b'MD5'), self.ae(auth[b'qop'], b'auth')
self.assertNotIn('stale', auth)
@ -199,14 +201,14 @@ class TestAuth(BaseTest):
# Check stale nonces
orig, r.auth_controller.max_age_seconds = r.auth_controller.max_age_seconds, -1
auth = parse_http_dict(test(conn, '/closed', headers={
'Authorization':digest(**args)},status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
'Authorization':digest(**args)},status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
self.assertIn('stale', auth)
r.auth_controller.max_age_seconds = orig
ok_test(conn, digest(**args))
def fail_test(conn, modify, **kw):
kw['body'] = kw.get('body', b'')
kw['status'] = kw.get('status', httplib.UNAUTHORIZED)
kw['status'] = kw.get('status', http_client.UNAUTHORIZED)
args['modify'] = modify
return test(conn, '/closed', headers={'Authorization':digest(**args)}, **kw)
@ -258,13 +260,13 @@ class TestAuth(BaseTest):
warnings = []
server.loop.log.warn = lambda *args, **kwargs: warnings.append(' '.join(args))
self.ae((httplib.OK, b'closed'), request())
self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y'))
self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y'))
self.ae(httplib.FORBIDDEN, request('x', 'y')[0])
self.ae(httplib.FORBIDDEN, request()[0])
self.ae((http_client.OK, b'closed'), request())
self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y'))
self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y'))
self.ae(http_client.FORBIDDEN, request('x', 'y')[0])
self.ae(http_client.FORBIDDEN, request()[0])
time.sleep(ban_for * 60 + 0.01)
self.ae((httplib.OK, b'closed'), request())
self.ae((http_client.OK, b'closed'), request())
# }}}
def test_android_auth_workaround(self): # {{{
@ -277,28 +279,28 @@ class TestAuth(BaseTest):
# First check that unauth access fails
conn.request('GET', '/android')
r = conn.getresponse()
self.ae(r.status, httplib.UNAUTHORIZED)
self.ae(r.status, http_client.UNAUTHORIZED)
auth_handler = HTTPDigestAuthHandler()
url = 'http://localhost:%d%s' % (server.address[1], '/android')
auth_handler.add_password(realm=REALM, uri=url, user='testuser', passwd='testpw')
cj = cookielib.CookieJar()
cj = CookieJar()
cookie_handler = HTTPCookieProcessor(cj)
r = build_opener(auth_handler, cookie_handler).open(url)
self.ae(r.getcode(), httplib.OK)
self.ae(r.getcode(), http_client.OK)
cookies = tuple(cj)
self.ae(len(cookies), 1)
cookie = cookies[0]
self.assertIn(b':', cookie.value)
self.ae(cookie.path, b'/android')
r = build_opener(cookie_handler).open(url)
self.ae(r.getcode(), httplib.OK)
self.ae(r.getcode(), http_client.OK)
self.ae(r.read(), b'android')
# Test that a replay attack against a different URL does not work
try:
build_opener(cookie_handler).open(url+'2')
assert ('Replay attack succeeded')
except HTTPError as e:
self.ae(e.code, httplib.UNAUTHORIZED)
self.ae(e.code, http_client.UNAUTHORIZED)
# }}}

5
src/calibre/srv/tests/base.py
View File

@ -7,12 +7,13 @@ __license__ = 'GPL v3'
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import unittest, time, httplib, shutil, gc, tempfile, atexit, os
import unittest, time, shutil, gc, tempfile, atexit, os
from io import BytesIO
from functools import partial
from threading import Thread
from calibre.srv.utils import ServerLog
from polyglot import http_client
rmtree = partial(shutil.rmtree, ignore_errors=True)
@ -120,7 +121,7 @@ class TestServer(Thread):
timeout = self.loop.opts.timeout
if interface is None:
interface = self.address[0]
return httplib.HTTPConnection(interface, self.address[1], strict=True, timeout=timeout)
return http_client.HTTPConnection(interface, self.address[1], strict=True, timeout=timeout)
def change_handler(self, handler):
from calibre.srv.http_response import create_http_handler

43
src/calibre/srv/tests/content.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, zlib, json, binascii, time, os
import zlib, json, binascii, time, os
from io import BytesIO
from calibre.ebooks.metadata.epub import get_metadata
@ -14,6 +14,7 @@ from calibre.ebooks.metadata.opf2 import OPF
from calibre.srv.tests.base import LibraryBaseTest
from calibre.utils.imghdr import identify
from calibre.utils.shared_file import share_open
from polyglot import http_client
def setUpModule():
@ -32,7 +33,7 @@ class ContentTest(LibraryBaseTest):
def missing(url, body=b''):
conn.request('GET', url)
r = conn.getresponse()
self.ae(r.status, httplib.NOT_FOUND)
self.ae(r.status, http_client.NOT_FOUND)
self.ae(r.read(), body)
for prefix in ('static', 'icon'):
@ -51,7 +52,7 @@ class ContentTest(LibraryBaseTest):
raw = P(src, data=True)
conn.request('GET', url)
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
data = r.read()
if sz is None:
self.ae(data, raw)
@ -60,7 +61,7 @@ class ContentTest(LibraryBaseTest):
test_response(r)
conn.request('GET', url, headers={'If-None-Match':r.getheader('ETag')})
r = conn.getresponse()
self.ae(r.status, httplib.NOT_MODIFIED)
self.ae(r.status, http_client.NOT_MODIFIED)
self.ae(b'', r.read())
test('content-server/empty.html', '/static/empty.html')
@ -85,7 +86,7 @@ class ContentTest(LibraryBaseTest):
# Test various invalid parameters
def bad(*args):
r, data = get(*args)
self.ae(r.status, httplib.NOT_FOUND)
self.ae(r.status, http_client.NOT_FOUND)
bad('xxx', 1)
bad('fmt1', 10)
bad('fmt1', 1, 'zzzz')
@ -103,7 +104,7 @@ class ContentTest(LibraryBaseTest):
# Test fetching of format with metadata update
raw = P('quick_start/eng.epub', data=True)
r, data = get('epub', 1)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
etag = r.getheader('ETag')
self.assertIsNotNone(etag)
self.ae(r.getheader('Used-Cache'), 'no')
@ -145,39 +146,39 @@ class ContentTest(LibraryBaseTest):
os.utime(cpath, (t, t))
r, data = get('cover', 1)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(data, db.cover(1))
self.ae(r.getheader('Used-Cache'), 'no')
self.ae(r.getheader('Content-Type'), 'image/jpeg')
r, data = get('cover', 1)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(data, db.cover(1))
self.ae(r.getheader('Used-Cache'), 'yes')
r, data = get('cover', 3)
self.ae(r.status, httplib.OK) # Auto generated cover
self.ae(r.status, http_client.OK) # Auto generated cover
r, data = get('thumb', 1)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(identify(data), ('jpeg', 60, 60))
self.ae(r.getheader('Used-Cache'), 'no')
r, data = get('thumb', 1)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.getheader('Used-Cache'), 'yes')
r, data = get('thumb', 1, q='sz=100')
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(identify(data), ('jpeg', 100, 100))
self.ae(r.getheader('Used-Cache'), 'no')
r, data = get('thumb', 1, q='sz=100x100')
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.getheader('Used-Cache'), 'yes')
change_cover(1, 1)
r, data = get('thumb', 1, q='sz=100')
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(identify(data), ('jpeg', 100, 100))
self.ae(r.getheader('Used-Cache'), 'no')
# Test file sharing in cache
r, data = get('cover', 2)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no')
path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8')
@ -185,7 +186,7 @@ class ContentTest(LibraryBaseTest):
# Now force an update
change_cover(1)
r, data = get('cover', 2)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no')
path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8')
@ -193,7 +194,7 @@ class ContentTest(LibraryBaseTest):
# Do it again
change_cover(2)
r, data = get('cover', 2)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no')
self.ae(f.read(), fdata)
@ -201,7 +202,7 @@ class ContentTest(LibraryBaseTest):
# Test serving of metadata as opf
r, data = get('opf', 1)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.getheader('Content-Type'), 'application/oebps-package+xml; charset=UTF-8')
self.assertIsNotNone(r.getheader('Last-Modified'))
opf = OPF(BytesIO(data), populate_spine=False, try_to_guess_cover=False)
@ -209,17 +210,17 @@ class ContentTest(LibraryBaseTest):
self.ae(db.field_for('authors', 1), tuple(opf.authors))
conn.request('GET', '/get/opf/1', headers={'Accept-Encoding':'gzip'})
r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip')
self.ae(r.status, http_client.OK), self.ae(r.getheader('Content-Encoding'), 'gzip')
raw = r.read()
self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data)
# Test serving metadata as json
r, data = get('json', 1)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(db.field_for('title', 1), json.loads(data)['title'])
conn.request('GET', '/get/json/1', headers={'Accept-Encoding':'gzip'})
r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip')
self.ae(r.status, http_client.OK), self.ae(r.getheader('Content-Encoding'), 'gzip')
raw = r.read()
self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data)

79
src/calibre/srv/tests/http.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, hashlib, zlib, string, time, os
import hashlib, zlib, string, time, os
from io import BytesIO
from tempfile import NamedTemporaryFile
@ -15,6 +15,7 @@ from calibre.srv.tests.base import BaseTest, TestServer
from calibre.srv.utils import eintr_retry_call
from calibre.utils.monotonic import monotonic
from polyglot.builtins import iteritems, range
from polyglot import http_client
is_ci = os.environ.get('CI', '').lower() == 'true'
@ -94,7 +95,7 @@ class TestHTTP(BaseTest):
def test(al, q):
conn.request('GET', '/', headers={'Accept-Language': al})
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
q += get_translator(q)[-1].ugettext('Unknown')
self.ae(r.read(), q)
@ -136,7 +137,7 @@ class TestHTTP(BaseTest):
def raw_send(conn, raw):
conn.send(raw)
conn._HTTPConnection__state = httplib._CS_REQ_SENT
conn._HTTPConnection__state = http_client._CS_REQ_SENT
return conn.getresponse()
base_timeout = 0.5 if is_ci else 0.1
@ -144,31 +145,31 @@ class TestHTTP(BaseTest):
with TestServer(handler, timeout=base_timeout, max_header_line_size=100./1024, max_request_body_size=100./(1024*1024)) as server:
conn = server.connect()
r = raw_send(conn, b'hello\n')
self.ae(r.status, httplib.BAD_REQUEST)
self.ae(r.status, http_client.BAD_REQUEST)
self.ae(r.read(), b'HTTP requires CRLF line terminators')
r = raw_send(conn, b'\r\nGET /index.html HTTP/1.1\r\n\r\n')
self.ae(r.status, httplib.NOT_FOUND), self.ae(r.read(), b'Requested resource not found')
self.ae(r.status, http_client.NOT_FOUND), self.ae(r.read(), b'Requested resource not found')
r = raw_send(conn, b'\r\n\r\nGET /index.html HTTP/1.1\r\n\r\n')
self.ae(r.status, httplib.BAD_REQUEST)
self.ae(r.status, http_client.BAD_REQUEST)
self.ae(r.read(), b'Multiple leading empty lines not allowed')
r = raw_send(conn, b'hello world\r\n')
self.ae(r.status, httplib.BAD_REQUEST)
self.ae(r.status, http_client.BAD_REQUEST)
self.ae(r.read(), b'Malformed Request-Line')
r = raw_send(conn, b'x' * 200)
self.ae(r.status, httplib.BAD_REQUEST)
self.ae(r.status, http_client.BAD_REQUEST)
self.ae(r.read(), b'')
r = raw_send(conn, b'XXX /index.html HTTP/1.1\r\n\r\n')
self.ae(r.status, httplib.BAD_REQUEST), self.ae(r.read(), b'Unknown HTTP method')
self.ae(r.status, http_client.BAD_REQUEST), self.ae(r.read(), b'Unknown HTTP method')
# Test 404
conn.request('HEAD', '/moose')
r = conn.getresponse()
self.ae(r.status, httplib.NOT_FOUND)
self.ae(r.status, http_client.NOT_FOUND)
self.assertIsNotNone(r.getheader('Date', None))
self.ae(r.getheader('Content-Length'), str(len(body)))
self.ae(r.getheader('Content-Type'), 'text/plain; charset=UTF-8')
@ -176,7 +177,7 @@ class TestHTTP(BaseTest):
self.ae(r.read(), '')
conn.request('GET', '/choose')
r = conn.getresponse()
self.ae(r.status, httplib.NOT_FOUND)
self.ae(r.status, http_client.NOT_FOUND)
self.ae(r.read(), b'Requested resource not found')
# Test 500
@ -186,7 +187,7 @@ class TestHTTP(BaseTest):
conn = server.connect()
conn.request('GET', '/test/')
r = conn.getresponse()
self.ae(r.status, httplib.INTERNAL_SERVER_ERROR)
self.ae(r.status, http_client.INTERNAL_SERVER_ERROR)
server.loop.log.filter_level = orig
# Test 301
@ -196,7 +197,7 @@ class TestHTTP(BaseTest):
conn = server.connect()
conn.request('GET', '/')
r = conn.getresponse()
self.ae(r.status, httplib.MOVED_PERMANENTLY)
self.ae(r.status, http_client.MOVED_PERMANENTLY)
self.ae(r.getheader('Location'), '/somewhere-else')
self.ae('', r.read())
@ -206,26 +207,26 @@ class TestHTTP(BaseTest):
# Test simple GET
conn.request('GET', '/test/')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read(), b'test')
# Test TRACE
lines = ['TRACE /xxx HTTP/1.1', 'Test: value', 'Xyz: abc, def', '', '']
r = raw_send(conn, ('\r\n'.join(lines)).encode('ascii'))
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read().decode('utf-8'), '\n'.join(lines[:-2]))
# Test POST with simple body
conn.request('POST', '/test', 'body')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody')
# Test POST with chunked transfer encoding
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'4\r\nbody\r\na\r\n1234567890\r\n0\r\n\r\n')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody1234567890')
# Test various incorrect input
@ -233,39 +234,39 @@ class TestHTTP(BaseTest):
conn.request('GET', '/test' + ('a' * 200))
r = conn.getresponse()
self.ae(r.status, httplib.BAD_REQUEST)
self.ae(r.status, http_client.BAD_REQUEST)
conn = server.connect()
conn.request('GET', '/test', ('a' * 200))
r = conn.getresponse()
self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE)
self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE)
conn = server.connect()
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'x\r\nbody\r\n0\r\n\r\n')
r = conn.getresponse()
self.ae(r.status, httplib.BAD_REQUEST)
self.ae(r.status, http_client.BAD_REQUEST)
self.assertIn(b'not a valid chunk size', r.read())
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'4\r\nbody\r\n200\r\n\r\n')
r = conn.getresponse()
self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE)
self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE)
conn.request('POST', '/test', body='a'*200)
r = conn.getresponse()
self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE)
self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE)
conn = server.connect()
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'3\r\nbody\r\n0\r\n\r\n')
r = conn.getresponse()
self.ae(r.status, httplib.BAD_REQUEST), self.ae(r.read(), b'Chunk does not have trailing CRLF')
self.ae(r.status, http_client.BAD_REQUEST), self.ae(r.read(), b'Chunk does not have trailing CRLF')
conn = server.connect(timeout=base_timeout * 5)
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'30\r\nbody\r\n0\r\n\r\n')
r = conn.getresponse()
self.ae(r.status, httplib.REQUEST_TIMEOUT)
self.ae(r.status, http_client.REQUEST_TIMEOUT)
self.assertIn(b'', r.read())
server.log.filter_level = orig_level
@ -273,14 +274,14 @@ class TestHTTP(BaseTest):
# Test pipelining
responses = []
for i in range(10):
conn._HTTPConnection__state = httplib._CS_IDLE
conn._HTTPConnection__state = http_client._CS_IDLE
conn.request('GET', '/%d'%i)
responses.append(conn.response_class(conn.sock, strict=conn.strict, method=conn._method))
for i in range(10):
r = responses[i]
r.begin()
self.ae(r.read(), ('%d' % i).encode('ascii'))
conn._HTTPConnection__state = httplib._CS_IDLE
conn._HTTPConnection__state = http_client._CS_IDLE
# Test closing
server.loop.opts.timeout = 10 # ensure socket is not closed because of timeout
@ -319,12 +320,12 @@ class TestHTTP(BaseTest):
conn = server.connect()
conn.request('GET', '/an_etagged_path')
r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.read(), b'an_etagged_path')
self.ae(r.status, http_client.OK), self.ae(r.read(), b'an_etagged_path')
etag = r.getheader('ETag')
self.ae(etag, '"%s"' % hashlib.sha1('an_etagged_path').hexdigest())
conn.request('GET', '/an_etagged_path', headers={'If-None-Match':etag})
r = conn.getresponse()
self.ae(r.status, httplib.NOT_MODIFIED)
self.ae(r.status, http_client.NOT_MODIFIED)
self.ae(r.read(), b'')
# Test gzip
@ -334,7 +335,7 @@ class TestHTTP(BaseTest):
conn.request('GET', '/an_etagged_path', headers={'Accept-Encoding':'gzip'})
r = conn.getresponse()
self.ae(str(len(raw)), r.getheader('Calibre-Uncompressed-Length'))
self.ae(r.status, httplib.OK), self.ae(zlib.decompress(r.read(), 16+zlib.MAX_WBITS), raw)
self.ae(r.status, http_client.OK), self.ae(zlib.decompress(r.read(), 16+zlib.MAX_WBITS), raw)
# Test dynamic etagged content
num_calls = [0]
@ -346,13 +347,13 @@ class TestHTTP(BaseTest):
conn = server.connect()
conn.request('GET', '/an_etagged_path')
r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.read(), b'data')
self.ae(r.status, http_client.OK), self.ae(r.read(), b'data')
etag = r.getheader('ETag')
self.ae(etag, b'"xxx"')
self.ae(r.getheader('Content-Length'), '4')
conn.request('GET', '/an_etagged_path', headers={'If-None-Match':etag})
r = conn.getresponse()
self.ae(r.status, httplib.NOT_MODIFIED)
self.ae(r.status, http_client.NOT_MODIFIED)
self.ae(r.read(), b'')
self.ae(num_calls[0], 1)
@ -368,11 +369,11 @@ class TestHTTP(BaseTest):
self.ae(r.getheader('Content-Type'), guess_type(f.name)[0])
self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes')
self.ae(int(r.getheader('Content-Length')), len(fdata))
self.ae(r.status, httplib.OK), self.ae(r.read(), fdata)
self.ae(r.status, http_client.OK), self.ae(r.read(), fdata)
conn.request('GET', '/test', headers={'Range':'bytes=2-25'})
r = conn.getresponse()
self.ae(r.status, httplib.PARTIAL_CONTENT)
self.ae(r.status, http_client.PARTIAL_CONTENT)
self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes')
self.ae(type('')(r.getheader('Content-Range')), 'bytes 2-25/%d' % len(fdata))
self.ae(int(r.getheader('Content-Length')), 24)
@ -380,27 +381,27 @@ class TestHTTP(BaseTest):
conn.request('GET', '/test', headers={'Range':'bytes=100000-'})
r = conn.getresponse()
self.ae(r.status, httplib.REQUESTED_RANGE_NOT_SATISFIABLE)
self.ae(r.status, http_client.REQUESTED_RANGE_NOT_SATISFIABLE)
self.ae(type('')(r.getheader('Content-Range')), 'bytes */%d' % len(fdata))
conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':etag})
r = conn.getresponse()
self.ae(r.status, httplib.PARTIAL_CONTENT), self.ae(r.read(), fdata[25:51])
self.ae(r.status, http_client.PARTIAL_CONTENT), self.ae(r.read(), fdata[25:51])
self.ae(int(r.getheader('Content-Length')), 26)
conn.request('GET', '/test', headers={'Range':'bytes=0-1000000'})
r = conn.getresponse()
self.ae(r.status, httplib.PARTIAL_CONTENT), self.ae(r.read(), fdata)
self.ae(r.status, http_client.PARTIAL_CONTENT), self.ae(r.read(), fdata)
conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':'"nomatch"'})
r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.read(), fdata)
self.ae(r.status, http_client.OK), self.ae(r.read(), fdata)
self.assertFalse(r.getheader('Content-Range'))
self.ae(int(r.getheader('Content-Length')), len(fdata))
conn.request('GET', '/test', headers={'Range':'bytes=0-25,26-50'})
r = conn.getresponse()
self.ae(r.status, httplib.PARTIAL_CONTENT)
self.ae(r.status, http_client.PARTIAL_CONTENT)
clen = int(r.getheader('Content-Length'))
data = r.read()
self.ae(clen, len(data))
@ -415,7 +416,7 @@ class TestHTTP(BaseTest):
conn = server.connect(timeout=1)
conn.request('GET', '/test')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
rdata = r.read()
self.ae(len(data), len(rdata))
self.ae(hashlib.sha1(data).hexdigest(), hashlib.sha1(rdata).hexdigest())

13
src/calibre/srv/tests/loop.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, ssl, os, socket, time
import ssl, os, socket, time
from collections import namedtuple
from unittest import skipIf
from glob import glob
@ -18,6 +18,7 @@ from calibre.ptempfile import TemporaryDirectory
from calibre.utils.certgen import create_server_cert
from calibre.utils.monotonic import monotonic
from polyglot.builtins import range
from polyglot import http_client
is_ci = os.environ.get('CI', '').lower() == 'true'
@ -92,7 +93,7 @@ class LoopTest(BaseTest):
conn.request('GET', '/')
with self.assertRaises(socket.timeout):
res = conn.getresponse()
if str(res.status) == str(httplib.REQUEST_TIMEOUT):
if str(res.status) == str(http_client.REQUEST_TIMEOUT):
raise socket.timeout('Timeout')
raise Exception('Got unexpected response: code: %s %s headers: %r data: %r' % (
res.status, res.reason, res.getheaders(), res.read()))
@ -135,7 +136,7 @@ class LoopTest(BaseTest):
conn = server.connect(interface='127.0.0.1')
conn.request('GET', '/test', 'body')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody')
def test_ring_buffer(self):
@ -203,10 +204,10 @@ class LoopTest(BaseTest):
create_server_cert(address, ca_file, cert_file, key_file, key_size=1024)
ctx = ssl.create_default_context(cafile=ca_file)
with TestServer(lambda data:(data.path[0] + data.read()), ssl_certfile=cert_file, ssl_keyfile=key_file, listen_on=address, port=0) as server:
conn = httplib.HTTPSConnection(address, server.address[1], strict=True, context=ctx)
conn = http_client.HTTPSConnection(address, server.address[1], strict=True, context=ctx)
conn.request('GET', '/test', 'body')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody')
cert = conn.sock.getpeercert()
subject = dict(x[0] for x in cert['subject'])
@ -226,7 +227,7 @@ class LoopTest(BaseTest):
conn = server.connect()
conn.request('GET', '/test', 'body')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody')
self.ae(server.loop.bound_address[1], port)

4
src/calibre/srv/utils.py
View File

@ -7,9 +7,7 @@ __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import errno, socket, select, os, time
from Cookie import SimpleCookie
from contextlib import closing
import repr as reprlib
from email.utils import formatdate
from operator import itemgetter
from binascii import hexlify, unhexlify
@ -23,6 +21,8 @@ from calibre.utils.socket_inheritance import set_socket_inherit
from calibre.utils.logging import ThreadSafeLog
from calibre.utils.shared_file import share_open, raise_winerror
from polyglot.builtins import iteritems, map, unicode_type, range
from polyglot import reprlib
from polyglot.http_cookie import SimpleCookie
from polyglot.urllib import parse_qs, quote as urlquote
HTTP1 = 'HTTP/1.0'

7
src/calibre/srv/web_socket.py
View File

@ -5,7 +5,7 @@
from __future__ import (unicode_literals, division, absolute_import,
print_function)
import httplib, os, weakref, socket
import os, weakref, socket
from base64 import standard_b64encode
from collections import deque
from hashlib import sha1
@ -19,6 +19,7 @@ from calibre.srv.http_response import HTTPConnection, create_http_handler
from calibre.srv.utils import DESIRED_SEND_BUFFER_SIZE
from calibre.utils.speedups import ReadOnlyFileBuffer
from polyglot.queue import Queue, Empty
from polyglot import http_client
speedup, err = plugins['speedup']
if not speedup:
raise RuntimeError('Failed to load speedup module with error: ' + err)
@ -286,9 +287,9 @@ class WebSocketConnection(HTTPConnection):
except Exception:
ver_ok = False
if not ver_ok:
return self.simple_response(httplib.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver)
return self.simple_response(http_client.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver)
if self.method != 'GET':
return self.simple_response(httplib.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method)
return self.simple_response(http_client.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method)
response = HANDSHAKE_STR % standard_b64encode(sha1(key + GUID_STR).digest())
self.optimize_for_sending_packet()

29
src/calibre/test_build.py
View File

@ -73,6 +73,29 @@ class BuildTest(unittest.TestCase):
from html5_parser import parse
parse('<p>xxx')
def test_imports(self):
import importlib
exclude = ['dbus_export.demo', 'dbus_export.gtk', 'upstream']
if not iswindows:
exclude.extend(['iphlpapi', 'windows', 'winreg', 'winusb'])
if not isosx:
exclude.append('osx')
if not islinux:
exclude.extend(['dbus', 'linux'])
base = os.path.dirname(__file__)
trimpath = len(os.path.dirname(base)) + 1
for root, dirs, files in os.walk(base):
for dir in dirs:
if not os.path.isfile(os.path.join(root, dir, '__init__.py')):
dirs.remove(dir)
for file in files:
file, ext = os.path.splitext(file)
if ext != '.py':
continue
name = '.'.join(root[trimpath:].split(os.path.sep) + [file])
if not any(x for x in exclude if x in name):
importlib.import_module(name)
def test_plugins(self):
exclusions = set()
if is_ci:
@ -99,7 +122,7 @@ class BuildTest(unittest.TestCase):
from calibre.utils.cleantext import test_clean_xml_chars
test_clean_xml_chars()
from lxml import etree
raw = '<a/>'
raw = b'<a/>'
root = etree.fromstring(raw)
self.assertEqual(etree.tostring(root), raw)
@ -175,7 +198,7 @@ class BuildTest(unittest.TestCase):
# it should just work because the hard-coded paths of the Qt
# installation should work. If they do not, then it is a distro
# problem.
fmts = set(map(unicode_type, QImageReader.supportedImageFormats()))
fmts = set(map(lambda x: x.data().decode('utf-8'), QImageReader.supportedImageFormats()))
testf = {'jpg', 'png', 'svg', 'ico', 'gif'}
self.assertEqual(testf.intersection(fmts), testf, "Qt doesn't seem to be able to load some of its image plugins. Available plugins: %s" % fmts)
data = P('images/blank.png', allow_user_override=False, data=True)
@ -254,7 +277,7 @@ class BuildTest(unittest.TestCase):
def test_netifaces(self):
import netifaces
self.assertGreaterEqual(netifaces.interfaces(), 1, 'netifaces could find no network interfaces')
self.assertGreaterEqual(len(netifaces.interfaces()), 1, 'netifaces could find no network interfaces')
def test_psutil(self):
import psutil

8
src/calibre/utils/browser.py
View File

@ -5,11 +5,13 @@ __license__ = 'GPL v3'
__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import copy, httplib, ssl
from cookielib import CookieJar, Cookie
import copy, ssl
from mechanize import Browser as B, HTTPSHandler
from polyglot import http_client
from polyglot.http_cookie import CookieJar, Cookie
class ModernHTTPSHandler(HTTPSHandler):
@ -24,7 +26,7 @@ class ModernHTTPSHandler(HTTPSHandler):
def conn_factory(hostport, **kw):
kw['context'] = self.ssl_context
return httplib.HTTPSConnection(hostport, **kw)
return http_client.HTTPSConnection(hostport, **kw)
return self.do_open(conn_factory, req)

5
src/calibre/utils/cleantext.py
View File

@ -2,8 +2,9 @@ __license__ = 'GPL 3'
__copyright__ = '2010, sengian <sengian1@gmail.com>'
__docformat__ = 'restructuredtext en'
import re, htmlentitydefs
import re
from polyglot.builtins import codepoint_to_chr, map, range
from polyglot.html_entities import name2codepoint
from calibre.constants import plugins, preferred_encoding
try:
@ -80,7 +81,7 @@ def unescape(text, rm=False, rchar=u''):
else:
# named entity
try:
text = codepoint_to_chr(htmlentitydefs.name2codepoint[text[1:-1]])
text = codepoint_to_chr(name2codepoint[text[1:-1]])
except KeyError:
pass
if rm:

21
src/calibre/utils/https.py
View File

@ -10,7 +10,7 @@ import ssl, socket, re
from contextlib import closing
from calibre import get_proxies
from calibre.constants import ispy3
from polyglot import http_client
from polyglot.urllib import urlsplit
has_ssl_verify = hasattr(ssl, 'create_default_context') and hasattr(ssl, '_create_unverified_context')
@ -19,19 +19,14 @@ class HTTPError(ValueError):
def __init__(self, url, code):
msg = '%s returned an unsupported http response code: %d (%s)' % (
url, code, httplib.responses.get(code, None))
url, code, http_client.responses.get(code, None))
ValueError.__init__(self, msg)
self.code = code
self.url = url
if ispy3:
import http.client as httplib
else:
import httplib
if has_ssl_verify:
class HTTPSConnection(httplib.HTTPSConnection):
class HTTPSConnection(http_client.HTTPSConnection):
def __init__(self, ssl_version, *args, **kwargs):
cafile = kwargs.pop('cert_file', None)
@ -39,7 +34,7 @@ if has_ssl_verify:
kwargs['context'] = ssl._create_unverified_context()
else:
kwargs['context'] = ssl.create_default_context(cafile=cafile)
httplib.HTTPSConnection.__init__(self, *args, **kwargs)
http_client.HTTPSConnection.__init__(self, *args, **kwargs)
else:
# Check certificate hostname {{{
# Implementation taken from python 3
@ -136,10 +131,10 @@ else:
"subjectAltName fields were found")
# }}}
class HTTPSConnection(httplib.HTTPSConnection):
class HTTPSConnection(http_client.HTTPSConnection):
def __init__(self, ssl_version, *args, **kwargs):
httplib.HTTPSConnection.__init__(self, *args, **kwargs)
http_client.HTTPSConnection.__init__(self, *args, **kwargs)
self.calibre_ssl_version = ssl_version
def connect(self):
@ -204,7 +199,7 @@ def get_https_resource_securely(
path += '?' + p.query
c.request('GET', path, headers=headers or {})
response = c.getresponse()
if response.status in (httplib.MOVED_PERMANENTLY, httplib.FOUND, httplib.SEE_OTHER):
if response.status in (http_client.MOVED_PERMANENTLY, http_client.FOUND, http_client.SEE_OTHER):
if max_redirects <= 0:
raise ValueError('Too many redirects, giving up')
newurl = response.getheader('Location', None)
@ -212,7 +207,7 @@ def get_https_resource_securely(
raise ValueError('%s returned a redirect response with no Location header' % url)
return get_https_resource_securely(
newurl, cacerts=cacerts, timeout=timeout, max_redirects=max_redirects-1, ssl_version=ssl_version, get_response=get_response)
if response.status != httplib.OK:
if response.status != http_client.OK:
raise HTTPError(url, response.status)
if get_response:
return response

3
src/calibre/utils/smtp.py
View File

@ -98,8 +98,7 @@ def get_mx(host, verbose=0):
if verbose:
print('Find mail exchanger for', host)
answers = list(dns.resolver.query(host, 'MX'))
answers.sort(cmp=lambda x, y: cmp(int(getattr(x, 'preference', sys.maxint)),
int(getattr(y, 'preference', sys.maxint))))
answers.sort(key=lambda x: int(getattr(x, 'preference', sys.maxint)))
return [str(x.exchange) for x in answers if hasattr(x, 'exchange')]

22
src/calibre/utils/unrar.py
View File

@ -114,20 +114,20 @@ def test_basic():
b"Rar!\x1a\x07\x00\xcf\x90s\x00\x00\r\x00\x00\x00\x00\x00\x00\x00\x14\xe7z\x00\x80#\x00\x17\x00\x00\x00\r\x00\x00\x00\x03\xc2\xb3\x96o\x00\x00\x00\x00\x1d3\x03\x00\x00\x00\x00\x00CMT\x0c\x00\x8b\xec\x8e\xef\x14\xf6\xe6h\x04\x17\xff\xcd\x0f\xffk9b\x11]^\x80\xd3dt \x90+\x00\x14\x00\x00\x00\x08\x00\x00\x00\x03\xf1\x84\x93\\\xb9]yA\x1d3\t\x00\xa4\x81\x00\x001\\sub-one\x00\xc0\x0c\x00\x8f\xec\x89\xfe.JM\x86\x82\x0c_\xfd\xfd\xd7\x11\x1a\xef@\x9eHt \x80'\x00\x0e\x00\x00\x00\x04\x00\x00\x00\x03\x9f\xa8\x17\xf8\xaf]yA\x1d3\x07\x00\xa4\x81\x00\x00one.txt\x00\x08\xbf\x08\xae\xf3\xca\x87\xfeo\xfe\xd2n\x80-Ht \x82:\x00\x18\x00\x00\x00\x10\x00\x00\x00\x03\xa86\x81\xdf\xf9fyA\x1d3\x1a\x00\xa4\x81\x00\x00\xe8\xaf\xb6\xe6\xaf\x94\xe5\xb1\x81.txt\x00\x8bh\xf6\xd4kA\\.\x00txt\x0c\x00\x8b\xec\x8e\xef\x14\xf6\xe2l\x91\x189\xff\xdf\xfe\xc2\xd3:g\x9a\x19F=cYt \x928\x00\x11\x00\x00\x00\x08\x00\x00\x00\x03\x7f\xd6\xb6\x7f\xeafyA\x1d3\x16\x00\xa4\x81\x00\x00F\xc3\xbc\xc3\x9fe.txt\x00\x01\x00F\xfc\xdfe\x00.txt\x00\xc0<D\xfe\xc8\xef\xbc\xd1\x04I?\xfd\xff\xdbF)]\xe8\xb9\xe1t \x90/\x00\x13\x00\x00\x00\x08\x00\x00\x00\x03\x1a$\x932\xc2]yA\x1d3\r\x00\xa4\x81\x00\x002\\sub-two.txt\x00\xc0\x10\x00S\xec\xcb\x7f\x8b\xa5(\x0b\x01\xcb\xef\xdf\xf6t\x89\x97z\x0eft \x90)\x00\r\x00\x00\x00\r\x00\x00\x00\x03c\x89K\xd3\xc8fyA\x140\x07\x00\xff\xa1\x00\x00symlink\x00\xc02/sub-two.txt\xeb\x86t\xe0\x90#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\xb9]yA\x140\x01\x00\xedA\x00\x001\x00\xc0\xe0Dt\xe0\x90#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\xc2]yA\x140\x01\x00\xedA\x00\x002\x00\xc0u\xa1t \x80,\x00\r\x00\x00\x00\r\x00\x00\x00\x03T\xea\x04\xca\xe6\x84yA\x140\x0c\x00\xa4\x81\x00\x00uncompresseduncompressed\n\xda\x10t \x900\x00\x0e\x00\x00\x00\x04\x00\x00\x00\x035K.\xa6\x18\x85yA\x1d5\x0e\x00\xa4\x81\x00\x00max-compressed\x00\xc0\x00\x08\xbf\x08\xae\xf2\xcc\x01s\xf8\xff\xec\x96\xe8\xc4={\x00@\x07\x00") # noqa }}}
tdata = {
u'1': b'',
u'1/sub-one': b'sub-one\n',
u'2': b'',
u'2/sub-two.txt': b'sub-two\n',
u'F\xfc\xdfe.txt': b'unicode\n',
u'max-compressed': b'max\n',
u'one.txt': b'one\n',
u'symlink': b'2/sub-two.txt',
u'uncompressed': b'uncompressed\n',
u'\u8bf6\u6bd4\u5c41.txt': b'chinese unicode\n'}
'1': b'',
'1/sub-one': b'sub-one\n',
'2': b'',
'2/sub-two.txt': b'sub-two\n',
'F\xfc\xdfe.txt': b'unicode\n',
'max-compressed': b'max\n',
'one.txt': b'one\n',
'symlink': b'2/sub-two.txt',
'uncompressed': b'uncompressed\n',
'\u8bf6\u6bd4\u5c41.txt': b'chinese unicode\n'}
def do_test(stream):
c = comment(stream)
if c != b'some comment\n':
if c != 'some comment\n':
raise ValueError('Comment not read: %r != %r' % (c, b'some comment\n'))
if set(names(stream)) != {
'1/sub-one', 'one.txt', '2/sub-two.txt', '诶比屁.txt', 'Füße.txt',

10
src/calibre/web/feeds/news.py
View File

@ -9,7 +9,7 @@ __docformat__ = "restructuredtext en"
import os, time, traceback, re, sys, io
from collections import defaultdict
from contextlib import nested, closing
from contextlib import closing
from calibre import (browser, __appname__, iswindows, force_unicode,
@ -760,7 +760,7 @@ class BasicNewsRecipe(Recipe):
in index are not in weights, they are assumed to have a weight of 0.
'''
weights = defaultdict(lambda: 0, weights)
index.sort(cmp=lambda x, y: cmp(weights[x], weights[y]))
index.sort(key=lambda x: weights[x])
return index
def parse_index(self):
@ -1097,7 +1097,7 @@ class BasicNewsRecipe(Recipe):
if bn:
img = os.path.join(imgdir, 'feed_image_%d%s'%(self.image_counter, os.path.splitext(bn)))
try:
with nested(open(img, 'wb'), closing(self.browser.open(feed.image_url))) as (fi, r):
with open(img, 'wb') as fi, closing(self.browser.open(feed.image_url)) as r:
fi.write(r.read())
self.image_counter += 1
feed.image_url = img
@ -1346,7 +1346,7 @@ class BasicNewsRecipe(Recipe):
with open(mpath, 'wb') as mfile:
mfile.write(open(mu, 'rb').read())
else:
with nested(open(mpath, 'wb'), closing(self.browser.open(mu))) as (mfile, r):
with open(mpath, 'wb') as mfile, closing(self.browser.open(mu)) as r:
mfile.write(r.read())
self.report_progress(1, _('Masthead image downloaded'))
self.prepare_masthead_image(mpath, outfile)
@ -1564,7 +1564,7 @@ class BasicNewsRecipe(Recipe):
opf.create_spine(entries)
opf.set_toc(toc)
with nested(open(opf_path, 'wb'), open(ncx_path, 'wb')) as (opf_file, ncx_file):
with open(opf_path, 'wb') as opf_file, open(ncx_path, 'wb') as ncx_file:
opf.render(opf_file, ncx_file)
def article_downloaded(self, request, result):

2
src/calibre/web/fetch/simple.py
View File

@ -18,7 +18,6 @@ import threading
import time
import traceback
from base64 import b64decode
from httplib import responses
from calibre import browser, relpath, unicode_path
from calibre.constants import filesystem_encoding, iswindows
@ -31,6 +30,7 @@ from calibre.utils.imghdr import what
from calibre.utils.logging import Log
from calibre.web.fetch.utils import rescale_image
from polyglot.builtins import unicode_type
from polyglot.http_client import responses
from polyglot.urllib import (
URLError, quote, url2pathname, urljoin, urlparse, urlsplit, urlunparse,
urlunsplit

10
src/polyglot/html_entities.py Normal file
View File

@ -0,0 +1,10 @@
#!/usr/bin/env python2
# vim:fileencoding=utf-8
# License: GPL v3 Copyright: 2019, Eli Schwartz <eschwartz@archlinux.org>
from polyglot.builtins import is_py3
if is_py3:
from html.entities import name2codepoint
else:
from htmlentitydefs import name2codepoint

22
src/polyglot/http_client.py Normal file
View File

@ -0,0 +1,22 @@
#!/usr/bin/env python2
# vim:fileencoding=utf-8
# License: GPL v3 Copyright: 2019, Eli Schwartz <eschwartz@archlinux.org>
from polyglot.builtins import is_py3
if is_py3:
from http.client import (responses, HTTPConnection, HTTPSConnection,
BAD_REQUEST, FOUND, FORBIDDEN, HTTP_VERSION_NOT_SUPPORTED,
INTERNAL_SERVER_ERROR, METHOD_NOT_ALLOWED, MOVED_PERMANENTLY,
NOT_FOUND, NOT_IMPLEMENTED, NOT_MODIFIED, OK, PARTIAL_CONTENT,
PRECONDITION_FAILED, REQUEST_ENTITY_TOO_LARGE, REQUEST_URI_TOO_LONG,
REQUESTED_RANGE_NOT_SATISFIABLE, REQUEST_TIMEOUT, SEE_OTHER,
SERVICE_UNAVAILABLE, UNAUTHORIZED, _CS_IDLE, _CS_REQ_SENT)
else:
from httplib import (responses, HTTPConnection, HTTPSConnection,
BAD_REQUEST, FOUND, FORBIDDEN, HTTP_VERSION_NOT_SUPPORTED,
INTERNAL_SERVER_ERROR, METHOD_NOT_ALLOWED, MOVED_PERMANENTLY,
NOT_FOUND, NOT_IMPLEMENTED, NOT_MODIFIED, OK, PARTIAL_CONTENT,
PRECONDITION_FAILED, REQUEST_ENTITY_TOO_LARGE, REQUEST_URI_TOO_LONG,
REQUESTED_RANGE_NOT_SATISFIABLE, REQUEST_TIMEOUT, SEE_OTHER,
SERVICE_UNAVAILABLE, UNAUTHORIZED, _CS_IDLE, _CS_REQ_SENT)

12
src/polyglot/http_cookie.py Normal file
View File

@ -0,0 +1,12 @@
#!/usr/bin/env python2
# vim:fileencoding=utf-8
# License: GPL v3 Copyright: 2019, Eli Schwartz <eschwartz@archlinux.org>
from polyglot.builtins import is_py3
if is_py3:
from http.cookies import SimpleCookie # noqa
from http.cookiejar import CookieJar, Cookie # noqa
else:
from Cookie import SimpleCookie # noqa
from cookielib import CookieJar, Cookie # noqa

10
src/polyglot/reprlib.py Executable file
View File

@ -0,0 +1,10 @@
#!/usr/bin/env python2
# vim:fileencoding=utf-8
# License: GPL v3 Copyright: 2019, Eli Schwartz <eschwartz@archlinux.org>
from polyglot.builtins import is_py3
if is_py3:
from reprlib import repr
else:
from repr import repr