Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-06-23 15:30:45 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'py3' of https://github.com/eli-schwartz/calibre

Browse Source
This commit is contained in:
Kovid Goyal 2019-03-20 20:08:13 +05:30
commit 1ea8b07078
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
100 changed files with 206 additions and 295 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/calibre/__init__.py
View File

@ -320,7 +320,7 @@ def extract(path, dir):
def get_proxies(debug=True):
from urllib import getproxies
from polyglot.urllib import getproxies
proxies = getproxies()
for key, proxy in list(proxies.items()):
if not proxy or '..' in proxy or key == 'auto':
@ -382,10 +382,10 @@ def get_proxy_info(proxy_scheme, proxy_string):
is not available in the string. If an exception occurs parsing the string
this method returns None.
'''
import urlparse
from polyglot.urllib import urlparse
try:
proxy_url = u'%s://%s'%(proxy_scheme, proxy_string)
urlinfo = urlparse.urlparse(proxy_url)
urlinfo = urlparse(proxy_url)
ans = {
u'scheme': urlinfo.scheme,
u'hostname': urlinfo.hostname,

3
src/calibre/db/cli/main.py
View File

@ -8,8 +8,6 @@ import httplib
import json
import os
import sys
from urllib import urlencode
from urlparse import urlparse, urlunparse
from calibre import browser, prints
from calibre.constants import __appname__, __version__, iswindows
@ -19,6 +17,7 @@ from calibre.utils.config import OptionParser, prefs
from calibre.utils.localization import localize_user_manual_link
from calibre.utils.lock import singleinstance
from calibre.utils.serialize import MSGPACK_MIME
from polyglot.urllib import urlencode, urlparse, urlunparse
COMMANDS = (
'list', 'add', 'remove', 'add_format', 'remove_format', 'show_metadata',

3
src/calibre/db/delete_service.py
View File

@ -8,11 +8,11 @@ __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
import os, tempfile, shutil, errno, time, atexit
from threading import Thread
from Queue import Queue
from calibre.ptempfile import remove_dir
from calibre.utils.filenames import remove_dir_if_empty
from calibre.utils.recycle_bin import delete_tree, delete_file
from polyglot.queue import Queue
class DeleteService(Thread):
@ -159,4 +159,3 @@ def has_jobs():
if __ds is not None:
return (not __ds.requests.empty()) or __ds.requests.unfinished_tasks
return False

10
src/calibre/devices/smart_device_app/driver.py
View File

@ -11,7 +11,6 @@ import socket, select, json, os, traceback, time, sys, random
import posixpath
from collections import defaultdict
import hashlib, threading
import Queue
from functools import wraps
from errno import EAGAIN, EINTR
@ -39,6 +38,7 @@ from calibre.utils.mdns import (publish as publish_zeroconf, unpublish as
unpublish_zeroconf, get_all_ips)
from calibre.utils.socket_inheritance import set_socket_inherit
from polyglot.builtins import unicode_type
from polyglot import queue
def synchronous(tlockname):
@ -103,7 +103,7 @@ class ConnectionListener(Thread):
{'otherDevice': d.get_gui_name()})
self.driver._send_byte_string(device_socket, (b'%d' % len(s)) + s)
sock.close()
except Queue.Empty:
except queue.Empty:
pass
if getattr(self.driver, 'broadcast_socket', None) is not None:
@ -148,7 +148,7 @@ class ConnectionListener(Thread):
try:
self.driver.connection_queue.put_nowait(device_socket)
except Queue.Full:
except queue.Full:
self._close_socket(device_socket)
device_socket = None
self.driver._debug('driver is not answering')
@ -993,7 +993,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
raise
except:
pass
except Queue.Empty:
except queue.Empty:
self.is_connected = False
return self if self.is_connected else None
return None
@ -1969,7 +1969,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
message = 'attaching port to broadcast socket failed. This is not fatal.'
self._debug(message)
self.connection_queue = Queue.Queue(1)
self.connection_queue = queue.Queue(1)
self.connection_listener = ConnectionListener(self)
self.connection_listener.start()
return message

2
src/calibre/ebooks/comic/input.py
View File

@ -8,7 +8,6 @@ Based on ideas from comiclrf created by FangornUK.
'''
import os, traceback, time
from Queue import Empty
from calibre import extract, prints, walk
from calibre.constants import filesystem_encoding
@ -17,6 +16,7 @@ from calibre.utils.icu import numeric_sort_key
from calibre.utils.ipc.server import Server
from calibre.utils.ipc.job import ParallelJob
from polyglot.builtins import unicode_type
from polyglot.queue import Empty
# If the specified screen has either dimension larger than this value, no image
# rescaling is done (we assume that it is a tablet output profile)

2
src/calibre/ebooks/conversion/plugins/chm_input.py
View File

@ -108,7 +108,7 @@ class CHMInput(InputFormatPlugin):
def _create_html_root(self, hhcpath, log, encoding):
from lxml import html
from urllib import unquote as _unquote
from polyglot.urllib import unquote as _unquote
from calibre.ebooks.oeb.base import urlquote
from calibre.ebooks.chardet import xml_to_unicode
hhcdata = self._read_file(hhcpath)

2
src/calibre/ebooks/conversion/plugins/html_input.py
View File

@ -247,7 +247,7 @@ class HTMLInput(InputFormatPlugin):
return link, frag
def resource_adder(self, link_, base=None):
from urllib import quote
from polyglot.urllib import quote
link, frag = self.link_to_local_path(link_, base=base)
if link is None:
return link_

4
src/calibre/ebooks/conversion/plugins/html_output.py
View File

@ -47,7 +47,7 @@ class HTMLOutput(OutputFormatPlugin):
Generate table of contents
'''
from lxml import etree
from urllib import unquote
from polyglot.urllib import unquote
from calibre.ebooks.oeb.base import element
from calibre.utils.cleantext import clean_xml_chars
@ -86,7 +86,7 @@ class HTMLOutput(OutputFormatPlugin):
from lxml import etree
from calibre.utils import zipfile
from templite import Templite
from urllib import unquote
from polyglot.urllib import unquote
from calibre.ebooks.html.meta import EasyMeta
# read template files

2
src/calibre/ebooks/conversion/plugins/oeb_output.py
View File

@ -21,7 +21,7 @@ class OEBOutput(OutputFormatPlugin):
recommendations = {('pretty_print', True, OptionRecommendation.HIGH)}
def convert(self, oeb_book, output_path, input_plugin, opts, log):
from urllib import unquote
from polyglot.urllib import unquote
from lxml import etree
self.log, self.opts = log, opts

2
src/calibre/ebooks/docx/writer/links.py
View File

@ -8,9 +8,9 @@ __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import posixpath, re
from uuid import uuid4
from urlparse import urlparse
from calibre.utils.filenames import ascii_text
from polyglot.urllib import urlparse
def start_text(tag, prefix_len=0, top_level=True):

2
src/calibre/ebooks/html/input.py
View File

@ -13,13 +13,13 @@ Input plugin for HTML or OPF ebooks.
'''
import os, re, sys, errno as gerrno
from urlparse import urlparse, urlunparse
from calibre.ebooks.oeb.base import urlunquote
from calibre.ebooks.chardet import detect_xml_encoding
from calibre.constants import iswindows
from calibre import unicode_path, as_unicode, replace_entities
from polyglot.builtins import unicode_type
from polyglot.urllib import urlparse, urlunparse
class Link(object):

2
src/calibre/ebooks/htmlz/oeb2html.py
View File

@ -15,7 +15,6 @@ import re
from functools import partial
from lxml import html
from urlparse import urldefrag
from calibre import prepare_string_for_xml
from calibre.ebooks.oeb.base import (
@ -23,6 +22,7 @@ from calibre.ebooks.oeb.base import (
from calibre.ebooks.oeb.stylizer import Stylizer
from calibre.utils.logging import default_log
from polyglot.builtins import unicode_type, string_or_bytes
from polyglot.urllib import urldefrag
SELF_CLOSING_TAGS = {'area', 'base', 'basefont', 'br', 'hr', 'input', 'img', 'link', 'meta'}

3
src/calibre/ebooks/lit/reader.py
View File

@ -9,8 +9,6 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net> ' \
'and Marshall T. Vandegrift <llasram@gmail.com>'
import io, struct, os, functools, re
from urlparse import urldefrag
from urllib import unquote as urlunquote
from lxml import etree
@ -22,6 +20,7 @@ from calibre.ebooks.oeb.reader import OEBReader
from calibre.ebooks import DRMError
from calibre import plugins
from polyglot.builtins import codepoint_to_chr, unicode_type, string_or_bytes, range
from polyglot.urllib import unquote as urlunquote, urldefrag
lzx, lxzerror = plugins['lzx']
msdes, msdeserror = plugins['msdes']

5
src/calibre/ebooks/lit/writer.py
View File

@ -17,8 +17,6 @@ import copy
import uuid
import functools
import numbers
from urlparse import urldefrag
from urllib import unquote as urlunquote
from lxml import etree
from calibre.ebooks.lit.reader import DirectoryEntry
import calibre.ebooks.lit.maps as maps
@ -33,6 +31,7 @@ from calibre import plugins
msdes, msdeserror = plugins['msdes']
import calibre.ebooks.lit.mssha1 as mssha1
from polyglot.builtins import codepoint_to_chr, unicode_type, string_or_bytes, range
from polyglot.urllib import urldefrag, unquote
__all__ = ['LitWriter']
@ -521,7 +520,7 @@ class LitWriter(object):
media_type = XHTML_MIME
elif media_type in OEB_STYLES:
media_type = CSS_MIME
href = urlunquote(item.href)
href = unquote(item.href)
item.offset = offset \
if state in ('linear', 'nonlinear') else 0
data.write(pack('<I', item.offset))

3
src/calibre/ebooks/lrf/html/convert_from.py
View File

@ -9,8 +9,6 @@ and to Falstaff for pylrs.
"""
import os, re, sys, copy, glob, tempfile
from collections import deque
from urllib import unquote
from urlparse import urlparse
from math import ceil, floor
from functools import partial
@ -37,6 +35,7 @@ from calibre.devices.interface import DevicePlugin as Device
from calibre.ebooks.lrf.html.color_map import lrs_color
from calibre.ebooks.chardet import xml_to_unicode
from polyglot.builtins import unicode_type
from polyglot.urllib import unquote, urlparse
def update_css(ncss, ocss):

8
src/calibre/ebooks/metadata/__init__.py
View File

@ -9,13 +9,11 @@ Provides abstraction for metadata reading.writing from a variety of ebook format
"""
import os, sys, re
from urlparse import urlparse
from calibre import relpath, guess_type, remove_bracketed_text, prints, force_unicode
from calibre.utils.config_base import tweaks
from polyglot.builtins import codepoint_to_chr, unicode_type
from polyglot.urllib import quote, unquote, urlparse
try:
_author_pat = re.compile(tweaks['authors_split_regex'])
@ -205,7 +203,6 @@ class Resource(object):
'''
def __init__(self, href_or_path, basedir=os.getcwdu(), is_path=True):
from urllib import unquote
self._href = None
self._basedir = basedir
self.path = None
@ -243,7 +240,6 @@ class Resource(object):
`basedir`: If None, the basedir of this resource is used (see :method:`set_basedir`).
If this resource has no basedir, then the current working directory is used as the basedir.
'''
from urllib import quote
if basedir is None:
if self._basedir:
basedir = self._basedir

7
src/calibre/ebooks/metadata/kdl.py
View File

@ -6,7 +6,7 @@ __license__ = 'GPL v3'
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import re, urllib, urlparse, socket
import re, socket
from mechanize import URLError
@ -15,6 +15,7 @@ from calibre import browser
from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre.ebooks.chardet import xml_to_unicode
from polyglot.builtins import codepoint_to_chr, unicode_type
from polyglot.urllib import parse_qs, quote_plus
URL = \
"http://ww2.kdl.org/libcat/WhatsNext.asp?AuthorLastName={0}&AuthorFirstName=&SeriesName=&BookTitle={1}&CategoryID=0&cmdSearch=Search&Search=1&grouping="
@ -32,7 +33,7 @@ def get_series(title, authors, timeout=60):
if isinstance(title, unicode_type):
title = title.encode('utf-8')
title = urllib.quote_plus(title)
title = quote_plus(title)
author = authors[0].strip()
if not author:
@ -64,7 +65,7 @@ def get_series(title, authors, timeout=60):
if a is None:
return mi
href = a['href'].partition('?')[-1]
data = urlparse.parse_qs(href)
data = parse_qs(href)
series = data.get('SeriesName', [])
if not series:
return mi

3
src/calibre/ebooks/metadata/opf2.py
View File

@ -9,8 +9,6 @@ lxml based OPF parser.
'''
import re, sys, unittest, functools, os, uuid, glob, io, json, copy
from urllib import unquote
from urlparse import urlparse
from lxml import etree
@ -26,6 +24,7 @@ from calibre import prints, guess_type
from calibre.utils.cleantext import clean_ascii_chars, clean_xml_chars
from calibre.utils.config import tweaks
from polyglot.builtins import unicode_type, range
from polyglot.urllib import unquote, urlparse
pretty_print_opf = False

2
src/calibre/ebooks/metadata/search_internet.py
View File

@ -4,7 +4,7 @@
from __future__ import absolute_import, division, print_function, unicode_literals
from urllib import quote_plus
from polyglot.urllib import quote_plus
AUTHOR_SEARCHES = {
'goodreads':

3
src/calibre/ebooks/metadata/sources/cli.py
View File

@ -21,6 +21,7 @@ from calibre.ebooks.metadata.sources.base import create_log
from calibre.ebooks.metadata.sources.identify import identify
from calibre.ebooks.metadata.sources.covers import download_cover
from calibre.ebooks.metadata.sources.update import patch_plugins
from polyglot.builtins import unicode_type
def option_parser():
@ -99,7 +100,7 @@ def main(args=sys.argv):
log = buf.getvalue()
result = (metadata_to_opf(result) if opts.opf else
type(u'')(result).encode('utf-8'))
unicode_type(result).encode('utf-8'))
if opts.verbose:
print (log, file=sys.stderr)

3
src/calibre/ebooks/metadata/sources/identify.py
View File

@ -27,6 +27,7 @@ from calibre.utils.html2text import html2text
from calibre.utils.icu import lower
from calibre.utils.date import UNDEFINED_DATE
from calibre.utils.formatter import EvalFormatter
from polyglot.builtins import unicode_type
# Download worker {{{
@ -471,7 +472,7 @@ def identify(log, abort, # {{{
for r in presults:
log('\n\n---')
try:
log(type(u'')(r))
log(unicode_type(r))
except TypeError:
log(repr(r))
if plog:

2
src/calibre/ebooks/metadata/sources/test.py
View File

@ -8,7 +8,6 @@ __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import os, tempfile, time
from Queue import Queue, Empty
from threading import Event
from calibre.customize.ui import all_metadata_plugins
@ -16,6 +15,7 @@ from calibre import prints, sanitize_file_name2
from calibre.ebooks.metadata import check_isbn
from calibre.ebooks.metadata.sources.base import create_log, get_cached_cover_urls
from calibre.ebooks.metadata.sources.prefs import msprefs
from polyglot.queue import Queue, Empty
def isbn_test(isbn):

2
src/calibre/ebooks/metadata/sources/worker.py
View File

@ -6,7 +6,6 @@ from __future__ import absolute_import, division, print_function, unicode_litera
import os
from collections import Counter
from io import BytesIO
from Queue import Empty, Queue
from threading import Event, Thread
from calibre.customize.ui import metadata_plugins
@ -18,6 +17,7 @@ from calibre.ebooks.metadata.sources.identify import identify, msprefs
from calibre.ebooks.metadata.sources.update import patch_plugins
from calibre.utils.date import as_utc
from calibre.utils.logging import GUILog
from polyglot.queue import Empty, Queue
def merge_result(oldmi, newmi, ensure_fields=None):

3
src/calibre/ebooks/metadata/toc.py
View File

@ -4,8 +4,6 @@ __license__ = 'GPL v3'
__copyright__ = '2010, Kovid Goyal <kovid at kovidgoyal.net>'
import os, glob, re, functools
from urlparse import urlparse
from urllib import unquote
from collections import Counter
from lxml import etree
@ -16,6 +14,7 @@ from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre.ebooks.chardet import xml_to_unicode
from calibre.utils.cleantext import clean_xml_chars
from polyglot.builtins import unicode_type
from polyglot.urllib import unquote, urlparse
NCX_NS = "http://www.daisy.org/z3986/2005/ncx/"
CALIBRE_NS = "http://calibre.kovidgoyal.net/2009/metadata"

2
src/calibre/ebooks/mobi/reader/mobi8.py
View File

@ -10,7 +10,6 @@ __docformat__ = 'restructuredtext en'
import struct, re, os
from collections import namedtuple
from itertools import repeat, izip
from urlparse import urldefrag
from uuid import uuid4
from lxml import etree
@ -26,6 +25,7 @@ from calibre.ebooks.mobi.utils import read_font_record
from calibre.ebooks.oeb.parse_utils import parse_html
from calibre.ebooks.oeb.base import XPath, XHTML, xml2text
from polyglot.builtins import range
from polyglot.urllib import urldefrag
Part = namedtuple('Part',
'num type filename start end aid')

2
src/calibre/ebooks/mobi/writer2/serializer.py
View File

@ -11,7 +11,6 @@ import re
import unicodedata
from collections import defaultdict
from io import BytesIO
from urlparse import urldefrag
from calibre.ebooks.mobi.mobiml import MBP_NS
from calibre.ebooks.mobi.utils import is_guide_ref_start
@ -19,6 +18,7 @@ from calibre.ebooks.oeb.base import (
OEB_DOCS, XHTML, XHTML_NS, XML_NS, namespace, prefixname, urlnormalize
)
from polyglot.builtins import unicode_type, string_or_bytes
from polyglot.urllib import urldefrag
class Buf(BytesIO):

3
src/calibre/ebooks/oeb/base.py
View File

@ -9,8 +9,6 @@ __docformat__ = 'restructuredtext en'
import os, re, logging
from collections import defaultdict
from itertools import count
from urlparse import urldefrag, urlparse, urlunparse, urljoin
from urllib import unquote
from lxml import etree, html
from calibre.constants import filesystem_encoding, __version__, ispy3
@ -23,6 +21,7 @@ from calibre.ebooks.oeb.parse_utils import (barename, XHTML_NS, RECOVER_PARSER,
from calibre.utils.cleantext import clean_xml_chars
from calibre.utils.short_uuid import uuid4
from polyglot.builtins import unicode_type, string_or_bytes, range
from polyglot.urllib import unquote, urldefrag, urljoin, urlparse, urlunparse
XML_NS = 'http://www.w3.org/XML/1998/namespace'
OEB_DOC_NS = 'http://openebook.org/namespaces/oeb-document/1.0/'

6
src/calibre/ebooks/oeb/display/test-cfi/run_rapydscript.py
View File

@ -8,9 +8,10 @@ __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import os, shutil, tempfile
import SimpleHTTPServer
import SocketServer
from polyglot.http_server import SimpleHTTPRequestHandler
def run_devel_server():
base = os.path.dirname(os.path.abspath(__file__))
@ -27,7 +28,7 @@ def run_devel_server():
with lopen('cfi-test.pyj', 'rb') as f, lopen('cfi-test.js', 'wb') as js:
js.write(compile_pyj(f.read()).encode('utf-8'))
PORT = 8000
Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
Handler = SimpleHTTPRequestHandler
httpd = SocketServer.TCPServer(("", PORT), Handler)
print('Serving CFI test at http://localhost:%d' % PORT)
try:
@ -37,4 +38,3 @@ def run_devel_server():
if __name__ == '__main__':
run_devel_server()

7
src/calibre/ebooks/oeb/polish/check/links.py
View File

@ -8,10 +8,7 @@ __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
import os
from collections import defaultdict
from urlparse import urlparse
from polyglot.builtins import map
from threading import Thread
from Queue import Queue, Empty
from calibre import browser
from calibre.ebooks.oeb.base import OEB_DOCS, OEB_STYLES, urlunquote, XHTML_MIME
@ -21,7 +18,9 @@ from calibre.ebooks.oeb.polish.replace import remove_links_to
from calibre.ebooks.oeb.polish.cover import get_raster_cover_name
from calibre.ebooks.oeb.polish.utils import guess_type, actual_case_for_name, corrected_case_for_name
from calibre.ebooks.oeb.polish.check.base import BaseError, WARN, INFO
from polyglot.builtins import range
from polyglot.builtins import map, range
from polyglot.urllib import urlparse
from polyglot.queue import Queue, Empty
class BadLink(BaseError):

2
src/calibre/ebooks/oeb/polish/container.py
View File

@ -17,7 +17,6 @@ from collections import defaultdict
from polyglot.builtins import unicode_type, zip
from io import BytesIO
from itertools import count
from urlparse import urlparse
from css_parser import getUrls, replaceUrls
from lxml import etree
@ -55,6 +54,7 @@ from calibre.utils.filenames import hardlink_file, nlinks_file
from calibre.utils.ipc.simple_worker import WorkerError, fork_job
from calibre.utils.logging import default_log
from calibre.utils.zipfile import ZipFile
from polyglot.urllib import urlparse
exists, join, relpath = os.path.exists, os.path.join, os.path.relpath

3
src/calibre/ebooks/oeb/polish/download.py
View File

@ -17,14 +17,13 @@ from functools import partial
from io import BytesIO
from multiprocessing.dummy import Pool
from tempfile import NamedTemporaryFile
from urllib2 import urlopen
from urlparse import urlparse
from calibre import as_unicode, sanitize_file_name2
from calibre.ebooks.oeb.base import OEB_DOCS, OEB_STYLES, barename, iterlinks
from calibre.ebooks.oeb.polish.utils import guess_type
from calibre.ptempfile import TemporaryDirectory
from calibre.web import get_download_filename_from_response
from polyglot.urllib import urlopen, urlparse
def is_external(url):

2
src/calibre/ebooks/oeb/polish/images.py
View File

@ -7,10 +7,10 @@ from __future__ import (unicode_literals, division, absolute_import,
import os
from functools import partial
from threading import Thread, Event
from Queue import Queue, Empty
from calibre import detect_ncpus, human_readable, force_unicode, filesystem_encoding
from polyglot.builtins import range
from polyglot.queue import Queue, Empty
class Worker(Thread):

2
src/calibre/ebooks/oeb/polish/replace.py
View File

@ -10,13 +10,13 @@ __docformat__ = 'restructuredtext en'
import codecs, shutil, os, posixpath
from polyglot.builtins import map
from functools import partial
from urlparse import urlparse, urlunparse
from collections import Counter, defaultdict
from calibre import sanitize_file_name_unicode
from calibre.ebooks.chardet import strip_encoding_declarations
from calibre.ebooks.oeb.polish.css import iter_declarations, remove_property_value
from calibre.ebooks.oeb.polish.utils import extract
from polyglot.urllib import urlparse, urlunparse
class LinkReplacer(object):

2
src/calibre/ebooks/oeb/polish/split.py
View File

@ -8,12 +8,12 @@ __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
import copy, os, re
from polyglot.builtins import map, string_or_bytes, range
from urlparse import urlparse
from calibre.ebooks.oeb.base import barename, XPNSMAP, XPath, OPF, XHTML, OEB_DOCS
from calibre.ebooks.oeb.polish.errors import MalformedMarkup
from calibre.ebooks.oeb.polish.toc import node_from_loc
from calibre.ebooks.oeb.polish.replace import LinkRebaser
from polyglot.urllib import urlparse
class AbortError(ValueError):

2
src/calibre/ebooks/oeb/polish/toc.py
View File

@ -8,7 +8,6 @@ __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import re
from urlparse import urlparse
from collections import Counter, OrderedDict
from functools import partial
from polyglot.builtins import map, unicode_type
@ -26,6 +25,7 @@ from calibre.ebooks.oeb.polish.opf import set_guide_item, get_book_language
from calibre.ebooks.oeb.polish.pretty import pretty_html_tree
from calibre.translations.dynamic import translate
from calibre.utils.localization import get_lang, canonicalize_lang, lang_as_iso639_1
from polyglot.urllib import urlparse
ns = etree.FunctionNamespace('calibre_xpath_extensions')
ns.prefix = 'calibre'

5
src/calibre/ebooks/oeb/reader.py
View File

@ -9,8 +9,6 @@ __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
import sys, os, uuid, copy, re, io
from itertools import izip
from urlparse import urldefrag, urlparse
from urllib import unquote as urlunquote
from collections import defaultdict
from lxml import etree
@ -31,6 +29,7 @@ from calibre.ptempfile import TemporaryDirectory
from calibre.constants import __appname__, __version__
from calibre import guess_type, xml_replace_entities
from polyglot.builtins import unicode_type
from polyglot.urllib import unquote, urldefrag, urlparse
__all__ = ['OEBReader']
@ -632,7 +631,7 @@ class OEBReader(object):
with TemporaryDirectory('_html_cover') as tdir:
writer = OEBWriter()
writer(self.oeb, tdir)
path = os.path.join(tdir, urlunquote(hcover.href))
path = os.path.join(tdir, unquote(hcover.href))
data = render_html_svg_workaround(path, self.logger)
if not data:
data = ''

2
src/calibre/ebooks/oeb/transforms/cover.py
View File

@ -6,12 +6,12 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import textwrap
from urllib import unquote
from lxml import etree
from calibre import guess_type
from calibre.utils.imghdr import identify
from polyglot.builtins import unicode_type
from polyglot.urllib import unquote
class CoverManager(object):

2
src/calibre/ebooks/oeb/transforms/filenames.py
View File

@ -6,11 +6,11 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import posixpath
from urlparse import urldefrag, urlparse
from lxml import etree
from calibre.ebooks.oeb.base import rewrite_links, urlnormalize
from polyglot.urllib import urldefrag, urlparse
class RenameFiles(object): # {{{

2
src/calibre/ebooks/oeb/transforms/rasterize.py
View File

@ -7,7 +7,6 @@ __license__ = 'GPL v3'
__copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
import os, re
from urlparse import urldefrag
from PyQt5.Qt import (
Qt, QByteArray, QBuffer, QIODevice, QColor, QImage, QPainter, QSvgRenderer)
@ -19,6 +18,7 @@ from calibre.ebooks.oeb.stylizer import Stylizer
from calibre.ptempfile import PersistentTemporaryFile
from calibre.utils.imghdr import what
from polyglot.builtins import unicode_type
from polyglot.urllib import urldefrag
IMAGE_TAGS = {XHTML('img'), XHTML('object')}
KEEP_ATTRS = {'class', 'style', 'width', 'height', 'align'}

2
src/calibre/ebooks/oeb/transforms/structure.py
View File

@ -9,12 +9,12 @@ __docformat__ = 'restructuredtext en'
import re, uuid
from lxml import etree
from urlparse import urlparse
from collections import OrderedDict, Counter
from calibre.ebooks.oeb.base import XPNSMAP, TOC, XHTML, xml2text, barename
from calibre.ebooks import ConversionError
from polyglot.builtins import unicode_type
from polyglot.urllib import urlparse
def XPath(x):

3
src/calibre/ebooks/oeb/transforms/trimmanifest.py
View File

@ -6,10 +6,9 @@ from __future__ import with_statement
__license__ = 'GPL v3'
__copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
from urlparse import urldefrag
from calibre.ebooks.oeb.base import CSS_MIME, OEB_DOCS
from calibre.ebooks.oeb.base import urlnormalize, iterlinks
from polyglot.urllib import urldefrag
class ManifestTrimmer(object):

3
src/calibre/ebooks/pdf/render/links.py
View File

@ -8,10 +8,9 @@ __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import os
from urlparse import urlparse
from urllib2 import unquote
from calibre.ebooks.pdf.render.common import Array, Name, Dictionary, String, UTF16String, current_log
from polyglot.urllib import unquote, urlparse
class Destination(Array):

4
src/calibre/ebooks/rb/reader.py
View File

@ -7,7 +7,6 @@ __docformat__ = 'restructuredtext en'
import os
import struct
import zlib
from urllib import unquote as urlunquote
from calibre import CurrentDir
from calibre.ebooks.rb import HEADER
@ -15,6 +14,7 @@ from calibre.ebooks.rb import RocketBookError
from calibre.ebooks.metadata.rb import get_metadata
from calibre.ebooks.metadata.opf2 import OPFCreator
from polyglot.builtins import range
from polyglot.urllib import unquote
class RBToc(list):
@ -64,7 +64,7 @@ class Reader(object):
toc = RBToc()
for i in range(pages):
name = urlunquote(self.stream.read(32).strip('\x00'))
name = unquote(self.stream.read(32).strip('\x00'))
size, offset, flags = self.read_i32(), self.read_i32(), self.read_i32()
toc.append(RBToc.Item(name=name, size=size, offset=offset, flags=flags))

9
src/calibre/ebooks/textile/functions.py
View File

@ -62,9 +62,9 @@ POSSIBILITY OF SUCH DAMAGE.
import re
import uuid
from urlparse import urlparse
from calibre.utils.smartypants import smartyPants
from polyglot.urllib import urlopen, urlparse
def _normalize_newlines(string):
@ -95,14 +95,9 @@ def getimagesize(url):
except ImportError:
return None
try:
import urllib2
except ImportError:
return None
try:
p = ImageFile.Parser()
f = urllib2.urlopen(url)
f = urlopen(url)
while True:
s = f.read(1024)
if not s:

4
src/calibre/gui2/__init__.py
View File

@ -3,7 +3,6 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
""" The GUI """
import glob
import os
import Queue
import signal
import sys
import threading
@ -36,6 +35,7 @@ from calibre.utils.date import UNDEFINED_DATE
from calibre.utils.file_type_icons import EXT_MAP
from calibre.utils.localization import get_lang
from polyglot.builtins import unicode_type, string_or_bytes, range
from polyglot import queue
try:
NO_URL_FORMATTING = QUrl.None_
@ -502,7 +502,7 @@ class FunctionDispatcher(QObject):
if not queued:
typ = Qt.AutoConnection if queued is None else Qt.DirectConnection
self.dispatch_signal.connect(self.dispatch, type=typ)
self.q = Queue.Queue()
self.q = queue.Queue()
self.lock = threading.Lock()
def __call__(self, *args, **kwargs):

2
src/calibre/gui2/add.py
View File

@ -9,7 +9,6 @@ __copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
import shutil, os, weakref, traceback, tempfile, time
from threading import Thread
from collections import OrderedDict
from Queue import Empty
from io import BytesIO
from polyglot.builtins import map, unicode_type, string_or_bytes
@ -30,6 +29,7 @@ from calibre.ptempfile import PersistentTemporaryDirectory
from calibre.utils import join_with_timeout
from calibre.utils.config import prefs
from calibre.utils.ipc.pool import Pool, Failure
from polyglot.queue import Empty
def validate_source(source, parent=None): # {{{

17
src/calibre/gui2/device.py
View File

@ -3,7 +3,7 @@ __license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
# Imports {{{
import os, traceback, Queue, time, io, re, sys, weakref
import os, traceback, time, io, re, sys, weakref
from threading import Thread, Event
from PyQt5.Qt import (
@ -35,6 +35,7 @@ from calibre.utils.img import scale_image
from calibre.library.save_to_disk import find_plugboard
from calibre.ptempfile import PersistentTemporaryFile, force_unicode as filename_to_unicode
from polyglot.builtins import unicode_type, string_or_bytes
from polyglot import queue
# }}}
@ -153,8 +154,8 @@ class DeviceManager(Thread): # {{{
self.sleep_time = sleep_time
self.connected_slot = connected_slot
self.allow_connect_slot = allow_connect_slot
self.jobs = Queue.Queue(0)
self.job_steps = Queue.Queue(0)
self.jobs = queue.Queue(0)
self.job_steps = queue.Queue(0)
self.keep_going = True
self.job_manager = job_manager
self.reported_errors = set([])
@ -163,7 +164,7 @@ class DeviceManager(Thread): # {{{
self.connected_device = None
self.connected_device_kind = None
self.ejected_devices = set([])
self.mount_connection_requests = Queue.Queue(0)
self.mount_connection_requests = queue.Queue(0)
self.open_feedback_slot = open_feedback_slot
self.open_feedback_only_once_seen = set()
self.after_callback_feedback_slot = after_callback_feedback_slot
@ -241,7 +242,7 @@ class DeviceManager(Thread): # {{{
try:
job = self.jobs.get_nowait()
job.abort(Exception(_('Device no longer connected.')))
except Queue.Empty:
except queue.Empty:
break
try:
self.connected_device.post_yank_cleanup()
@ -358,13 +359,13 @@ class DeviceManager(Thread): # {{{
if not self.job_steps.empty():
try:
return self.job_steps.get_nowait()
except Queue.Empty:
except queue.Empty:
pass
if not self.jobs.empty():
try:
return self.jobs.get_nowait()
except Queue.Empty:
except queue.Empty:
pass
def run_startup(self, dev):
@ -391,7 +392,7 @@ class DeviceManager(Thread): # {{{
try:
(kls,device_kind, folder_path) = \
self.mount_connection_requests.get_nowait()
except Queue.Empty:
except queue.Empty:
break
if kls is not None:
try:

12
src/calibre/gui2/dnd.py
View File

@ -7,10 +7,8 @@ __license__ = 'GPL v3'
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import posixpath, os, urllib, re
from urlparse import urlparse
import posixpath, os, re
from threading import Thread
from Queue import Queue, Empty
from PyQt5.Qt import QPixmap, Qt, QDialog, QLabel, QVBoxLayout, \
QDialogButtonBox, QProgressBar, QTimer, QUrl, QImageReader
@ -21,6 +19,8 @@ from calibre import browser, as_unicode, prints
from calibre.gui2 import error_dialog
from calibre.utils.imghdr import what
from polyglot.builtins import unicode_type
from polyglot.urllib import unquote, urlparse
from polyglot.queue import Queue, Empty
def image_extensions():
@ -162,7 +162,7 @@ def path_from_qurl(qurl):
raw = bytes(qurl.toEncoded(
QUrl.PreferLocalFile | QUrl.RemoveScheme | QUrl.RemovePassword | QUrl.RemoveUserInfo |
QUrl.RemovePort | QUrl.RemoveAuthority | QUrl.RemoveQuery | QUrl.RemoveFragment))
ans = urllib.unquote(raw).decode('utf-8', 'replace')
ans = unquote(raw).decode('utf-8', 'replace')
if iswindows and ans.startswith('/'):
ans = ans[1:]
return ans
@ -232,7 +232,7 @@ def dnd_get_image(md, image_exts=None):
paths = [path_from_qurl(u) for u in urls]
# First look for a local file
images = [xi for xi in paths if
posixpath.splitext(urllib.unquote(xi))[1][1:].lower() in
posixpath.splitext(unquote(xi))[1][1:].lower() in
image_exts]
images = [xi for xi in images if os.path.exists(xi)]
p = QPixmap()
@ -278,7 +278,7 @@ def dnd_get_files(md, exts, allow_all_extensions=False, filter_exts=()):
if allow_all_extensions and ext and ext not in filter_exts:
return True
return ext in exts and ext not in filter_exts
local_files = [p for p in local_files if is_ok(urllib.unquote(p))]
local_files = [p for p in local_files if is_ok(unquote(p))]
local_files = [x for x in local_files if os.path.exists(x)]
if local_files:
return local_files, None

2
src/calibre/gui2/icon_theme.py
View File

@ -10,7 +10,6 @@ import os, errno, json, importlib, math, httplib, bz2, shutil, sys
from itertools import count
from io import BytesIO
from polyglot.builtins import map
from Queue import Queue, Empty
from threading import Thread, Event
from multiprocessing.pool import ThreadPool
@ -39,6 +38,7 @@ from calibre.utils.img import image_from_data, Canvas, optimize_png, optimize_jp
from calibre.utils.zipfile import ZipFile, ZIP_STORED
from calibre.utils.filenames import atomic_rename
from lzma.xz import compress, decompress
from polyglot.queue import Queue, Empty
IMAGE_EXTENSIONS = {'png', 'jpg', 'jpeg'}
THEME_COVER = 'icon-theme-cover.jpg'

2
src/calibre/gui2/jobs.py
View File

@ -8,7 +8,6 @@ Job management.
'''
import re, time
from Queue import Empty, Queue
from PyQt5.Qt import (QAbstractTableModel, QModelIndex, Qt, QPainter,
QTimer, pyqtSignal, QIcon, QDialog, QAbstractItemDelegate, QApplication,
@ -31,6 +30,7 @@ from calibre.gui2.widgets2 import Dialog
from calibre.utils.search_query_parser import SearchQueryParser, ParseException
from calibre.utils.icu import lower
from polyglot.builtins import unicode_type, range
from polyglot.queue import Empty, Queue
class AdaptSQP(SearchQueryParser):

2
src/calibre/gui2/library/alternate_views.py
View File

@ -9,7 +9,6 @@ __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
import itertools, operator, os, math
from types import MethodType
from threading import Event, Thread
from Queue import LifoQueue
from functools import wraps, partial
from textwrap import wrap
@ -31,6 +30,7 @@ from calibre.gui2.gestures import GestureManager
from calibre.gui2.library.caches import CoverCache, ThumbnailCache
from calibre.utils.config import prefs, tweaks
from polyglot.builtins import unicode_type, range
from polyglot.queue import LifoQueue
CM_TO_INCH = 0.393701
CACHE_FORMAT = 'PPM'

2
src/calibre/gui2/metadata/single_download.py
View File

@ -13,7 +13,6 @@ DEBUG_DIALOG = False
import os, time
from threading import Thread, Event
from operator import attrgetter
from Queue import Queue, Empty
from io import BytesIO
from PyQt5.Qt import (
@ -41,6 +40,7 @@ from calibre.utils.config import tweaks
from calibre.utils.ipc.simple_worker import fork_job, WorkerError
from calibre.ptempfile import TemporaryDirectory
from polyglot.builtins import unicode_type, range
from polyglot.queue import Queue, Empty
# }}}

2
src/calibre/gui2/qt_file_dialogs.py
View File

@ -5,13 +5,13 @@
from __future__ import absolute_import, division, print_function, unicode_literals
import os
from urllib import unquote
from PyQt5.Qt import QFileDialog, QObject
from calibre.gui2.linux_file_dialogs import dialog_name, image_extensions
from calibre.utils.filenames import expanduser
from polyglot.builtins import unicode_type, string_or_bytes
from polyglot.urllib import unquote
def select_initial_dir(q):

2
src/calibre/gui2/save.py
View File

@ -9,7 +9,6 @@ __copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
import traceback, errno, os, time, shutil
from collections import namedtuple, defaultdict
from tempfile import SpooledTemporaryFile
from Queue import Empty
from PyQt5.Qt import QObject, Qt, pyqtSignal
@ -26,6 +25,7 @@ from calibre.utils.formatter_functions import load_user_template_functions
from calibre.utils.ipc.pool import Pool, Failure
from calibre.library.save_to_disk import sanitize_args, get_path_components, find_plugboard, plugboard_save_to_disk_value
from polyglot.builtins import unicode_type
from polyglot.queue import Empty
BookId = namedtuple('BookId', 'title authors')

2
src/calibre/gui2/store/loader.py
View File

@ -11,12 +11,12 @@ import sys, time, io, re
from zlib import decompressobj
from collections import OrderedDict
from threading import Thread
from urllib import urlencode
from calibre import prints
from calibre.constants import numeric_version, DEBUG
from calibre.gui2.store import StorePlugin
from calibre.utils.config import JSONConfig
from polyglot.urllib import urlencode
class VersionMismatch(ValueError):

2
src/calibre/gui2/store/search/download_thread.py
View File

@ -9,12 +9,12 @@ __docformat__ = 'restructuredtext en'
import traceback, base64
from contextlib import closing
from threading import Thread
from Queue import Queue
from calibre import browser
from calibre.constants import DEBUG
from calibre.utils.img import scale_image
from polyglot.builtins import range
from polyglot.queue import Queue
class GenericDownloadThreadPool(object):

2
src/calibre/gui2/store/web_control.py
View File

@ -7,7 +7,6 @@ __copyright__ = '2011, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en'
import os
from urlparse import urlparse
from PyQt5.Qt import QNetworkCookieJar, QNetworkProxy
from PyQt5.QtWebKitWidgets import QWebView, QWebPage
@ -20,6 +19,7 @@ from calibre.ptempfile import PersistentTemporaryFile
from calibre.utils.filenames import ascii_filename
from calibre.web import get_download_filename
from polyglot.builtins import unicode_type
from polyglot.urllib import urlparse
class NPWebView(QWebView):

4
src/calibre/gui2/threaded_jobs.py
View File

@ -9,11 +9,11 @@ __docformat__ = 'restructuredtext en'
import os, time, tempfile, json
from threading import Thread, RLock, Event
from Queue import Queue
from calibre.utils.ipc.job import BaseJob
from calibre.utils.logging import GUILog
from calibre.ptempfile import base_dir
from polyglot.queue import Queue
class ThreadedJob(BaseJob):
@ -245,5 +245,3 @@ class ThreadedJobServer(Thread):
queued_types.append(job.type)
ans.append(job)
return ans

2
src/calibre/gui2/tweak_book/boss.py
View File

@ -9,7 +9,6 @@ import shutil
import sys
import tempfile
from functools import partial, wraps
from urlparse import urlparse
from PyQt5.Qt import (
QApplication, QCheckBox, QDialog, QDialogButtonBox, QGridLayout, QIcon,
@ -71,6 +70,7 @@ from calibre.utils.icu import numeric_sort_key
from calibre.utils.imghdr import identify
from calibre.utils.tdir_in_cache import tdir_in_cache
from polyglot.builtins import iteritems, string_or_bytes
from polyglot.urllib import urlparse
_diff_dialogs = []
last_used_transform_rules = []

2
src/calibre/gui2/tweak_book/completion/worker.py
View File

@ -8,7 +8,6 @@ __copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
import os, sys
from threading import Thread, Event, RLock
from Queue import Queue
from contextlib import closing
from collections import namedtuple
@ -17,6 +16,7 @@ from calibre.gui2.tweak_book.completion.basic import Request
from calibre.gui2.tweak_book.completion.utils import DataError
from calibre.utils.ipc import eintr_retry_call
from calibre.utils.serialize import msgpack_loads, msgpack_dumps
from polyglot.queue import Queue
COMPLETION_REQUEST = 'completion request'
CLEAR_REQUEST = 'clear request'

4
src/calibre/gui2/tweak_book/preview.py
View File

@ -11,9 +11,7 @@ from bisect import bisect_right
from base64 import b64encode
from polyglot.builtins import map, unicode_type
from threading import Thread
from Queue import Queue, Empty
from functools import partial
from urlparse import urlparse
from PyQt5.Qt import (
QWidget, QVBoxLayout, QApplication, QSize, QNetworkAccessManager, QMenu, QIcon,
@ -31,6 +29,8 @@ from calibre.gui2.viewer.documentview import apply_settings
from calibre.gui2.viewer.config import config
from calibre.gui2.widgets2 import HistoryLineEdit2
from calibre.utils.ipc.simple_worker import offload_worker
from polyglot.urllib import urlparse
from polyglot.queue import Queue, Empty
shutdown = object()

2
src/calibre/gui2/tweak_book/save.py
View File

@ -8,7 +8,6 @@ __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
import shutil, os, errno
from threading import Thread
from Queue import LifoQueue, Empty
from PyQt5.Qt import (QObject, pyqtSignal, QLabel, QWidget, QHBoxLayout, Qt)
@ -18,6 +17,7 @@ from calibre.gui2.progress_indicator import ProgressIndicator
from calibre.utils import join_with_timeout
from calibre.utils.filenames import atomic_rename, format_permissions
from calibre.utils.ipc import RC
from polyglot.queue import LifoQueue, Empty
def save_dir_container(container, path):

2
src/calibre/gui2/ui.py
View File

@ -10,7 +10,6 @@ __docformat__ = 'restructuredtext en'
'''The main GUI'''
import collections, os, sys, textwrap, time, gc, errno, re
from Queue import Queue, Empty
from threading import Thread
from collections import OrderedDict
from io import BytesIO
@ -54,6 +53,7 @@ from calibre.gui2.open_with import register_keyboard_shortcuts
from calibre.library import current_library_name
from calibre.srv.library_broker import GuiLibraryBroker
from polyglot.builtins import unicode_type, string_or_bytes
from polyglot.queue import Queue, Empty
class Listener(Thread): # {{{

2
src/calibre/gui2/viewer/main.py
View File

@ -420,7 +420,7 @@ class EbookViewer(MainWindow):
at_start=True)
def lookup(self, word):
from urllib import quote
from polyglot.urllib import quote
word = word.replace(u'\u00ad', '')
word = quote(word.encode('utf-8'))
lang = canonicalize_lang(self.view.current_language) or get_lang() or 'en'

2
src/calibre/library/sqlite.py
View File

@ -11,7 +11,6 @@ import sqlite3 as sqlite, traceback, time, uuid, sys, os
import repr as reprlib
from sqlite3 import IntegrityError, OperationalError
from threading import Thread
from Queue import Queue
from threading import RLock
from datetime import datetime
from functools import partial
@ -23,6 +22,7 @@ from calibre.constants import iswindows, DEBUG, plugins
from calibre.utils.icu import sort_key
from calibre import prints
from polyglot.builtins import unicode_type
from polyglot.queue import Queue
from dateutil.tz import tzoffset

2
src/calibre/srv/auto_reload.py
View File

@ -8,7 +8,6 @@ __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import os, sys, subprocess, signal, time, errno, socket, ssl
from threading import Thread, Lock
from Queue import Queue, Empty
from calibre.constants import islinux, iswindows, isosx
from calibre.srv.http_response import create_http_handler
@ -18,6 +17,7 @@ from calibre.srv.standalone import create_option_parser
from calibre.srv.utils import create_sock_pair
from calibre.srv.web_socket import DummyHandler
from calibre.utils.monotonic import monotonic
from polyglot.queue import Queue, Empty
MAX_RETRIES = 10

2
src/calibre/srv/content.py
View File

@ -12,7 +12,6 @@ from io import BytesIO
from threading import Lock
from polyglot.builtins import map
from functools import partial
from urllib import quote
from calibre import fit_image, sanitize_file_name_unicode
from calibre.constants import config_dir, iswindows
@ -30,6 +29,7 @@ from calibre.utils.date import timestampfromdt
from calibre.utils.img import scale_image, image_from_data
from calibre.utils.filenames import ascii_filename, atomic_rename
from calibre.utils.shared_file import share_open
from polyglot.urllib import quote
plugboard_content_server_value = 'content_server'
plugboard_content_server_formats = ['epub', 'mobi', 'azw3']

2
src/calibre/srv/http_request.py
View File

@ -8,13 +8,13 @@ __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import re, httplib, repr as reprlib
from io import BytesIO, DEFAULT_BUFFER_SIZE
from urllib import unquote
from calibre import as_unicode, force_unicode
from calibre.ptempfile import SpooledTemporaryFile
from calibre.srv.errors import HTTPSimpleResponse
from calibre.srv.loop import Connection, READ, WRITE
from calibre.srv.utils import MultiDict, HTTP1, HTTP11, Accumulator
from polyglot.urllib import unquote
protocol_map = {(1, 0):HTTP1, (1, 1):HTTP11}
quoted_slash = re.compile(br'%2[fF]')

2
src/calibre/srv/jobs.py
View File

@ -9,11 +9,11 @@ from itertools import count
from collections import namedtuple, deque
from functools import partial
from threading import RLock, Thread, Event
from Queue import Queue, Empty
from calibre import detect_ncpus, force_unicode
from calibre.utils.monotonic import monotonic
from calibre.utils.ipc.simple_worker import fork_job, WorkerError
from polyglot.queue import Queue, Empty
StartEvent = namedtuple('StartEvent', 'job_id name module function args kwargs callback data')
DoneEvent = namedtuple('DoneEvent', 'job_id')

2
src/calibre/srv/legacy.py
View File

@ -5,7 +5,6 @@
from __future__ import absolute_import, division, print_function, unicode_literals
from functools import partial
from urllib import urlencode
from lxml.html import tostring
from lxml.html.builder import E as E_
@ -21,6 +20,7 @@ from calibre.srv.utils import get_library_data, http_date
from calibre.utils.cleantext import clean_xml_chars
from calibre.utils.date import dt_as_local, is_date_undefined, timestampfromdt
from polyglot.builtins import string_or_bytes
from polyglot.urllib import urlencode
# /mobile {{{

2
src/calibre/srv/loop.py
View File

@ -8,7 +8,6 @@ __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import ssl, socket, select, os, traceback
from io import BytesIO
from Queue import Empty, Full
from functools import partial
from calibre import as_unicode
@ -26,6 +25,7 @@ from calibre.utils.logging import ThreadSafeLog
from calibre.utils.monotonic import monotonic
from calibre.utils.mdns import get_external_ip
from polyglot.builtins import range
from polyglot.queue import Empty, Full
READ, WRITE, RDWR, WAIT = 'READ', 'WRITE', 'RDWR', 'WAIT'
WAKEUP, JOB_DONE = bytes(bytearray(range(2)))

2
src/calibre/srv/metadata.py
View File

@ -10,7 +10,6 @@ from collections import namedtuple
from datetime import datetime, time
from functools import partial
from threading import Lock
from urllib import quote
from calibre.constants import config_dir
from calibre.db.categories import Tag
@ -24,6 +23,7 @@ from calibre.utils.localization import calibre_langcode_to_name
from calibre.library.comments import comments_to_html, markdown
from calibre.library.field_metadata import category_icon_map
from polyglot.builtins import range
from polyglot.urllib import quote
IGNORED_FIELDS = frozenset('cover ondevice path marked au_map size'.split())

2
src/calibre/srv/opds.py
View File

@ -8,7 +8,6 @@ __docformat__ = 'restructuredtext en'
import hashlib, binascii
from functools import partial
from collections import OrderedDict, namedtuple
from urllib import urlencode
from lxml import etree, html
from lxml.builder import ElementMaker
@ -28,6 +27,7 @@ from calibre.srv.errors import HTTPNotFound, HTTPInternalServerError
from calibre.srv.routes import endpoint
from calibre.srv.utils import get_library_data, http_date, Offsets
from polyglot.builtins import unicode_type
from polyglot.urllib import urlencode
def hexlify(x):

2
src/calibre/srv/pool.py
View File

@ -7,11 +7,11 @@ __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import sys
from Queue import Queue, Full
from threading import Thread
from calibre.utils.monotonic import monotonic
from polyglot.builtins import range
from polyglot.queue import Queue, Full
class Worker(Thread):

3
src/calibre/srv/render_book.py
View File

@ -10,8 +10,6 @@ from collections import defaultdict, OrderedDict
from itertools import count
from functools import partial
from polyglot.builtins import map, unicode_type
from urlparse import urlparse
from urllib import quote
from css_parser import replaceUrls
from css_parser.css import CSSRule
@ -30,6 +28,7 @@ from calibre.ebooks.oeb.polish.toc import get_toc, get_landmarks
from calibre.ebooks.oeb.polish.utils import guess_type
from calibre.utils.short_uuid import uuid4
from calibre.utils.logging import default_log
from polyglot.urllib import quote, urlparse
RENDER_VERSION = 1

2
src/calibre/srv/routes.py
View File

@ -7,7 +7,6 @@ __license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, sys, inspect, re, time, numbers, json as jsonlib, textwrap
from urllib import quote as urlquote
from itertools import izip
from operator import attrgetter
@ -15,6 +14,7 @@ from calibre.srv.errors import HTTPSimpleResponse, HTTPNotFound, RouteError
from calibre.srv.utils import http_date
from calibre.utils.serialize import msgpack_dumps, json_dumps, MSGPACK_MIME
from polyglot.builtins import unicode_type, range
from polyglot.urllib import quote as urlquote
default_methods = frozenset(('HEAD', 'GET'))

2
src/calibre/srv/tests/ajax.py
View File

@ -9,11 +9,11 @@ __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, zlib, json, base64, os
from io import BytesIO
from functools import partial
from urllib import urlencode, quote
from httplib import OK, NOT_FOUND, FORBIDDEN
from calibre.ebooks.metadata.meta import get_metadata
from calibre.srv.tests.base import LibraryBaseTest
from polyglot.urllib import urlencode, quote
def make_request(conn, url, headers={}, prefix='/ajax', username=None, password=None, method='GET', data=None):

20
src/calibre/srv/tests/auth.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, base64, urllib2, subprocess, os, cookielib, time
import httplib, base64, subprocess, os, cookielib, time
from collections import namedtuple
try:
from distutils.spawn import find_executable
@ -17,6 +17,8 @@ from calibre.ptempfile import TemporaryDirectory
from calibre.srv.errors import HTTPForbidden
from calibre.srv.tests.base import BaseTest, TestServer
from calibre.srv.routes import endpoint, Router
from polyglot.urllib import (build_opener, HTTPBasicAuthHandler,
HTTPCookieProcessor, HTTPDigestAuthHandler, HTTPError)
REALM = 'calibre-test'
@ -50,10 +52,10 @@ def router(prefer_basic_auth=False, ban_for=0, ban_after=5):
def urlopen(server, path='/closed', un='testuser', pw='testpw', method='digest'):
auth_handler = urllib2.HTTPBasicAuthHandler() if method == 'basic' else urllib2.HTTPDigestAuthHandler()
auth_handler = HTTPBasicAuthHandler() if method == 'basic' else HTTPDigestAuthHandler()
url = 'http://localhost:%d%s' % (server.address[1], path)
auth_handler.add_password(realm=REALM, uri=url, user=un, passwd=pw)
return urllib2.build_opener(auth_handler).open(url)
return build_opener(auth_handler).open(url)
def digest(un, pw, nonce=None, uri=None, method='GET', nc=1, qop='auth', realm=REALM, cnonce=None, algorithm='MD5', body=b'', modify=lambda x:None):
@ -276,26 +278,26 @@ class TestAuth(BaseTest):
r = conn.getresponse()
self.ae(r.status, httplib.UNAUTHORIZED)
auth_handler = urllib2.HTTPDigestAuthHandler()
auth_handler = HTTPDigestAuthHandler()
url = 'http://localhost:%d%s' % (server.address[1], '/android')
auth_handler.add_password(realm=REALM, uri=url, user='testuser', passwd='testpw')
cj = cookielib.CookieJar()
cookie_handler = urllib2.HTTPCookieProcessor(cj)
r = urllib2.build_opener(auth_handler, cookie_handler).open(url)
cookie_handler = HTTPCookieProcessor(cj)
r = build_opener(auth_handler, cookie_handler).open(url)
self.ae(r.getcode(), httplib.OK)
cookies = tuple(cj)
self.ae(len(cookies), 1)
cookie = cookies[0]
self.assertIn(b':', cookie.value)
self.ae(cookie.path, b'/android')
r = urllib2.build_opener(cookie_handler).open(url)
r = build_opener(cookie_handler).open(url)
self.ae(r.getcode(), httplib.OK)
self.ae(r.read(), b'android')
# Test that a replay attack against a different URL does not work
try:
urllib2.build_opener(cookie_handler).open(url+'2')
build_opener(cookie_handler).open(url+'2')
assert ('Replay attack succeeded')
except urllib2.HTTPError as e:
except HTTPError as e:
self.ae(e.code, httplib.UNAUTHORIZED)
# }}}

3
src/calibre/srv/utils.py
View File

@ -9,12 +9,10 @@ __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import errno, socket, select, os, time
from Cookie import SimpleCookie
from contextlib import closing
from urlparse import parse_qs
import repr as reprlib
from email.utils import formatdate
from operator import itemgetter
from polyglot.builtins import map, unicode_type, range
from urllib import quote as urlquote
from binascii import hexlify, unhexlify
from calibre import prints
@ -25,6 +23,7 @@ from calibre.utils.localization import get_translator
from calibre.utils.socket_inheritance import set_socket_inherit
from calibre.utils.logging import ThreadSafeLog
from calibre.utils.shared_file import share_open, raise_winerror
from polyglot.urllib import parse_qs, quote as urlquote
HTTP1 = 'HTTP/1.0'
HTTP11 = 'HTTP/1.1'

2
src/calibre/srv/web_socket.py
View File

@ -9,7 +9,6 @@ import httplib, os, weakref, socket
from base64 import standard_b64encode
from collections import deque
from hashlib import sha1
from Queue import Queue, Empty
from struct import unpack_from, pack, error as struct_error
from threading import Lock
@ -19,6 +18,7 @@ from calibre.srv.loop import ServerLoop, HandleInterrupt, WRITE, READ, RDWR, Con
from calibre.srv.http_response import HTTPConnection, create_http_handler
from calibre.srv.utils import DESIRED_SEND_BUFFER_SIZE
from calibre.utils.speedups import ReadOnlyFileBuffer
from polyglot.queue import Queue, Empty
speedup, err = plugins['speedup']
if not speedup:
raise RuntimeError('Failed to load speedup module with error: ' + err)

6
src/calibre/translations/__init__.py
View File

@ -4,7 +4,9 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
'''
Manage translation of user visible strings.
'''
import shutil, tarfile, re, os, subprocess, urllib2
import shutil, tarfile, re, os, subprocess
from polyglot.urllib import urlopen
language_codes = {
'aa':'Afar','ab':'Abkhazian','af':'Afrikaans','am':'Amharic','ar':'Arabic','as':'Assamese','ay':'Aymara','az':'Azerbaijani',
@ -38,7 +40,7 @@ language_codes = {
def import_from_launchpad(url):
f = open('/tmp/launchpad_export.tar.gz', 'wb')
shutil.copyfileobj(urllib2.urlopen(url), f)
shutil.copyfileobj(urlopen(url), f)
f.close()
tf = tarfile.open('/tmp/launchpad_export.tar.gz', 'r:gz')
next = tf.next()

5
src/calibre/utils/https.py
View File

@ -11,6 +11,7 @@ from contextlib import closing
from calibre import get_proxies
from calibre.constants import ispy3
from polyglot.urllib import urlsplit
has_ssl_verify = hasattr(ssl, 'create_default_context') and hasattr(ssl, '_create_unverified_context')
@ -25,11 +26,9 @@ class HTTPError(ValueError):
if ispy3:
from urllib.parse import urlparse
import http.client as httplib
else:
import httplib
from urlparse import urlsplit as urlparse
if has_ssl_verify:
class HTTPSConnection(httplib.HTTPSConnection):
@ -175,7 +174,7 @@ def get_https_resource_securely(
cert_file = None
if cacerts is not None:
cert_file = P(cacerts, allow_user_override=False)
p = urlparse(url)
p = urlsplit(url)
if p.scheme != 'https':
raise ValueError('URL %s scheme must be https, not %r' % (url, p.scheme))

2
src/calibre/utils/ipc/job.py
View File

@ -9,10 +9,10 @@ __docformat__ = 'restructuredtext en'
_count = 0
import time, io
from Queue import Queue, Empty
from calibre import prints
from calibre.constants import DEBUG
from polyglot.queue import Queue, Empty
class BaseJob(object):

2
src/calibre/utils/ipc/pool.py
View File

@ -9,7 +9,6 @@ __copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
import os, sys
from threading import Thread
from collections import namedtuple
from Queue import Queue
from calibre import detect_ncpus, as_unicode, prints
from calibre.constants import iswindows, DEBUG
@ -17,6 +16,7 @@ from calibre.ptempfile import PersistentTemporaryFile
from calibre.utils import join_with_timeout
from calibre.utils.ipc import eintr_retry_call
from calibre.utils.serialize import msgpack_dumps, msgpack_loads, pickle_dumps, pickle_loads
from polyglot.queue import Queue
Job = namedtuple('Job', 'id module func args kwargs')
Result = namedtuple('Result', 'value err traceback')

2
src/calibre/utils/ipc/server.py
View File

@ -17,7 +17,6 @@ from binascii import hexlify
from collections import deque
from math import ceil
from multiprocessing.connection import Listener, arbitrary_address
from Queue import Empty, Queue
from threading import RLock, Thread
from calibre import detect_ncpus as cpu_count
@ -28,6 +27,7 @@ from calibre.utils.ipc.launch import Worker
from calibre.utils.ipc.worker import PARALLEL_FUNCS
from calibre.utils.serialize import msgpack_dumps, pickle_loads
from polyglot.builtins import string_or_bytes, environ_item
from polyglot.queue import Empty, Queue
_counter = 0

2
src/calibre/utils/ipc/worker.py
View File

@ -10,7 +10,6 @@ __docformat__ = 'restructuredtext en'
import os, sys, importlib
from multiprocessing.connection import Client
from threading import Thread
from Queue import Queue
from contextlib import closing
from binascii import unhexlify
from zipimport import ZipImportError
@ -19,6 +18,7 @@ from calibre import prints
from calibre.constants import iswindows, isosx
from calibre.utils.ipc import eintr_retry_call
from calibre.utils.serialize import msgpack_loads, pickle_dumps
from polyglot.queue import Queue
PARALLEL_FUNCS = {
'lrfviewer' :

2
src/calibre/utils/linux_trash.py
View File

@ -22,9 +22,9 @@ from __future__ import (unicode_literals, division, absolute_import,
import os, stat
import os.path as op
from datetime import datetime
from urllib import quote
from polyglot.builtins import unicode_type
from polyglot.urllib import quote
FILES_DIR = 'files'
INFO_DIR = 'info'

9
src/calibre/utils/localization.py
View File

@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en'
import os, locale, re, io
from gettext import GNUTranslations, NullTranslations
from polyglot.builtins import unicode_type
from polyglot.builtins import is_py3, unicode_type
_available_translations = None
@ -244,6 +244,9 @@ def set_translators():
set_translators.lang = t.info().get('language')
except Exception:
pass
if is_py3:
t.install(names=('ngettext',))
else:
t.install(unicode=True, names=('ngettext',))
# Now that we have installed a translator, we have to retranslate the help
# for the global prefs object as it was instantiated in get_lang(), before
@ -500,7 +503,7 @@ def localize_user_manual_link(url):
stats = user_manual_stats()
if stats.get(lc, 0) < 0.3:
return url
from urlparse import urlparse, urlunparse
from polyglot.urllib import urlparse, urlunparse
parts = urlparse(url)
path = re.sub(r'/generated/[a-z]+/', '/generated/%s/' % lc, parts.path or '')
path = '/%s%s' % (lc, path)
@ -525,7 +528,7 @@ def localize_website_link(url):
langs = website_languages()
if lc == 'en' or lc not in langs:
return url
from urlparse import urlparse, urlunparse
from polyglot.urllib import urlparse, urlunparse
parts = urlparse(url)
path = '/{}{}'.format(lc, parts.path)
parts = list(parts)

2
src/calibre/utils/matcher.py
View File

@ -9,7 +9,6 @@ import atexit, os, sys
from math import ceil
from unicodedata import normalize
from threading import Thread, Lock
from Queue import Queue
from operator import itemgetter
from collections import OrderedDict
from itertools import islice
@ -20,6 +19,7 @@ from polyglot.builtins import map, unicode_type, range
from calibre import detect_ncpus as cpu_count, as_unicode
from calibre.constants import plugins, filesystem_encoding
from calibre.utils.icu import primary_sort_key, primary_find, primary_collator
from polyglot.queue import Queue
DEFAULT_LEVEL1 = '/'
DEFAULT_LEVEL2 = '-_ 0123456789'

4
src/calibre/utils/open_with/osx.py
View File

@ -20,9 +20,9 @@ application_locations = ('/Applications', '~/Applications', '~/Desktop')
def generate_public_uti_map():
from lxml import etree
import urllib
from polyglot.urllib import urlopen
from html5_parser import parse
raw = urllib.urlopen(
raw = urlopen(
'https://developer.apple.com/library/ios/documentation/Miscellaneous/Reference/UTIRef/Articles/System-DeclaredUniformTypeIdentifiers.html').read()
root = parse(raw)
tables = root.xpath('//table')[0::2]

4
src/calibre/utils/opensearch/query.py
View File

@ -6,8 +6,7 @@ __license__ = 'GPL 3'
__copyright__ = '2006, Ed Summers <ehs@pobox.com>'
__docformat__ = 'restructuredtext en'
from urlparse import urlparse, urlunparse, parse_qs
from urllib import urlencode
from polyglot.urllib import parse_qs, urlencode, urlparse, urlunparse
class Query(object):
@ -72,4 +71,3 @@ class Query(object):
def has_macro(self, macro):
return macro in self.macro_map

2
src/calibre/utils/rapydscript.py
View File

@ -14,7 +14,6 @@ import subprocess
import sys
from functools import partial
from io import BytesIO
from Queue import Empty, Queue
from threading import Thread, local
from calibre import force_unicode
@ -24,6 +23,7 @@ from calibre.utils.terminal import ANSIStream
from duktape import Context, JSError, to_python
from lzma.xz import compress, decompress
from polyglot.builtins import range
from polyglot.queue import Empty, Queue
COMPILER_PATH = 'rapydscript/compiler.js.xz'

4
src/calibre/utils/serve_coffee.py
View File

@ -18,9 +18,9 @@ if sys.version_info.major > 2:
file=sys.stderr)
raise SystemExit(1)
import time, BaseHTTPServer, os, sys, re, SocketServer
import time, os, sys, re, SocketServer
from threading import Lock, local
from SimpleHTTPServer import SimpleHTTPRequestHandler
from polyglot.http_server import BaseHTTPServer, SimpleHTTPRequestHandler
# Compiler {{{

90
src/calibre/utils/sftp.py
View File

@ -1,90 +0,0 @@
#!/usr/bin/env python2
from __future__ import print_function
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
__docformat__ = 'restructuredtext en'
'''
Create a file handle to a remote file over the sftp protocol.
'''
import sys, socket, getpass
from urlparse import urlparse
from binascii import hexlify
import paramiko
def agent_auth(transport, username):
"""
Attempt to authenticate to the given transport using any of the private
keys available from an SSH agent.
"""
agent = paramiko.Agent()
agent_keys = agent.get_keys()
if len(agent_keys) == 0:
return
for key in agent_keys:
print('Trying ssh-agent key %s' % hexlify(key.get_fingerprint()), end=' ')
try:
transport.auth_publickey(username, key)
print('... success!')
return True
except paramiko.SSHException:
print('... failed.')
return False
def portable_getpass(username, hostname, retry):
return getpass.getpass('%sPlease enter the password for %s on %s: '%(
'Incorrect password. ' if retry else '', username, hostname))
def password_auth(transport, username, hostname, getpw=portable_getpass):
for i in range(3):
pw = getpw(username, hostname, i>0)
transport.auth_password(username, pw)
if transport.is_authenticated():
return True
return False
def connect_to_url(url, getpw=portable_getpass, mode='r+', bufsize=-1):
protocol, host, path = urlparse(url)[:3]
if protocol != 'sftp':
raise ValueError(_('URL must have the scheme sftp'))
try:
username, host = host.split('@')
except:
raise ValueError(_('host must be of the form user@hostname'))
port = 22
if ':' in host:
host, port = host.split(':')
port = int(port)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((host, port))
t = paramiko.Transport(sock)
try:
t.start_client()
except:
raise Exception(_('Failed to negotiate SSH session: ') + str(t.get_exception()))
if not agent_auth(t, username):
if not password_auth(t, username, host, getpw):
raise ValueError(_('Failed to authenticate with server: %s')%url)
sftp = paramiko.SFTPClient.from_transport(t)
return sftp, sftp.open(path, mode=mode, bufsize=bufsize)
def main(args=sys.argv):
f = connect_to_url(args[1])[-1]
print(f.read())
f.seek(0, 2)
print(f.tell())
f.close()
return 0
if __name__ == '__main__':
sys.exit(main())

10
src/calibre/utils/threadpool.py
View File

@ -48,7 +48,7 @@ __license__ = 'Python license'
# standard library modules
import threading
import Queue
from polyglot import queue
# exceptions
@ -75,7 +75,7 @@ class WorkerThread(threading.Thread):
def __init__(self, requestsQueue, resultsQueue, **kwds):
"""Set up thread in daemonic mode and start it immediatedly.
requestsQueue and resultQueue are instances of Queue.Queue passed
requestsQueue and resultQueue are instances of queue.Queue passed
by the ThreadPool class when it creates a new worker thread.
"""
@ -174,8 +174,8 @@ class ThreadPool:
more work requests in it (see putRequest method).
"""
self.requestsQueue = Queue.Queue(q_size)
self.resultsQueue = Queue.Queue()
self.requestsQueue = queue.Queue(q_size)
self.resultsQueue = queue.Queue()
self.workers = []
self.workRequests = {}
self.createWorkers(num_workers)
@ -223,7 +223,7 @@ class ThreadPool:
(request.exception and request.exc_callback):
request.callback(request, result)
del self.workRequests[request.requestID]
except Queue.Empty:
except queue.Empty:
break
def wait(self, sleep=0):

7
src/calibre/web/__init__.py
View File

@ -7,12 +7,11 @@ class Recipe(object):
def get_download_filename_from_response(response):
from urlparse import urlparse
from urllib2 import unquote as urllib2_unquote
from polyglot.urllib import unquote, urlparse
filename = last_part_name = ''
try:
purl = urlparse(response.geturl())
last_part_name = urllib2_unquote(purl.path.split('/')[-1])
last_part_name = unquote(purl.path.split('/')[-1])
disposition = response.info().get('Content-disposition', '')
for p in disposition.split(';'):
if 'filename' in p:
@ -25,7 +24,7 @@ def get_download_filename_from_response(response):
filename = filename[1:]
if filename[-1] in ('\'', '"'):
filename = filename[:-1]
filename = urllib2_unquote(filename)
filename = unquote(filename)
break
except Exception:
import traceback

7
src/calibre/web/feeds/news.py
View File

@ -7,7 +7,7 @@ Defines various abstract base classes that can be subclassed to create powerful
__docformat__ = "restructuredtext en"
import os, time, traceback, re, urlparse, sys, io
import os, time, traceback, re, sys, io
from collections import defaultdict
from functools import partial
from contextlib import nested, closing
@ -32,6 +32,7 @@ from calibre.utils.img import save_cover_data_to, add_borders_to_image, image_to
from calibre.utils.localization import canonicalize_lang
from calibre.utils.logging import ThreadSafeWrapper
from polyglot.builtins import unicode_type, string_or_bytes
from polyglot.urllib import urlparse, urlsplit
class LoginFailed(ValueError):
@ -652,7 +653,7 @@ class BasicNewsRecipe(Recipe):
download an article.
'''
try:
parts = urlparse.urlparse(url)
parts = urlparse(url)
except Exception:
self.log.error('Failed to parse url: %r, ignoring' % url)
return frozenset()
@ -1092,7 +1093,7 @@ class BasicNewsRecipe(Recipe):
if feed.image_url in self.image_map:
feed.image_url = self.image_map[feed.image_url]
else:
bn = urlparse.urlsplit(feed.image_url).path
bn = urlsplit(feed.image_url).path
if bn:
bn = bn.rpartition('/')[-1]
if bn:

37
src/calibre/web/fetch/simple.py
View File

@ -7,8 +7,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
Fetch a webpage and its links recursively. The webpages are saved to disk in
UTF-8 encoding with any charset declarations removed.
'''
import sys, socket, os, urlparse, re, time, urllib2, threading, traceback
from urllib import url2pathname, quote
import sys, socket, os, re, time, threading, traceback
from httplib import responses
from base64 import b64decode
@ -25,6 +24,8 @@ from calibre.utils.img import image_from_data, image_to_data
from calibre.utils.imghdr import what
from calibre.web.fetch.utils import rescale_image
from polyglot.builtins import unicode_type
from polyglot.urllib import (quote, URLError, url2pathname, urljoin,
urlparse, urlsplit, urlunsplit)
class AbortArticle(Exception):
@ -57,7 +58,7 @@ bad_url_counter = 0
def basename(url):
try:
parts = urlparse.urlsplit(url)
parts = urlsplit(url)
path = url2pathname(parts.path)
res = os.path.basename(path)
except:
@ -261,16 +262,16 @@ class RecursiveFetcher(object):
# handles quoting automatically, but leaving it
# in case it breaks something
if re.search(r'\s+', url) is not None:
purl = list(urlparse.urlparse(url))
purl = list(urlparse(url))
for i in range(2, 6):
purl[i] = quote(purl[i])
url = urlparse.urlunparse(purl)
url = urlunparse(purl)
open_func = getattr(self.browser, 'open_novisit', self.browser.open)
try:
with closing(open_func(url, timeout=self.timeout)) as f:
data = response(f.read()+f.read())
data.newurl = f.geturl()
except urllib2.URLError as err:
except URLError as err:
if hasattr(err, 'code') and err.code in responses:
raise FetchError(responses[err.code])
if getattr(err, 'reason', [0])[0] == 104 or \
@ -325,8 +326,8 @@ class RecursiveFetcher(object):
for c, tag in enumerate(soup.findAll(lambda tag: tag.name.lower()in ['link', 'style'] and tag.has_key('type') and tag['type'].lower() == 'text/css')): # noqa
if tag.has_key('href'): # noqa
iurl = tag['href']
if not urlparse.urlsplit(iurl).scheme:
iurl = urlparse.urljoin(baseurl, iurl, False)
if not urlsplit(iurl).scheme:
iurl = urljoin(baseurl, iurl, False)
with self.stylemap_lock:
if self.stylemap.has_key(iurl): # noqa
tag['href'] = self.stylemap[iurl]
@ -348,8 +349,8 @@ class RecursiveFetcher(object):
m = self.__class__.CSS_IMPORT_PATTERN.search(src)
if m:
iurl = m.group(1)
if not urlparse.urlsplit(iurl).scheme:
iurl = urlparse.urljoin(baseurl, iurl, False)
if not urlsplit(iurl).scheme:
iurl = urljoin(baseurl, iurl, False)
with self.stylemap_lock:
if self.stylemap.has_key(iurl): # noqa
ns.replaceWith(src.replace(m.group(1), self.stylemap[iurl]))
@ -386,8 +387,8 @@ class RecursiveFetcher(object):
else:
if callable(self.image_url_processor):
iurl = self.image_url_processor(baseurl, iurl)
if not urlparse.urlsplit(iurl).scheme:
iurl = urlparse.urljoin(baseurl, iurl, False)
if not urlsplit(iurl).scheme:
iurl = urljoin(baseurl, iurl, False)
with self.imagemap_lock:
if self.imagemap.has_key(iurl): # noqa
tag['src'] = self.imagemap[iurl]
@ -443,11 +444,11 @@ class RecursiveFetcher(object):
def absurl(self, baseurl, tag, key, filter=True):
iurl = tag[key]
parts = urlparse.urlsplit(iurl)
parts = urlsplit(iurl)
if not parts.netloc and not parts.path and not parts.query:
return None
if not parts.scheme:
iurl = urlparse.urljoin(baseurl, iurl, False)
iurl = urljoin(baseurl, iurl, False)
if not self.is_link_ok(iurl):
self.log.debug('Skipping invalid link:', iurl)
return None
@ -457,12 +458,12 @@ class RecursiveFetcher(object):
return iurl
def normurl(self, url):
parts = list(urlparse.urlsplit(url))
parts = list(urlsplit(url))
parts[4] = ''
return urlparse.urlunsplit(parts)
return urlunsplit(parts)
def localize_link(self, tag, key, path):
parts = urlparse.urlsplit(tag[key])
parts = urlsplit(tag[key])
suffix = ('#'+parts.fragment) if parts.fragment else ''
tag[key] = path+suffix
@ -547,7 +548,7 @@ class RecursiveFetcher(object):
if newbaseurl and not newbaseurl.startswith('/'):
for atag in soup.findAll('a', href=lambda x: x and x.startswith('/')):
atag['href'] = urlparse.urljoin(newbaseurl, atag['href'], True)
atag['href'] = urljoin(newbaseurl, atag['href'], True)
if callable(self.postprocess_html_ext):
soup = self.postprocess_html_ext(soup,
c==0 and recursion_level==0 and not getattr(self, 'called_first', False),

10
src/polyglot/queue.py Normal file
View File

@ -0,0 +1,10 @@
#!/usr/bin/env python2
# vim:fileencoding=utf-8
# License: GPL v3 Copyright: 2019, Eli Schwartz <eschwartz@archlinux.org>
from polyglot.builtins import is_py3
if is_py3:
from queue import * # noqa
else:
from Queue import * # noqa

20
src/polyglot/urllib.py
View File

@ -2,14 +2,22 @@
# vim:fileencoding=utf-8
# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
from __future__ import absolute_import, division, print_function, unicode_literals
from __future__ import absolute_import
from polyglot.builtins import is_py3
if is_py3:
from urllib.request import urlopen, Request # noqa
from urllib.parse import urlencode # noqa
from urllib.request import (build_opener, getproxies, install_opener, # noqa
HTTPBasicAuthHandler, HTTPCookieProcessor, HTTPDigestAuthHandler, # noqa
url2pathname, urlopen, Request) # noqa
from urllib.parse import (parse_qs, quote, unquote, quote_plus, urldefrag, # noqa
urlencode, urljoin, urlparse, urlunparse, urlsplit, urlunsplit) # noqa
from urllib.error import HTTPError, URLError # noqa
else:
from urllib import urlencode # noqa
from urllib2 import urlopen, Request # noqa
from urllib import (getproxies, quote, unquote, quote_plus, url2pathname, # noqa
urlencode) # noqa
from urllib2 import (build_opener, install_opener, HTTPBasicAuthHandler, # noqa
HTTPCookieProcessor, HTTPDigestAuthHandler, HTTPError, URLError, # noqa
urlopen, Request) # noqa
from urlparse import (parse_qs, urldefrag, urljoin, urlparse, urlunparse, # noqa
urlsplit, urlunsplit) # noqa