python3: add more urllib wrappers

src/calibre/__init__.py

@@ -320,7 +320,7 @@ def extract(path, dir):
 
 
 def get_proxies(debug=True):
-    from urllib import getproxies
+    from polyglot.urllib import getproxies
     proxies = getproxies()
     for key, proxy in list(proxies.items()):
         if not proxy or '..' in proxy or key == 'auto':
@@ -382,10 +382,10 @@ def get_proxy_info(proxy_scheme, proxy_string):
     is not available in the string. If an exception occurs parsing the string
     this method returns None.
     '''
-    import urlparse
+    from polyglot.urllib import urlparse
     try:
         proxy_url = u'%s://%s'%(proxy_scheme, proxy_string)
-        urlinfo = urlparse.urlparse(proxy_url)
+        urlinfo = urlparse(proxy_url)
         ans = {
             u'scheme': urlinfo.scheme,
             u'hostname': urlinfo.hostname,

src/calibre/db/cli/main.py

@@ -8,8 +8,6 @@ import httplib
 import json
 import os
 import sys
-from urllib import urlencode
-from urlparse import urlparse, urlunparse
 
 from calibre import browser, prints
 from calibre.constants import __appname__, __version__, iswindows
@@ -19,6 +17,7 @@ from calibre.utils.config import OptionParser, prefs
 from calibre.utils.localization import localize_user_manual_link
 from calibre.utils.lock import singleinstance
 from calibre.utils.serialize import MSGPACK_MIME
+from polyglot.urllib import urlencode, urlparse, urlunparse
 
 COMMANDS = (
     'list', 'add', 'remove', 'add_format', 'remove_format', 'show_metadata',

src/calibre/ebooks/conversion/plugins/chm_input.py

@@ -108,7 +108,7 @@ class CHMInput(InputFormatPlugin):
 
     def _create_html_root(self, hhcpath, log, encoding):
         from lxml import html
-        from urllib import unquote as _unquote
+        from polyglot.urllib import unquote as _unquote
         from calibre.ebooks.oeb.base import urlquote
         from calibre.ebooks.chardet import xml_to_unicode
         hhcdata = self._read_file(hhcpath)

src/calibre/ebooks/conversion/plugins/html_input.py

@@ -247,7 +247,7 @@ class HTMLInput(InputFormatPlugin):
         return link, frag
 
     def resource_adder(self, link_, base=None):
-        from urllib import quote
+        from polyglot.urllib import quote
         link, frag = self.link_to_local_path(link_, base=base)
         if link is None:
             return link_

src/calibre/ebooks/conversion/plugins/html_output.py

@@ -47,7 +47,7 @@ class HTMLOutput(OutputFormatPlugin):
         Generate table of contents
         '''
         from lxml import etree
-        from urllib import unquote
+        from polyglot.urllib import unquote
 
         from calibre.ebooks.oeb.base import element
         from calibre.utils.cleantext import clean_xml_chars
@@ -86,7 +86,7 @@ class HTMLOutput(OutputFormatPlugin):
         from lxml import etree
         from calibre.utils import zipfile
         from templite import Templite
-        from urllib import unquote
+        from polyglot.urllib import unquote
         from calibre.ebooks.html.meta import EasyMeta
 
         # read template files

src/calibre/ebooks/conversion/plugins/oeb_output.py

@@ -21,7 +21,7 @@ class OEBOutput(OutputFormatPlugin):
     recommendations = {('pretty_print', True, OptionRecommendation.HIGH)}
 
     def convert(self, oeb_book, output_path, input_plugin, opts, log):
-        from urllib import unquote
+        from polyglot.urllib import unquote
         from lxml import etree
 
         self.log, self.opts = log, opts

src/calibre/ebooks/docx/writer/links.py

@@ -8,9 +8,9 @@ __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import posixpath, re
 from uuid import uuid4
-from urlparse import urlparse
 
 from calibre.utils.filenames import ascii_text
+from polyglot.urllib import urlparse
 
 
 def start_text(tag, prefix_len=0, top_level=True):

src/calibre/ebooks/html/input.py

@@ -13,13 +13,13 @@ Input plugin for HTML or OPF ebooks.
 '''
 
 import os, re, sys,  errno as gerrno
-from urlparse import urlparse, urlunparse
 
 from calibre.ebooks.oeb.base import urlunquote
 from calibre.ebooks.chardet import detect_xml_encoding
 from calibre.constants import iswindows
 from calibre import unicode_path, as_unicode, replace_entities
 from polyglot.builtins import unicode_type
+from polyglot.urllib import urlparse, urlunparse
 
 
 class Link(object):

src/calibre/ebooks/htmlz/oeb2html.py

@@ -15,7 +15,6 @@ import re
 
 from functools import partial
 from lxml import html
-from urlparse import urldefrag
 
 from calibre import prepare_string_for_xml
 from calibre.ebooks.oeb.base import (
@@ -23,6 +22,7 @@ from calibre.ebooks.oeb.base import (
 from calibre.ebooks.oeb.stylizer import Stylizer
 from calibre.utils.logging import default_log
 from polyglot.builtins import unicode_type, string_or_bytes
+from polyglot.urllib import urldefrag
 
 SELF_CLOSING_TAGS = {'area', 'base', 'basefont', 'br', 'hr', 'input', 'img', 'link', 'meta'}
 

src/calibre/ebooks/lit/reader.py

@@ -9,8 +9,6 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net> ' \
     'and Marshall T. Vandegrift <llasram@gmail.com>'
 
 import io, struct, os, functools, re
-from urlparse import urldefrag
-from urllib import unquote as urlunquote
 
 from lxml import etree
 
@@ -22,6 +20,7 @@ from calibre.ebooks.oeb.reader import OEBReader
 from calibre.ebooks import DRMError
 from calibre import plugins
 from polyglot.builtins import codepoint_to_chr, unicode_type, string_or_bytes, range
+from polyglot.urllib import unquote as urlunquote, urldefrag
 
 lzx, lxzerror = plugins['lzx']
 msdes, msdeserror = plugins['msdes']

src/calibre/ebooks/lit/writer.py

@@ -17,8 +17,6 @@ import copy
 import uuid
 import functools
 import numbers
-from urlparse import urldefrag
-from urllib import unquote as urlunquote
 from lxml import etree
 from calibre.ebooks.lit.reader import DirectoryEntry
 import calibre.ebooks.lit.maps as maps
@@ -33,6 +31,7 @@ from calibre import plugins
 msdes, msdeserror = plugins['msdes']
 import calibre.ebooks.lit.mssha1 as mssha1
 from polyglot.builtins import codepoint_to_chr, unicode_type, string_or_bytes, range
+from polyglot.urllib import urldefrag, unquote
 
 __all__ = ['LitWriter']
 
@@ -521,7 +520,7 @@ class LitWriter(object):
                     media_type = XHTML_MIME
                 elif media_type in OEB_STYLES:
                     media_type = CSS_MIME
-                href = urlunquote(item.href)
+                href = unquote(item.href)
                 item.offset = offset \
                     if state in ('linear', 'nonlinear') else 0
                 data.write(pack('<I', item.offset))

src/calibre/ebooks/lrf/html/convert_from.py

@@ -9,8 +9,6 @@ and to Falstaff for pylrs.
 """
 import os, re, sys, copy, glob, tempfile
 from collections import deque
-from urllib import unquote
-from urlparse import urlparse
 from math import ceil, floor
 from functools import partial
 
@@ -37,6 +35,7 @@ from calibre.devices.interface import DevicePlugin as Device
 from calibre.ebooks.lrf.html.color_map import lrs_color
 from calibre.ebooks.chardet import xml_to_unicode
 from polyglot.builtins import unicode_type
+from polyglot.urllib import unquote, urlparse
 
 
 def update_css(ncss, ocss):

src/calibre/ebooks/metadata/__init__.py

@@ -9,13 +9,11 @@ Provides abstraction for metadata reading.writing from a variety of ebook format
 """
 import os, sys, re
 
-from urlparse import urlparse
-
 from calibre import relpath, guess_type, remove_bracketed_text, prints, force_unicode
-
 from calibre.utils.config_base import tweaks
-
 from polyglot.builtins import codepoint_to_chr, unicode_type
+from polyglot.urllib import quote, unquote, urlparse
+
 
 try:
     _author_pat = re.compile(tweaks['authors_split_regex'])
@@ -205,7 +203,6 @@ class Resource(object):
     '''
 
     def __init__(self, href_or_path, basedir=os.getcwdu(), is_path=True):
-        from urllib import unquote
         self._href = None
         self._basedir = basedir
         self.path = None
@@ -243,7 +240,6 @@ class Resource(object):
         `basedir`: If None, the basedir of this resource is used (see :method:`set_basedir`).
         If this resource has no basedir, then the current working directory is used as the basedir.
         '''
-        from urllib import quote
         if basedir is None:
             if self._basedir:
                 basedir = self._basedir

src/calibre/ebooks/metadata/kdl.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import re, urllib, urlparse, socket
+import re, socket
 
 from mechanize import URLError
 
@@ -15,6 +15,7 @@ from calibre import browser
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.ebooks.chardet import xml_to_unicode
 from polyglot.builtins import codepoint_to_chr, unicode_type
+from polyglot.urllib import parse_qs, quote_plus
 
 URL = \
 "http://ww2.kdl.org/libcat/WhatsNext.asp?AuthorLastName={0}&AuthorFirstName=&SeriesName=&BookTitle={1}&CategoryID=0&cmdSearch=Search&Search=1&grouping="
@@ -32,7 +33,7 @@ def get_series(title, authors, timeout=60):
     if isinstance(title, unicode_type):
         title = title.encode('utf-8')
 
-    title = urllib.quote_plus(title)
+    title = quote_plus(title)
 
     author = authors[0].strip()
     if not author:
@@ -64,7 +65,7 @@ def get_series(title, authors, timeout=60):
     if a is None:
         return mi
     href = a['href'].partition('?')[-1]
-    data = urlparse.parse_qs(href)
+    data = parse_qs(href)
     series = data.get('SeriesName', [])
     if not series:
         return mi

src/calibre/ebooks/metadata/opf2.py

@@ -9,8 +9,6 @@ lxml based OPF parser.
 '''
 
 import re, sys, unittest, functools, os, uuid, glob, io, json, copy
-from urllib import unquote
-from urlparse import urlparse
 
 from lxml import etree
 
@@ -26,6 +24,7 @@ from calibre import prints, guess_type
 from calibre.utils.cleantext import clean_ascii_chars, clean_xml_chars
 from calibre.utils.config import tweaks
 from polyglot.builtins import unicode_type, range
+from polyglot.urllib import unquote, urlparse
 
 pretty_print_opf = False
 

src/calibre/ebooks/metadata/search_internet.py

@@ -4,7 +4,7 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-from urllib import quote_plus
+from polyglot.urllib import quote_plus
 
 AUTHOR_SEARCHES = {
     'goodreads':

src/calibre/ebooks/metadata/toc.py

@@ -4,8 +4,6 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import os, glob, re, functools
-from urlparse import urlparse
-from urllib import unquote
 from collections import Counter
 
 from lxml import etree
@@ -16,6 +14,7 @@ from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.utils.cleantext import clean_xml_chars
 from polyglot.builtins import unicode_type
+from polyglot.urllib import unquote, urlparse
 
 NCX_NS = "http://www.daisy.org/z3986/2005/ncx/"
 CALIBRE_NS = "http://calibre.kovidgoyal.net/2009/metadata"

src/calibre/ebooks/mobi/reader/mobi8.py

@@ -10,7 +10,6 @@ __docformat__ = 'restructuredtext en'
 import struct, re, os
 from collections import namedtuple
 from itertools import repeat, izip
-from urlparse import urldefrag
 from uuid import uuid4
 
 from lxml import etree
@@ -26,6 +25,7 @@ from calibre.ebooks.mobi.utils import read_font_record
 from calibre.ebooks.oeb.parse_utils import parse_html
 from calibre.ebooks.oeb.base import XPath, XHTML, xml2text
 from polyglot.builtins import range
+from polyglot.urllib import urldefrag
 
 Part = namedtuple('Part',
     'num type filename start end aid')

src/calibre/ebooks/mobi/writer2/serializer.py

@@ -11,7 +11,6 @@ import re
 import unicodedata
 from collections import defaultdict
 from io import BytesIO
-from urlparse import urldefrag
 
 from calibre.ebooks.mobi.mobiml import MBP_NS
 from calibre.ebooks.mobi.utils import is_guide_ref_start
@@ -19,6 +18,7 @@ from calibre.ebooks.oeb.base import (
     OEB_DOCS, XHTML, XHTML_NS, XML_NS, namespace, prefixname, urlnormalize
 )
 from polyglot.builtins import unicode_type, string_or_bytes
+from polyglot.urllib import urldefrag
 
 
 class Buf(BytesIO):

src/calibre/ebooks/oeb/base.py

@@ -9,8 +9,6 @@ __docformat__ = 'restructuredtext en'
 import os, re, logging
 from collections import defaultdict
 from itertools import count
-from urlparse import urldefrag, urlparse, urlunparse, urljoin
-from urllib import unquote
 
 from lxml import etree, html
 from calibre.constants import filesystem_encoding, __version__, ispy3
@@ -23,6 +21,7 @@ from calibre.ebooks.oeb.parse_utils import (barename, XHTML_NS, RECOVER_PARSER,
 from calibre.utils.cleantext import clean_xml_chars
 from calibre.utils.short_uuid import uuid4
 from polyglot.builtins import unicode_type, string_or_bytes, range
+from polyglot.urllib import unquote, urldefrag, urljoin, urlparse, urlunparse
 
 XML_NS       = 'http://www.w3.org/XML/1998/namespace'
 OEB_DOC_NS   = 'http://openebook.org/namespaces/oeb-document/1.0/'

src/calibre/ebooks/oeb/polish/check/links.py

@@ -8,8 +8,6 @@ __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import os
 from collections import defaultdict
-from urlparse import urlparse
-from polyglot.builtins import map
 from threading import Thread
 from Queue import Queue, Empty
 
@@ -21,7 +19,8 @@ from calibre.ebooks.oeb.polish.replace import remove_links_to
 from calibre.ebooks.oeb.polish.cover import get_raster_cover_name
 from calibre.ebooks.oeb.polish.utils import guess_type, actual_case_for_name, corrected_case_for_name
 from calibre.ebooks.oeb.polish.check.base import BaseError, WARN, INFO
-from polyglot.builtins import range
+from polyglot.builtins import map, range
+from polyglot.urllib import urlparse
 
 
 class BadLink(BaseError):

src/calibre/ebooks/oeb/polish/container.py

@@ -17,7 +17,6 @@ from collections import defaultdict
 from polyglot.builtins import unicode_type, zip
 from io import BytesIO
 from itertools import count
-from urlparse import urlparse
 
 from css_parser import getUrls, replaceUrls
 from lxml import etree
@@ -55,6 +54,7 @@ from calibre.utils.filenames import hardlink_file, nlinks_file
 from calibre.utils.ipc.simple_worker import WorkerError, fork_job
 from calibre.utils.logging import default_log
 from calibre.utils.zipfile import ZipFile
+from polyglot.urllib import urlparse
 
 exists, join, relpath = os.path.exists, os.path.join, os.path.relpath
 

src/calibre/ebooks/oeb/polish/download.py

@@ -17,14 +17,13 @@ from functools import partial
 from io import BytesIO
 from multiprocessing.dummy import Pool
 from tempfile import NamedTemporaryFile
-from urllib2 import urlopen
-from urlparse import urlparse
 
 from calibre import as_unicode, sanitize_file_name2
 from calibre.ebooks.oeb.base import OEB_DOCS, OEB_STYLES, barename, iterlinks
 from calibre.ebooks.oeb.polish.utils import guess_type
 from calibre.ptempfile import TemporaryDirectory
 from calibre.web import get_download_filename_from_response
+from polyglot.urllib import urlopen, urlparse
 
 
 def is_external(url):

src/calibre/ebooks/oeb/polish/replace.py

@@ -10,13 +10,13 @@ __docformat__ = 'restructuredtext en'
 import codecs, shutil, os, posixpath
 from polyglot.builtins import map
 from functools import partial
-from urlparse import urlparse, urlunparse
 from collections import Counter, defaultdict
 
 from calibre import sanitize_file_name_unicode
 from calibre.ebooks.chardet import strip_encoding_declarations
 from calibre.ebooks.oeb.polish.css import iter_declarations, remove_property_value
 from calibre.ebooks.oeb.polish.utils import extract
+from polyglot.urllib import urlparse, urlunparse
 
 
 class LinkReplacer(object):

src/calibre/ebooks/oeb/polish/split.py

@@ -8,12 +8,12 @@ __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import copy, os, re
 from polyglot.builtins import map, string_or_bytes, range
-from urlparse import urlparse
 
 from calibre.ebooks.oeb.base import barename, XPNSMAP, XPath, OPF, XHTML, OEB_DOCS
 from calibre.ebooks.oeb.polish.errors import MalformedMarkup
 from calibre.ebooks.oeb.polish.toc import node_from_loc
 from calibre.ebooks.oeb.polish.replace import LinkRebaser
+from polyglot.urllib import urlparse
 
 
 class AbortError(ValueError):

src/calibre/ebooks/oeb/polish/toc.py

@@ -8,7 +8,6 @@ __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import re
-from urlparse import urlparse
 from collections import Counter, OrderedDict
 from functools import partial
 from polyglot.builtins import map, unicode_type
@@ -26,6 +25,7 @@ from calibre.ebooks.oeb.polish.opf import set_guide_item, get_book_language
 from calibre.ebooks.oeb.polish.pretty import pretty_html_tree
 from calibre.translations.dynamic import translate
 from calibre.utils.localization import get_lang, canonicalize_lang, lang_as_iso639_1
+from polyglot.urllib import urlparse
 
 ns = etree.FunctionNamespace('calibre_xpath_extensions')
 ns.prefix = 'calibre'

src/calibre/ebooks/oeb/reader.py

@@ -9,8 +9,6 @@ __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
 
 import sys, os, uuid, copy, re, io
 from itertools import izip
-from urlparse import urldefrag, urlparse
-from urllib import unquote as urlunquote
 from collections import defaultdict
 
 from lxml import etree
@@ -31,6 +29,7 @@ from calibre.ptempfile import TemporaryDirectory
 from calibre.constants import __appname__, __version__
 from calibre import guess_type, xml_replace_entities
 from polyglot.builtins import unicode_type
+from polyglot.urllib import unquote, urldefrag, urlparse
 
 __all__ = ['OEBReader']
 
@@ -632,7 +631,7 @@ class OEBReader(object):
         with TemporaryDirectory('_html_cover') as tdir:
             writer = OEBWriter()
             writer(self.oeb, tdir)
-            path = os.path.join(tdir, urlunquote(hcover.href))
+            path = os.path.join(tdir, unquote(hcover.href))
             data = render_html_svg_workaround(path, self.logger)
             if not data:
                 data = ''

src/calibre/ebooks/oeb/transforms/cover.py

@@ -6,12 +6,12 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import textwrap
-from urllib import unquote
 
 from lxml import etree
 from calibre import guess_type
 from calibre.utils.imghdr import identify
 from polyglot.builtins import unicode_type
+from polyglot.urllib import unquote
 
 
 class CoverManager(object):

src/calibre/ebooks/oeb/transforms/filenames.py

@@ -6,11 +6,11 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import posixpath
-from urlparse import urldefrag, urlparse
 
 from lxml import etree
 
 from calibre.ebooks.oeb.base import rewrite_links, urlnormalize
+from polyglot.urllib import urldefrag, urlparse
 
 
 class RenameFiles(object):  # {{{

src/calibre/ebooks/oeb/transforms/rasterize.py

@@ -7,7 +7,6 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
 
 import os, re
-from urlparse import urldefrag
 
 from PyQt5.Qt import (
     Qt, QByteArray, QBuffer, QIODevice, QColor, QImage, QPainter, QSvgRenderer)
@@ -19,6 +18,7 @@ from calibre.ebooks.oeb.stylizer import Stylizer
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.utils.imghdr import what
 from polyglot.builtins import unicode_type
+from polyglot.urllib import urldefrag
 
 IMAGE_TAGS = {XHTML('img'), XHTML('object')}
 KEEP_ATTRS = {'class', 'style', 'width', 'height', 'align'}

src/calibre/ebooks/oeb/transforms/structure.py

@@ -9,12 +9,12 @@ __docformat__ = 'restructuredtext en'
 import re, uuid
 
 from lxml import etree
-from urlparse import urlparse
 from collections import OrderedDict, Counter
 
 from calibre.ebooks.oeb.base import XPNSMAP, TOC, XHTML, xml2text, barename
 from calibre.ebooks import ConversionError
 from polyglot.builtins import unicode_type
+from polyglot.urllib import urlparse
 
 
 def XPath(x):

src/calibre/ebooks/oeb/transforms/trimmanifest.py

@@ -6,10 +6,9 @@ from __future__ import with_statement
 __license__   = 'GPL v3'
 __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
 
-from urlparse import urldefrag
-
 from calibre.ebooks.oeb.base import CSS_MIME, OEB_DOCS
 from calibre.ebooks.oeb.base import urlnormalize, iterlinks
+from polyglot.urllib import urldefrag
 
 
 class ManifestTrimmer(object):

src/calibre/ebooks/pdf/render/links.py

@@ -8,10 +8,9 @@ __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import os
-from urlparse import urlparse
-from urllib2 import unquote
 
 from calibre.ebooks.pdf.render.common import Array, Name, Dictionary, String, UTF16String, current_log
+from polyglot.urllib import unquote, urlparse
 
 
 class Destination(Array):

src/calibre/ebooks/rb/reader.py

@@ -7,7 +7,6 @@ __docformat__ = 'restructuredtext en'
 import os
 import struct
 import zlib
-from urllib import unquote as urlunquote
 
 from calibre import CurrentDir
 from calibre.ebooks.rb import HEADER
@@ -15,6 +14,7 @@ from calibre.ebooks.rb import RocketBookError
 from calibre.ebooks.metadata.rb import get_metadata
 from calibre.ebooks.metadata.opf2 import OPFCreator
 from polyglot.builtins import range
+from polyglot.urllib import unquote
 
 
 class RBToc(list):
@@ -64,7 +64,7 @@ class Reader(object):
 
         toc = RBToc()
         for i in range(pages):
-            name = urlunquote(self.stream.read(32).strip('\x00'))
+            name = unquote(self.stream.read(32).strip('\x00'))
             size, offset, flags = self.read_i32(), self.read_i32(), self.read_i32()
             toc.append(RBToc.Item(name=name, size=size, offset=offset, flags=flags))
 

src/calibre/ebooks/textile/functions.py

@@ -62,9 +62,9 @@ POSSIBILITY OF SUCH DAMAGE.
 
 import re
 import uuid
-from urlparse import urlparse
 
 from calibre.utils.smartypants import smartyPants
+from polyglot.urllib import urlopen, urlparse
 
 
 def _normalize_newlines(string):
@@ -95,14 +95,9 @@ def getimagesize(url):
         except ImportError:
             return None
 
-    try:
-        import urllib2
-    except ImportError:
-        return None
-
     try:
         p = ImageFile.Parser()
-        f = urllib2.urlopen(url)
+        f = urlopen(url)
         while True:
             s = f.read(1024)
             if not s:

src/calibre/gui2/dnd.py

@@ -7,8 +7,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import posixpath, os, urllib, re
+import posixpath, os, re
-from urlparse import urlparse
 from threading import Thread
 from Queue import Queue, Empty
 
@@ -21,6 +20,7 @@ from calibre import browser, as_unicode, prints
 from calibre.gui2 import error_dialog
 from calibre.utils.imghdr import what
 from polyglot.builtins import unicode_type
+from polyglot.urllib import unquote, urlparse
 
 
 def image_extensions():
@@ -162,7 +162,7 @@ def path_from_qurl(qurl):
     raw = bytes(qurl.toEncoded(
         QUrl.PreferLocalFile | QUrl.RemoveScheme | QUrl.RemovePassword | QUrl.RemoveUserInfo |
         QUrl.RemovePort | QUrl.RemoveAuthority | QUrl.RemoveQuery | QUrl.RemoveFragment))
-    ans = urllib.unquote(raw).decode('utf-8', 'replace')
+    ans = unquote(raw).decode('utf-8', 'replace')
     if iswindows and ans.startswith('/'):
         ans = ans[1:]
     return ans
@@ -232,7 +232,7 @@ def dnd_get_image(md, image_exts=None):
     paths = [path_from_qurl(u) for u in urls]
     # First look for a local file
     images = [xi for xi in paths if
-            posixpath.splitext(urllib.unquote(xi))[1][1:].lower() in
+            posixpath.splitext(unquote(xi))[1][1:].lower() in
             image_exts]
     images = [xi for xi in images if os.path.exists(xi)]
     p = QPixmap()
@@ -278,7 +278,7 @@ def dnd_get_files(md, exts, allow_all_extensions=False, filter_exts=()):
         if allow_all_extensions and ext and ext not in filter_exts:
             return True
         return ext in exts and ext not in filter_exts
-    local_files = [p for p in local_files if is_ok(urllib.unquote(p))]
+    local_files = [p for p in local_files if is_ok(unquote(p))]
     local_files = [x for x in local_files if os.path.exists(x)]
     if local_files:
         return local_files, None

src/calibre/gui2/qt_file_dialogs.py

@@ -5,13 +5,13 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import os
-from urllib import unquote
 
 from PyQt5.Qt import QFileDialog, QObject
 
 from calibre.gui2.linux_file_dialogs import dialog_name, image_extensions
 from calibre.utils.filenames import expanduser
 from polyglot.builtins import unicode_type, string_or_bytes
+from polyglot.urllib import unquote
 
 
 def select_initial_dir(q):

src/calibre/gui2/store/loader.py

@@ -11,12 +11,12 @@ import sys, time, io, re
 from zlib import decompressobj
 from collections import OrderedDict
 from threading import Thread
-from urllib import urlencode
 
 from calibre import prints
 from calibre.constants import numeric_version, DEBUG
 from calibre.gui2.store import StorePlugin
 from calibre.utils.config import JSONConfig
+from polyglot.urllib import urlencode
 
 
 class VersionMismatch(ValueError):

src/calibre/gui2/store/web_control.py

@@ -7,7 +7,6 @@ __copyright__ = '2011, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
 import os
-from urlparse import urlparse
 
 from PyQt5.Qt import QNetworkCookieJar, QNetworkProxy
 from PyQt5.QtWebKitWidgets import QWebView, QWebPage
@@ -20,6 +19,7 @@ from calibre.ptempfile import PersistentTemporaryFile
 from calibre.utils.filenames import ascii_filename
 from calibre.web import get_download_filename
 from polyglot.builtins import unicode_type
+from polyglot.urllib import urlparse
 
 
 class NPWebView(QWebView):

src/calibre/gui2/tweak_book/boss.py

@@ -9,7 +9,6 @@ import shutil
 import sys
 import tempfile
 from functools import partial, wraps
-from urlparse import urlparse
 
 from PyQt5.Qt import (
     QApplication, QCheckBox, QDialog, QDialogButtonBox, QGridLayout, QIcon,
@@ -71,6 +70,7 @@ from calibre.utils.icu import numeric_sort_key
 from calibre.utils.imghdr import identify
 from calibre.utils.tdir_in_cache import tdir_in_cache
 from polyglot.builtins import iteritems, string_or_bytes
+from polyglot.urllib import urlparse
 
 _diff_dialogs = []
 last_used_transform_rules = []

src/calibre/gui2/tweak_book/preview.py

@@ -13,7 +13,6 @@ from polyglot.builtins import map, unicode_type
 from threading import Thread
 from Queue import Queue, Empty
 from functools import partial
-from urlparse import urlparse
 
 from PyQt5.Qt import (
     QWidget, QVBoxLayout, QApplication, QSize, QNetworkAccessManager, QMenu, QIcon,
@@ -31,6 +30,7 @@ from calibre.gui2.viewer.documentview import apply_settings
 from calibre.gui2.viewer.config import config
 from calibre.gui2.widgets2 import HistoryLineEdit2
 from calibre.utils.ipc.simple_worker import offload_worker
+from polyglot.urllib import urlparse
 
 shutdown = object()
 

src/calibre/gui2/viewer/main.py

@@ -420,7 +420,7 @@ class EbookViewer(MainWindow):
                 at_start=True)
 
     def lookup(self, word):
-        from urllib import quote
+        from polyglot.urllib import quote
         word = word.replace(u'\u00ad', '')
         word = quote(word.encode('utf-8'))
         lang = canonicalize_lang(self.view.current_language) or get_lang() or 'en'

src/calibre/srv/content.py

@@ -12,7 +12,6 @@ from io import BytesIO
 from threading import Lock
 from polyglot.builtins import map
 from functools import partial
-from urllib import quote
 
 from calibre import fit_image, sanitize_file_name_unicode
 from calibre.constants import config_dir, iswindows
@@ -30,6 +29,7 @@ from calibre.utils.date import timestampfromdt
 from calibre.utils.img import scale_image, image_from_data
 from calibre.utils.filenames import ascii_filename, atomic_rename
 from calibre.utils.shared_file import share_open
+from polyglot.urllib import quote
 
 plugboard_content_server_value = 'content_server'
 plugboard_content_server_formats = ['epub', 'mobi', 'azw3']

src/calibre/srv/http_request.py

@@ -8,13 +8,13 @@ __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import re, httplib, repr as reprlib
 from io import BytesIO, DEFAULT_BUFFER_SIZE
-from urllib import unquote
 
 from calibre import as_unicode, force_unicode
 from calibre.ptempfile import SpooledTemporaryFile
 from calibre.srv.errors import HTTPSimpleResponse
 from calibre.srv.loop import Connection, READ, WRITE
 from calibre.srv.utils import MultiDict, HTTP1, HTTP11, Accumulator
+from polyglot.urllib import unquote
 
 protocol_map = {(1, 0):HTTP1, (1, 1):HTTP11}
 quoted_slash = re.compile(br'%2[fF]')

src/calibre/srv/legacy.py

@@ -5,7 +5,6 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 from functools import partial
-from urllib import urlencode
 
 from lxml.html import tostring
 from lxml.html.builder import E as E_
@@ -21,6 +20,7 @@ from calibre.srv.utils import get_library_data, http_date
 from calibre.utils.cleantext import clean_xml_chars
 from calibre.utils.date import dt_as_local, is_date_undefined, timestampfromdt
 from polyglot.builtins import string_or_bytes
+from polyglot.urllib import urlencode
 
 # /mobile {{{
 

src/calibre/srv/metadata.py

@@ -10,7 +10,6 @@ from collections import namedtuple
 from datetime import datetime, time
 from functools import partial
 from threading import Lock
-from urllib import quote
 
 from calibre.constants import config_dir
 from calibre.db.categories import Tag
@@ -24,6 +23,7 @@ from calibre.utils.localization import calibre_langcode_to_name
 from calibre.library.comments import comments_to_html, markdown
 from calibre.library.field_metadata import category_icon_map
 from polyglot.builtins import range
+from polyglot.urllib import quote
 
 IGNORED_FIELDS = frozenset('cover ondevice path marked au_map size'.split())
 

src/calibre/srv/opds.py

@@ -8,7 +8,6 @@ __docformat__ = 'restructuredtext en'
 import hashlib, binascii
 from functools import partial
 from collections import OrderedDict, namedtuple
-from urllib import urlencode
 
 from lxml import etree, html
 from lxml.builder import ElementMaker
@@ -28,6 +27,7 @@ from calibre.srv.errors import HTTPNotFound, HTTPInternalServerError
 from calibre.srv.routes import endpoint
 from calibre.srv.utils import get_library_data, http_date, Offsets
 from polyglot.builtins import unicode_type
+from polyglot.urllib import urlencode
 
 
 def hexlify(x):

src/calibre/srv/render_book.py

@@ -10,8 +10,6 @@ from collections import defaultdict, OrderedDict
 from itertools import count
 from functools import partial
 from polyglot.builtins import map, unicode_type
-from urlparse import urlparse
-from urllib import quote
 
 from css_parser import replaceUrls
 from css_parser.css import CSSRule
@@ -30,6 +28,7 @@ from calibre.ebooks.oeb.polish.toc import get_toc, get_landmarks
 from calibre.ebooks.oeb.polish.utils import guess_type
 from calibre.utils.short_uuid import uuid4
 from calibre.utils.logging import default_log
+from polyglot.urllib import quote, urlparse
 
 RENDER_VERSION = 1
 

src/calibre/srv/routes.py

@@ -7,7 +7,6 @@ __license__ = 'GPL v3'
 __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import httplib, sys, inspect, re, time, numbers, json as jsonlib, textwrap
-from urllib import quote as urlquote
 from itertools import izip
 from operator import attrgetter
 
@@ -15,6 +14,7 @@ from calibre.srv.errors import HTTPSimpleResponse, HTTPNotFound, RouteError
 from calibre.srv.utils import http_date
 from calibre.utils.serialize import msgpack_dumps, json_dumps, MSGPACK_MIME
 from polyglot.builtins import unicode_type, range
+from polyglot.urllib import quote as urlquote
 
 default_methods = frozenset(('HEAD', 'GET'))
 

src/calibre/srv/tests/ajax.py

@@ -9,11 +9,11 @@ __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
 import httplib, zlib, json, base64, os
 from io import BytesIO
 from functools import partial
-from urllib import urlencode, quote
 from httplib import OK, NOT_FOUND, FORBIDDEN
 
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.srv.tests.base import LibraryBaseTest
+from polyglot.urllib import urlencode, quote
 
 
 def make_request(conn, url, headers={}, prefix='/ajax', username=None, password=None, method='GET', data=None):

src/calibre/srv/tests/auth.py

@@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
 __license__ = 'GPL v3'
 __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
 
-import httplib, base64, urllib2, subprocess, os, cookielib, time
+import httplib, base64, subprocess, os, cookielib, time
 from collections import namedtuple
 try:
     from distutils.spawn import find_executable
@@ -17,6 +17,8 @@ from calibre.ptempfile import TemporaryDirectory
 from calibre.srv.errors import HTTPForbidden
 from calibre.srv.tests.base import BaseTest, TestServer
 from calibre.srv.routes import endpoint, Router
+from polyglot.urllib import (build_opener, HTTPBasicAuthHandler,
+        HTTPCookieProcessor, HTTPDigestAuthHandler, HTTPError)
 
 REALM = 'calibre-test'
 
@@ -50,10 +52,10 @@ def router(prefer_basic_auth=False, ban_for=0, ban_after=5):
 
 
 def urlopen(server, path='/closed', un='testuser', pw='testpw', method='digest'):
-    auth_handler = urllib2.HTTPBasicAuthHandler() if method == 'basic' else urllib2.HTTPDigestAuthHandler()
+    auth_handler = HTTPBasicAuthHandler() if method == 'basic' else HTTPDigestAuthHandler()
     url = 'http://localhost:%d%s' % (server.address[1], path)
     auth_handler.add_password(realm=REALM, uri=url, user=un, passwd=pw)
-    return urllib2.build_opener(auth_handler).open(url)
+    return build_opener(auth_handler).open(url)
 
 
 def digest(un, pw, nonce=None, uri=None, method='GET', nc=1, qop='auth', realm=REALM, cnonce=None, algorithm='MD5', body=b'', modify=lambda x:None):
@@ -276,26 +278,26 @@ class TestAuth(BaseTest):
             r = conn.getresponse()
             self.ae(r.status, httplib.UNAUTHORIZED)
 
-            auth_handler = urllib2.HTTPDigestAuthHandler()
+            auth_handler = HTTPDigestAuthHandler()
             url = 'http://localhost:%d%s' % (server.address[1], '/android')
             auth_handler.add_password(realm=REALM, uri=url, user='testuser', passwd='testpw')
             cj = cookielib.CookieJar()
-            cookie_handler = urllib2.HTTPCookieProcessor(cj)
+            cookie_handler = HTTPCookieProcessor(cj)
-            r = urllib2.build_opener(auth_handler, cookie_handler).open(url)
+            r = build_opener(auth_handler, cookie_handler).open(url)
             self.ae(r.getcode(), httplib.OK)
             cookies = tuple(cj)
             self.ae(len(cookies), 1)
             cookie = cookies[0]
             self.assertIn(b':', cookie.value)
             self.ae(cookie.path, b'/android')
-            r = urllib2.build_opener(cookie_handler).open(url)
+            r = build_opener(cookie_handler).open(url)
             self.ae(r.getcode(), httplib.OK)
             self.ae(r.read(), b'android')
             # Test that a replay attack against a different URL does not work
             try:
-                urllib2.build_opener(cookie_handler).open(url+'2')
+                build_opener(cookie_handler).open(url+'2')
                 assert ('Replay attack succeeded')
-            except urllib2.HTTPError as e:
+            except HTTPError as e:
                 self.ae(e.code, httplib.UNAUTHORIZED)
 
     # }}}

src/calibre/srv/utils.py

@@ -9,12 +9,10 @@ __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
 import errno, socket, select, os, time
 from Cookie import SimpleCookie
 from contextlib import closing
-from urlparse import parse_qs
 import repr as reprlib
 from email.utils import formatdate
 from operator import itemgetter
 from polyglot.builtins import map, unicode_type, range
-from urllib import quote as urlquote
 from binascii import hexlify, unhexlify
 
 from calibre import prints
@@ -25,6 +23,7 @@ from calibre.utils.localization import get_translator
 from calibre.utils.socket_inheritance import set_socket_inherit
 from calibre.utils.logging import ThreadSafeLog
 from calibre.utils.shared_file import share_open, raise_winerror
+from polyglot.urllib import parse_qs, quote as urlquote
 
 HTTP1  = 'HTTP/1.0'
 HTTP11 = 'HTTP/1.1'

src/calibre/translations/__init__.py

@@ -4,7 +4,9 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 '''
 Manage translation of user visible strings.
 '''
-import shutil, tarfile, re, os, subprocess, urllib2
+import shutil, tarfile, re, os, subprocess
+
+from polyglot.urllib import urlopen
 
 language_codes = {
     'aa':'Afar','ab':'Abkhazian','af':'Afrikaans','am':'Amharic','ar':'Arabic','as':'Assamese','ay':'Aymara','az':'Azerbaijani',
@@ -38,7 +40,7 @@ language_codes = {
 
 def import_from_launchpad(url):
     f = open('/tmp/launchpad_export.tar.gz', 'wb')
-    shutil.copyfileobj(urllib2.urlopen(url), f)
+    shutil.copyfileobj(urlopen(url), f)
     f.close()
     tf = tarfile.open('/tmp/launchpad_export.tar.gz', 'r:gz')
     next = tf.next()

src/calibre/utils/https.py

@@ -11,6 +11,7 @@ from contextlib import closing
 
 from calibre import get_proxies
 from calibre.constants import ispy3
+from polyglot.urllib import urlsplit
 has_ssl_verify = hasattr(ssl, 'create_default_context') and hasattr(ssl, '_create_unverified_context')
 
 
@@ -25,11 +26,9 @@ class HTTPError(ValueError):
 
 
 if ispy3:
-    from urllib.parse import urlparse
     import http.client as httplib
 else:
     import httplib
-    from urlparse import urlsplit as urlparse
 
 if has_ssl_verify:
     class HTTPSConnection(httplib.HTTPSConnection):
@@ -175,7 +174,7 @@ def get_https_resource_securely(
     cert_file = None
     if cacerts is not None:
         cert_file = P(cacerts, allow_user_override=False)
-    p = urlparse(url)
+    p = urlsplit(url)
     if p.scheme != 'https':
         raise ValueError('URL %s scheme must be https, not %r' % (url, p.scheme))
 

src/calibre/utils/linux_trash.py

@@ -22,9 +22,9 @@ from __future__ import (unicode_literals, division, absolute_import,
 import os, stat
 import os.path as op
 from datetime import datetime
-from urllib import quote
 
 from polyglot.builtins import unicode_type
+from polyglot.urllib import quote
 
 FILES_DIR = 'files'
 INFO_DIR = 'info'

src/calibre/utils/localization.py

@@ -503,7 +503,7 @@ def localize_user_manual_link(url):
     stats = user_manual_stats()
     if stats.get(lc, 0) < 0.3:
         return url
-    from urlparse import urlparse, urlunparse
+    from polyglot.urllib import urlparse, urlunparse
     parts = urlparse(url)
     path = re.sub(r'/generated/[a-z]+/', '/generated/%s/' % lc, parts.path or '')
     path = '/%s%s' % (lc, path)
@@ -528,7 +528,7 @@ def localize_website_link(url):
     langs = website_languages()
     if lc == 'en' or lc not in langs:
         return url
-    from urlparse import urlparse, urlunparse
+    from polyglot.urllib import urlparse, urlunparse
     parts = urlparse(url)
     path = '/{}{}'.format(lc, parts.path)
     parts = list(parts)

src/calibre/utils/open_with/osx.py

@@ -20,9 +20,9 @@ application_locations = ('/Applications', '~/Applications', '~/Desktop')
 
 def generate_public_uti_map():
     from lxml import etree
-    import urllib
+    from polyglot.urllib import urlopen
     from html5_parser import parse
-    raw = urllib.urlopen(
+    raw = urlopen(
         'https://developer.apple.com/library/ios/documentation/Miscellaneous/Reference/UTIRef/Articles/System-DeclaredUniformTypeIdentifiers.html').read()
     root = parse(raw)
     tables = root.xpath('//table')[0::2]

src/calibre/utils/opensearch/query.py

@@ -6,8 +6,7 @@ __license__ = 'GPL 3'
 __copyright__ = '2006, Ed Summers <ehs@pobox.com>'
 __docformat__ = 'restructuredtext en'
 
-from urlparse import urlparse, urlunparse, parse_qs
+from polyglot.urllib import parse_qs, urlencode, urlparse, urlunparse
-from urllib import urlencode
 
 
 class Query(object):
@@ -72,4 +71,3 @@ class Query(object):
 
     def has_macro(self, macro):
         return macro in self.macro_map
-

src/calibre/web/__init__.py

@@ -7,12 +7,11 @@ class Recipe(object):
 
 
 def get_download_filename_from_response(response):
-    from urlparse import urlparse
+    from polyglot.urllib import unquote, urlparse
-    from urllib2 import unquote as urllib2_unquote
     filename = last_part_name = ''
     try:
         purl = urlparse(response.geturl())
-        last_part_name = urllib2_unquote(purl.path.split('/')[-1])
+        last_part_name = unquote(purl.path.split('/')[-1])
         disposition = response.info().get('Content-disposition', '')
         for p in disposition.split(';'):
             if 'filename' in p:
@@ -25,7 +24,7 @@ def get_download_filename_from_response(response):
                     filename = filename[1:]
                 if filename[-1] in ('\'', '"'):
                     filename = filename[:-1]
-                filename = urllib2_unquote(filename)
+                filename = unquote(filename)
                 break
     except Exception:
         import traceback

src/calibre/web/feeds/news.py

@@ -7,7 +7,7 @@ Defines various abstract base classes that can be subclassed to create powerful
 __docformat__ = "restructuredtext en"
 
 
-import os, time, traceback, re, urlparse, sys, io
+import os, time, traceback, re, sys, io
 from collections import defaultdict
 from functools import partial
 from contextlib import nested, closing
@@ -32,6 +32,7 @@ from calibre.utils.img import save_cover_data_to, add_borders_to_image, image_to
 from calibre.utils.localization import canonicalize_lang
 from calibre.utils.logging import ThreadSafeWrapper
 from polyglot.builtins import unicode_type, string_or_bytes
+from polyglot.urllib import urlparse, urlsplit
 
 
 class LoginFailed(ValueError):
@@ -652,7 +653,7 @@ class BasicNewsRecipe(Recipe):
                         download an article.
         '''
         try:
-            parts = urlparse.urlparse(url)
+            parts = urlparse(url)
         except Exception:
             self.log.error('Failed to parse url: %r, ignoring' % url)
             return frozenset()
@@ -1092,7 +1093,7 @@ class BasicNewsRecipe(Recipe):
             if feed.image_url in self.image_map:
                 feed.image_url = self.image_map[feed.image_url]
             else:
-                bn = urlparse.urlsplit(feed.image_url).path
+                bn = urlsplit(feed.image_url).path
                 if bn:
                     bn = bn.rpartition('/')[-1]
                     if bn:

src/calibre/web/fetch/simple.py

@@ -7,8 +7,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Fetch a webpage and its links recursively. The webpages are saved to disk in
 UTF-8 encoding with any charset declarations removed.
 '''
-import sys, socket, os, urlparse, re, time, urllib2, threading, traceback
+import sys, socket, os, re, time, threading, traceback
-from urllib import url2pathname, quote
 from httplib import responses
 from base64 import b64decode
 
@@ -25,6 +24,8 @@ from calibre.utils.img import image_from_data, image_to_data
 from calibre.utils.imghdr import what
 from calibre.web.fetch.utils import rescale_image
 from polyglot.builtins import unicode_type
+from polyglot.urllib import (quote, URLError, url2pathname, urljoin,
+        urlparse, urlsplit, urlunsplit)
 
 
 class AbortArticle(Exception):
@@ -57,7 +58,7 @@ bad_url_counter = 0
 
 def basename(url):
     try:
-        parts = urlparse.urlsplit(url)
+        parts = urlsplit(url)
         path = url2pathname(parts.path)
         res = os.path.basename(path)
     except:
@@ -261,16 +262,16 @@ class RecursiveFetcher(object):
         # handles quoting automatically, but leaving it
         # in case it breaks something
         if re.search(r'\s+', url) is not None:
-            purl = list(urlparse.urlparse(url))
+            purl = list(urlparse(url))
             for i in range(2, 6):
                 purl[i] = quote(purl[i])
-            url = urlparse.urlunparse(purl)
+            url = urlunparse(purl)
         open_func = getattr(self.browser, 'open_novisit', self.browser.open)
         try:
             with closing(open_func(url, timeout=self.timeout)) as f:
                 data = response(f.read()+f.read())
                 data.newurl = f.geturl()
-        except urllib2.URLError as err:
+        except URLError as err:
             if hasattr(err, 'code') and err.code in responses:
                 raise FetchError(responses[err.code])
             if getattr(err, 'reason', [0])[0] == 104 or \
@@ -325,8 +326,8 @@ class RecursiveFetcher(object):
         for c, tag in enumerate(soup.findAll(lambda tag: tag.name.lower()in ['link', 'style'] and tag.has_key('type') and tag['type'].lower() == 'text/css')):  # noqa
             if tag.has_key('href'):  # noqa
                 iurl = tag['href']
-                if not urlparse.urlsplit(iurl).scheme:
+                if not urlsplit(iurl).scheme:
-                    iurl = urlparse.urljoin(baseurl, iurl, False)
+                    iurl = urljoin(baseurl, iurl, False)
                 with self.stylemap_lock:
                     if self.stylemap.has_key(iurl):  # noqa
                         tag['href'] = self.stylemap[iurl]
@@ -348,8 +349,8 @@ class RecursiveFetcher(object):
                     m = self.__class__.CSS_IMPORT_PATTERN.search(src)
                     if m:
                         iurl = m.group(1)
-                        if not urlparse.urlsplit(iurl).scheme:
+                        if not urlsplit(iurl).scheme:
-                            iurl = urlparse.urljoin(baseurl, iurl, False)
+                            iurl = urljoin(baseurl, iurl, False)
                         with self.stylemap_lock:
                             if self.stylemap.has_key(iurl):  # noqa
                                 ns.replaceWith(src.replace(m.group(1), self.stylemap[iurl]))
@@ -386,8 +387,8 @@ class RecursiveFetcher(object):
             else:
                 if callable(self.image_url_processor):
                     iurl = self.image_url_processor(baseurl, iurl)
-                if not urlparse.urlsplit(iurl).scheme:
+                if not urlsplit(iurl).scheme:
-                    iurl = urlparse.urljoin(baseurl, iurl, False)
+                    iurl = urljoin(baseurl, iurl, False)
                 with self.imagemap_lock:
                     if self.imagemap.has_key(iurl):  # noqa
                         tag['src'] = self.imagemap[iurl]
@@ -443,11 +444,11 @@ class RecursiveFetcher(object):
 
     def absurl(self, baseurl, tag, key, filter=True):
         iurl = tag[key]
-        parts = urlparse.urlsplit(iurl)
+        parts = urlsplit(iurl)
         if not parts.netloc and not parts.path and not parts.query:
             return None
         if not parts.scheme:
-            iurl = urlparse.urljoin(baseurl, iurl, False)
+            iurl = urljoin(baseurl, iurl, False)
         if not self.is_link_ok(iurl):
             self.log.debug('Skipping invalid link:', iurl)
             return None
@@ -457,12 +458,12 @@ class RecursiveFetcher(object):
         return iurl
 
     def normurl(self, url):
-        parts = list(urlparse.urlsplit(url))
+        parts = list(urlsplit(url))
         parts[4] = ''
-        return urlparse.urlunsplit(parts)
+        return urlunsplit(parts)
 
     def localize_link(self, tag, key, path):
-        parts = urlparse.urlsplit(tag[key])
+        parts = urlsplit(tag[key])
         suffix = ('#'+parts.fragment) if parts.fragment else ''
         tag[key] = path+suffix
 
@@ -547,7 +548,7 @@ class RecursiveFetcher(object):
 
                     if newbaseurl and not newbaseurl.startswith('/'):
                         for atag in soup.findAll('a', href=lambda x: x and x.startswith('/')):
-                            atag['href'] = urlparse.urljoin(newbaseurl, atag['href'], True)
+                            atag['href'] = urljoin(newbaseurl, atag['href'], True)
                     if callable(self.postprocess_html_ext):
                         soup = self.postprocess_html_ext(soup,
                                 c==0 and recursion_level==0 and not getattr(self, 'called_first', False),

src/polyglot/urllib.py

@@ -2,14 +2,22 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
 
-from __future__ import absolute_import, division, print_function, unicode_literals
+from __future__ import absolute_import
-
 
 from polyglot.builtins import is_py3
 
 if is_py3:
-    from urllib.request import urlopen, Request  # noqa
+    from urllib.request import (build_opener, getproxies, install_opener,  # noqa
-    from urllib.parse import urlencode  # noqa
+            HTTPBasicAuthHandler, HTTPCookieProcessor, HTTPDigestAuthHandler,  # noqa
+            url2pathname, urlopen, Request)  # noqa
+    from urllib.parse import (parse_qs, quote, unquote, quote_plus, urldefrag,  # noqa
+            urlencode, urljoin, urlparse, urlunparse, urlsplit, urlunsplit)  # noqa
+    from urllib.error import HTTPError, URLError  # noqa
 else:
-    from urllib import urlencode  # noqa
+    from urllib import (getproxies, quote, unquote, quote_plus, url2pathname,  # noqa
-    from urllib2 import urlopen, Request  # noqa
+            urlencode)  # noqa
+    from urllib2 import (build_opener, install_opener, HTTPBasicAuthHandler,  # noqa
+            HTTPCookieProcessor, HTTPDigestAuthHandler, HTTPError, URLError,  # noqa
+            urlopen, Request)  # noqa
+    from urlparse import (parse_qs, urldefrag, urljoin, urlparse, urlunparse,  # noqa
+            urlsplit, urlunsplit)  # noqa