From 8594f52fe8e8bd54d3c8b45c409f8ede2e816fd1 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Sun, 17 Mar 2019 15:46:27 -0400 Subject: [PATCH] python3: add more urllib wrappers --- src/calibre/__init__.py | 6 +-- src/calibre/db/cli/main.py | 3 +- .../ebooks/conversion/plugins/chm_input.py | 2 +- .../ebooks/conversion/plugins/html_input.py | 2 +- .../ebooks/conversion/plugins/html_output.py | 4 +- .../ebooks/conversion/plugins/oeb_output.py | 2 +- src/calibre/ebooks/docx/writer/links.py | 2 +- src/calibre/ebooks/html/input.py | 2 +- src/calibre/ebooks/htmlz/oeb2html.py | 2 +- src/calibre/ebooks/lit/reader.py | 3 +- src/calibre/ebooks/lit/writer.py | 5 +-- src/calibre/ebooks/lrf/html/convert_from.py | 3 +- src/calibre/ebooks/metadata/__init__.py | 8 +--- src/calibre/ebooks/metadata/kdl.py | 7 ++-- src/calibre/ebooks/metadata/opf2.py | 3 +- .../ebooks/metadata/search_internet.py | 2 +- src/calibre/ebooks/metadata/toc.py | 3 +- src/calibre/ebooks/mobi/reader/mobi8.py | 2 +- src/calibre/ebooks/mobi/writer2/serializer.py | 2 +- src/calibre/ebooks/oeb/base.py | 3 +- src/calibre/ebooks/oeb/polish/check/links.py | 5 +-- src/calibre/ebooks/oeb/polish/container.py | 2 +- src/calibre/ebooks/oeb/polish/download.py | 3 +- src/calibre/ebooks/oeb/polish/replace.py | 2 +- src/calibre/ebooks/oeb/polish/split.py | 2 +- src/calibre/ebooks/oeb/polish/toc.py | 2 +- src/calibre/ebooks/oeb/reader.py | 5 +-- src/calibre/ebooks/oeb/transforms/cover.py | 2 +- .../ebooks/oeb/transforms/filenames.py | 2 +- .../ebooks/oeb/transforms/rasterize.py | 2 +- .../ebooks/oeb/transforms/structure.py | 2 +- .../ebooks/oeb/transforms/trimmanifest.py | 3 +- src/calibre/ebooks/pdf/render/links.py | 3 +- src/calibre/ebooks/rb/reader.py | 4 +- src/calibre/ebooks/textile/functions.py | 9 +---- src/calibre/gui2/dnd.py | 10 ++--- src/calibre/gui2/qt_file_dialogs.py | 2 +- src/calibre/gui2/store/loader.py | 2 +- src/calibre/gui2/store/web_control.py | 2 +- src/calibre/gui2/tweak_book/boss.py | 2 +- src/calibre/gui2/tweak_book/preview.py | 2 +- src/calibre/gui2/viewer/main.py | 2 +- src/calibre/srv/content.py | 2 +- src/calibre/srv/http_request.py | 2 +- src/calibre/srv/legacy.py | 2 +- src/calibre/srv/metadata.py | 2 +- src/calibre/srv/opds.py | 2 +- src/calibre/srv/render_book.py | 3 +- src/calibre/srv/routes.py | 2 +- src/calibre/srv/tests/ajax.py | 2 +- src/calibre/srv/tests/auth.py | 20 +++++----- src/calibre/srv/utils.py | 3 +- src/calibre/translations/__init__.py | 6 ++- src/calibre/utils/https.py | 5 +-- src/calibre/utils/linux_trash.py | 2 +- src/calibre/utils/localization.py | 4 +- src/calibre/utils/open_with/osx.py | 4 +- src/calibre/utils/opensearch/query.py | 4 +- src/calibre/web/__init__.py | 7 ++-- src/calibre/web/feeds/news.py | 7 ++-- src/calibre/web/fetch/simple.py | 37 ++++++++++--------- src/polyglot/urllib.py | 20 +++++++--- 62 files changed, 130 insertions(+), 142 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index f5d4a2e628..5f441311e4 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -320,7 +320,7 @@ def extract(path, dir): def get_proxies(debug=True): - from urllib import getproxies + from polyglot.urllib import getproxies proxies = getproxies() for key, proxy in list(proxies.items()): if not proxy or '..' in proxy or key == 'auto': @@ -382,10 +382,10 @@ def get_proxy_info(proxy_scheme, proxy_string): is not available in the string. If an exception occurs parsing the string this method returns None. ''' - import urlparse + from polyglot.urllib import urlparse try: proxy_url = u'%s://%s'%(proxy_scheme, proxy_string) - urlinfo = urlparse.urlparse(proxy_url) + urlinfo = urlparse(proxy_url) ans = { u'scheme': urlinfo.scheme, u'hostname': urlinfo.hostname, diff --git a/src/calibre/db/cli/main.py b/src/calibre/db/cli/main.py index 1a957e0820..9833d919d4 100644 --- a/src/calibre/db/cli/main.py +++ b/src/calibre/db/cli/main.py @@ -8,8 +8,6 @@ import httplib import json import os import sys -from urllib import urlencode -from urlparse import urlparse, urlunparse from calibre import browser, prints from calibre.constants import __appname__, __version__, iswindows @@ -19,6 +17,7 @@ from calibre.utils.config import OptionParser, prefs from calibre.utils.localization import localize_user_manual_link from calibre.utils.lock import singleinstance from calibre.utils.serialize import MSGPACK_MIME +from polyglot.urllib import urlencode, urlparse, urlunparse COMMANDS = ( 'list', 'add', 'remove', 'add_format', 'remove_format', 'show_metadata', diff --git a/src/calibre/ebooks/conversion/plugins/chm_input.py b/src/calibre/ebooks/conversion/plugins/chm_input.py index fc0af0552f..01b81bbbec 100644 --- a/src/calibre/ebooks/conversion/plugins/chm_input.py +++ b/src/calibre/ebooks/conversion/plugins/chm_input.py @@ -108,7 +108,7 @@ class CHMInput(InputFormatPlugin): def _create_html_root(self, hhcpath, log, encoding): from lxml import html - from urllib import unquote as _unquote + from polyglot.urllib import unquote as _unquote from calibre.ebooks.oeb.base import urlquote from calibre.ebooks.chardet import xml_to_unicode hhcdata = self._read_file(hhcpath) diff --git a/src/calibre/ebooks/conversion/plugins/html_input.py b/src/calibre/ebooks/conversion/plugins/html_input.py index fa5e211e4d..e04b82b221 100644 --- a/src/calibre/ebooks/conversion/plugins/html_input.py +++ b/src/calibre/ebooks/conversion/plugins/html_input.py @@ -247,7 +247,7 @@ class HTMLInput(InputFormatPlugin): return link, frag def resource_adder(self, link_, base=None): - from urllib import quote + from polyglot.urllib import quote link, frag = self.link_to_local_path(link_, base=base) if link is None: return link_ diff --git a/src/calibre/ebooks/conversion/plugins/html_output.py b/src/calibre/ebooks/conversion/plugins/html_output.py index 4c82974460..15d2f02c07 100644 --- a/src/calibre/ebooks/conversion/plugins/html_output.py +++ b/src/calibre/ebooks/conversion/plugins/html_output.py @@ -47,7 +47,7 @@ class HTMLOutput(OutputFormatPlugin): Generate table of contents ''' from lxml import etree - from urllib import unquote + from polyglot.urllib import unquote from calibre.ebooks.oeb.base import element from calibre.utils.cleantext import clean_xml_chars @@ -86,7 +86,7 @@ class HTMLOutput(OutputFormatPlugin): from lxml import etree from calibre.utils import zipfile from templite import Templite - from urllib import unquote + from polyglot.urllib import unquote from calibre.ebooks.html.meta import EasyMeta # read template files diff --git a/src/calibre/ebooks/conversion/plugins/oeb_output.py b/src/calibre/ebooks/conversion/plugins/oeb_output.py index 72518e264b..82ae031c4d 100644 --- a/src/calibre/ebooks/conversion/plugins/oeb_output.py +++ b/src/calibre/ebooks/conversion/plugins/oeb_output.py @@ -21,7 +21,7 @@ class OEBOutput(OutputFormatPlugin): recommendations = {('pretty_print', True, OptionRecommendation.HIGH)} def convert(self, oeb_book, output_path, input_plugin, opts, log): - from urllib import unquote + from polyglot.urllib import unquote from lxml import etree self.log, self.opts = log, opts diff --git a/src/calibre/ebooks/docx/writer/links.py b/src/calibre/ebooks/docx/writer/links.py index 6293be50c5..4e6315d738 100644 --- a/src/calibre/ebooks/docx/writer/links.py +++ b/src/calibre/ebooks/docx/writer/links.py @@ -8,9 +8,9 @@ __copyright__ = '2015, Kovid Goyal ' import posixpath, re from uuid import uuid4 -from urlparse import urlparse from calibre.utils.filenames import ascii_text +from polyglot.urllib import urlparse def start_text(tag, prefix_len=0, top_level=True): diff --git a/src/calibre/ebooks/html/input.py b/src/calibre/ebooks/html/input.py index 723fcc3688..6bd6183821 100644 --- a/src/calibre/ebooks/html/input.py +++ b/src/calibre/ebooks/html/input.py @@ -13,13 +13,13 @@ Input plugin for HTML or OPF ebooks. ''' import os, re, sys, errno as gerrno -from urlparse import urlparse, urlunparse from calibre.ebooks.oeb.base import urlunquote from calibre.ebooks.chardet import detect_xml_encoding from calibre.constants import iswindows from calibre import unicode_path, as_unicode, replace_entities from polyglot.builtins import unicode_type +from polyglot.urllib import urlparse, urlunparse class Link(object): diff --git a/src/calibre/ebooks/htmlz/oeb2html.py b/src/calibre/ebooks/htmlz/oeb2html.py index f27a7308d5..ba619d1842 100644 --- a/src/calibre/ebooks/htmlz/oeb2html.py +++ b/src/calibre/ebooks/htmlz/oeb2html.py @@ -15,7 +15,6 @@ import re from functools import partial from lxml import html -from urlparse import urldefrag from calibre import prepare_string_for_xml from calibre.ebooks.oeb.base import ( @@ -23,6 +22,7 @@ from calibre.ebooks.oeb.base import ( from calibre.ebooks.oeb.stylizer import Stylizer from calibre.utils.logging import default_log from polyglot.builtins import unicode_type, string_or_bytes +from polyglot.urllib import urldefrag SELF_CLOSING_TAGS = {'area', 'base', 'basefont', 'br', 'hr', 'input', 'img', 'link', 'meta'} diff --git a/src/calibre/ebooks/lit/reader.py b/src/calibre/ebooks/lit/reader.py index b487c673a1..4cef2e9083 100644 --- a/src/calibre/ebooks/lit/reader.py +++ b/src/calibre/ebooks/lit/reader.py @@ -9,8 +9,6 @@ __copyright__ = '2008, Kovid Goyal ' \ 'and Marshall T. Vandegrift ' import io, struct, os, functools, re -from urlparse import urldefrag -from urllib import unquote as urlunquote from lxml import etree @@ -22,6 +20,7 @@ from calibre.ebooks.oeb.reader import OEBReader from calibre.ebooks import DRMError from calibre import plugins from polyglot.builtins import codepoint_to_chr, unicode_type, string_or_bytes, range +from polyglot.urllib import unquote as urlunquote, urldefrag lzx, lxzerror = plugins['lzx'] msdes, msdeserror = plugins['msdes'] diff --git a/src/calibre/ebooks/lit/writer.py b/src/calibre/ebooks/lit/writer.py index b5256101b8..ecc69ab748 100644 --- a/src/calibre/ebooks/lit/writer.py +++ b/src/calibre/ebooks/lit/writer.py @@ -17,8 +17,6 @@ import copy import uuid import functools import numbers -from urlparse import urldefrag -from urllib import unquote as urlunquote from lxml import etree from calibre.ebooks.lit.reader import DirectoryEntry import calibre.ebooks.lit.maps as maps @@ -33,6 +31,7 @@ from calibre import plugins msdes, msdeserror = plugins['msdes'] import calibre.ebooks.lit.mssha1 as mssha1 from polyglot.builtins import codepoint_to_chr, unicode_type, string_or_bytes, range +from polyglot.urllib import urldefrag, unquote __all__ = ['LitWriter'] @@ -521,7 +520,7 @@ class LitWriter(object): media_type = XHTML_MIME elif media_type in OEB_STYLES: media_type = CSS_MIME - href = urlunquote(item.href) + href = unquote(item.href) item.offset = offset \ if state in ('linear', 'nonlinear') else 0 data.write(pack(' -from __future__ import absolute_import, division, print_function, unicode_literals - +from __future__ import absolute_import from polyglot.builtins import is_py3 if is_py3: - from urllib.request import urlopen, Request # noqa - from urllib.parse import urlencode # noqa + from urllib.request import (build_opener, getproxies, install_opener, # noqa + HTTPBasicAuthHandler, HTTPCookieProcessor, HTTPDigestAuthHandler, # noqa + url2pathname, urlopen, Request) # noqa + from urllib.parse import (parse_qs, quote, unquote, quote_plus, urldefrag, # noqa + urlencode, urljoin, urlparse, urlunparse, urlsplit, urlunsplit) # noqa + from urllib.error import HTTPError, URLError # noqa else: - from urllib import urlencode # noqa - from urllib2 import urlopen, Request # noqa + from urllib import (getproxies, quote, unquote, quote_plus, url2pathname, # noqa + urlencode) # noqa + from urllib2 import (build_opener, install_opener, HTTPBasicAuthHandler, # noqa + HTTPCookieProcessor, HTTPDigestAuthHandler, HTTPError, URLError, # noqa + urlopen, Request) # noqa + from urlparse import (parse_qs, urldefrag, urljoin, urlparse, urlunparse, # noqa + urlsplit, urlunsplit) # noqa