From 2d382274cd684d68c5288e2abc8b0a8aefdb5c97 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Sun, 17 Mar 2019 07:46:31 -0400 Subject: [PATCH 1/6] py3: gettext no longer has a unicode= argument This is unicode by default "since there's no point in translating to byte strings." Good riddance. --- src/calibre/utils/localization.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/calibre/utils/localization.py b/src/calibre/utils/localization.py index 7f42282a47..79eafd1754 100644 --- a/src/calibre/utils/localization.py +++ b/src/calibre/utils/localization.py @@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en' import os, locale, re, io from gettext import GNUTranslations, NullTranslations -from polyglot.builtins import unicode_type +from polyglot.builtins import is_py3, unicode_type _available_translations = None @@ -244,7 +244,10 @@ def set_translators(): set_translators.lang = t.info().get('language') except Exception: pass - t.install(unicode=True, names=('ngettext',)) + if is_py3: + t.install(names=('ngettext',)) + else: + t.install(unicode=True, names=('ngettext',)) # Now that we have installed a translator, we have to retranslate the help # for the global prefs object as it was instantiated in get_lang(), before # the translator was installed. From 1653c790db82e61a47d75ce25f9f7637fbaa6787 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 19 Mar 2019 14:54:36 -0400 Subject: [PATCH 2/6] remove dead code Initially added in commit c1ef643b8ed6c1740438692e345761a401fea1af but I cannot tell what it was ever used for. --- src/calibre/utils/sftp.py | 90 --------------------------------------- 1 file changed, 90 deletions(-) delete mode 100644 src/calibre/utils/sftp.py diff --git a/src/calibre/utils/sftp.py b/src/calibre/utils/sftp.py deleted file mode 100644 index 07a66917da..0000000000 --- a/src/calibre/utils/sftp.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python2 -from __future__ import print_function -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' -__docformat__ = 'restructuredtext en' - -''' -Create a file handle to a remote file over the sftp protocol. -''' - -import sys, socket, getpass -from urlparse import urlparse -from binascii import hexlify - -import paramiko - - -def agent_auth(transport, username): - """ - Attempt to authenticate to the given transport using any of the private - keys available from an SSH agent. - """ - - agent = paramiko.Agent() - agent_keys = agent.get_keys() - if len(agent_keys) == 0: - return - - for key in agent_keys: - print('Trying ssh-agent key %s' % hexlify(key.get_fingerprint()), end=' ') - try: - transport.auth_publickey(username, key) - print('... success!') - return True - except paramiko.SSHException: - print('... failed.') - return False - - -def portable_getpass(username, hostname, retry): - return getpass.getpass('%sPlease enter the password for %s on %s: '%( - 'Incorrect password. ' if retry else '', username, hostname)) - - -def password_auth(transport, username, hostname, getpw=portable_getpass): - for i in range(3): - pw = getpw(username, hostname, i>0) - transport.auth_password(username, pw) - if transport.is_authenticated(): - return True - return False - - -def connect_to_url(url, getpw=portable_getpass, mode='r+', bufsize=-1): - protocol, host, path = urlparse(url)[:3] - if protocol != 'sftp': - raise ValueError(_('URL must have the scheme sftp')) - try: - username, host = host.split('@') - except: - raise ValueError(_('host must be of the form user@hostname')) - port = 22 - if ':' in host: - host, port = host.split(':') - port = int(port) - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((host, port)) - t = paramiko.Transport(sock) - try: - t.start_client() - except: - raise Exception(_('Failed to negotiate SSH session: ') + str(t.get_exception())) - if not agent_auth(t, username): - if not password_auth(t, username, host, getpw): - raise ValueError(_('Failed to authenticate with server: %s')%url) - sftp = paramiko.SFTPClient.from_transport(t) - return sftp, sftp.open(path, mode=mode, bufsize=bufsize) - - -def main(args=sys.argv): - f = connect_to_url(args[1])[-1] - print(f.read()) - f.seek(0, 2) - print(f.tell()) - f.close() - return 0 - - -if __name__ == '__main__': - sys.exit(main()) From 8594f52fe8e8bd54d3c8b45c409f8ede2e816fd1 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Sun, 17 Mar 2019 15:46:27 -0400 Subject: [PATCH 3/6] python3: add more urllib wrappers --- src/calibre/__init__.py | 6 +-- src/calibre/db/cli/main.py | 3 +- .../ebooks/conversion/plugins/chm_input.py | 2 +- .../ebooks/conversion/plugins/html_input.py | 2 +- .../ebooks/conversion/plugins/html_output.py | 4 +- .../ebooks/conversion/plugins/oeb_output.py | 2 +- src/calibre/ebooks/docx/writer/links.py | 2 +- src/calibre/ebooks/html/input.py | 2 +- src/calibre/ebooks/htmlz/oeb2html.py | 2 +- src/calibre/ebooks/lit/reader.py | 3 +- src/calibre/ebooks/lit/writer.py | 5 +-- src/calibre/ebooks/lrf/html/convert_from.py | 3 +- src/calibre/ebooks/metadata/__init__.py | 8 +--- src/calibre/ebooks/metadata/kdl.py | 7 ++-- src/calibre/ebooks/metadata/opf2.py | 3 +- .../ebooks/metadata/search_internet.py | 2 +- src/calibre/ebooks/metadata/toc.py | 3 +- src/calibre/ebooks/mobi/reader/mobi8.py | 2 +- src/calibre/ebooks/mobi/writer2/serializer.py | 2 +- src/calibre/ebooks/oeb/base.py | 3 +- src/calibre/ebooks/oeb/polish/check/links.py | 5 +-- src/calibre/ebooks/oeb/polish/container.py | 2 +- src/calibre/ebooks/oeb/polish/download.py | 3 +- src/calibre/ebooks/oeb/polish/replace.py | 2 +- src/calibre/ebooks/oeb/polish/split.py | 2 +- src/calibre/ebooks/oeb/polish/toc.py | 2 +- src/calibre/ebooks/oeb/reader.py | 5 +-- src/calibre/ebooks/oeb/transforms/cover.py | 2 +- .../ebooks/oeb/transforms/filenames.py | 2 +- .../ebooks/oeb/transforms/rasterize.py | 2 +- .../ebooks/oeb/transforms/structure.py | 2 +- .../ebooks/oeb/transforms/trimmanifest.py | 3 +- src/calibre/ebooks/pdf/render/links.py | 3 +- src/calibre/ebooks/rb/reader.py | 4 +- src/calibre/ebooks/textile/functions.py | 9 +---- src/calibre/gui2/dnd.py | 10 ++--- src/calibre/gui2/qt_file_dialogs.py | 2 +- src/calibre/gui2/store/loader.py | 2 +- src/calibre/gui2/store/web_control.py | 2 +- src/calibre/gui2/tweak_book/boss.py | 2 +- src/calibre/gui2/tweak_book/preview.py | 2 +- src/calibre/gui2/viewer/main.py | 2 +- src/calibre/srv/content.py | 2 +- src/calibre/srv/http_request.py | 2 +- src/calibre/srv/legacy.py | 2 +- src/calibre/srv/metadata.py | 2 +- src/calibre/srv/opds.py | 2 +- src/calibre/srv/render_book.py | 3 +- src/calibre/srv/routes.py | 2 +- src/calibre/srv/tests/ajax.py | 2 +- src/calibre/srv/tests/auth.py | 20 +++++----- src/calibre/srv/utils.py | 3 +- src/calibre/translations/__init__.py | 6 ++- src/calibre/utils/https.py | 5 +-- src/calibre/utils/linux_trash.py | 2 +- src/calibre/utils/localization.py | 4 +- src/calibre/utils/open_with/osx.py | 4 +- src/calibre/utils/opensearch/query.py | 4 +- src/calibre/web/__init__.py | 7 ++-- src/calibre/web/feeds/news.py | 7 ++-- src/calibre/web/fetch/simple.py | 37 ++++++++++--------- src/polyglot/urllib.py | 20 +++++++--- 62 files changed, 130 insertions(+), 142 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index f5d4a2e628..5f441311e4 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -320,7 +320,7 @@ def extract(path, dir): def get_proxies(debug=True): - from urllib import getproxies + from polyglot.urllib import getproxies proxies = getproxies() for key, proxy in list(proxies.items()): if not proxy or '..' in proxy or key == 'auto': @@ -382,10 +382,10 @@ def get_proxy_info(proxy_scheme, proxy_string): is not available in the string. If an exception occurs parsing the string this method returns None. ''' - import urlparse + from polyglot.urllib import urlparse try: proxy_url = u'%s://%s'%(proxy_scheme, proxy_string) - urlinfo = urlparse.urlparse(proxy_url) + urlinfo = urlparse(proxy_url) ans = { u'scheme': urlinfo.scheme, u'hostname': urlinfo.hostname, diff --git a/src/calibre/db/cli/main.py b/src/calibre/db/cli/main.py index 1a957e0820..9833d919d4 100644 --- a/src/calibre/db/cli/main.py +++ b/src/calibre/db/cli/main.py @@ -8,8 +8,6 @@ import httplib import json import os import sys -from urllib import urlencode -from urlparse import urlparse, urlunparse from calibre import browser, prints from calibre.constants import __appname__, __version__, iswindows @@ -19,6 +17,7 @@ from calibre.utils.config import OptionParser, prefs from calibre.utils.localization import localize_user_manual_link from calibre.utils.lock import singleinstance from calibre.utils.serialize import MSGPACK_MIME +from polyglot.urllib import urlencode, urlparse, urlunparse COMMANDS = ( 'list', 'add', 'remove', 'add_format', 'remove_format', 'show_metadata', diff --git a/src/calibre/ebooks/conversion/plugins/chm_input.py b/src/calibre/ebooks/conversion/plugins/chm_input.py index fc0af0552f..01b81bbbec 100644 --- a/src/calibre/ebooks/conversion/plugins/chm_input.py +++ b/src/calibre/ebooks/conversion/plugins/chm_input.py @@ -108,7 +108,7 @@ class CHMInput(InputFormatPlugin): def _create_html_root(self, hhcpath, log, encoding): from lxml import html - from urllib import unquote as _unquote + from polyglot.urllib import unquote as _unquote from calibre.ebooks.oeb.base import urlquote from calibre.ebooks.chardet import xml_to_unicode hhcdata = self._read_file(hhcpath) diff --git a/src/calibre/ebooks/conversion/plugins/html_input.py b/src/calibre/ebooks/conversion/plugins/html_input.py index fa5e211e4d..e04b82b221 100644 --- a/src/calibre/ebooks/conversion/plugins/html_input.py +++ b/src/calibre/ebooks/conversion/plugins/html_input.py @@ -247,7 +247,7 @@ class HTMLInput(InputFormatPlugin): return link, frag def resource_adder(self, link_, base=None): - from urllib import quote + from polyglot.urllib import quote link, frag = self.link_to_local_path(link_, base=base) if link is None: return link_ diff --git a/src/calibre/ebooks/conversion/plugins/html_output.py b/src/calibre/ebooks/conversion/plugins/html_output.py index 4c82974460..15d2f02c07 100644 --- a/src/calibre/ebooks/conversion/plugins/html_output.py +++ b/src/calibre/ebooks/conversion/plugins/html_output.py @@ -47,7 +47,7 @@ class HTMLOutput(OutputFormatPlugin): Generate table of contents ''' from lxml import etree - from urllib import unquote + from polyglot.urllib import unquote from calibre.ebooks.oeb.base import element from calibre.utils.cleantext import clean_xml_chars @@ -86,7 +86,7 @@ class HTMLOutput(OutputFormatPlugin): from lxml import etree from calibre.utils import zipfile from templite import Templite - from urllib import unquote + from polyglot.urllib import unquote from calibre.ebooks.html.meta import EasyMeta # read template files diff --git a/src/calibre/ebooks/conversion/plugins/oeb_output.py b/src/calibre/ebooks/conversion/plugins/oeb_output.py index 72518e264b..82ae031c4d 100644 --- a/src/calibre/ebooks/conversion/plugins/oeb_output.py +++ b/src/calibre/ebooks/conversion/plugins/oeb_output.py @@ -21,7 +21,7 @@ class OEBOutput(OutputFormatPlugin): recommendations = {('pretty_print', True, OptionRecommendation.HIGH)} def convert(self, oeb_book, output_path, input_plugin, opts, log): - from urllib import unquote + from polyglot.urllib import unquote from lxml import etree self.log, self.opts = log, opts diff --git a/src/calibre/ebooks/docx/writer/links.py b/src/calibre/ebooks/docx/writer/links.py index 6293be50c5..4e6315d738 100644 --- a/src/calibre/ebooks/docx/writer/links.py +++ b/src/calibre/ebooks/docx/writer/links.py @@ -8,9 +8,9 @@ __copyright__ = '2015, Kovid Goyal ' import posixpath, re from uuid import uuid4 -from urlparse import urlparse from calibre.utils.filenames import ascii_text +from polyglot.urllib import urlparse def start_text(tag, prefix_len=0, top_level=True): diff --git a/src/calibre/ebooks/html/input.py b/src/calibre/ebooks/html/input.py index 723fcc3688..6bd6183821 100644 --- a/src/calibre/ebooks/html/input.py +++ b/src/calibre/ebooks/html/input.py @@ -13,13 +13,13 @@ Input plugin for HTML or OPF ebooks. ''' import os, re, sys, errno as gerrno -from urlparse import urlparse, urlunparse from calibre.ebooks.oeb.base import urlunquote from calibre.ebooks.chardet import detect_xml_encoding from calibre.constants import iswindows from calibre import unicode_path, as_unicode, replace_entities from polyglot.builtins import unicode_type +from polyglot.urllib import urlparse, urlunparse class Link(object): diff --git a/src/calibre/ebooks/htmlz/oeb2html.py b/src/calibre/ebooks/htmlz/oeb2html.py index f27a7308d5..ba619d1842 100644 --- a/src/calibre/ebooks/htmlz/oeb2html.py +++ b/src/calibre/ebooks/htmlz/oeb2html.py @@ -15,7 +15,6 @@ import re from functools import partial from lxml import html -from urlparse import urldefrag from calibre import prepare_string_for_xml from calibre.ebooks.oeb.base import ( @@ -23,6 +22,7 @@ from calibre.ebooks.oeb.base import ( from calibre.ebooks.oeb.stylizer import Stylizer from calibre.utils.logging import default_log from polyglot.builtins import unicode_type, string_or_bytes +from polyglot.urllib import urldefrag SELF_CLOSING_TAGS = {'area', 'base', 'basefont', 'br', 'hr', 'input', 'img', 'link', 'meta'} diff --git a/src/calibre/ebooks/lit/reader.py b/src/calibre/ebooks/lit/reader.py index b487c673a1..4cef2e9083 100644 --- a/src/calibre/ebooks/lit/reader.py +++ b/src/calibre/ebooks/lit/reader.py @@ -9,8 +9,6 @@ __copyright__ = '2008, Kovid Goyal ' \ 'and Marshall T. Vandegrift ' import io, struct, os, functools, re -from urlparse import urldefrag -from urllib import unquote as urlunquote from lxml import etree @@ -22,6 +20,7 @@ from calibre.ebooks.oeb.reader import OEBReader from calibre.ebooks import DRMError from calibre import plugins from polyglot.builtins import codepoint_to_chr, unicode_type, string_or_bytes, range +from polyglot.urllib import unquote as urlunquote, urldefrag lzx, lxzerror = plugins['lzx'] msdes, msdeserror = plugins['msdes'] diff --git a/src/calibre/ebooks/lit/writer.py b/src/calibre/ebooks/lit/writer.py index b5256101b8..ecc69ab748 100644 --- a/src/calibre/ebooks/lit/writer.py +++ b/src/calibre/ebooks/lit/writer.py @@ -17,8 +17,6 @@ import copy import uuid import functools import numbers -from urlparse import urldefrag -from urllib import unquote as urlunquote from lxml import etree from calibre.ebooks.lit.reader import DirectoryEntry import calibre.ebooks.lit.maps as maps @@ -33,6 +31,7 @@ from calibre import plugins msdes, msdeserror = plugins['msdes'] import calibre.ebooks.lit.mssha1 as mssha1 from polyglot.builtins import codepoint_to_chr, unicode_type, string_or_bytes, range +from polyglot.urllib import urldefrag, unquote __all__ = ['LitWriter'] @@ -521,7 +520,7 @@ class LitWriter(object): media_type = XHTML_MIME elif media_type in OEB_STYLES: media_type = CSS_MIME - href = urlunquote(item.href) + href = unquote(item.href) item.offset = offset \ if state in ('linear', 'nonlinear') else 0 data.write(pack(' -from __future__ import absolute_import, division, print_function, unicode_literals - +from __future__ import absolute_import from polyglot.builtins import is_py3 if is_py3: - from urllib.request import urlopen, Request # noqa - from urllib.parse import urlencode # noqa + from urllib.request import (build_opener, getproxies, install_opener, # noqa + HTTPBasicAuthHandler, HTTPCookieProcessor, HTTPDigestAuthHandler, # noqa + url2pathname, urlopen, Request) # noqa + from urllib.parse import (parse_qs, quote, unquote, quote_plus, urldefrag, # noqa + urlencode, urljoin, urlparse, urlunparse, urlsplit, urlunsplit) # noqa + from urllib.error import HTTPError, URLError # noqa else: - from urllib import urlencode # noqa - from urllib2 import urlopen, Request # noqa + from urllib import (getproxies, quote, unquote, quote_plus, url2pathname, # noqa + urlencode) # noqa + from urllib2 import (build_opener, install_opener, HTTPBasicAuthHandler, # noqa + HTTPCookieProcessor, HTTPDigestAuthHandler, HTTPError, URLError, # noqa + urlopen, Request) # noqa + from urlparse import (parse_qs, urldefrag, urljoin, urlparse, urlunparse, # noqa + urlsplit, urlunsplit) # noqa From ea6a210e70ac890066146ddfb9edbb61f546671f Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 19 Mar 2019 17:08:50 -0400 Subject: [PATCH 4/6] python3: wire up more code to polyglot.http_server --- src/calibre/ebooks/oeb/display/test-cfi/run_rapydscript.py | 6 +++--- src/calibre/utils/serve_coffee.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/calibre/ebooks/oeb/display/test-cfi/run_rapydscript.py b/src/calibre/ebooks/oeb/display/test-cfi/run_rapydscript.py index 887708f454..0ca3698b23 100644 --- a/src/calibre/ebooks/oeb/display/test-cfi/run_rapydscript.py +++ b/src/calibre/ebooks/oeb/display/test-cfi/run_rapydscript.py @@ -8,9 +8,10 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' import os, shutil, tempfile -import SimpleHTTPServer import SocketServer +from polyglot.http_server import SimpleHTTPRequestHandler + def run_devel_server(): base = os.path.dirname(os.path.abspath(__file__)) @@ -27,7 +28,7 @@ def run_devel_server(): with lopen('cfi-test.pyj', 'rb') as f, lopen('cfi-test.js', 'wb') as js: js.write(compile_pyj(f.read()).encode('utf-8')) PORT = 8000 - Handler = SimpleHTTPServer.SimpleHTTPRequestHandler + Handler = SimpleHTTPRequestHandler httpd = SocketServer.TCPServer(("", PORT), Handler) print('Serving CFI test at http://localhost:%d' % PORT) try: @@ -37,4 +38,3 @@ def run_devel_server(): if __name__ == '__main__': run_devel_server() - diff --git a/src/calibre/utils/serve_coffee.py b/src/calibre/utils/serve_coffee.py index 2239860932..887df3a323 100644 --- a/src/calibre/utils/serve_coffee.py +++ b/src/calibre/utils/serve_coffee.py @@ -18,9 +18,9 @@ if sys.version_info.major > 2: file=sys.stderr) raise SystemExit(1) -import time, BaseHTTPServer, os, sys, re, SocketServer +import time, os, sys, re, SocketServer from threading import Lock, local -from SimpleHTTPServer import SimpleHTTPRequestHandler +from polyglot.http_server import BaseHTTPServer, SimpleHTTPRequestHandler # Compiler {{{ From 83b055122ade5869b09ad1ae48354c0a18722775 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 19 Mar 2019 17:11:00 -0400 Subject: [PATCH 5/6] python3: add polyglot.queue wrapper --- src/calibre/db/delete_service.py | 3 +-- src/calibre/devices/smart_device_app/driver.py | 10 +++++----- src/calibre/ebooks/comic/input.py | 2 +- src/calibre/ebooks/metadata/sources/test.py | 2 +- src/calibre/ebooks/metadata/sources/worker.py | 2 +- src/calibre/ebooks/oeb/polish/check/links.py | 2 +- src/calibre/ebooks/oeb/polish/images.py | 2 +- src/calibre/gui2/__init__.py | 4 ++-- src/calibre/gui2/add.py | 2 +- src/calibre/gui2/device.py | 17 +++++++++-------- src/calibre/gui2/dnd.py | 2 +- src/calibre/gui2/icon_theme.py | 2 +- src/calibre/gui2/jobs.py | 2 +- src/calibre/gui2/library/alternate_views.py | 2 +- src/calibre/gui2/metadata/single_download.py | 2 +- src/calibre/gui2/save.py | 2 +- .../gui2/store/search/download_thread.py | 2 +- src/calibre/gui2/threaded_jobs.py | 4 +--- .../gui2/tweak_book/completion/worker.py | 2 +- src/calibre/gui2/tweak_book/preview.py | 2 +- src/calibre/gui2/tweak_book/save.py | 2 +- src/calibre/gui2/ui.py | 2 +- src/calibre/library/sqlite.py | 2 +- src/calibre/srv/auto_reload.py | 2 +- src/calibre/srv/jobs.py | 2 +- src/calibre/srv/loop.py | 2 +- src/calibre/srv/pool.py | 2 +- src/calibre/srv/web_socket.py | 2 +- src/calibre/utils/ipc/job.py | 2 +- src/calibre/utils/ipc/pool.py | 2 +- src/calibre/utils/ipc/server.py | 2 +- src/calibre/utils/ipc/worker.py | 2 +- src/calibre/utils/matcher.py | 2 +- src/calibre/utils/rapydscript.py | 2 +- src/calibre/utils/threadpool.py | 10 +++++----- src/polyglot/queue.py | 10 ++++++++++ 36 files changed, 62 insertions(+), 54 deletions(-) create mode 100644 src/polyglot/queue.py diff --git a/src/calibre/db/delete_service.py b/src/calibre/db/delete_service.py index c33dda1f7c..6698f4680c 100644 --- a/src/calibre/db/delete_service.py +++ b/src/calibre/db/delete_service.py @@ -8,11 +8,11 @@ __copyright__ = '2013, Kovid Goyal ' import os, tempfile, shutil, errno, time, atexit from threading import Thread -from Queue import Queue from calibre.ptempfile import remove_dir from calibre.utils.filenames import remove_dir_if_empty from calibre.utils.recycle_bin import delete_tree, delete_file +from polyglot.queue import Queue class DeleteService(Thread): @@ -159,4 +159,3 @@ def has_jobs(): if __ds is not None: return (not __ds.requests.empty()) or __ds.requests.unfinished_tasks return False - diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index 7d60bf2916..7d52dc76c8 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -11,7 +11,6 @@ import socket, select, json, os, traceback, time, sys, random import posixpath from collections import defaultdict import hashlib, threading -import Queue from functools import wraps from errno import EAGAIN, EINTR @@ -39,6 +38,7 @@ from calibre.utils.mdns import (publish as publish_zeroconf, unpublish as unpublish_zeroconf, get_all_ips) from calibre.utils.socket_inheritance import set_socket_inherit from polyglot.builtins import unicode_type +from polyglot import queue def synchronous(tlockname): @@ -103,7 +103,7 @@ class ConnectionListener(Thread): {'otherDevice': d.get_gui_name()}) self.driver._send_byte_string(device_socket, (b'%d' % len(s)) + s) sock.close() - except Queue.Empty: + except queue.Empty: pass if getattr(self.driver, 'broadcast_socket', None) is not None: @@ -148,7 +148,7 @@ class ConnectionListener(Thread): try: self.driver.connection_queue.put_nowait(device_socket) - except Queue.Full: + except queue.Full: self._close_socket(device_socket) device_socket = None self.driver._debug('driver is not answering') @@ -993,7 +993,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): raise except: pass - except Queue.Empty: + except queue.Empty: self.is_connected = False return self if self.is_connected else None return None @@ -1969,7 +1969,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): message = 'attaching port to broadcast socket failed. This is not fatal.' self._debug(message) - self.connection_queue = Queue.Queue(1) + self.connection_queue = queue.Queue(1) self.connection_listener = ConnectionListener(self) self.connection_listener.start() return message diff --git a/src/calibre/ebooks/comic/input.py b/src/calibre/ebooks/comic/input.py index a636069fbb..fb6db0d152 100755 --- a/src/calibre/ebooks/comic/input.py +++ b/src/calibre/ebooks/comic/input.py @@ -8,7 +8,6 @@ Based on ideas from comiclrf created by FangornUK. ''' import os, traceback, time -from Queue import Empty from calibre import extract, prints, walk from calibre.constants import filesystem_encoding @@ -17,6 +16,7 @@ from calibre.utils.icu import numeric_sort_key from calibre.utils.ipc.server import Server from calibre.utils.ipc.job import ParallelJob from polyglot.builtins import unicode_type +from polyglot.queue import Empty # If the specified screen has either dimension larger than this value, no image # rescaling is done (we assume that it is a tablet output profile) diff --git a/src/calibre/ebooks/metadata/sources/test.py b/src/calibre/ebooks/metadata/sources/test.py index fec33ffeab..fcd4df3a42 100644 --- a/src/calibre/ebooks/metadata/sources/test.py +++ b/src/calibre/ebooks/metadata/sources/test.py @@ -8,7 +8,6 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' import os, tempfile, time -from Queue import Queue, Empty from threading import Event from calibre.customize.ui import all_metadata_plugins @@ -16,6 +15,7 @@ from calibre import prints, sanitize_file_name2 from calibre.ebooks.metadata import check_isbn from calibre.ebooks.metadata.sources.base import create_log, get_cached_cover_urls from calibre.ebooks.metadata.sources.prefs import msprefs +from polyglot.queue import Queue, Empty def isbn_test(isbn): diff --git a/src/calibre/ebooks/metadata/sources/worker.py b/src/calibre/ebooks/metadata/sources/worker.py index b8fb28d8e2..436d7014eb 100644 --- a/src/calibre/ebooks/metadata/sources/worker.py +++ b/src/calibre/ebooks/metadata/sources/worker.py @@ -6,7 +6,6 @@ from __future__ import absolute_import, division, print_function, unicode_litera import os from collections import Counter from io import BytesIO -from Queue import Empty, Queue from threading import Event, Thread from calibre.customize.ui import metadata_plugins @@ -18,6 +17,7 @@ from calibre.ebooks.metadata.sources.identify import identify, msprefs from calibre.ebooks.metadata.sources.update import patch_plugins from calibre.utils.date import as_utc from calibre.utils.logging import GUILog +from polyglot.queue import Empty, Queue def merge_result(oldmi, newmi, ensure_fields=None): diff --git a/src/calibre/ebooks/oeb/polish/check/links.py b/src/calibre/ebooks/oeb/polish/check/links.py index 53179a06a8..b581bc6418 100644 --- a/src/calibre/ebooks/oeb/polish/check/links.py +++ b/src/calibre/ebooks/oeb/polish/check/links.py @@ -9,7 +9,6 @@ __copyright__ = '2013, Kovid Goyal ' import os from collections import defaultdict from threading import Thread -from Queue import Queue, Empty from calibre import browser from calibre.ebooks.oeb.base import OEB_DOCS, OEB_STYLES, urlunquote, XHTML_MIME @@ -21,6 +20,7 @@ from calibre.ebooks.oeb.polish.utils import guess_type, actual_case_for_name, co from calibre.ebooks.oeb.polish.check.base import BaseError, WARN, INFO from polyglot.builtins import map, range from polyglot.urllib import urlparse +from polyglot.queue import Queue, Empty class BadLink(BaseError): diff --git a/src/calibre/ebooks/oeb/polish/images.py b/src/calibre/ebooks/oeb/polish/images.py index 8ae9f9c413..680fb47af0 100644 --- a/src/calibre/ebooks/oeb/polish/images.py +++ b/src/calibre/ebooks/oeb/polish/images.py @@ -7,10 +7,10 @@ from __future__ import (unicode_literals, division, absolute_import, import os from functools import partial from threading import Thread, Event -from Queue import Queue, Empty from calibre import detect_ncpus, human_readable, force_unicode, filesystem_encoding from polyglot.builtins import range +from polyglot.queue import Queue, Empty class Worker(Thread): diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 56fbb4e074..d16d949c43 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -3,7 +3,6 @@ __copyright__ = '2008, Kovid Goyal ' """ The GUI """ import glob import os -import Queue import signal import sys import threading @@ -36,6 +35,7 @@ from calibre.utils.date import UNDEFINED_DATE from calibre.utils.file_type_icons import EXT_MAP from calibre.utils.localization import get_lang from polyglot.builtins import unicode_type, string_or_bytes, range +from polyglot import queue try: NO_URL_FORMATTING = QUrl.None_ @@ -502,7 +502,7 @@ class FunctionDispatcher(QObject): if not queued: typ = Qt.AutoConnection if queued is None else Qt.DirectConnection self.dispatch_signal.connect(self.dispatch, type=typ) - self.q = Queue.Queue() + self.q = queue.Queue() self.lock = threading.Lock() def __call__(self, *args, **kwargs): diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index 482b735745..7aa6130644 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -9,7 +9,6 @@ __copyright__ = '2014, Kovid Goyal ' import shutil, os, weakref, traceback, tempfile, time from threading import Thread from collections import OrderedDict -from Queue import Empty from io import BytesIO from polyglot.builtins import map, unicode_type, string_or_bytes @@ -30,6 +29,7 @@ from calibre.ptempfile import PersistentTemporaryDirectory from calibre.utils import join_with_timeout from calibre.utils.config import prefs from calibre.utils.ipc.pool import Pool, Failure +from polyglot.queue import Empty def validate_source(source, parent=None): # {{{ diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 456a0dea3b..7414e4102f 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -3,7 +3,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' # Imports {{{ -import os, traceback, Queue, time, io, re, sys, weakref +import os, traceback, time, io, re, sys, weakref from threading import Thread, Event from PyQt5.Qt import ( @@ -35,6 +35,7 @@ from calibre.utils.img import scale_image from calibre.library.save_to_disk import find_plugboard from calibre.ptempfile import PersistentTemporaryFile, force_unicode as filename_to_unicode from polyglot.builtins import unicode_type, string_or_bytes +from polyglot import queue # }}} @@ -153,8 +154,8 @@ class DeviceManager(Thread): # {{{ self.sleep_time = sleep_time self.connected_slot = connected_slot self.allow_connect_slot = allow_connect_slot - self.jobs = Queue.Queue(0) - self.job_steps = Queue.Queue(0) + self.jobs = queue.Queue(0) + self.job_steps = queue.Queue(0) self.keep_going = True self.job_manager = job_manager self.reported_errors = set([]) @@ -163,7 +164,7 @@ class DeviceManager(Thread): # {{{ self.connected_device = None self.connected_device_kind = None self.ejected_devices = set([]) - self.mount_connection_requests = Queue.Queue(0) + self.mount_connection_requests = queue.Queue(0) self.open_feedback_slot = open_feedback_slot self.open_feedback_only_once_seen = set() self.after_callback_feedback_slot = after_callback_feedback_slot @@ -241,7 +242,7 @@ class DeviceManager(Thread): # {{{ try: job = self.jobs.get_nowait() job.abort(Exception(_('Device no longer connected.'))) - except Queue.Empty: + except queue.Empty: break try: self.connected_device.post_yank_cleanup() @@ -358,13 +359,13 @@ class DeviceManager(Thread): # {{{ if not self.job_steps.empty(): try: return self.job_steps.get_nowait() - except Queue.Empty: + except queue.Empty: pass if not self.jobs.empty(): try: return self.jobs.get_nowait() - except Queue.Empty: + except queue.Empty: pass def run_startup(self, dev): @@ -391,7 +392,7 @@ class DeviceManager(Thread): # {{{ try: (kls,device_kind, folder_path) = \ self.mount_connection_requests.get_nowait() - except Queue.Empty: + except queue.Empty: break if kls is not None: try: diff --git a/src/calibre/gui2/dnd.py b/src/calibre/gui2/dnd.py index 71c47ca44c..2b6201792f 100644 --- a/src/calibre/gui2/dnd.py +++ b/src/calibre/gui2/dnd.py @@ -9,7 +9,6 @@ __docformat__ = 'restructuredtext en' import posixpath, os, re from threading import Thread -from Queue import Queue, Empty from PyQt5.Qt import QPixmap, Qt, QDialog, QLabel, QVBoxLayout, \ QDialogButtonBox, QProgressBar, QTimer, QUrl, QImageReader @@ -21,6 +20,7 @@ from calibre.gui2 import error_dialog from calibre.utils.imghdr import what from polyglot.builtins import unicode_type from polyglot.urllib import unquote, urlparse +from polyglot.queue import Queue, Empty def image_extensions(): diff --git a/src/calibre/gui2/icon_theme.py b/src/calibre/gui2/icon_theme.py index 95fc1dcaca..495a5c2969 100644 --- a/src/calibre/gui2/icon_theme.py +++ b/src/calibre/gui2/icon_theme.py @@ -10,7 +10,6 @@ import os, errno, json, importlib, math, httplib, bz2, shutil, sys from itertools import count from io import BytesIO from polyglot.builtins import map -from Queue import Queue, Empty from threading import Thread, Event from multiprocessing.pool import ThreadPool @@ -39,6 +38,7 @@ from calibre.utils.img import image_from_data, Canvas, optimize_png, optimize_jp from calibre.utils.zipfile import ZipFile, ZIP_STORED from calibre.utils.filenames import atomic_rename from lzma.xz import compress, decompress +from polyglot.queue import Queue, Empty IMAGE_EXTENSIONS = {'png', 'jpg', 'jpeg'} THEME_COVER = 'icon-theme-cover.jpg' diff --git a/src/calibre/gui2/jobs.py b/src/calibre/gui2/jobs.py index 59b6e74be1..8f835be45b 100644 --- a/src/calibre/gui2/jobs.py +++ b/src/calibre/gui2/jobs.py @@ -8,7 +8,6 @@ Job management. ''' import re, time -from Queue import Empty, Queue from PyQt5.Qt import (QAbstractTableModel, QModelIndex, Qt, QPainter, QTimer, pyqtSignal, QIcon, QDialog, QAbstractItemDelegate, QApplication, @@ -31,6 +30,7 @@ from calibre.gui2.widgets2 import Dialog from calibre.utils.search_query_parser import SearchQueryParser, ParseException from calibre.utils.icu import lower from polyglot.builtins import unicode_type, range +from polyglot.queue import Empty, Queue class AdaptSQP(SearchQueryParser): diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 1f11a5d84c..bab59f60d1 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -9,7 +9,6 @@ __copyright__ = '2013, Kovid Goyal ' import itertools, operator, os, math from types import MethodType from threading import Event, Thread -from Queue import LifoQueue from functools import wraps, partial from textwrap import wrap @@ -31,6 +30,7 @@ from calibre.gui2.gestures import GestureManager from calibre.gui2.library.caches import CoverCache, ThumbnailCache from calibre.utils.config import prefs, tweaks from polyglot.builtins import unicode_type, range +from polyglot.queue import LifoQueue CM_TO_INCH = 0.393701 CACHE_FORMAT = 'PPM' diff --git a/src/calibre/gui2/metadata/single_download.py b/src/calibre/gui2/metadata/single_download.py index 56c2c3f641..fe51133577 100644 --- a/src/calibre/gui2/metadata/single_download.py +++ b/src/calibre/gui2/metadata/single_download.py @@ -13,7 +13,6 @@ DEBUG_DIALOG = False import os, time from threading import Thread, Event from operator import attrgetter -from Queue import Queue, Empty from io import BytesIO from PyQt5.Qt import ( @@ -41,6 +40,7 @@ from calibre.utils.config import tweaks from calibre.utils.ipc.simple_worker import fork_job, WorkerError from calibre.ptempfile import TemporaryDirectory from polyglot.builtins import unicode_type, range +from polyglot.queue import Queue, Empty # }}} diff --git a/src/calibre/gui2/save.py b/src/calibre/gui2/save.py index 1c268c7fe7..d7a1553496 100644 --- a/src/calibre/gui2/save.py +++ b/src/calibre/gui2/save.py @@ -9,7 +9,6 @@ __copyright__ = '2014, Kovid Goyal ' import traceback, errno, os, time, shutil from collections import namedtuple, defaultdict from tempfile import SpooledTemporaryFile -from Queue import Empty from PyQt5.Qt import QObject, Qt, pyqtSignal @@ -26,6 +25,7 @@ from calibre.utils.formatter_functions import load_user_template_functions from calibre.utils.ipc.pool import Pool, Failure from calibre.library.save_to_disk import sanitize_args, get_path_components, find_plugboard, plugboard_save_to_disk_value from polyglot.builtins import unicode_type +from polyglot.queue import Empty BookId = namedtuple('BookId', 'title authors') diff --git a/src/calibre/gui2/store/search/download_thread.py b/src/calibre/gui2/store/search/download_thread.py index 5df4dd0966..b41c4fefe1 100644 --- a/src/calibre/gui2/store/search/download_thread.py +++ b/src/calibre/gui2/store/search/download_thread.py @@ -9,12 +9,12 @@ __docformat__ = 'restructuredtext en' import traceback, base64 from contextlib import closing from threading import Thread -from Queue import Queue from calibre import browser from calibre.constants import DEBUG from calibre.utils.img import scale_image from polyglot.builtins import range +from polyglot.queue import Queue class GenericDownloadThreadPool(object): diff --git a/src/calibre/gui2/threaded_jobs.py b/src/calibre/gui2/threaded_jobs.py index 77f66dec4f..aef553133d 100644 --- a/src/calibre/gui2/threaded_jobs.py +++ b/src/calibre/gui2/threaded_jobs.py @@ -9,11 +9,11 @@ __docformat__ = 'restructuredtext en' import os, time, tempfile, json from threading import Thread, RLock, Event -from Queue import Queue from calibre.utils.ipc.job import BaseJob from calibre.utils.logging import GUILog from calibre.ptempfile import base_dir +from polyglot.queue import Queue class ThreadedJob(BaseJob): @@ -245,5 +245,3 @@ class ThreadedJobServer(Thread): queued_types.append(job.type) ans.append(job) return ans - - diff --git a/src/calibre/gui2/tweak_book/completion/worker.py b/src/calibre/gui2/tweak_book/completion/worker.py index 13959df159..4a0e7b3a66 100644 --- a/src/calibre/gui2/tweak_book/completion/worker.py +++ b/src/calibre/gui2/tweak_book/completion/worker.py @@ -8,7 +8,6 @@ __copyright__ = '2014, Kovid Goyal ' import os, sys from threading import Thread, Event, RLock -from Queue import Queue from contextlib import closing from collections import namedtuple @@ -17,6 +16,7 @@ from calibre.gui2.tweak_book.completion.basic import Request from calibre.gui2.tweak_book.completion.utils import DataError from calibre.utils.ipc import eintr_retry_call from calibre.utils.serialize import msgpack_loads, msgpack_dumps +from polyglot.queue import Queue COMPLETION_REQUEST = 'completion request' CLEAR_REQUEST = 'clear request' diff --git a/src/calibre/gui2/tweak_book/preview.py b/src/calibre/gui2/tweak_book/preview.py index 4d9c2b4288..aa655d7682 100644 --- a/src/calibre/gui2/tweak_book/preview.py +++ b/src/calibre/gui2/tweak_book/preview.py @@ -11,7 +11,6 @@ from bisect import bisect_right from base64 import b64encode from polyglot.builtins import map, unicode_type from threading import Thread -from Queue import Queue, Empty from functools import partial from PyQt5.Qt import ( @@ -31,6 +30,7 @@ from calibre.gui2.viewer.config import config from calibre.gui2.widgets2 import HistoryLineEdit2 from calibre.utils.ipc.simple_worker import offload_worker from polyglot.urllib import urlparse +from polyglot.queue import Queue, Empty shutdown = object() diff --git a/src/calibre/gui2/tweak_book/save.py b/src/calibre/gui2/tweak_book/save.py index 52ce1ac6e0..1b706b51b7 100644 --- a/src/calibre/gui2/tweak_book/save.py +++ b/src/calibre/gui2/tweak_book/save.py @@ -8,7 +8,6 @@ __copyright__ = '2013, Kovid Goyal ' import shutil, os, errno from threading import Thread -from Queue import LifoQueue, Empty from PyQt5.Qt import (QObject, pyqtSignal, QLabel, QWidget, QHBoxLayout, Qt) @@ -18,6 +17,7 @@ from calibre.gui2.progress_indicator import ProgressIndicator from calibre.utils import join_with_timeout from calibre.utils.filenames import atomic_rename, format_permissions from calibre.utils.ipc import RC +from polyglot.queue import LifoQueue, Empty def save_dir_container(container, path): diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 4bdbecded6..10bd4106c3 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -10,7 +10,6 @@ __docformat__ = 'restructuredtext en' '''The main GUI''' import collections, os, sys, textwrap, time, gc, errno, re -from Queue import Queue, Empty from threading import Thread from collections import OrderedDict from io import BytesIO @@ -54,6 +53,7 @@ from calibre.gui2.open_with import register_keyboard_shortcuts from calibre.library import current_library_name from calibre.srv.library_broker import GuiLibraryBroker from polyglot.builtins import unicode_type, string_or_bytes +from polyglot.queue import Queue, Empty class Listener(Thread): # {{{ diff --git a/src/calibre/library/sqlite.py b/src/calibre/library/sqlite.py index 5fd140ce90..d876e96ccf 100644 --- a/src/calibre/library/sqlite.py +++ b/src/calibre/library/sqlite.py @@ -11,7 +11,6 @@ import sqlite3 as sqlite, traceback, time, uuid, sys, os import repr as reprlib from sqlite3 import IntegrityError, OperationalError from threading import Thread -from Queue import Queue from threading import RLock from datetime import datetime from functools import partial @@ -23,6 +22,7 @@ from calibre.constants import iswindows, DEBUG, plugins from calibre.utils.icu import sort_key from calibre import prints from polyglot.builtins import unicode_type +from polyglot.queue import Queue from dateutil.tz import tzoffset diff --git a/src/calibre/srv/auto_reload.py b/src/calibre/srv/auto_reload.py index ca3827adee..e986846cac 100644 --- a/src/calibre/srv/auto_reload.py +++ b/src/calibre/srv/auto_reload.py @@ -8,7 +8,6 @@ __copyright__ = '2015, Kovid Goyal ' import os, sys, subprocess, signal, time, errno, socket, ssl from threading import Thread, Lock -from Queue import Queue, Empty from calibre.constants import islinux, iswindows, isosx from calibre.srv.http_response import create_http_handler @@ -18,6 +17,7 @@ from calibre.srv.standalone import create_option_parser from calibre.srv.utils import create_sock_pair from calibre.srv.web_socket import DummyHandler from calibre.utils.monotonic import monotonic +from polyglot.queue import Queue, Empty MAX_RETRIES = 10 diff --git a/src/calibre/srv/jobs.py b/src/calibre/srv/jobs.py index 3398cd42a0..d529ba3f6c 100644 --- a/src/calibre/srv/jobs.py +++ b/src/calibre/srv/jobs.py @@ -9,11 +9,11 @@ from itertools import count from collections import namedtuple, deque from functools import partial from threading import RLock, Thread, Event -from Queue import Queue, Empty from calibre import detect_ncpus, force_unicode from calibre.utils.monotonic import monotonic from calibre.utils.ipc.simple_worker import fork_job, WorkerError +from polyglot.queue import Queue, Empty StartEvent = namedtuple('StartEvent', 'job_id name module function args kwargs callback data') DoneEvent = namedtuple('DoneEvent', 'job_id') diff --git a/src/calibre/srv/loop.py b/src/calibre/srv/loop.py index 3dd74ccb00..180603c42a 100644 --- a/src/calibre/srv/loop.py +++ b/src/calibre/srv/loop.py @@ -8,7 +8,6 @@ __copyright__ = '2015, Kovid Goyal ' import ssl, socket, select, os, traceback from io import BytesIO -from Queue import Empty, Full from functools import partial from calibre import as_unicode @@ -26,6 +25,7 @@ from calibre.utils.logging import ThreadSafeLog from calibre.utils.monotonic import monotonic from calibre.utils.mdns import get_external_ip from polyglot.builtins import range +from polyglot.queue import Empty, Full READ, WRITE, RDWR, WAIT = 'READ', 'WRITE', 'RDWR', 'WAIT' WAKEUP, JOB_DONE = bytes(bytearray(range(2))) diff --git a/src/calibre/srv/pool.py b/src/calibre/srv/pool.py index 507aef42e5..c83d769fab 100644 --- a/src/calibre/srv/pool.py +++ b/src/calibre/srv/pool.py @@ -7,11 +7,11 @@ __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' import sys -from Queue import Queue, Full from threading import Thread from calibre.utils.monotonic import monotonic from polyglot.builtins import range +from polyglot.queue import Queue, Full class Worker(Thread): diff --git a/src/calibre/srv/web_socket.py b/src/calibre/srv/web_socket.py index 7305737417..de6f72ecdb 100644 --- a/src/calibre/srv/web_socket.py +++ b/src/calibre/srv/web_socket.py @@ -9,7 +9,6 @@ import httplib, os, weakref, socket from base64 import standard_b64encode from collections import deque from hashlib import sha1 -from Queue import Queue, Empty from struct import unpack_from, pack, error as struct_error from threading import Lock @@ -19,6 +18,7 @@ from calibre.srv.loop import ServerLoop, HandleInterrupt, WRITE, READ, RDWR, Con from calibre.srv.http_response import HTTPConnection, create_http_handler from calibre.srv.utils import DESIRED_SEND_BUFFER_SIZE from calibre.utils.speedups import ReadOnlyFileBuffer +from polyglot.queue import Queue, Empty speedup, err = plugins['speedup'] if not speedup: raise RuntimeError('Failed to load speedup module with error: ' + err) diff --git a/src/calibre/utils/ipc/job.py b/src/calibre/utils/ipc/job.py index 73829f83bb..70cd4612c6 100644 --- a/src/calibre/utils/ipc/job.py +++ b/src/calibre/utils/ipc/job.py @@ -9,10 +9,10 @@ __docformat__ = 'restructuredtext en' _count = 0 import time, io -from Queue import Queue, Empty from calibre import prints from calibre.constants import DEBUG +from polyglot.queue import Queue, Empty class BaseJob(object): diff --git a/src/calibre/utils/ipc/pool.py b/src/calibre/utils/ipc/pool.py index aff69e4354..ea1f1c5cef 100644 --- a/src/calibre/utils/ipc/pool.py +++ b/src/calibre/utils/ipc/pool.py @@ -9,7 +9,6 @@ __copyright__ = '2014, Kovid Goyal ' import os, sys from threading import Thread from collections import namedtuple -from Queue import Queue from calibre import detect_ncpus, as_unicode, prints from calibre.constants import iswindows, DEBUG @@ -17,6 +16,7 @@ from calibre.ptempfile import PersistentTemporaryFile from calibre.utils import join_with_timeout from calibre.utils.ipc import eintr_retry_call from calibre.utils.serialize import msgpack_dumps, msgpack_loads, pickle_dumps, pickle_loads +from polyglot.queue import Queue Job = namedtuple('Job', 'id module func args kwargs') Result = namedtuple('Result', 'value err traceback') diff --git a/src/calibre/utils/ipc/server.py b/src/calibre/utils/ipc/server.py index 58269ce4a5..7bc3d33b42 100644 --- a/src/calibre/utils/ipc/server.py +++ b/src/calibre/utils/ipc/server.py @@ -17,7 +17,6 @@ from binascii import hexlify from collections import deque from math import ceil from multiprocessing.connection import Listener, arbitrary_address -from Queue import Empty, Queue from threading import RLock, Thread from calibre import detect_ncpus as cpu_count @@ -28,6 +27,7 @@ from calibre.utils.ipc.launch import Worker from calibre.utils.ipc.worker import PARALLEL_FUNCS from calibre.utils.serialize import msgpack_dumps, pickle_loads from polyglot.builtins import string_or_bytes, environ_item +from polyglot.queue import Empty, Queue _counter = 0 diff --git a/src/calibre/utils/ipc/worker.py b/src/calibre/utils/ipc/worker.py index c4ca19cd13..a244b5e3f5 100644 --- a/src/calibre/utils/ipc/worker.py +++ b/src/calibre/utils/ipc/worker.py @@ -10,7 +10,6 @@ __docformat__ = 'restructuredtext en' import os, sys, importlib from multiprocessing.connection import Client from threading import Thread -from Queue import Queue from contextlib import closing from binascii import unhexlify from zipimport import ZipImportError @@ -19,6 +18,7 @@ from calibre import prints from calibre.constants import iswindows, isosx from calibre.utils.ipc import eintr_retry_call from calibre.utils.serialize import msgpack_loads, pickle_dumps +from polyglot.queue import Queue PARALLEL_FUNCS = { 'lrfviewer' : diff --git a/src/calibre/utils/matcher.py b/src/calibre/utils/matcher.py index 7da1be1742..091f736c51 100644 --- a/src/calibre/utils/matcher.py +++ b/src/calibre/utils/matcher.py @@ -9,7 +9,6 @@ import atexit, os, sys from math import ceil from unicodedata import normalize from threading import Thread, Lock -from Queue import Queue from operator import itemgetter from collections import OrderedDict from itertools import islice @@ -20,6 +19,7 @@ from polyglot.builtins import map, unicode_type, range from calibre import detect_ncpus as cpu_count, as_unicode from calibre.constants import plugins, filesystem_encoding from calibre.utils.icu import primary_sort_key, primary_find, primary_collator +from polyglot.queue import Queue DEFAULT_LEVEL1 = '/' DEFAULT_LEVEL2 = '-_ 0123456789' diff --git a/src/calibre/utils/rapydscript.py b/src/calibre/utils/rapydscript.py index ee0cf8aa07..67db41e87e 100644 --- a/src/calibre/utils/rapydscript.py +++ b/src/calibre/utils/rapydscript.py @@ -14,7 +14,6 @@ import subprocess import sys from functools import partial from io import BytesIO -from Queue import Empty, Queue from threading import Thread, local from calibre import force_unicode @@ -24,6 +23,7 @@ from calibre.utils.terminal import ANSIStream from duktape import Context, JSError, to_python from lzma.xz import compress, decompress from polyglot.builtins import range +from polyglot.queue import Empty, Queue COMPILER_PATH = 'rapydscript/compiler.js.xz' diff --git a/src/calibre/utils/threadpool.py b/src/calibre/utils/threadpool.py index 9d6503173a..8ed716743c 100644 --- a/src/calibre/utils/threadpool.py +++ b/src/calibre/utils/threadpool.py @@ -48,7 +48,7 @@ __license__ = 'Python license' # standard library modules import threading -import Queue +from polyglot import queue # exceptions @@ -75,7 +75,7 @@ class WorkerThread(threading.Thread): def __init__(self, requestsQueue, resultsQueue, **kwds): """Set up thread in daemonic mode and start it immediatedly. - requestsQueue and resultQueue are instances of Queue.Queue passed + requestsQueue and resultQueue are instances of queue.Queue passed by the ThreadPool class when it creates a new worker thread. """ @@ -174,8 +174,8 @@ class ThreadPool: more work requests in it (see putRequest method). """ - self.requestsQueue = Queue.Queue(q_size) - self.resultsQueue = Queue.Queue() + self.requestsQueue = queue.Queue(q_size) + self.resultsQueue = queue.Queue() self.workers = [] self.workRequests = {} self.createWorkers(num_workers) @@ -223,7 +223,7 @@ class ThreadPool: (request.exception and request.exc_callback): request.callback(request, result) del self.workRequests[request.requestID] - except Queue.Empty: + except queue.Empty: break def wait(self, sleep=0): diff --git a/src/polyglot/queue.py b/src/polyglot/queue.py new file mode 100644 index 0000000000..17323ed37b --- /dev/null +++ b/src/polyglot/queue.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +# License: GPL v3 Copyright: 2019, Eli Schwartz + +from polyglot.builtins import is_py3 + +if is_py3: + from queue import * # noqa +else: + from Queue import * # noqa From e828a4bd985bb152a423c1b0b80523c394719db9 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 19 Mar 2019 17:40:01 -0400 Subject: [PATCH 6/6] python3: more use of unicode_type These metadata sources files are part of the infrastructure, not the parts which are downloaded. --- src/calibre/ebooks/metadata/sources/cli.py | 3 ++- src/calibre/ebooks/metadata/sources/identify.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/cli.py b/src/calibre/ebooks/metadata/sources/cli.py index d7d5153cf9..61b250e1db 100644 --- a/src/calibre/ebooks/metadata/sources/cli.py +++ b/src/calibre/ebooks/metadata/sources/cli.py @@ -21,6 +21,7 @@ from calibre.ebooks.metadata.sources.base import create_log from calibre.ebooks.metadata.sources.identify import identify from calibre.ebooks.metadata.sources.covers import download_cover from calibre.ebooks.metadata.sources.update import patch_plugins +from polyglot.builtins import unicode_type def option_parser(): @@ -99,7 +100,7 @@ def main(args=sys.argv): log = buf.getvalue() result = (metadata_to_opf(result) if opts.opf else - type(u'')(result).encode('utf-8')) + unicode_type(result).encode('utf-8')) if opts.verbose: print (log, file=sys.stderr) diff --git a/src/calibre/ebooks/metadata/sources/identify.py b/src/calibre/ebooks/metadata/sources/identify.py index 5fc4a5bfb0..44cdba029b 100644 --- a/src/calibre/ebooks/metadata/sources/identify.py +++ b/src/calibre/ebooks/metadata/sources/identify.py @@ -27,6 +27,7 @@ from calibre.utils.html2text import html2text from calibre.utils.icu import lower from calibre.utils.date import UNDEFINED_DATE from calibre.utils.formatter import EvalFormatter +from polyglot.builtins import unicode_type # Download worker {{{ @@ -471,7 +472,7 @@ def identify(log, abort, # {{{ for r in presults: log('\n\n---') try: - log(type(u'')(r)) + log(unicode_type(r)) except TypeError: log(repr(r)) if plog: