diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 1456b62638..358a7ee4bf 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -320,6 +320,28 @@ def get_parsed_proxy(typ='http', debug=True): prints('Using http proxy', str(ans)) return ans +def get_proxy_info(proxy_scheme, proxy_string): + ''' + Parse all proxy information from a proxy string (as returned by + get_proxies). The returned dict will have members set to None when the info + is not available in the string. If an exception occurs parsing the string + this method returns None. + ''' + import urlparse + try: + proxy_url = u'%s://%s'%(proxy_scheme, proxy_string) + urlinfo = urlparse.urlparse(proxy_url) + ans = { + u'scheme': urlinfo.scheme, + u'hostname': urlinfo.hostname, + u'port': urlinfo.port, + u'username': urlinfo.username, + u'password': urlinfo.password, + } + except: + return None + return ans + USER_AGENT = 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.13) Gecko/20101210 Gentoo Firefox/3.6.13' USER_AGENT_MOBILE = 'Mozilla/5.0 (Windows; U; Windows CE 5.1; rv:1.8.1a3) Gecko/20060610 Minimo/0.016' diff --git a/src/calibre/constants.py b/src/calibre/constants.py index 425dbdd9dd..9a429768b1 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -33,6 +33,7 @@ islinux = not(iswindows or isosx or isbsd) isfrozen = hasattr(sys, 'frozen') isunix = isosx or islinux isportable = os.environ.get('CALIBRE_PORTABLE_BUILD', None) is not None +ispy3 = sys.version_info[0] > 2 try: preferred_encoding = locale.getpreferredencoding() diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 0e123bee8b..7d4a39fe4e 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -753,15 +753,24 @@ def open_local_file(path): url = QUrl.fromLocalFile(path) open_url(url) -def is_ok_to_use_qt(): +def must_use_qt(): global gui_thread, _store_app if (islinux or isbsd) and ':' not in os.environ.get('DISPLAY', ''): - return False + raise RuntimeError('X server required. If you are running on a' + ' headless machine, use xvfb') if _store_app is None and QApplication.instance() is None: _store_app = QApplication([]) if gui_thread is None: gui_thread = QThread.currentThread() - return gui_thread is QThread.currentThread() + if gui_thread is not QThread.currentThread(): + raise RuntimeError('Cannot use Qt in non GUI thread') + +def is_ok_to_use_qt(): + try: + must_use_qt() + except RuntimeError: + return False + return True def is_gui_thread(): global gui_thread diff --git a/src/calibre/web/jsbrowser/__init__.py b/src/calibre/web/jsbrowser/__init__.py new file mode 100644 index 0000000000..6e45b0e9e6 --- /dev/null +++ b/src/calibre/web/jsbrowser/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + diff --git a/src/calibre/web/jsbrowser/browser.py b/src/calibre/web/jsbrowser/browser.py new file mode 100644 index 0000000000..6a87594030 --- /dev/null +++ b/src/calibre/web/jsbrowser/browser.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os, pprint + +from PyQt4.Qt import (QObject, QNetworkAccessManager, QNetworkDiskCache, + QNetworkProxy, QNetworkProxyFactory) +from PyQt4.QtWebKit import QWebPage + +from calibre import USER_AGENT, prints, get_proxies, get_proxy_info +from calibre.constants import ispy3, config_dir +from calibre.utils.logging import ThreadSafeLog +from calibre.gui2 import must_use_qt + +class WebPage(QWebPage): # {{{ + + def __init__(self, log, + confirm_callback=None, + prompt_callback=None, + user_agent=USER_AGENT, + parent=None): + QWebPage.__init__(self, parent) + + self.log = log + self.user_agent = user_agent if user_agent else USER_AGENT + self.confirm_callback = confirm_callback + self.prompt_callback = prompt_callback + self.setForwardUnsupportedContent(True) + self.unsupportedContent.connect(self.on_unsupported_content) + + def userAgentForUrl(self, url): + return self.user_agent + + def javaScriptAlert(self, frame, msg): + if self.view() is not None: + return QWebPage.javaScriptAlert(self, frame, msg) + prints('JSBrowser alert():', unicode(msg)) + + def javaScriptConfirm(self, frame, msg): + if self.view() is not None: + return QWebPage.javaScriptConfirm(self, frame, msg) + if self.confirm_callback is not None: + return self.confirm_callback(unicode(msg)) + return True + + def javaScriptConsoleMessage(self, msg, lineno, source_id): + prints('JSBrowser msg():%s:%s:'%(unicode(source_id), lineno), unicode(msg)) + + def javaScriptPrompt(self, frame, msg, default_value, *args): + if self.view() is not None: + return QWebPage.javaScriptPrompt(self, frame, msg, default_value, + *args) + if self.prompt_callback is None: + return (False, default_value) if ispy3 else False + value = self.prompt_callback(unicode(msg), unicode(default_value)) + ok = value is not None + if ispy3: + return ok, value + if ok: + result = args[0] + result.clear() + result.append(value) + return ok + + def shouldInterruptJavaScript(self): + if self.view() is not None: + return QWebPage.shouldInterruptJavaScript(self) + return True + + def on_unsupported_content(self, reply): + self.log.warn('Unsupported content, ignoring: %s'%reply.url()) + +# }}} + +class ProxyFactory(QNetworkProxyFactory): # {{{ + + def __init__(self, log): + QNetworkProxyFactory.__init__(self) + proxies = get_proxies() + self.proxies = {} + for scheme, proxy_string in proxies.iteritems(): + scheme = scheme.lower() + info = get_proxy_info(scheme, proxy_string) + if info is None: continue + hn, port = info['hostname'], info['port'] + if not hn or not port: continue + log.debug('JSBrowser using proxy:', pprint.pformat(info)) + pt = {'socks5':QNetworkProxy.Socks5Proxy}.get(scheme, + QNetworkProxy.HttpProxy) + proxy = QNetworkProxy(pt, hn, port) + un, pw = info['username'], info['password'] + if un: + proxy.setUser(un) + if pw: + proxy.setPassword(pw) + self.proxies[scheme] = proxy + + self.default_proxy = QNetworkProxy(QNetworkProxy.DefaultProxy) + + def queryProxy(self, query): + scheme = unicode(query.protocolTag()).lower() + return [self.proxies.get(scheme, self.default_proxy)] +# }}} + +class NetworkAccessManager(QNetworkAccessManager): # {{{ + + OPERATION_NAMES = { getattr(QNetworkAccessManager, '%sOperation'%x) : + x.upper() for x in ('Head', 'Get', 'Put', 'Post', 'Delete', + 'Custom') + } + + def __init__(self, log, use_disk_cache=True, parent=None): + QNetworkAccessManager.__init__(self, parent) + self.log = log + if use_disk_cache: + self.cache = QNetworkDiskCache(self) + self.cache.setCacheDirectory(os.path.join(config_dir, 'caches', + 'jsbrowser')) + self.setCache(self.cache) + self.sslErrors.connect(self.on_ssl_errors) + self.pf = ProxyFactory(log) + self.setProxyFactory(self.pf) + self.finished.connect(self.on_finished) + + def on_ssl_errors(self, reply, errors): + reply.ignoreSslErrors() + + def createRequest(self, operation, request, data): + url = unicode(request.url().toString()) + operation_name = self.OPERATION_NAMES[operation] + debug = [] + debug.append(('Request: %s %s' % (operation_name, url))) + for h in request.rawHeaderList(): + try: + d = ' %s: %s' % (h, request.rawHeader(h)) + except: + d = ' %r: %r' % (h, request.rawHeader(h)) + debug.append(d) + + if data is not None: + raw = data.peek(1024) + try: + raw = raw.decode('utf-8') + except: + raw = repr(raw) + debug.append(' Request data: %s'%raw) + + self.log.debug('\n'.join(debug)) + return QNetworkAccessManager.createRequest(self, operation, request, + data) + + def on_finished(self, reply): + reply_url = unicode(reply.url().toString()) + + if reply.error(): + self.log.warn("Reply error: %s - %d (%s)" % + (reply_url, reply.error(), reply.errorString())) + else: + debug = [] + debug.append("Reply successful: %s" % reply_url) + for h in reply.rawHeaderList(): + try: + d = ' %s: %s' % (h, reply.rawHeader(h)) + except: + d = ' %r: %r' % (h, reply.rawHeader(h)) + debug.append(d) + self.log.debug('\n'.join(debug)) +# }}} + +class Browser(QObject): + + def __init__(self, + # Logging. If None, uses a default log, which does not output + # debugging info + log = None, + # Receives a string and returns True/False. By default, returns + # True for all strings + confirm_callback=None, + + # Prompt callback. Receives a msg string and a default value + # string. Should return the user input value or None if the user + # canceled the prompt. By default returns None. + prompt_callback=None, + + # User agent to be used + user_agent=USER_AGENT, + + # If True a disk cache is used + use_disk_cache=True, + + # Verbosity + verbosity = 0 + ): + must_use_qt() + QObject.__init__(self) + + if log is None: + log = ThreadSafeLog() + if verbosity: + log.filter_level = log.DEBUG + + self.jquery_lib = P('content_server/jquery.js', data=True, + allow_user_override=False).decode('utf-8') + self.simulate_lib = P('jquery.simulate.js', data=True, + allow_user_override=False).decode('utf-8') + + self.page = WebPage(log, confirm_callback=confirm_callback, + prompt_callback=prompt_callback, user_agent=user_agent, + parent=self) + self.nam = NetworkAccessManager(log, use_disk_cache=use_disk_cache, parent=self) + self.page.setNetworkAccessManager(self.nam) + + def visit(self, url): + pass +