From 1e2cd173f23a34207dba6a84b95db5a22e431af8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 16 Aug 2024 19:46:04 +0530 Subject: [PATCH] Fix recipe browser tests --- src/calibre/scraper/qt_backend.py | 5 ++++- src/calibre/scraper/test_fetch_backend.py | 20 +++++++++++++------- src/calibre/scraper/webengine_backend.py | 8 ++++++-- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/calibre/scraper/qt_backend.py b/src/calibre/scraper/qt_backend.py index 9e6c867894..e48bacd157 100644 --- a/src/calibre/scraper/qt_backend.py +++ b/src/calibre/scraper/qt_backend.py @@ -68,7 +68,7 @@ class CookieJar(QNetworkCookieJar): c = QNetworkCookie(c) c.normalize(url) ans.append(c) - return ans + super().cookiesForUrl(url) + return super().cookiesForUrl(url) + ans def too_slow_or_timed_out(timeout: float, last_activity_at: float, created_at: float, downloaded_bytes: int, now: float) -> bool: @@ -206,6 +206,9 @@ class FetchBackend(QNetworkAccessManager): rq.setTransferTimeout(int(timeout * 1000)) rq.setRawHeader(b'User-Agent', self.current_user_agent().encode()) for (name, val) in req['headers']: + ex = rq.rawHeader(name) + if len(ex): + val = bytes(ex).decode() + ', ' + val rq.setRawHeader(name.encode(), val.encode()) qmethod = req['method'].lower() data_path = req['data_path'] diff --git a/src/calibre/scraper/test_fetch_backend.py b/src/calibre/scraper/test_fetch_backend.py index 6f4086365f..c960c7a4b4 100644 --- a/src/calibre/scraper/test_fetch_backend.py +++ b/src/calibre/scraper/test_fetch_backend.py @@ -12,6 +12,7 @@ from lxml.html import fromstring, tostring from calibre.utils.resources import get_path as P +from .qt import Browser, WebEngineBrowser from .simple import Overseer skip = '' @@ -55,6 +56,9 @@ class Handler(http.server.BaseHTTPRequestHandler): def do_GET(self): if self.test_obj.dont_send_response: return + if self.path == '/favicon.ico': + self.send_response(http.HTTPStatus.NOT_FOUND) + return if self.path == '/redirect': self.send_response(http.HTTPStatus.FOUND) self.send_header('Location', '/redirected') @@ -103,18 +107,16 @@ class TestFetchBackend(unittest.TestCase): self.server_thread.join(5) def test_recipe_browser_qt(self): - from .qt import Browser self.do_recipe_browser_test(Browser) def test_recipe_browser_webengine(self): - from .qt import WebEngineBrowser self.do_recipe_browser_test(WebEngineBrowser) - def do_recipe_browser_test(self, Browser): + def do_recipe_browser_test(self, browser_class): from urllib.error import URLError from urllib.request import Request - br = Browser(user_agent='test-ua', headers=(('th', '1'),), start_worker=True) + br = browser_class(user_agent='test-ua', headers=(('th', '1'),), start_worker=True) def u(path=''): return f'http://localhost:{self.port}{path}' @@ -161,13 +163,17 @@ class TestFetchBackend(unittest.TestCase): self.assertTrue(r['final_url'].endswith('/redirected')) self.ae(r['headers']['User-Agent'], ['test-ua']) r = get(headers={'th': '2', 'tc': '1'}) - self.ae(r['headers']['Th'], ['2']) - self.ae(r['headers']['Tc'], ['1']) + if browser_class is Browser: + self.ae(r['headers']['Th'], ['1, 2']) + self.ae(r['headers']['Tc'], ['1']) + else: + self.ae(r['headers']['th'], ['1, 2']) + self.ae(r['headers']['tc'], ['1']) br.set_simple_cookie('cook', 'ie') br.set_user_agent('man in black') r = get() self.ae(r['headers']['User-Agent'], ['man in black']) - self.ae(r['headers']['Cookie'], ['cook=ie; sc=1']) + self.ae(r['headers']['Cookie'], ['sc=1; cook=ie']) finally: br.shutdown() diff --git a/src/calibre/scraper/webengine_backend.py b/src/calibre/scraper/webengine_backend.py index 5a23623c6e..15b837faa7 100644 --- a/src/calibre/scraper/webengine_backend.py +++ b/src/calibre/scraper/webengine_backend.py @@ -12,7 +12,7 @@ from contextlib import suppress from http import HTTPStatus from time import monotonic -from qt.core import QApplication, QByteArray, QNetworkCookie, QObject, Qt, QTimer, QUrl, pyqtSignal +from qt.core import QApplication, QByteArray, QNetworkCookie, QObject, Qt, QTimer, QUrl, pyqtSignal, sip from qt.webengine import QWebEnginePage, QWebEngineScript from calibre.scraper.qt_backend import Request, too_slow_or_timed_out @@ -94,11 +94,15 @@ class Worker(QWebEnginePage): msg = json.loads(message.partition(' ')[2]) t = msg.get('type') if t == 'messages_available': - self.runjs('window.get_messages()', self.messages_dispatch.emit) + self.runjs('window.get_messages()', self.dispatch_messages) else: print(f'{source_id}:{line_num}:{message}') return + def dispatch_messages(self, messages: list) -> None: + if not sip.isdeleted(self): + self.messages_dispatch.emit(messages) + def runjs(self, js: str, callback) -> None: self.runJavaScript(js, QWebEngineScript.ScriptWorldId.ApplicationWorld, callback)