Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-08 10:44:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

Fix recipe browser tests

Browse Source
This commit is contained in:
Kovid Goyal 2024-08-16 19:46:04 +05:30
parent 6b907d8ec2
commit 1e2cd173f2
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
3 changed files with 23 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/calibre/scraper/qt_backend.py
View File

@ -68,7 +68,7 @@ class CookieJar(QNetworkCookieJar):
c = QNetworkCookie(c) c = QNetworkCookie(c)
c.normalize(url) c.normalize(url)
ans.append(c) ans.append(c)
return ans + super().cookiesForUrl(url) return super().cookiesForUrl(url) + ans
def too_slow_or_timed_out(timeout: float, last_activity_at: float, created_at: float, downloaded_bytes: int, now: float) -> bool: def too_slow_or_timed_out(timeout: float, last_activity_at: float, created_at: float, downloaded_bytes: int, now: float) -> bool:
@ -206,6 +206,9 @@ class FetchBackend(QNetworkAccessManager):
rq.setTransferTimeout(int(timeout * 1000)) rq.setTransferTimeout(int(timeout * 1000))
rq.setRawHeader(b'User-Agent', self.current_user_agent().encode()) rq.setRawHeader(b'User-Agent', self.current_user_agent().encode())
for (name, val) in req['headers']: for (name, val) in req['headers']:
ex = rq.rawHeader(name)
if len(ex):
val = bytes(ex).decode() + ', ' + val
rq.setRawHeader(name.encode(), val.encode()) rq.setRawHeader(name.encode(), val.encode())
qmethod = req['method'].lower() qmethod = req['method'].lower()
data_path = req['data_path'] data_path = req['data_path']

18
src/calibre/scraper/test_fetch_backend.py
View File

@ -12,6 +12,7 @@ from lxml.html import fromstring, tostring
from calibre.utils.resources import get_path as P from calibre.utils.resources import get_path as P
from .qt import Browser, WebEngineBrowser
from .simple import Overseer from .simple import Overseer
skip = '' skip = ''
@ -55,6 +56,9 @@ class Handler(http.server.BaseHTTPRequestHandler):
def do_GET(self): def do_GET(self):
if self.test_obj.dont_send_response: if self.test_obj.dont_send_response:
return return
if self.path == '/favicon.ico':
self.send_response(http.HTTPStatus.NOT_FOUND)
return
if self.path == '/redirect': if self.path == '/redirect':
self.send_response(http.HTTPStatus.FOUND) self.send_response(http.HTTPStatus.FOUND)
self.send_header('Location', '/redirected') self.send_header('Location', '/redirected')
@ -103,18 +107,16 @@ class TestFetchBackend(unittest.TestCase):
self.server_thread.join(5) self.server_thread.join(5)
def test_recipe_browser_qt(self): def test_recipe_browser_qt(self):
from .qt import Browser
self.do_recipe_browser_test(Browser) self.do_recipe_browser_test(Browser)
def test_recipe_browser_webengine(self): def test_recipe_browser_webengine(self):
from .qt import WebEngineBrowser
self.do_recipe_browser_test(WebEngineBrowser) self.do_recipe_browser_test(WebEngineBrowser)
def do_recipe_browser_test(self, Browser): def do_recipe_browser_test(self, browser_class):
from urllib.error import URLError from urllib.error import URLError
from urllib.request import Request from urllib.request import Request
br = Browser(user_agent='test-ua', headers=(('th', '1'),), start_worker=True) br = browser_class(user_agent='test-ua', headers=(('th', '1'),), start_worker=True)
def u(path=''): def u(path=''):
return f'http://localhost:{self.port}{path}' return f'http://localhost:{self.port}{path}'
@ -161,13 +163,17 @@ class TestFetchBackend(unittest.TestCase):
self.assertTrue(r['final_url'].endswith('/redirected')) self.assertTrue(r['final_url'].endswith('/redirected'))
self.ae(r['headers']['User-Agent'], ['test-ua']) self.ae(r['headers']['User-Agent'], ['test-ua'])
r = get(headers={'th': '2', 'tc': '1'}) r = get(headers={'th': '2', 'tc': '1'})
self.ae(r['headers']['Th'], ['2']) if browser_class is Browser:
self.ae(r['headers']['Th'], ['1, 2'])
self.ae(r['headers']['Tc'], ['1']) self.ae(r['headers']['Tc'], ['1'])
else:
self.ae(r['headers']['th'], ['1, 2'])
self.ae(r['headers']['tc'], ['1'])
br.set_simple_cookie('cook', 'ie') br.set_simple_cookie('cook', 'ie')
br.set_user_agent('man in black') br.set_user_agent('man in black')
r = get() r = get()
self.ae(r['headers']['User-Agent'], ['man in black']) self.ae(r['headers']['User-Agent'], ['man in black'])
self.ae(r['headers']['Cookie'], ['cook=ie; sc=1']) self.ae(r['headers']['Cookie'], ['sc=1; cook=ie'])
finally: finally:
br.shutdown() br.shutdown()

8
src/calibre/scraper/webengine_backend.py
View File

@ -12,7 +12,7 @@ from contextlib import suppress
from http import HTTPStatus from http import HTTPStatus
from time import monotonic from time import monotonic
from qt.core import QApplication, QByteArray, QNetworkCookie, QObject, Qt, QTimer, QUrl, pyqtSignal from qt.core import QApplication, QByteArray, QNetworkCookie, QObject, Qt, QTimer, QUrl, pyqtSignal, sip
from qt.webengine import QWebEnginePage, QWebEngineScript from qt.webengine import QWebEnginePage, QWebEngineScript
from calibre.scraper.qt_backend import Request, too_slow_or_timed_out from calibre.scraper.qt_backend import Request, too_slow_or_timed_out
@ -94,11 +94,15 @@ class Worker(QWebEnginePage):
msg = json.loads(message.partition(' ')[2]) msg = json.loads(message.partition(' ')[2])
t = msg.get('type') t = msg.get('type')
if t == 'messages_available': if t == 'messages_available':
self.runjs('window.get_messages()', self.messages_dispatch.emit) self.runjs('window.get_messages()', self.dispatch_messages)
else: else:
print(f'{source_id}:{line_num}:{message}') print(f'{source_id}:{line_num}:{message}')
return return
def dispatch_messages(self, messages: list) -> None:
if not sip.isdeleted(self):
self.messages_dispatch.emit(messages)
def runjs(self, js: str, callback) -> None: def runjs(self, js: str, callback) -> None:
self.runJavaScript(js, QWebEngineScript.ScriptWorldId.ApplicationWorld, callback) self.runJavaScript(js, QWebEngineScript.ScriptWorldId.ApplicationWorld, callback)