Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-08 02:34:06 -04:00
Code Issues Packages Projects Releases Wiki Activity

Fix recipe browser tests

Browse Source
This commit is contained in:
Kovid Goyal 2024-08-16 19:46:04 +05:30
parent 6b907d8ec2
commit 1e2cd173f2
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
3 changed files with 23 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/calibre/scraper/qt_backend.py
View File

@ -68,7 +68,7 @@ class CookieJar(QNetworkCookieJar):
c = QNetworkCookie(c)
c.normalize(url)
ans.append(c)
return ans + super().cookiesForUrl(url)
return super().cookiesForUrl(url) + ans
def too_slow_or_timed_out(timeout: float, last_activity_at: float, created_at: float, downloaded_bytes: int, now: float) -> bool:
@ -206,6 +206,9 @@ class FetchBackend(QNetworkAccessManager):
rq.setTransferTimeout(int(timeout * 1000))
rq.setRawHeader(b'User-Agent', self.current_user_agent().encode())
for (name, val) in req['headers']:
ex = rq.rawHeader(name)
if len(ex):
val = bytes(ex).decode() + ', ' + val
rq.setRawHeader(name.encode(), val.encode())
qmethod = req['method'].lower()
data_path = req['data_path']

20
src/calibre/scraper/test_fetch_backend.py
View File

@ -12,6 +12,7 @@ from lxml.html import fromstring, tostring
from calibre.utils.resources import get_path as P
from .qt import Browser, WebEngineBrowser
from .simple import Overseer
skip = ''
@ -55,6 +56,9 @@ class Handler(http.server.BaseHTTPRequestHandler):
def do_GET(self):
if self.test_obj.dont_send_response:
return
if self.path == '/favicon.ico':
self.send_response(http.HTTPStatus.NOT_FOUND)
return
if self.path == '/redirect':
self.send_response(http.HTTPStatus.FOUND)
self.send_header('Location', '/redirected')
@ -103,18 +107,16 @@ class TestFetchBackend(unittest.TestCase):
self.server_thread.join(5)
def test_recipe_browser_qt(self):
from .qt import Browser
self.do_recipe_browser_test(Browser)
def test_recipe_browser_webengine(self):
from .qt import WebEngineBrowser
self.do_recipe_browser_test(WebEngineBrowser)
def do_recipe_browser_test(self, Browser):
def do_recipe_browser_test(self, browser_class):
from urllib.error import URLError
from urllib.request import Request
br = Browser(user_agent='test-ua', headers=(('th', '1'),), start_worker=True)
br = browser_class(user_agent='test-ua', headers=(('th', '1'),), start_worker=True)
def u(path=''):
return f'http://localhost:{self.port}{path}'
@ -161,13 +163,17 @@ class TestFetchBackend(unittest.TestCase):
self.assertTrue(r['final_url'].endswith('/redirected'))
self.ae(r['headers']['User-Agent'], ['test-ua'])
r = get(headers={'th': '2', 'tc': '1'})
self.ae(r['headers']['Th'], ['2'])
self.ae(r['headers']['Tc'], ['1'])
if browser_class is Browser:
self.ae(r['headers']['Th'], ['1, 2'])
self.ae(r['headers']['Tc'], ['1'])
else:
self.ae(r['headers']['th'], ['1, 2'])
self.ae(r['headers']['tc'], ['1'])
br.set_simple_cookie('cook', 'ie')
br.set_user_agent('man in black')
r = get()
self.ae(r['headers']['User-Agent'], ['man in black'])
self.ae(r['headers']['Cookie'], ['cook=ie; sc=1'])
self.ae(r['headers']['Cookie'], ['sc=1; cook=ie'])
finally:
br.shutdown()

8
src/calibre/scraper/webengine_backend.py
View File

@ -12,7 +12,7 @@ from contextlib import suppress
from http import HTTPStatus
from time import monotonic
from qt.core import QApplication, QByteArray, QNetworkCookie, QObject, Qt, QTimer, QUrl, pyqtSignal
from qt.core import QApplication, QByteArray, QNetworkCookie, QObject, Qt, QTimer, QUrl, pyqtSignal, sip
from qt.webengine import QWebEnginePage, QWebEngineScript
from calibre.scraper.qt_backend import Request, too_slow_or_timed_out
@ -94,11 +94,15 @@ class Worker(QWebEnginePage):
msg = json.loads(message.partition(' ')[2])
t = msg.get('type')
if t == 'messages_available':
self.runjs('window.get_messages()', self.messages_dispatch.emit)
self.runjs('window.get_messages()', self.dispatch_messages)
else:
print(f'{source_id}:{line_num}:{message}')
return
def dispatch_messages(self, messages: list) -> None:
if not sip.isdeleted(self):
self.messages_dispatch.emit(messages)
def runjs(self, js: str, callback) -> None:
self.runJavaScript(js, QWebEngineScript.ScriptWorldId.ApplicationWorld, callback)