From 68cd95a2c5c6e65f61899afbc419c3f32523b4c4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 16 Jul 2022 12:19:07 +0530 Subject: [PATCH] Consolidate setting of webengine storage paths in one place Qt6 uses the persistentstoragepath to cache data even for off the record profiles. So we need to set both that and the cache path. And do it for the default profile, and every other profile we create. Sigh. --- src/calibre/ebooks/oeb/polish/check/css.py | 3 ++- src/calibre/ebooks/pdf/html_writer.py | 3 ++- src/calibre/gui2/store/web_store.py | 21 +++++++++++++++------ src/calibre/gui2/toc/location.py | 3 ++- src/calibre/gui2/tweak_book/preview.py | 5 ++++- src/calibre/gui2/viewer/lookup.py | 15 ++++++++------- src/calibre/gui2/viewer/web_view.py | 5 +++-- src/calibre/scraper/simple_backend.py | 6 ++---- src/calibre/test_build.py | 2 ++ src/calibre/utils/rapydscript.py | 7 +++++-- src/calibre/utils/webengine.py | 17 +++++++++++++---- 11 files changed, 58 insertions(+), 29 deletions(-) diff --git a/src/calibre/ebooks/oeb/polish/check/css.py b/src/calibre/ebooks/oeb/polish/check/css.py index ba9890a26d..ffe4f2bc21 100644 --- a/src/calibre/ebooks/oeb/polish/check/css.py +++ b/src/calibre/ebooks/oeb/polish/check/css.py @@ -16,7 +16,7 @@ from qt.webengine import ( from calibre import detect_ncpus as cpu_count, prints from calibre.ebooks.oeb.polish.check.base import ERROR, WARN, BaseError from calibre.gui2 import must_use_qt -from calibre.utils.webengine import secure_webengine +from calibre.utils.webengine import secure_webengine, setup_profile class CSSParseError(BaseError): @@ -116,6 +116,7 @@ def create_profile(): ans = getattr(create_profile, 'ans', None) if ans is None: ans = create_profile.ans = QWebEngineProfile(QApplication.instance()) + setup_profile(ans) s = QWebEngineScript() s.setName('csslint.js') s.setSourceCode(csslint_js()) diff --git a/src/calibre/ebooks/pdf/html_writer.py b/src/calibre/ebooks/pdf/html_writer.py index fdd91cf7e4..b1d16f0db2 100644 --- a/src/calibre/ebooks/pdf/html_writer.py +++ b/src/calibre/ebooks/pdf/html_writer.py @@ -49,7 +49,7 @@ from calibre.utils.podofo import ( dedup_type3_fonts, get_podofo, remove_unused_fonts, set_metadata_implementation ) from calibre.utils.short_uuid import uuid4 -from calibre.utils.webengine import secure_webengine, send_reply +from calibre.utils.webengine import secure_webengine, send_reply, setup_profile from polyglot.builtins import as_bytes, iteritems from polyglot.urllib import urlparse @@ -352,6 +352,7 @@ class RenderManager(QObject): self.has_maths = {} self.interceptor.log = self.log = log ans = QWebEngineProfile(QApplication.instance()) + setup_profile(ans) self.url_handler = UrlSchemeHandler(container, parent=ans) ans.installUrlSchemeHandler(QByteArray(FAKE_PROTOCOL.encode('ascii')), self.url_handler) ua = 'calibre-pdf-output ' + __version__ diff --git a/src/calibre/gui2/store/web_store.py b/src/calibre/gui2/store/web_store.py index 14263f39b0..c1b1825c9c 100644 --- a/src/calibre/gui2/store/web_store.py +++ b/src/calibre/gui2/store/web_store.py @@ -9,10 +9,12 @@ from qt.core import ( QApplication, QHBoxLayout, QIcon, QLabel, QProgressBar, QPushButton, QSize, QUrl, QVBoxLayout, QWidget, pyqtSignal ) -from qt.webengine import QWebEngineDownloadRequest, QWebEngineView +from qt.webengine import ( + QWebEngineDownloadRequest, QWebEnginePage, QWebEngineProfile, QWebEngineView +) from calibre import random_user_agent, url_slash_cleaner -from calibre.constants import STORE_DIALOG_APP_UID, cache_dir, islinux, iswindows +from calibre.constants import STORE_DIALOG_APP_UID, islinux, iswindows from calibre.ebooks import BOOK_EXTENSIONS from calibre.gui2 import ( Application, choose_save_file, error_dialog, gprefs, info_dialog, set_app_uid @@ -21,6 +23,7 @@ from calibre.gui2.dialogs.confirm_delete import confirm from calibre.gui2.listener import send_message_in_process from calibre.gui2.main_window import MainWindow from calibre.ptempfile import PersistentTemporaryDirectory, reset_base_dir +from calibre.utils.webengine import setup_profile from polyglot.binary import as_base64_bytes, from_base64_bytes from polyglot.builtins import string_or_bytes @@ -75,6 +78,14 @@ class DownloadProgress(QWidget): self.setVisible(False) +def create_profile(): + ans = getattr(create_profile, 'ans', None) + if ans is None: + ans = create_profile.ans = setup_profile(QWebEngineProfile('web_store', QApplication.instance())) + ans.setHttpUserAgent(random_user_agent(allow_ie=False)) + return ans + + class Central(QWidget): home = pyqtSignal() @@ -83,10 +94,8 @@ class Central(QWidget): QWidget.__init__(self, parent) self.l = l = QVBoxLayout(self) self.view = v = QWebEngineView(self) - profile = v.page().profile() - profile.setCachePath(os.path.join(cache_dir(), 'web_store', 'hc')) - profile.setPersistentStoragePath(os.path.join(cache_dir(), 'web_store', 'ps')) - profile.setHttpUserAgent(random_user_agent(allow_ie=False)) + self._page = QWebEnginePage(create_profile(), v) + v.setPage(self._page) v.loadStarted.connect(self.load_started) v.loadProgress.connect(self.load_progress) v.loadFinished.connect(self.load_finished) diff --git a/src/calibre/gui2/toc/location.py b/src/calibre/gui2/toc/location.py index c1ea20a676..fa825e6416 100644 --- a/src/calibre/gui2/toc/location.py +++ b/src/calibre/gui2/toc/location.py @@ -24,7 +24,7 @@ from calibre.gui2 import error_dialog, gprefs, is_dark_theme, question_dialog from calibre.gui2.palette import dark_color, dark_link_color, dark_text_color from calibre.utils.logging import default_log from calibre.utils.short_uuid import uuid4 -from calibre.utils.webengine import secure_webengine, send_reply +from calibre.utils.webengine import secure_webengine, send_reply, setup_profile from polyglot.builtins import as_bytes @@ -143,6 +143,7 @@ class Page(QWebEnginePage): # {{{ self.current_frag = None self.com_id = str(uuid4()) profile = QWebEngineProfile(QApplication.instance()) + setup_profile(profile) # store these globally as they need to be destructed after the QWebEnginePage current_container.url_handler = UrlSchemeHandler(parent=profile) current_container.interceptor = RequestInterceptor(profile) diff --git a/src/calibre/gui2/tweak_book/preview.py b/src/calibre/gui2/tweak_book/preview.py index 10e36b6229..fa7d1b1015 100644 --- a/src/calibre/gui2/tweak_book/preview.py +++ b/src/calibre/gui2/tweak_book/preview.py @@ -37,7 +37,8 @@ from calibre.gui2.webengine import RestartingWebEngineView from calibre.gui2.widgets2 import HistoryLineEdit2 from calibre.utils.ipc.simple_worker import offload_worker from calibre.utils.webengine import ( - Bridge, create_script, from_js, insert_scripts, secure_webengine, to_js + Bridge, create_script, from_js, insert_scripts, secure_webengine, setup_profile, + to_js ) from polyglot.builtins import iteritems from polyglot.queue import Empty, Queue @@ -294,6 +295,7 @@ def create_profile(): ans = getattr(create_profile, 'ans', None) if ans is None: ans = QWebEngineProfile(QApplication.instance()) + setup_profile(ans) ua = 'calibre-editor-preview ' + __version__ ans.setHttpUserAgent(ua) if is_running_from_develop: @@ -391,6 +393,7 @@ class Inspector(QWidget): def visibility_changed(self, visible): if visible and self.view is None: self.view = QWebEngineView(self.view_to_debug) + setup_profile(self.view.page().profile()) self.view_to_debug.page().setDevToolsPage(self.view.page()) self.layout.addWidget(self.view) diff --git a/src/calibre/gui2/viewer/lookup.py b/src/calibre/gui2/viewer/lookup.py index 5b2acaac8c..c4adbbe5db 100644 --- a/src/calibre/gui2/viewer/lookup.py +++ b/src/calibre/gui2/viewer/lookup.py @@ -2,24 +2,24 @@ # License: GPL v3 Copyright: 2019, Kovid Goyal -import os import sys import textwrap from qt.core import ( - QApplication, QCheckBox, QComboBox, QDialog, QDialogButtonBox, QFormLayout, QAbstractItemView, - QHBoxLayout, QIcon, QLabel, QLineEdit, QListWidget, QListWidgetItem, QPushButton, - QSize, Qt, QTimer, QUrl, QVBoxLayout, QWidget, pyqtSignal + QAbstractItemView, QApplication, QCheckBox, QComboBox, QDialog, QDialogButtonBox, + QFormLayout, QHBoxLayout, QIcon, QLabel, QLineEdit, QListWidget, QListWidgetItem, + QPushButton, QSize, Qt, QTimer, QUrl, QVBoxLayout, QWidget, pyqtSignal ) from qt.webengine import ( QWebEnginePage, QWebEngineProfile, QWebEngineScript, QWebEngineView ) from calibre import prints, random_user_agent -from calibre.constants import cache_dir from calibre.gui2 import error_dialog from calibre.gui2.viewer.web_view import apply_font_settings, vprefs -from calibre.utils.webengine import create_script, insert_scripts, secure_webengine from calibre.gui2.widgets2 import Dialog +from calibre.utils.webengine import ( + create_script, insert_scripts, secure_webengine, setup_profile +) vprefs.defaults['lookup_locations'] = [ { @@ -191,7 +191,7 @@ def create_profile(): if ans is None: ans = QWebEngineProfile('viewer-lookup', QApplication.instance()) ans.setHttpUserAgent(random_user_agent(allow_ie=False)) - ans.setCachePath(os.path.join(cache_dir(), 'ev2vl')) + setup_profile(ans) js = P('lookup.js', data=True, allow_user_override=False) insert_scripts(ans, create_script('lookup.js', js, injection_point=QWebEngineScript.InjectionPoint.DocumentCreation)) s = ans.settings() @@ -305,6 +305,7 @@ class Lookup(QWidget): self._devtools_page = QWebEnginePage() self._devtools_view = QWebEngineView(self) self._devtools_view.setPage(self._devtools_page) + setup_profile(self._devtools_page.profile()) self._page.setDevToolsPage(self._devtools_page) self._devtools_dialog = d = QDialog(self) d.setWindowTitle('Inspect Lookup page') diff --git a/src/calibre/gui2/viewer/web_view.py b/src/calibre/gui2/viewer/web_view.py index c2c009ea28..9ec216ed3c 100644 --- a/src/calibre/gui2/viewer/web_view.py +++ b/src/calibre/gui2/viewer/web_view.py @@ -34,7 +34,7 @@ from calibre.utils.serialize import json_loads from calibre.utils.shared_file import share_open from calibre.utils.webengine import ( Bridge, create_script, from_js, insert_scripts, secure_webengine, send_reply, - to_js + to_js, setup_profile ) from polyglot.builtins import as_bytes, iteritems from polyglot.functools import lru_cache @@ -183,7 +183,7 @@ class UrlSchemeHandler(QWebEngineUrlSchemeHandler): def create_profile(): ans = getattr(create_profile, 'ans', None) if ans is None: - ans = QWebEngineProfile(QApplication.instance()) + ans = setup_profile(QWebEngineProfile(QApplication.instance())) osname = 'windows' if iswindows else ('macos' if ismacos else 'linux') # DO NOT change the user agent as it is used to workaround # Qt bugs see workaround_qt_bug() in ajax.pyj @@ -399,6 +399,7 @@ class Inspector(QWidget): def visibility_changed(self, visible): if visible and self.view is None: self.view = QWebEngineView(self.view_to_debug) + setup_profile(self.view.page().profile()) self.view_to_debug.page().setDevToolsPage(self.view.page()) self.layout.addWidget(self.view) diff --git a/src/calibre/scraper/simple_backend.py b/src/calibre/scraper/simple_backend.py index ca389cea6f..025f6571b9 100644 --- a/src/calibre/scraper/simple_backend.py +++ b/src/calibre/scraper/simple_backend.py @@ -3,7 +3,6 @@ # License: GPL v3 Copyright: 2022, Kovid Goyal import json -import os import secrets import sys import time @@ -11,8 +10,7 @@ from functools import lru_cache from qt.core import QApplication, QEventLoop, QUrl from qt.webengine import QWebEnginePage, QWebEngineProfile, QWebEngineSettings -from calibre.constants import cache_dir -from calibre.utils.webengine import create_script, insert_scripts +from calibre.utils.webengine import create_script, insert_scripts, setup_profile def canonicalize_qurl(qurl): @@ -29,9 +27,9 @@ def create_profile(cache_name='', allow_js=False): from calibre.utils.random_ua import random_common_chrome_user_agent if cache_name: ans = QWebEngineProfile(cache_name, QApplication.instance()) - ans.setCachePath(os.path.join(cache_dir(), 'scraper', cache_name)) else: ans = QWebEngineProfile(QApplication.instance()) + setup_profile(ans) ans.setHttpUserAgent(random_common_chrome_user_agent()) ans.setHttpCacheMaximumSize(0) # managed by webengine s = ans.settings() diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index c0f5029216..e8364ab9ee 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -320,6 +320,7 @@ class BuildTest(unittest.TestCase): test() from calibre.gui2 import ensure_app, destroy_app + from calibre.utils.webengine import setup_profile display_env_var = os.environ.pop('DISPLAY', None) try: ensure_app() @@ -333,6 +334,7 @@ class BuildTest(unittest.TestCase): na = QNetworkAccessManager() self.assertTrue(hasattr(na, 'sslErrors'), 'Qt not compiled with openssl') p = QWebEnginePage() + setup_profile(p.profile()) def callback(result): callback.result = result diff --git a/src/calibre/utils/rapydscript.py b/src/calibre/utils/rapydscript.py index 1768a6898c..747ee74272 100644 --- a/src/calibre/utils/rapydscript.py +++ b/src/calibre/utils/rapydscript.py @@ -59,7 +59,7 @@ def compiler(): from calibre import walk from calibre.gui2 import must_use_qt - from calibre.utils.webengine import secure_webengine, setup_default_profile + from calibre.utils.webengine import secure_webengine, setup_default_profile, setup_profile must_use_qt() setup_default_profile() @@ -126,6 +126,7 @@ document.title = 'compiler initialized'; def __init__(self): super().__init__() + setup_profile(self.profile()) self.errors = [] secure_webengine(self) script = compiler_script @@ -342,9 +343,10 @@ def run_rapydscript_tests(): from calibre.gui2 import must_use_qt from calibre.gui2.viewer.web_view import send_reply from calibre.utils.webengine import ( - create_script, insert_scripts, secure_webengine + create_script, insert_scripts, secure_webengine, setup_default_profile, setup_profile ) must_use_qt() + setup_default_profile() scheme = QWebEngineUrlScheme(FAKE_PROTOCOL.encode('ascii')) scheme.setSyntax(QWebEngineUrlScheme.Syntax.Host) scheme.setFlags(QWebEngineUrlScheme.Flag.SecureScheme) @@ -388,6 +390,7 @@ def run_rapydscript_tests(): def __init__(self): profile = QWebEngineProfile(QApplication.instance()) profile.setHttpUserAgent('calibre-tester') + setup_profile(profile) insert_scripts(profile, create_script('test-rapydscript.js', js, on_subframes=False)) url_handler = UrlSchemeHandler(profile) profile.installUrlSchemeHandler(QByteArray(FAKE_PROTOCOL.encode('ascii')), url_handler) diff --git a/src/calibre/utils/webengine.py b/src/calibre/utils/webengine.py index 3d036c67e8..1b6b3bbfcd 100644 --- a/src/calibre/utils/webengine.py +++ b/src/calibre/utils/webengine.py @@ -10,11 +10,20 @@ from qt.webengine import QWebEngineScript, QWebEngineSettings, QWebEngineProfile from calibre.constants import cache_dir, SPECIAL_TITLE_FOR_WEBENGINE_COMMS +def setup_profile(profile): + # Qt uses persistent storage path to store cached GPU data even for OTR profiles + base = os.path.abspath(os.path.join(cache_dir(), 'qwe', profile.storageName() or 'dp')) + cp = os.path.join(base, 'c') + if profile.cachePath() != cp: + profile.setCachePath(cp) + sp = os.path.join(base, 'sp') + if profile.persistentStoragePath() != sp: + profile.setPersistentStoragePath(sp) + return profile + + def setup_default_profile(): - p = QWebEngineProfile.defaultProfile() - q = os.path.join(cache_dir(), 'qt-webeng') - if p.cachePath() != q: - p.setCachePath(q) + return setup_profile(QWebEngineProfile.defaultProfile()) def send_reply(rq, mime_type, data):