Consolidate setting of webengine storage paths in one place

Qt6 uses the persistentstoragepath to cache data even for off the record
profiles. So we need to set both that and the cache path. And do it for
the default profile, and every other profile we create. Sigh.

src/calibre/ebooks/oeb/polish/check/css.py

@@ -16,7 +16,7 @@ from qt.webengine import (
 from calibre import detect_ncpus as cpu_count, prints
 from calibre.ebooks.oeb.polish.check.base import ERROR, WARN, BaseError
 from calibre.gui2 import must_use_qt
-from calibre.utils.webengine import secure_webengine
+from calibre.utils.webengine import secure_webengine, setup_profile
 
 
 class CSSParseError(BaseError):
@@ -116,6 +116,7 @@ def create_profile():
     ans = getattr(create_profile, 'ans', None)
     if ans is None:
         ans = create_profile.ans = QWebEngineProfile(QApplication.instance())
+        setup_profile(ans)
         s = QWebEngineScript()
         s.setName('csslint.js')
         s.setSourceCode(csslint_js())

src/calibre/ebooks/pdf/html_writer.py

@@ -49,7 +49,7 @@ from calibre.utils.podofo import (
     dedup_type3_fonts, get_podofo, remove_unused_fonts, set_metadata_implementation
 )
 from calibre.utils.short_uuid import uuid4
-from calibre.utils.webengine import secure_webengine, send_reply
+from calibre.utils.webengine import secure_webengine, send_reply, setup_profile
 from polyglot.builtins import as_bytes, iteritems
 from polyglot.urllib import urlparse
 
@@ -352,6 +352,7 @@ class RenderManager(QObject):
         self.has_maths = {}
         self.interceptor.log = self.log = log
         ans = QWebEngineProfile(QApplication.instance())
+        setup_profile(ans)
         self.url_handler = UrlSchemeHandler(container, parent=ans)
         ans.installUrlSchemeHandler(QByteArray(FAKE_PROTOCOL.encode('ascii')), self.url_handler)
         ua = 'calibre-pdf-output ' + __version__

src/calibre/gui2/store/web_store.py

@@ -9,10 +9,12 @@ from qt.core import (
     QApplication, QHBoxLayout, QIcon, QLabel, QProgressBar, QPushButton, QSize, QUrl,
     QVBoxLayout, QWidget, pyqtSignal
 )
-from qt.webengine import QWebEngineDownloadRequest, QWebEngineView
+from qt.webengine import (
+    QWebEngineDownloadRequest, QWebEnginePage, QWebEngineProfile, QWebEngineView
+)
 
 from calibre import random_user_agent, url_slash_cleaner
-from calibre.constants import STORE_DIALOG_APP_UID, cache_dir, islinux, iswindows
+from calibre.constants import STORE_DIALOG_APP_UID, islinux, iswindows
 from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.gui2 import (
     Application, choose_save_file, error_dialog, gprefs, info_dialog, set_app_uid
@@ -21,6 +23,7 @@ from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.listener import send_message_in_process
 from calibre.gui2.main_window import MainWindow
 from calibre.ptempfile import PersistentTemporaryDirectory, reset_base_dir
+from calibre.utils.webengine import setup_profile
 from polyglot.binary import as_base64_bytes, from_base64_bytes
 from polyglot.builtins import string_or_bytes
 
@@ -75,6 +78,14 @@ class DownloadProgress(QWidget):
             self.setVisible(False)
 
 
+def create_profile():
+    ans = getattr(create_profile, 'ans', None)
+    if ans is None:
+        ans = create_profile.ans = setup_profile(QWebEngineProfile('web_store', QApplication.instance()))
+        ans.setHttpUserAgent(random_user_agent(allow_ie=False))
+    return ans
+
+
 class Central(QWidget):
 
     home = pyqtSignal()
@@ -83,10 +94,8 @@ class Central(QWidget):
         QWidget.__init__(self, parent)
         self.l = l = QVBoxLayout(self)
         self.view = v = QWebEngineView(self)
-        profile = v.page().profile()
+        self._page = QWebEnginePage(create_profile(), v)
-        profile.setCachePath(os.path.join(cache_dir(), 'web_store', 'hc'))
+        v.setPage(self._page)
-        profile.setPersistentStoragePath(os.path.join(cache_dir(), 'web_store', 'ps'))
-        profile.setHttpUserAgent(random_user_agent(allow_ie=False))
         v.loadStarted.connect(self.load_started)
         v.loadProgress.connect(self.load_progress)
         v.loadFinished.connect(self.load_finished)

src/calibre/gui2/toc/location.py

@@ -24,7 +24,7 @@ from calibre.gui2 import error_dialog, gprefs, is_dark_theme, question_dialog
 from calibre.gui2.palette import dark_color, dark_link_color, dark_text_color
 from calibre.utils.logging import default_log
 from calibre.utils.short_uuid import uuid4
-from calibre.utils.webengine import secure_webengine, send_reply
+from calibre.utils.webengine import secure_webengine, send_reply, setup_profile
 from polyglot.builtins import as_bytes
 
 
@@ -143,6 +143,7 @@ class Page(QWebEnginePage):  # {{{
         self.current_frag = None
         self.com_id = str(uuid4())
         profile = QWebEngineProfile(QApplication.instance())
+        setup_profile(profile)
         # store these globally as they need to be destructed after the QWebEnginePage
         current_container.url_handler = UrlSchemeHandler(parent=profile)
         current_container.interceptor = RequestInterceptor(profile)

src/calibre/gui2/tweak_book/preview.py

@@ -37,7 +37,8 @@ from calibre.gui2.webengine import RestartingWebEngineView
 from calibre.gui2.widgets2 import HistoryLineEdit2
 from calibre.utils.ipc.simple_worker import offload_worker
 from calibre.utils.webengine import (
-    Bridge, create_script, from_js, insert_scripts, secure_webengine, to_js
+    Bridge, create_script, from_js, insert_scripts, secure_webengine, setup_profile,
+    to_js
 )
 from polyglot.builtins import iteritems
 from polyglot.queue import Empty, Queue
@@ -294,6 +295,7 @@ def create_profile():
     ans = getattr(create_profile, 'ans', None)
     if ans is None:
         ans = QWebEngineProfile(QApplication.instance())
+        setup_profile(ans)
         ua = 'calibre-editor-preview ' + __version__
         ans.setHttpUserAgent(ua)
         if is_running_from_develop:
@@ -391,6 +393,7 @@ class Inspector(QWidget):
     def visibility_changed(self, visible):
         if visible and self.view is None:
             self.view = QWebEngineView(self.view_to_debug)
+            setup_profile(self.view.page().profile())
             self.view_to_debug.page().setDevToolsPage(self.view.page())
             self.layout.addWidget(self.view)
 

src/calibre/gui2/viewer/lookup.py

@@ -2,24 +2,24 @@
 # License: GPL v3 Copyright: 2019, Kovid Goyal <kovid at kovidgoyal.net>
 
 
-import os
 import sys
 import textwrap
 from qt.core import (
-    QApplication, QCheckBox, QComboBox, QDialog, QDialogButtonBox, QFormLayout, QAbstractItemView,
+    QAbstractItemView, QApplication, QCheckBox, QComboBox, QDialog, QDialogButtonBox,
-    QHBoxLayout, QIcon, QLabel, QLineEdit, QListWidget, QListWidgetItem, QPushButton,
+    QFormLayout, QHBoxLayout, QIcon, QLabel, QLineEdit, QListWidget, QListWidgetItem,
-    QSize, Qt, QTimer, QUrl, QVBoxLayout, QWidget, pyqtSignal
+    QPushButton, QSize, Qt, QTimer, QUrl, QVBoxLayout, QWidget, pyqtSignal
 )
 from qt.webengine import (
     QWebEnginePage, QWebEngineProfile, QWebEngineScript, QWebEngineView
 )
 
 from calibre import prints, random_user_agent
-from calibre.constants import cache_dir
 from calibre.gui2 import error_dialog
 from calibre.gui2.viewer.web_view import apply_font_settings, vprefs
-from calibre.utils.webengine import create_script, insert_scripts, secure_webengine
 from calibre.gui2.widgets2 import Dialog
+from calibre.utils.webengine import (
+    create_script, insert_scripts, secure_webengine, setup_profile
+)
 
 vprefs.defaults['lookup_locations'] = [
     {
@@ -191,7 +191,7 @@ def create_profile():
     if ans is None:
         ans = QWebEngineProfile('viewer-lookup', QApplication.instance())
         ans.setHttpUserAgent(random_user_agent(allow_ie=False))
-        ans.setCachePath(os.path.join(cache_dir(), 'ev2vl'))
+        setup_profile(ans)
         js = P('lookup.js', data=True, allow_user_override=False)
         insert_scripts(ans, create_script('lookup.js', js, injection_point=QWebEngineScript.InjectionPoint.DocumentCreation))
         s = ans.settings()
@@ -305,6 +305,7 @@ class Lookup(QWidget):
             self._devtools_page = QWebEnginePage()
             self._devtools_view = QWebEngineView(self)
             self._devtools_view.setPage(self._devtools_page)
+            setup_profile(self._devtools_page.profile())
             self._page.setDevToolsPage(self._devtools_page)
             self._devtools_dialog = d = QDialog(self)
             d.setWindowTitle('Inspect Lookup page')

src/calibre/gui2/viewer/web_view.py

@@ -34,7 +34,7 @@ from calibre.utils.serialize import json_loads
 from calibre.utils.shared_file import share_open
 from calibre.utils.webengine import (
     Bridge, create_script, from_js, insert_scripts, secure_webengine, send_reply,
-    to_js
+    to_js, setup_profile
 )
 from polyglot.builtins import as_bytes, iteritems
 from polyglot.functools import lru_cache
@@ -183,7 +183,7 @@ class UrlSchemeHandler(QWebEngineUrlSchemeHandler):
 def create_profile():
     ans = getattr(create_profile, 'ans', None)
     if ans is None:
-        ans = QWebEngineProfile(QApplication.instance())
+        ans = setup_profile(QWebEngineProfile(QApplication.instance()))
         osname = 'windows' if iswindows else ('macos' if ismacos else 'linux')
         # DO NOT change the user agent as it is used to workaround
         # Qt bugs see workaround_qt_bug() in ajax.pyj
@@ -399,6 +399,7 @@ class Inspector(QWidget):
     def visibility_changed(self, visible):
         if visible and self.view is None:
             self.view = QWebEngineView(self.view_to_debug)
+            setup_profile(self.view.page().profile())
             self.view_to_debug.page().setDevToolsPage(self.view.page())
             self.layout.addWidget(self.view)
 

src/calibre/scraper/simple_backend.py

@@ -3,7 +3,6 @@
 # License: GPL v3 Copyright: 2022, Kovid Goyal <kovid at kovidgoyal.net>
 
 import json
-import os
 import secrets
 import sys
 import time
@@ -11,8 +10,7 @@ from functools import lru_cache
 from qt.core import QApplication, QEventLoop, QUrl
 from qt.webengine import QWebEnginePage, QWebEngineProfile, QWebEngineSettings
 
-from calibre.constants import cache_dir
+from calibre.utils.webengine import create_script, insert_scripts, setup_profile
-from calibre.utils.webengine import create_script, insert_scripts
 
 
 def canonicalize_qurl(qurl):
@@ -29,9 +27,9 @@ def create_profile(cache_name='', allow_js=False):
     from calibre.utils.random_ua import random_common_chrome_user_agent
     if cache_name:
         ans = QWebEngineProfile(cache_name, QApplication.instance())
-        ans.setCachePath(os.path.join(cache_dir(), 'scraper', cache_name))
     else:
         ans = QWebEngineProfile(QApplication.instance())
+    setup_profile(ans)
     ans.setHttpUserAgent(random_common_chrome_user_agent())
     ans.setHttpCacheMaximumSize(0)  # managed by webengine
     s = ans.settings()

src/calibre/test_build.py

@@ -320,6 +320,7 @@ class BuildTest(unittest.TestCase):
         test()
 
         from calibre.gui2 import ensure_app, destroy_app
+        from calibre.utils.webengine import setup_profile
         display_env_var = os.environ.pop('DISPLAY', None)
         try:
             ensure_app()
@@ -333,6 +334,7 @@ class BuildTest(unittest.TestCase):
             na = QNetworkAccessManager()
             self.assertTrue(hasattr(na, 'sslErrors'), 'Qt not compiled with openssl')
             p = QWebEnginePage()
+            setup_profile(p.profile())
 
             def callback(result):
                 callback.result = result

src/calibre/utils/rapydscript.py

@@ -59,7 +59,7 @@ def compiler():
 
     from calibre import walk
     from calibre.gui2 import must_use_qt
-    from calibre.utils.webengine import secure_webengine, setup_default_profile
+    from calibre.utils.webengine import secure_webengine, setup_default_profile, setup_profile
     must_use_qt()
     setup_default_profile()
 
@@ -126,6 +126,7 @@ document.title = 'compiler initialized';
 
         def __init__(self):
             super().__init__()
+            setup_profile(self.profile())
             self.errors = []
             secure_webengine(self)
             script = compiler_script
@@ -342,9 +343,10 @@ def run_rapydscript_tests():
     from calibre.gui2 import must_use_qt
     from calibre.gui2.viewer.web_view import send_reply
     from calibre.utils.webengine import (
-        create_script, insert_scripts, secure_webengine
+        create_script, insert_scripts, secure_webengine, setup_default_profile, setup_profile
     )
     must_use_qt()
+    setup_default_profile()
     scheme = QWebEngineUrlScheme(FAKE_PROTOCOL.encode('ascii'))
     scheme.setSyntax(QWebEngineUrlScheme.Syntax.Host)
     scheme.setFlags(QWebEngineUrlScheme.Flag.SecureScheme)
@@ -388,6 +390,7 @@ def run_rapydscript_tests():
         def __init__(self):
             profile = QWebEngineProfile(QApplication.instance())
             profile.setHttpUserAgent('calibre-tester')
+            setup_profile(profile)
             insert_scripts(profile, create_script('test-rapydscript.js', js, on_subframes=False))
             url_handler = UrlSchemeHandler(profile)
             profile.installUrlSchemeHandler(QByteArray(FAKE_PROTOCOL.encode('ascii')), url_handler)

src/calibre/utils/webengine.py

@@ -10,11 +10,20 @@ from qt.webengine import QWebEngineScript, QWebEngineSettings, QWebEngineProfile
 from calibre.constants import cache_dir, SPECIAL_TITLE_FOR_WEBENGINE_COMMS
 
 
+def setup_profile(profile):
+    # Qt uses persistent storage path to store cached GPU data even for OTR profiles
+    base = os.path.abspath(os.path.join(cache_dir(), 'qwe', profile.storageName() or 'dp'))
+    cp = os.path.join(base, 'c')
+    if profile.cachePath() != cp:
+        profile.setCachePath(cp)
+    sp = os.path.join(base, 'sp')
+    if profile.persistentStoragePath() != sp:
+        profile.setPersistentStoragePath(sp)
+    return profile
+
+
 def setup_default_profile():
-    p = QWebEngineProfile.defaultProfile()
+    return setup_profile(QWebEngineProfile.defaultProfile())
-    q = os.path.join(cache_dir(), 'qt-webeng')
-    if p.cachePath() != q:
-        p.setCachePath(q)
 
 
 def send_reply(rq, mime_type, data):