diff --git a/docs/admin/architecture.rst b/docs/admin/architecture.rst index d0d40715d..b310ebba0 100644 --- a/docs/admin/architecture.rst +++ b/docs/admin/architecture.rst @@ -29,8 +29,8 @@ up and maintained by the scripts from our :ref:`toolboxing`. Reference architecture of a public SearXNG setup. -The reference installation activates ``server.limiter``, ``server.image_proxy`` -and ``ui.static_use_hash`` (:origin:`/etc/searxng/settings.yml +The reference installation activates ``server.limiter`` and +``server.image_proxy`` (:origin:`/etc/searxng/settings.yml `) .. literalinclude:: ../../utils/templates/etc/searxng/settings.yml diff --git a/docs/admin/installation-searxng.rst b/docs/admin/installation-searxng.rst index 73797456a..0db119245 100644 --- a/docs/admin/installation-searxng.rst +++ b/docs/admin/installation-searxng.rst @@ -86,7 +86,6 @@ below. This setup: - enables :ref:`limiter ` to protect against bots - enables :ref:`image proxy ` for better privacy -- enables :ref:`cache busting ` to save bandwidth Modify the ``/etc/searxng/settings.yml`` to your needs: @@ -129,4 +128,3 @@ configuration file. If everything works fine, hit ``[CTRL-C]`` to stop the *webapp* and disable the debug option in ``settings.yml``. You can now exit SearXNG user bash session (enter exit command twice). At this point SearXNG is not demonized; uwsgi allows this. - diff --git a/docs/admin/installation-uwsgi.rst b/docs/admin/installation-uwsgi.rst index a2152409e..78a2746f5 100644 --- a/docs/admin/installation-uwsgi.rst +++ b/docs/admin/installation-uwsgi.rst @@ -181,10 +181,7 @@ uWSGI setup Create the configuration ini-file according to your distribution and restart the uwsgi application. As shown below, the :ref:`installation scripts` installs by -default: - -- a uWSGI setup that listens on a socket and -- enables :ref:`cache busting `. +default a uWSGI setup that listens on a socket. .. tabs:: diff --git a/docs/admin/settings/settings_ui.rst b/docs/admin/settings/settings_ui.rst index 0e0235594..222e0f2c5 100644 --- a/docs/admin/settings/settings_ui.rst +++ b/docs/admin/settings/settings_ui.rst @@ -10,7 +10,6 @@ .. code:: yaml ui: - static_use_hash: false default_locale: "" query_in_title: false infinite_scroll: false @@ -23,11 +22,6 @@ hotkeys: default url_formatting: pretty -.. _static_use_hash: - -``static_use_hash`` : ``$SEARXNG_STATIC_USE_HASH`` - Enables `cache busting`_ of static files. - ``default_locale`` : SearXNG interface language. If blank, the locale is detected by using the browser language. If it doesn't work, or you are deploying a language diff --git a/docs/admin/update-searxng.rst b/docs/admin/update-searxng.rst index d4a197603..69638225e 100644 --- a/docs/admin/update-searxng.rst +++ b/docs/admin/update-searxng.rst @@ -58,10 +58,6 @@ and then, to name just a few: - Bot protection has been switched from filtron to SearXNG's :ref:`limiter `, this requires a :ref:`Valkey ` database. -- To save bandwidth :ref:`cache busting ` has been implemented. - To get in use, the ``static-expires`` needs to be set in the :ref:`uwsgi - setup`. - To stay tuned and get in use of the new features, instance maintainers have to update the SearXNG code regularly (see :ref:`update searxng`). As the above examples show, this is not always enough, sometimes services have to be set up diff --git a/searx/settings.yml b/searx/settings.yml index fb85f0ff4..1b6739927 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -119,8 +119,6 @@ valkey: ui: # Custom static path - leave it blank if you didn't change static_path: "" - # Is overwritten by ${SEARXNG_STATIC_USE_HASH}. - static_use_hash: false # Custom templates path - leave it blank if you didn't change templates_path: "" # query_in_title: When true, the result page's titles contains the query diff --git a/searx/settings_defaults.py b/searx/settings_defaults.py index 7e785e4d2..5ec5ae0a6 100644 --- a/searx/settings_defaults.py +++ b/searx/settings_defaults.py @@ -194,7 +194,6 @@ SCHEMA = { }, 'ui': { 'static_path': SettingsDirectoryValue(str, os.path.join(searx_dir, 'static')), - 'static_use_hash': SettingsValue(bool, False, 'SEARXNG_STATIC_USE_HASH'), 'templates_path': SettingsDirectoryValue(str, os.path.join(searx_dir, 'templates')), 'default_theme': SettingsValue(str, 'simple'), 'default_locale': SettingsValue(str, ''), diff --git a/searx/webapp.py b/searx/webapp.py index 15f79f151..cf7c58680 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -76,7 +76,6 @@ from searx.engines import ( from searx import webutils from searx.webutils import ( highlight_content, - get_static_files, get_result_templates, get_themes, exception_classname_to_text, @@ -131,7 +130,6 @@ warnings.simplefilter("always") # about static logger.debug('static directory is %s', settings['ui']['static_path']) -static_files = get_static_files(settings['ui']['static_path']) # about templates logger.debug('templates directory is %s', settings['ui']['templates_path']) @@ -239,25 +237,44 @@ def get_result_template(theme_name: str, template_name: str): return 'result_templates/' + template_name +_STATIC_FILES: list[str] = [] + + def custom_url_for(endpoint: str, **values): - suffix = "" - if endpoint == 'static' and values.get('filename'): - file_hash = static_files.get(values['filename']) - if not file_hash: + global _STATIC_FILES # pylint: disable=global-statement + if not _STATIC_FILES: + _STATIC_FILES = webutils.get_static_file_list() + + if endpoint == "static" and values.get("filename"): + + # We need to verify the "filename" argument: in the jinja templates + # there could be call like: + # url_for('static', filename='img/favicon.png') + # which should map to: + # static/themes//img/favicon.png + + arg_filename = values["filename"] + if arg_filename not in _STATIC_FILES: # try file in the current theme - theme_name = sxng_request.preferences.get_value('theme') - filename_with_theme = "themes/{}/{}".format(theme_name, values['filename']) - file_hash = static_files.get(filename_with_theme) - if file_hash: - values['filename'] = filename_with_theme - if get_setting('ui.static_use_hash') and file_hash: - suffix = "?" + file_hash - if endpoint == 'info' and 'locale' not in values: - locale = sxng_request.preferences.get_value('locale') - if infopage.INFO_PAGES.get_page(values['pagename'], locale) is None: + theme_name = sxng_request.preferences.get_value("theme") + arg_filename = f"themes/{theme_name}/{arg_filename}" + if arg_filename in _STATIC_FILES: + values["filename"] = arg_filename + + if endpoint == "info" and "locale" not in values: + + # We need to verify the "locale" argument: in the jinja templates there + # could be call like: + # url_for('info', pagename='about') + # which should map to: + # info//about + + locale = sxng_request.preferences.get_value("locale") + if infopage.INFO_PAGES.get_page(values["pagename"], locale) is None: locale = infopage.INFO_PAGES.locale_default - values['locale'] = locale - return url_for(endpoint, **values) + suffix + values["locale"] = locale + + return url_for(endpoint, **values) def image_proxify(url: str): diff --git a/searx/webutils.py b/searx/webutils.py index 2a2da3797..d32038482 100644 --- a/searx/webutils.py +++ b/searx/webutils.py @@ -12,14 +12,15 @@ import re import itertools import json from datetime import datetime, timedelta -from typing import Iterable, List, Tuple, Dict, TYPE_CHECKING +from typing import Iterable, List, Tuple, TYPE_CHECKING from io import StringIO from codecs import getincrementalencoder from flask_babel import gettext, format_date # type: ignore -from searx import logger, settings +from searx import logger, get_setting + from searx.engines import DEFAULT_CATEGORY if TYPE_CHECKING: @@ -177,30 +178,22 @@ def get_themes(templates_path): return os.listdir(templates_path) -def get_hash_for_file(file: pathlib.Path) -> str: - m = hashlib.sha1() - with file.open('rb') as f: - m.update(f.read()) - return m.hexdigest() +def get_static_file_list() -> list[str]: + file_list = [] + static_path = pathlib.Path(str(get_setting("ui.static_path"))) - -def get_static_files(static_path: str) -> Dict[str, str]: - static_files: Dict[str, str] = {} - static_path_path = pathlib.Path(static_path) - - def walk(path: pathlib.Path): - for file in path.iterdir(): - if file.name.startswith('.'): + def _walk(path: pathlib.Path): + for f in path.iterdir(): + if f.name.startswith('.'): # ignore hidden file continue - if file.is_file(): - static_files[str(file.relative_to(static_path_path))] = get_hash_for_file(file) - if file.is_dir() and file.name not in ('node_modules', 'src'): - # ignore "src" and "node_modules" directories - walk(file) + if f.is_file(): + file_list.append(str(f.relative_to(static_path))) + if f.is_dir(): + _walk(f) - walk(static_path_path) - return static_files + _walk(static_path) + return file_list def get_result_templates(templates_path): @@ -331,7 +324,7 @@ def group_engines_in_tab(engines: Iterable[Engine]) -> List[Tuple[str, Iterable[ def engine_sort_key(engine): return (engine.about.get('language', ''), engine.name) - tabs = list(settings['categories_as_tabs'].keys()) + tabs = list(get_setting('categories_as_tabs').keys()) subgroups = itertools.groupby(sorted(engines, key=get_subgroup), get_subgroup) sorted_groups = sorted(((name, list(engines)) for name, engines in subgroups), key=group_sort_key) diff --git a/tests/unit/test_webapp.py b/tests/unit/test_webapp.py index 35a41302c..91c6309fc 100644 --- a/tests/unit/test_webapp.py +++ b/tests/unit/test_webapp.py @@ -25,9 +25,6 @@ class ViewsTestCase(SearxTestCase): # pylint: disable=too-many-public-methods pass self.setattr4test(searx.search.processors, 'initialize_processor', dummy) - # remove sha for the static file so the tests don't have to care about - # the changing URLs - self.setattr4test(searx.webapp, 'static_files', {}) # set some defaults test_results = [ diff --git a/utils/templates/etc/httpd/sites-available/searxng.conf b/utils/templates/etc/httpd/sites-available/searxng.conf index 5278640c3..004066200 100644 --- a/utils/templates/etc/httpd/sites-available/searxng.conf +++ b/utils/templates/etc/httpd/sites-available/searxng.conf @@ -33,9 +33,6 @@ LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so -# uWSGI serves the static files and in settings.yml we use:: -# -# ui: -# static_use_hash: true +# To serve the static files via the HTTP server # # Alias ${SEARXNG_URL_PATH}/static/ ${SEARXNG_STATIC}/ diff --git a/utils/templates/etc/httpd/sites-available/searxng.conf:socket b/utils/templates/etc/httpd/sites-available/searxng.conf:socket index b55ea7560..81bd7e4df 100644 --- a/utils/templates/etc/httpd/sites-available/searxng.conf:socket +++ b/utils/templates/etc/httpd/sites-available/searxng.conf:socket @@ -33,9 +33,6 @@ LoadModule proxy_uwsgi_module ${APACHE_MODULES}/mod_proxy_uwsgi.so -# uWSGI serves the static files and in settings.yml we use:: -# -# ui: -# static_use_hash: true +# To serve the static files via the HTTP server # # Alias ${SEARXNG_URL_PATH}/static/ ${SEARXNG_STATIC}/ diff --git a/utils/templates/etc/nginx/default.apps-available/searxng.conf b/utils/templates/etc/nginx/default.apps-available/searxng.conf index 7225a8f96..27a8c9bcc 100644 --- a/utils/templates/etc/nginx/default.apps-available/searxng.conf +++ b/utils/templates/etc/nginx/default.apps-available/searxng.conf @@ -19,10 +19,7 @@ location ${SEARXNG_URL_PATH} { } -# uWSGI serves the static files and in settings.yml we use:: -# -# ui: -# static_use_hash: true +# To serve the static files via the HTTP server # # location ${SEARXNG_URL_PATH}/static/ { # alias ${SEARXNG_STATIC}/; diff --git a/utils/templates/etc/nginx/default.apps-available/searxng.conf:socket b/utils/templates/etc/nginx/default.apps-available/searxng.conf:socket index 7a74eab48..9ff5658c4 100644 --- a/utils/templates/etc/nginx/default.apps-available/searxng.conf:socket +++ b/utils/templates/etc/nginx/default.apps-available/searxng.conf:socket @@ -16,10 +16,7 @@ location ${SEARXNG_URL_PATH} { uwsgi_param HTTP_X_FORWARDED_FOR \$proxy_add_x_forwarded_for; } -# uWSGI serves the static files and in settings.yml we use:: -# -# ui: -# static_use_hash: true +# To serve the static files via the HTTP server # # location ${SEARXNG_URL_PATH}/static/ { # alias ${SEARXNG_STATIC}/; diff --git a/utils/templates/etc/searxng/settings.yml b/utils/templates/etc/searxng/settings.yml index b5da38ebd..5e29825bf 100644 --- a/utils/templates/etc/searxng/settings.yml +++ b/utils/templates/etc/searxng/settings.yml @@ -25,9 +25,6 @@ valkey: # URL to connect valkey database. Is overwritten by ${SEARXNG_VALKEY_URL}. url: valkey://localhost:6379/0 -ui: - static_use_hash: true - # preferences: # lock: # - autocomplete diff --git a/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini b/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini index 12543cf5b..3388e432f 100644 --- a/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini +++ b/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini @@ -75,11 +75,7 @@ pythonpath = ${SEARXNG_SRC} http = ${SEARXNG_INTERNAL_HTTP} buffer-size = 8192 -# uWSGI serves the static files and in settings.yml we use:: -# -# ui: -# static_use_hash: true -# +# To serve the static files via the WSGI server static-map = /static=${SEARXNG_STATIC} static-gzip-all = True offload-threads = %k diff --git a/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini:socket b/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini:socket index 0e6a7529d..4a55f0ab6 100644 --- a/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini:socket +++ b/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini:socket @@ -72,11 +72,7 @@ pythonpath = ${SEARXNG_SRC} socket = ${SEARXNG_UWSGI_SOCKET} buffer-size = 8192 -# uWSGI serves the static files and in settings.yml we use:: -# -# ui: -# static_use_hash: true -# +# To serve the static files via the WSGI server static-map = /static=${SEARXNG_STATIC} static-gzip-all = True offload-threads = %k diff --git a/utils/templates/etc/uwsgi/apps-available/searxng.ini b/utils/templates/etc/uwsgi/apps-available/searxng.ini index 9887d7909..e47c74abc 100644 --- a/utils/templates/etc/uwsgi/apps-available/searxng.ini +++ b/utils/templates/etc/uwsgi/apps-available/searxng.ini @@ -78,11 +78,7 @@ pythonpath = ${SEARXNG_SRC} http = ${SEARXNG_INTERNAL_HTTP} buffer-size = 8192 -# uWSGI serves the static files and in settings.yml we use:: -# -# ui: -# static_use_hash: true -# +# To serve the static files via the WSGI server static-map = /static=${SEARXNG_STATIC} static-gzip-all = True offload-threads = %k diff --git a/utils/templates/etc/uwsgi/apps-available/searxng.ini:socket b/utils/templates/etc/uwsgi/apps-available/searxng.ini:socket index 9e2cfc273..fb96973ba 100644 --- a/utils/templates/etc/uwsgi/apps-available/searxng.ini:socket +++ b/utils/templates/etc/uwsgi/apps-available/searxng.ini:socket @@ -75,11 +75,7 @@ pythonpath = ${SEARXNG_SRC} socket = ${SEARXNG_UWSGI_SOCKET} buffer-size = 8192 -# uWSGI serves the static files and in settings.yml we use:: -# -# ui: -# static_use_hash: true -# +# To serve the static files via the WSGI server static-map = /static=${SEARXNG_STATIC} static-gzip-all = True offload-threads = %k