mirror of
https://github.com/searxng/searxng.git
synced 2025-11-10 16:46:43 -05:00
62b0b3f697
The time for the bootload is measured and recorded. To have an eye on the
bootload time is motivated by: In [1] it can be seen that the bootload of the
CURRENCIES cache takes about 2/3 of the time required to initialize SearXNG::
6.068 <module> searx/webapp.py:1
└─ 6.068 wrapper pyinstrument/context_manager.py:52
├─ 5.538 init searx/webapp.py:1373
│ ├─ 4.822 initialize searx/search/__init__.py:34
│ │ ├─ 4.631 ProcessorMap.init searx/search/processors/__init__.py:47
│ │ │ ├─ 4.607 OnlineCurrencyProcessor.initialize searx/search/processors/online_currency.py:55
│ │ │ │ └─ 4.607 CurrenciesDB.init searx/data/currencies.py:25
│ │ │ │ ├─ 4.601 CurrenciesDB.load searx/data/currencies.py:34
│ │ │ │ │ ├─ 4.572 ExpireCacheSQLite.set searx/cache.py:334
In the example, the CurrenciesDB.init call takes 4.6 seconds... on my laptop,
CurrenciesDB.init takes only 0.7 seconds. The absolute numerical values depend
on external conditions, where I already find 4-5 seconds very long. Test::
$ rm /tmp/sxng_cache_*.db*
$ make run 2>&1 | grep "searx.data.CURRENCIES"
DEBUG searx.data : init searx.data.CURRENCIES
DEBUG searx.data : init searx.data.CURRENCIES added 9089 items in 0.7623255252838135 sec.
[1] https://github.com/searxng/searxng/issues/5223#issuecomment-3323083411
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
68 lines
2.3 KiB
Python
68 lines
2.3 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
"""Simple implementation to store currencies data in a SQL database."""
|
|
|
|
__all__ = ["CurrenciesDB"]
|
|
|
|
import typing as t
|
|
import json
|
|
import pathlib
|
|
import time
|
|
|
|
from .core import get_cache, log
|
|
|
|
|
|
@t.final
|
|
class CurrenciesDB:
|
|
# pylint: disable=missing-class-docstring
|
|
|
|
ctx_names: str = "data_currencies_names"
|
|
ctx_iso4217: str = "data_currencies_iso4217"
|
|
|
|
json_file: pathlib.Path = pathlib.Path(__file__).parent / "currencies.json"
|
|
|
|
def __init__(self):
|
|
self.cache = get_cache()
|
|
|
|
def init(self):
|
|
if self.cache.properties("currencies loaded") != "OK":
|
|
# To avoid parallel initializations, the property is set first
|
|
self.cache.properties.set("currencies loaded", "OK")
|
|
self.load()
|
|
# F I X M E:
|
|
# do we need a maintenance .. rember: database is stored
|
|
# in /tmp and will be rebuild during the reboot anyway
|
|
|
|
def load(self):
|
|
_start = time.time()
|
|
log.debug("init searx.data.CURRENCIES")
|
|
with open(self.json_file, encoding="utf-8") as f:
|
|
data_dict: dict[str, dict[str, str]] = json.load(f)
|
|
for key, value in data_dict["names"].items():
|
|
self.cache.set(key=key, value=value, ctx=self.ctx_names, expire=None)
|
|
for key, value in data_dict["iso4217"].items():
|
|
self.cache.set(key=key, value=value, ctx=self.ctx_iso4217, expire=None)
|
|
log.debug(
|
|
"init searx.data.CURRENCIES added %s items in %s sec.",
|
|
len(data_dict["names"]) + len(data_dict["iso4217"]),
|
|
time.time() - _start,
|
|
)
|
|
|
|
def name_to_iso4217(self, name: str) -> str | None:
|
|
self.init()
|
|
|
|
ret_val: str | list[str] | None = self.cache.get(key=name, default=None, ctx=self.ctx_names)
|
|
if isinstance(ret_val, list):
|
|
# if more alternatives, use the last in the list
|
|
ret_val = ret_val[-1]
|
|
return ret_val
|
|
|
|
def iso4217_to_name(self, iso4217: str, language: str) -> str | None:
|
|
self.init()
|
|
|
|
iso4217_languages: dict[str, str] = self.cache.get(key=iso4217, default={}, ctx=self.ctx_iso4217)
|
|
return iso4217_languages.get(language)
|
|
|
|
def is_iso4217(self, iso4217: str) -> bool:
|
|
item = self.cache.get(key=iso4217, default={}, ctx=self.ctx_iso4217)
|
|
return bool(item)
|