Cutlery/searxng
1
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2025-11-04 03:27:06 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1642 from dalf/start-speed

Browse Source 
Speed up start time and decrease CPU load
This commit is contained in:
Alexandre Flament 2019-07-23 13:18:29 +02:00 committed by GitHub
commit d0dd296424
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 19 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
searx/engines/soundcloud.py
View File

@ -28,8 +28,10 @@ categories = ['music']
paging = True paging = True
# search-url # search-url
url = 'https://api.soundcloud.com/' # missing attribute: user_id, app_version, app_locale
url = 'https://api-v2.soundcloud.com/'
search_url = url + 'search?{query}'\ search_url = url + 'search?{query}'\
'&variant_ids='\
'&facet=model'\ '&facet=model'\
'&limit=20'\ '&limit=20'\
'&offset={offset}'\ '&offset={offset}'\
@ -57,7 +59,7 @@ def get_client_id():
# gets app_js and searches for the clientid # gets app_js and searches for the clientid
response = http_get(app_js_url) response = http_get(app_js_url)
if response.ok: if response.ok:
cids = cid_re.search(response.text) cids = cid_re.search(response.content.decode("utf-8"))
if cids is not None and len(cids.groups()): if cids is not None and len(cids.groups()):
return cids.groups()[0] return cids.groups()[0]
logger.warning("Unable to fetch guest client_id from SoundCloud, check parser!") logger.warning("Unable to fetch guest client_id from SoundCloud, check parser!")

21
searx/utils.py
View File

@ -47,6 +47,8 @@ blocked_tags = ('script',
useragents = json.loads(open(os.path.dirname(os.path.realpath(__file__)) useragents = json.loads(open(os.path.dirname(os.path.realpath(__file__))
+ "/data/useragents.json", 'r', encoding='utf-8').read()) + "/data/useragents.json", 'r', encoding='utf-8').read())
lang_to_lc_cache = dict()
def searx_useragent(): def searx_useragent():
return 'searx/{searx_version} {suffix}'.format( return 'searx/{searx_version} {suffix}'.format(
@ -183,7 +185,7 @@ def get_resources_directory(searx_directory, subdirectory, resources_directory):
if not resources_directory: if not resources_directory:
resources_directory = os.path.join(searx_directory, subdirectory) resources_directory = os.path.join(searx_directory, subdirectory)
if not os.path.isdir(resources_directory): if not os.path.isdir(resources_directory):
raise Exception(directory + " is not a directory") raise Exception(resources_directory + " is not a directory")
return resources_directory return resources_directory
@ -314,6 +316,17 @@ def is_valid_lang(lang):
return False return False
def _get_lang_to_lc_dict(lang_list):
key = str(lang_list)
value = lang_to_lc_cache.get(key, None)
if value is None:
value = dict()
for lc in lang_list:
value.setdefault(lc.split('-')[0], lc)
lang_to_lc_cache[key] = value
return value
# auxiliary function to match lang_code in lang_list # auxiliary function to match lang_code in lang_list
def _match_language(lang_code, lang_list=[], custom_aliases={}): def _match_language(lang_code, lang_list=[], custom_aliases={}):
# replace language code with a custom alias if necessary # replace language code with a custom alias if necessary
@ -334,11 +347,7 @@ def _match_language(lang_code, lang_list=[], custom_aliases={}):
return new_code return new_code
# try to get the any supported country for this language # try to get the any supported country for this language
for lc in lang_list: return _get_lang_to_lc_dict(lang_list).get(lang_code, None)
if lang_code == lc.split('-')[0]:
return lc
return None
# get the language code from lang_list that best matches locale_code # get the language code from lang_list that best matches locale_code