on master help sign in
Merge pull request #1642 from dalf/start-speed

Speed up start time and decrease CPU load

commitd0dd296424d306049582a5a1b2fa01cbe7d7ce20 authorAlexandre Flament <alex@al-f.net> dateTue 23 Jul 2019, 13:18 (7 years ago) parent554a21eda0ce58
2 files changed+19−8
searx/engines/soundcloud.py+4−2 searx/utils.py+15−6
searx/engines/soundcloud.py+4−2
@@ -28,8 +28,10 @@ categories = ['music']
2828paging = True
2929
3030# search-url
31−url = 'https://api.soundcloud.com/'
31+# missing attribute: user_id, app_version, app_locale
32+url = 'https://api-v2.soundcloud.com/'
3233search_url = url + 'search?{query}'\
34+ '&variant_ids='\
3335 '&facet=model'\
3436 '&limit=20'\
3537 '&offset={offset}'\
@@ -57,7 +59,7 @@ def get_client_id():
5759 # gets app_js and searches for the clientid
5860 response = http_get(app_js_url)
5961 if response.ok:
60− cids = cid_re.search(response.text)
62+ cids = cid_re.search(response.content.decode("utf-8"))
6163 if cids is not None and len(cids.groups()):
6264 return cids.groups()[0]
6365 logger.warning("Unable to fetch guest client_id from SoundCloud, check parser!")
searx/utils.py+15−6
@@ -47,6 +47,8 @@ blocked_tags = ('script',
4747useragents = json.loads(open(os.path.dirname(os.path.realpath(__file__))
4848 + "/data/useragents.json", 'r', encoding='utf-8').read())
4949
50+lang_to_lc_cache = dict()
51+
5052
5153def searx_useragent():
5254 return 'searx/{searx_version} {suffix}'.format(
@@ -183,7 +185,7 @@ def get_resources_directory(searx_directory, subdirectory, resources_directory):
183185 if not resources_directory:
184186 resources_directory = os.path.join(searx_directory, subdirectory)
185187 if not os.path.isdir(resources_directory):
186− raise Exception(directory + " is not a directory")
188+ raise Exception(resources_directory + " is not a directory")
187189 return resources_directory
188190
189191
@@ -314,6 +316,17 @@ def is_valid_lang(lang):
314316 return False
315317
316318
319+def _get_lang_to_lc_dict(lang_list):
320+ key = str(lang_list)
321+ value = lang_to_lc_cache.get(key, None)
322+ if value is None:
323+ value = dict()
324+ for lc in lang_list:
325+ value.setdefault(lc.split('-')[0], lc)
326+ lang_to_lc_cache[key] = value
327+ return value
328+
329+
317330# auxiliary function to match lang_code in lang_list
318331def _match_language(lang_code, lang_list=[], custom_aliases={}):
319332 # replace language code with a custom alias if necessary
@@ -334,11 +347,7 @@ def _match_language(lang_code, lang_list=[], custom_aliases={}):
334347 return new_code
335348
336349 # try to get the any supported country for this language
337− for lc in lang_list:
338− if lang_code == lc.split('-')[0]:
339− return lc
340−
341− return None
350+ return _get_lang_to_lc_dict(lang_list).get(lang_code, None)
342351
343352
344353# get the language code from lang_list that best matches locale_code