git.iko.soy/searx
on master help sign in
Merge pull request #2481 from dalf/mod-check

Mod check

commitf310305c54c3cd1d9fc74f09453294edbd2b5486 authorAdam Tauber <asciimoo@gmail.com> dateWed 20 Jan 2021, 18:48 (6 years ago) parent0495e1573c86f9
6 files changed+26−16
compare with master
unifiedsplit
requirements.txt+1−1 searx/search/checker/__main__.py+1 searx/search/checker/impl.py+10−5 searx/search/processors/online.py+3−3 searx/settings.yml+9−3 utils/searx.sh+2−4
requirements.txt+1−1
@@ -9,4 +9,4 @@ pygments==2.1.3
99python-dateutil==2.8.1
1010pyyaml==5.3.1
1111requests[socks]==2.25.1
12−pycld3==0.20
12+langdetect==1.0.8
searx/search/checker/__main__.py+1
@@ -74,6 +74,7 @@ def run(engine_name_list, verbose):
7474 stdout.write(f' {"found languages":15}: {" ".join(sorted(list(checker.test_results.languages)))}\n')
7575 for test_name, logs in checker.test_results.logs.items():
7676 for log in logs:
77+ log = map(lambda l: l if isinstance(l, str) else repr(l), log)
7778 stdout.write(f' {test_name:15}: {RED}{" ".join(log)}{RESET_SEQ}\n')
7879
7980
searx/search/checker/impl.py+10−5
@@ -9,7 +9,8 @@ from time import time
99from urllib.parse import urlparse
1010
1111import re
12−import cld3
12+from langdetect import detect_langs
13+from langdetect.lang_detect_exception import LangDetectException
1314import requests.exceptions
1415
1516from searx import poolrequests, logger
@@ -181,10 +182,14 @@ class ResultContainerTests:
181182 self.test_results.add_error(self.test_name, message, *args, '(' + sqstr + ')')
182183
183184 def _add_language(self, text: str) -> typing.Optional[str]:
184− r = cld3.get_language(str(text)) # pylint: disable=E1101
185− if r is not None and r.probability >= 0.98 and r.is_reliable:
186− self.languages.add(r.language)
187− self.test_results.add_language(r.language)
185+ try:
186+ r = detect_langs(str(text)) # pylint: disable=E1101
187+ except LangDetectException:
188+ return None
189+
190+ if len(r) > 0 and r[0].prob > 0.95:
191+ self.languages.add(r[0].lang)
192+ self.test_results.add_language(r[0].lang)
188193 return None
189194
190195 def _check_result(self, result):
searx/search/processors/online.py+3−3
@@ -239,14 +239,14 @@ class OnlineProcessor(EngineProcessor):
239239 'test': ['unique_results']
240240 }
241241
242− if getattr(self.engine, 'lang', False):
242+ if getattr(self.engine, 'supported_languages', []):
243243 tests['lang_fr'] = {
244244 'matrix': {'query': 'paris', 'lang': 'fr'},
245− 'result_container': ['not_empty', ('has_lang', 'fr')],
245+ 'result_container': ['not_empty', ('has_language', 'fr')],
246246 }
247247 tests['lang_en'] = {
248248 'matrix': {'query': 'paris', 'lang': 'en'},
249− 'result_container': ['not_empty', ('has_lang', 'en')],
249+ 'result_container': ['not_empty', ('has_language', 'en')],
250250 }
251251
252252 if getattr(self.engine, 'safesearch', False):
searx/settings.yml+9−3
@@ -105,11 +105,17 @@ outgoing: # communication with search engines
105105checker:
106106 # disable checker when in debug mode
107107 off_when_debug: True
108+
108109 # scheduling: interval or int
109110 # use "scheduling: False" to disable scheduling
110− scheduling:
111− start_after: [300, 1800] # delay to start the first run of the checker
112− every: [86400, 90000] # how often the checker runs
111+ # to activate the scheduler:
112+ # * uncomment "scheduling" section
113+ # * add "cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1" to your uwsgi.ini
114+
115+ # scheduling:
116+ # start_after: [300, 1800] # delay to start the first run of the checker
117+ # every: [86400, 90000] # how often the checker runs
118+
113119 # additional tests: only for the YAML anchors (see the engines section)
114120 additional_tests:
115121 rosebud: &test_rosebud
utils/searx.sh+2−4
@@ -46,7 +46,6 @@ SEARX_PACKAGES_debian="\
4646python3-dev python3-babel python3-venv
4747uwsgi uwsgi-plugin-python3
4848git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev
49−libprotobuf-dev protobuf-compiler
5049shellcheck"
5150
5251BUILD_PACKAGES_debian="\
@@ -59,7 +58,6 @@ SEARX_PACKAGES_arch="\
5958python python-pip python-lxml python-babel
6059uwsgi uwsgi-plugin-python
6160git base-devel libxml2
62−protobuf
6361shellcheck"
6462
6563BUILD_PACKAGES_arch="\
@@ -71,7 +69,7 @@ SEARX_PACKAGES_fedora="\
7169python python-pip python-lxml python-babel
7270uwsgi uwsgi-plugin-python3
7371git @development-tools libxml2
74−ShellCheck protobuf-compiler protobuf-devel"
72+ShellCheck"
7573
7674BUILD_PACKAGES_fedora="\
7775firefox graphviz graphviz-gd ImageMagick librsvg2-tools
@@ -84,7 +82,7 @@ SEARX_PACKAGES_centos="\
8482python36 python36-pip python36-lxml python-babel
8583uwsgi uwsgi-plugin-python3
8684git @development-tools libxml2
87−ShellCheck protobuf-compiler protobuf-devel"
85+ShellCheck"
8886
8987BUILD_PACKAGES_centos="\
9088firefox graphviz graphviz-gd ImageMagick librsvg2-tools