git.iko.soy/searx
on master help sign in
[mod] checker : replace pycld3 by langdetect
aa887ebAlexandre Flament
requirements.txt+1−1
compare with master
unifiedsplit
@@ -9,4 +9,4 @@ pygments==2.1.3
9python-dateutil==2.8.1
9python-dateutil==2.8.1
10pyyaml==5.3.1
10pyyaml==5.3.1
11requests[socks]==2.25.1
11requests[socks]==2.25.1
12−pycld3==0.20
12+langdetect==1.0.8
searx/search/checker/impl.py+10−5
@@ -9,7 +9,8 @@ from time import time
9from urllib.parse import urlparse
9from urllib.parse import urlparse
10
10
11import re
11import re
12−import cld3
12+from langdetect import detect_langs
13+from langdetect.lang_detect_exception import LangDetectException
13import requests.exceptions
14import requests.exceptions
14
15
15from searx import poolrequests, logger
16from searx import poolrequests, logger
@@ -181,10 +182,14 @@ class ResultContainerTests:
181 self.test_results.add_error(self.test_name, message, *args, '(' + sqstr + ')')
182 self.test_results.add_error(self.test_name, message, *args, '(' + sqstr + ')')
182
183
183 def _add_language(self, text: str) -> typing.Optional[str]:
184 def _add_language(self, text: str) -> typing.Optional[str]:
185+ try:
184− r = cld3.get_language(str(text)) # pylint: disable=E1101
186+ r = detect_langs(str(text)) # pylint: disable=E1101
185− if r is not None and r.probability >= 0.98 and r.is_reliable:
187+ except LangDetectException:
188+ return None
189+
190+ if len(r) > 0 and r[0].prob > 0.95:
186− self.languages.add(r.language)
191+ self.languages.add(r[0].lang)
187− self.test_results.add_language(r.language)
192+ self.test_results.add_language(r[0].lang)
188 return None
193 return None
189
194
190 def _check_result(self, result):
195 def _check_result(self, result):
utils/searx.sh+2−4
@@ -46,7 +46,6 @@ SEARX_PACKAGES_debian="\
46python3-dev python3-babel python3-venv
46python3-dev python3-babel python3-venv
47uwsgi uwsgi-plugin-python3
47uwsgi uwsgi-plugin-python3
48git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev
48git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev
49−libprotobuf-dev protobuf-compiler
50shellcheck"
49shellcheck"
51
50
52BUILD_PACKAGES_debian="\
51BUILD_PACKAGES_debian="\
@@ -59,7 +58,6 @@ SEARX_PACKAGES_arch="\
59python python-pip python-lxml python-babel
58python python-pip python-lxml python-babel
60uwsgi uwsgi-plugin-python
59uwsgi uwsgi-plugin-python
61git base-devel libxml2
60git base-devel libxml2
62−protobuf
63shellcheck"
61shellcheck"
64
62
65BUILD_PACKAGES_arch="\
63BUILD_PACKAGES_arch="\
@@ -71,7 +69,7 @@ SEARX_PACKAGES_fedora="\
71python python-pip python-lxml python-babel
69python python-pip python-lxml python-babel
72uwsgi uwsgi-plugin-python3
70uwsgi uwsgi-plugin-python3
73git @development-tools libxml2
71git @development-tools libxml2
74−ShellCheck protobuf-compiler protobuf-devel"
72+ShellCheck"
75
73
76BUILD_PACKAGES_fedora="\
74BUILD_PACKAGES_fedora="\
77firefox graphviz graphviz-gd ImageMagick librsvg2-tools
75firefox graphviz graphviz-gd ImageMagick librsvg2-tools
@@ -84,7 +82,7 @@ SEARX_PACKAGES_centos="\
84python36 python36-pip python36-lxml python-babel
82python36 python36-pip python36-lxml python-babel
85uwsgi uwsgi-plugin-python3
83uwsgi uwsgi-plugin-python3
86git @development-tools libxml2
84git @development-tools libxml2
87−ShellCheck protobuf-compiler protobuf-devel"
85+ShellCheck"
88
86
89BUILD_PACKAGES_centos="\
87BUILD_PACKAGES_centos="\
90firefox graphviz graphviz-gd ImageMagick librsvg2-tools
88firefox graphviz graphviz-gd ImageMagick librsvg2-tools