f310305 · searx · git.iko.soy

requirements.txt+1−1

compare with master

unifiedsplit

@@ -9,4 +9,4 @@ pygments==2.1.3

9python-dateutil==2.8.1

10pyyaml==5.3.1

11requests[socks]==2.25.1

12−pycld3==0.20

12+langdetect==1.0.8

searx/search/checker/__main__.py+1

@@ -74,6 +74,7 @@ def run(engine_name_list, verbose):

74 stdout.write(f' {"found languages":15}: {" ".join(sorted(list(checker.test_results.languages)))}\n')

75 for test_name, logs in checker.test_results.logs.items():

76 for log in logs:

77+ log = map(lambda l: l if isinstance(l, str) else repr(l), log)

77 stdout.write(f' {test_name:15}: {RED}{" ".join(log)}{RESET_SEQ}\n')

78 stdout.write(f' {test_name:15}: {RED}{" ".join(log)}{RESET_SEQ}\n')

78

79

80

searx/search/checker/impl.py+10−5

@@ -9,7 +9,8 @@ from time import time

9from urllib.parse import urlparse

10

11import re

12−import cld3

12+from langdetect import detect_langs

13+from langdetect.lang_detect_exception import LangDetectException

13import requests.exceptions

14import requests.exceptions

14

15

15from searx import poolrequests, logger

16from searx import poolrequests, logger

@@ -181,10 +182,14 @@ class ResultContainerTests:

181 self.test_results.add_error(self.test_name, message, *args, '(' + sqstr + ')')

182 self.test_results.add_error(self.test_name, message, *args, '(' + sqstr + ')')

182

183

183 def _add_language(self, text: str) -> typing.Optional[str]:

184 def _add_language(self, text: str) -> typing.Optional[str]:

185+ try:

184− r = cld3.get_language(str(text)) # pylint: disable=E1101

186+ r = detect_langs(str(text)) # pylint: disable=E1101

185− if r is not None and r.probability >= 0.98 and r.is_reliable:

187+ except LangDetectException:

188+ return None

189+

190+ if len(r) > 0 and r[0].prob > 0.95:

186− self.languages.add(r.language)

191+ self.languages.add(r[0].lang)

187− self.test_results.add_language(r.language)

192+ self.test_results.add_language(r[0].lang)

188 return None

193 return None

189

194

190 def _check_result(self, result):

195 def _check_result(self, result):

searx/search/processors/online.py+3−3

@@ -239,14 +239,14 @@ class OnlineProcessor(EngineProcessor):

239 'test': ['unique_results']

240 }

241

242− if getattr(self.engine, 'lang', False):

242+ if getattr(self.engine, 'supported_languages', []):

243 tests['lang_fr'] = {

244 'matrix': {'query': 'paris', 'lang': 'fr'},

245− 'result_container': ['not_empty', ('has_lang', 'fr')],

245+ 'result_container': ['not_empty', ('has_language', 'fr')],

246 }

247 tests['lang_en'] = {

248 'matrix': {'query': 'paris', 'lang': 'en'},

249− 'result_container': ['not_empty', ('has_lang', 'en')],

249+ 'result_container': ['not_empty', ('has_language', 'en')],

250 }

251

252 if getattr(self.engine, 'safesearch', False):

searx/settings.yml+9−3

@@ -105,11 +105,17 @@ outgoing: # communication with search engines

105checker:

106 # disable checker when in debug mode

107 off_when_debug: True

108+

108 # scheduling: interval or int

109 # scheduling: interval or int

109 # use "scheduling: False" to disable scheduling

110 # use "scheduling: False" to disable scheduling

111+ # to activate the scheduler:

112+ # * uncomment "scheduling" section

113+ # * add "cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1" to your uwsgi.ini

114+

110− scheduling:

115+ # scheduling:

111− start_after: [300, 1800] # delay to start the first run of the checker

116+ # start_after: [300, 1800] # delay to start the first run of the checker

112− every: [86400, 90000] # how often the checker runs

117+ # every: [86400, 90000] # how often the checker runs

118+

113 # additional tests: only for the YAML anchors (see the engines section)

119 # additional tests: only for the YAML anchors (see the engines section)

114 additional_tests:

120 additional_tests:

115 rosebud: &test_rosebud

121 rosebud: &test_rosebud

utils/searx.sh+2−4

@@ -46,7 +46,6 @@ SEARX_PACKAGES_debian="\

46python3-dev python3-babel python3-venv

47uwsgi uwsgi-plugin-python3

48git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev

49−libprotobuf-dev protobuf-compiler

50shellcheck"

49shellcheck"

51

50

52BUILD_PACKAGES_debian="\

51BUILD_PACKAGES_debian="\

@@ -59,7 +58,6 @@ SEARX_PACKAGES_arch="\

59python python-pip python-lxml python-babel

58python python-pip python-lxml python-babel

60uwsgi uwsgi-plugin-python

59uwsgi uwsgi-plugin-python

61git base-devel libxml2

60git base-devel libxml2

62−protobuf

63shellcheck"

61shellcheck"

64

62

65BUILD_PACKAGES_arch="\

63BUILD_PACKAGES_arch="\

@@ -71,7 +69,7 @@ SEARX_PACKAGES_fedora="\

71python python-pip python-lxml python-babel

69python python-pip python-lxml python-babel

72uwsgi uwsgi-plugin-python3

70uwsgi uwsgi-plugin-python3

73git @development-tools libxml2

71git @development-tools libxml2

74−ShellCheck protobuf-compiler protobuf-devel"

72+ShellCheck"

75

73

76BUILD_PACKAGES_fedora="\

74BUILD_PACKAGES_fedora="\

77firefox graphviz graphviz-gd ImageMagick librsvg2-tools

75firefox graphviz graphviz-gd ImageMagick librsvg2-tools

@@ -84,7 +82,7 @@ SEARX_PACKAGES_centos="\

84python36 python36-pip python36-lxml python-babel

82python36 python36-pip python36-lxml python-babel

85uwsgi uwsgi-plugin-python3

83uwsgi uwsgi-plugin-python3

86git @development-tools libxml2

84git @development-tools libxml2

87−ShellCheck protobuf-compiler protobuf-devel"

85+ShellCheck"

88

86

89BUILD_PACKAGES_centos="\

87BUILD_PACKAGES_centos="\

90firefox graphviz graphviz-gd ImageMagick librsvg2-tools

88firefox graphviz graphviz-gd ImageMagick librsvg2-tools