git.iko.soy/searx
on master help sign in
Merge pull request #2419 from dalf/checker
484dc99Alexandre Flament
Dockerfile+3
compare with master
unifiedsplit
@@ -41,6 +41,8 @@ RUN apk upgrade --no-cache \
41 openssl-dev \
41 openssl-dev \
42 tar \
42 tar \
43 git \
43 git \
44+ protoc \
45+ protobuf-dev \
44 && apk add --no-cache \
46 && apk add --no-cache \
45 ca-certificates \
47 ca-certificates \
46 su-exec \
48 su-exec \
@@ -53,6 +55,7 @@ RUN apk upgrade --no-cache \
53 uwsgi \
55 uwsgi \
54 uwsgi-python3 \
56 uwsgi-python3 \
55 brotli \
57 brotli \
58+ protobuf \
56 && pip3 install --upgrade pip \
59 && pip3 install --upgrade pip \
57 && pip3 install --no-cache -r requirements.txt \
60 && pip3 install --no-cache -r requirements.txt \
58 && apk del build-dependencies \
61 && apk del build-dependencies \
dockerfiles/uwsgi.ini+3
@@ -42,3 +42,6 @@ static-map = /static=/usr/local/searx/searx/static
42static-expires = /* 864000
42static-expires = /* 864000
43static-gzip-all = True
43static-gzip-all = True
44offload-threads = %k
44offload-threads = %k
45+
46+# Cache
47+cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1
requirements.txt+1
@@ -9,3 +9,4 @@ pygments==2.1.3
9python-dateutil==2.8.1
9python-dateutil==2.8.1
10pyyaml==5.3.1
10pyyaml==5.3.1
11requests[socks]==2.25.1
11requests[socks]==2.25.1
12+pycld3==0.20
searx/search/__init__.py+5−60
@@ -28,7 +28,9 @@ from searx.external_bang import get_bang_url
28from searx.results import ResultContainer
28from searx.results import ResultContainer
29from searx import logger
29from searx import logger
30from searx.plugins import plugins
30from searx.plugins import plugins
31+from searx.search.models import EngineRef, SearchQuery
31from searx.search.processors import processors, initialize as initialize_processors
32from searx.search.processors import processors, initialize as initialize_processors
33+from searx.search.checker import initialize as initialize_checker
32
34
33
35
34logger = logger.getChild('search')
36logger = logger.getChild('search')
@@ -45,68 +47,11 @@ else:
45 sys.exit(1)
47 sys.exit(1)
46
48
47
49
48−def initialize(settings_engines=None):
50+def initialize(settings_engines=None, enable_checker=False):
49 settings_engines = settings_engines or settings['engines']
51 settings_engines = settings_engines or settings['engines']
50 initialize_processors(settings_engines)
52 initialize_processors(settings_engines)
51−
53+ if enable_checker:
52−
54+ initialize_checker()
53−class EngineRef:
54−
55− __slots__ = 'name', 'category'
56−
57− def __init__(self, name: str, category: str):
58− self.name = name
59− self.category = category
60−
61− def __repr__(self):
62− return "EngineRef({!r}, {!r})".format(self.name, self.category)
63−
64− def __eq__(self, other):
65− return self.name == other.name and self.category == other.category
66−
67−
68−class SearchQuery:
69− """container for all the search parameters (query, language, etc...)"""
70−
71− __slots__ = 'query', 'engineref_list', 'lang', 'safesearch', 'pageno', 'time_range',\
72− 'timeout_limit', 'external_bang'
73−
74− def __init__(self,
75− query: str,
76− engineref_list: typing.List[EngineRef],
77− lang: str='all',
78− safesearch: int=0,
79− pageno: int=1,
80− time_range: typing.Optional[str]=None,
81− timeout_limit: typing.Optional[float]=None,
82− external_bang: typing.Optional[str]=None):
83− self.query = query
84− self.engineref_list = engineref_list
85− self.lang = lang
86− self.safesearch = safesearch
87− self.pageno = pageno
88− self.time_range = time_range
89− self.timeout_limit = timeout_limit
90− self.external_bang = external_bang
91−
92− @property
93− def categories(self):
94− return list(set(map(lambda engineref: engineref.category, self.engineref_list)))
95−
96− def __repr__(self):
97− return "SearchQuery({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})".\
98− format(self.query, self.engineref_list, self.lang, self.safesearch,
99− self.pageno, self.time_range, self.timeout_limit, self.external_bang)
100−
101− def __eq__(self, other):
102− return self.query == other.query\
103− and self.engineref_list == other.engineref_list\
104− and self.lang == other.lang\
105− and self.safesearch == other.safesearch\
106− and self.pageno == other.pageno\
107− and self.time_range == other.time_range\
108− and self.timeout_limit == other.timeout_limit\
109− and self.external_bang == other.external_bang
110
55
111
56
112class Search:
57class Search:
searx/search/checker/__init__.pynew file+4
@@ -0,0 +1,4 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+
3+from .impl import Checker
4+from .background import initialize, get_result
searx/search/checker/__main__.pynew file+94
@@ -0,0 +1,94 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+
3+import sys
4+import io
5+import os
6+import argparse
7+import logging
8+
9+import searx.search
10+import searx.search.checker
11+from searx.search import processors
12+from searx.engines import engine_shortcuts
13+
14+
15+# configure logging
16+root = logging.getLogger()
17+handler = logging.StreamHandler(sys.stdout)
18+for h in root.handlers:
19+ root.removeHandler(h)
20+root.addHandler(handler)
21+
22+# color only for a valid terminal
23+if sys.stdout.isatty() and os.environ.get('TERM') not in ['dumb', 'unknown']:
24+ RESET_SEQ = "\033[0m"
25+ COLOR_SEQ = "\033[1;%dm"
26+ BOLD_SEQ = "\033[1m"
27+ BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = map(lambda i: COLOR_SEQ % (30 + i), range(8))
28+else:
29+ RESET_SEQ = ""
30+ COLOR_SEQ = ""
31+ BOLD_SEQ = ""
32+ BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = "", "", "", "", "", "", "", ""
33+
34+# equivalent of 'python -u' (unbuffered stdout, stderr)
35+stdout = io.TextIOWrapper(open(sys.stdout.fileno(), 'wb', 0), write_through=True)
36+stderr = io.TextIOWrapper(open(sys.stderr.fileno(), 'wb', 0), write_through=True)
37+
38+
39+# iterator of processors
40+def iter_processor(engine_name_list):
41+ if len(engine_name_list) > 0:
42+ for name in engine_name_list:
43+ name = engine_shortcuts.get(name, name)
44+ processor = processors.get(name)
45+ if processor is not None:
46+ yield name, processor
47+ else:
48+ stdout.write(f'{BOLD_SEQ}Engine {name:30}{RESET_SEQ}{RED}Engine does not exist{RESET_SEQ}')
49+ else:
50+ for name, processor in searx.search.processors.items():
51+ yield name, processor
52+
53+
54+# actual check & display
55+def run(engine_name_list, verbose):
56+ searx.search.initialize()
57+ for name, processor in iter_processor(engine_name_list):
58+ stdout.write(f'{BOLD_SEQ}Engine {name:30}{RESET_SEQ}Checking\n')
59+ if not sys.stdout.isatty():
60+ stderr.write(f'{BOLD_SEQ}Engine {name:30}{RESET_SEQ}Checking\n')
61+ checker = searx.search.checker.Checker(processor)
62+ checker.run()
63+ if checker.test_results.succesfull:
64+ stdout.write(f'{BOLD_SEQ}Engine {name:30}{RESET_SEQ}{GREEN}OK{RESET_SEQ}\n')
65+ if verbose:
66+ stdout.write(f' {"found languages":15}: {" ".join(sorted(list(checker.test_results.languages)))}\n')
67+ else:
68+ stdout.write(f'{BOLD_SEQ}Engine {name:30}{RESET_SEQ}{RESET_SEQ}{RED}Error{RESET_SEQ}')
69+ if not verbose:
70+ errors = [test_name + ': ' + error for test_name, error in checker.test_results]
71+ stdout.write(f'{RED}Error {str(errors)}{RESET_SEQ}\n')
72+ else:
73+ stdout.write('\n')
74+ stdout.write(f' {"found languages":15}: {" ".join(sorted(list(checker.test_results.languages)))}\n')
75+ for test_name, logs in checker.test_results.logs.items():
76+ for log in logs:
77+ stdout.write(f' {test_name:15}: {RED}{" ".join(log)}{RESET_SEQ}\n')
78+
79+
80+# call by setup.py
81+def main():
82+ parser = argparse.ArgumentParser(description='Check searx engines.')
83+ parser.add_argument('engine_name_list', metavar='engine name', type=str, nargs='*',
84+ help='engines name or shortcut list. Empty for all engines.')
85+ parser.add_argument('--verbose', '-v',
86+ action='store_true', dest='verbose',
87+ help='Display details about the test results',
88+ default=False)
89+ args = parser.parse_args()
90+ run(args.engine_name_list, args.verbose)
91+
92+
93+if __name__ == '__main__':
94+ main()
searx/search/checker/background.pynew file+123
@@ -0,0 +1,123 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+
3+import json
4+import random
5+import time
6+import threading
7+import os
8+import signal
9+
10+from searx import logger, settings, searx_debug
11+from searx.exceptions import SearxSettingsException
12+from searx.search.processors import processors
13+from searx.search.checker import Checker
14+from searx.shared import schedule, storage
15+
16+
17+CHECKER_RESULT = 'CHECKER_RESULT'
18+running = threading.Lock()
19+
20+
21+def _get_interval(every, error_msg):
22+ if isinstance(every, int):
23+ every = (every, every)
24+ if not isinstance(every, (tuple, list))\
25+ or len(every) != 2\
26+ or not isinstance(every[0], int)\
27+ or not isinstance(every[1], int):
28+ raise SearxSettingsException(error_msg, None)
29+ return every
30+
31+
32+def _get_every():
33+ every = settings.get('checker', {}).get('scheduling', {}).get('every', (300, 1800))
34+ return _get_interval(every, 'checker.scheduling.every is not a int or list')
35+
36+
37+def get_result():
38+ serialized_result = storage.get_str(CHECKER_RESULT)
39+ if serialized_result is not None:
40+ return json.loads(serialized_result)
41+
42+
43+def _set_result(result, include_timestamp=True):
44+ if include_timestamp:
45+ result['timestamp'] = int(time.time() / 3600) * 3600
46+ storage.set_str(CHECKER_RESULT, json.dumps(result))
47+
48+
49+def run():
50+ if not running.acquire(blocking=False):
51+ return
52+ try:
53+ logger.info('Starting checker')
54+ result = {
55+ 'status': 'ok',
56+ 'engines': {}
57+ }
58+ for name, processor in processors.items():
59+ logger.debug('Checking %s engine', name)
60+ checker = Checker(processor)
61+ checker.run()
62+ if checker.test_results.succesfull:
63+ result['engines'][name] = {'success': True}
64+ else:
65+ result['engines'][name] = {'success': False, 'errors': checker.test_results.errors}
66+
67+ _set_result(result)
68+ logger.info('Check done')
69+ except Exception:
70+ _set_result({'status': 'error'})
71+ logger.exception('Error while running the checker')
72+ finally:
73+ running.release()
74+
75+
76+def _run_with_delay():
77+ every = _get_every()
78+ delay = random.randint(0, every[1] - every[0])
79+ logger.debug('Start checker in %i seconds', delay)
80+ time.sleep(delay)
81+ run()
82+
83+
84+def _start_scheduling():
85+ every = _get_every()
86+ if schedule(every[0], _run_with_delay):
87+ run()
88+
89+
90+def _signal_handler(signum, frame):
91+ t = threading.Thread(target=run)
92+ t.daemon = True
93+ t.start()
94+
95+
96+def initialize():
97+ logger.info('Send SIGUSR1 signal to pid %i to start the checker', os.getpid())
98+ signal.signal(signal.SIGUSR1, _signal_handler)
99+
100+ # disabled by default
101+ _set_result({'status': 'disabled'})
102+
103+ # special case when debug is activate
104+ if searx_debug and settings.get('checker', {}).get('off_when_debug', True):
105+ logger.info('debug mode: checker is disabled')
106+ return
107+
108+ # check value of checker.scheduling.every now
109+ scheduling = settings.get('checker', {}).get('scheduling', None)
110+ if scheduling is None or not scheduling:
111+ logger.info('Checker scheduler is disabled')
112+ return
113+
114+ #
115+ _set_result({'status': 'unknown'}, include_timestamp=False)
116+
117+ start_after = scheduling.get('start_after', (300, 1800))
118+ start_after = _get_interval(start_after, 'checker.scheduling.start_after is not a int or list')
119+ delay = random.randint(start_after[0], start_after[1])
120+ logger.info('Start checker in %i seconds', delay)
121+ t = threading.Timer(delay, _start_scheduling)
122+ t.daemon = True
123+ t.start()
searx/search/checker/impl.pynew file+406
@@ -0,0 +1,406 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+
3+import typing
4+import types
5+import functools
6+import itertools
7+from time import time
8+from urllib.parse import urlparse
9+
10+import re
11+import cld3
12+import requests.exceptions
13+
14+from searx import poolrequests, logger
15+from searx.results import ResultContainer
16+from searx.search.models import SearchQuery, EngineRef
17+from searx.search.processors import EngineProcessor
18+
19+
20+logger = logger.getChild('searx.search.checker')
21+
22+HTML_TAGS = [
23+ 'embed', 'iframe', 'object', 'param', 'picture', 'source', 'svg', 'math', 'canvas', 'noscript', 'script',
24+ 'del', 'ins', 'area', 'audio', 'img', 'map', 'track', 'video', 'a', 'abbr', 'b', 'bdi', 'bdo', 'br', 'cite',
25+ 'code', 'data', 'dfn', 'em', 'i', 'kdb', 'mark', 'q', 'rb', 'rp', 'rt', 'rtc', 'ruby', 's', 'samp', 'small',
26+ 'span', 'strong', 'sub', 'sup', 'time', 'u', 'var', 'wbr', 'style', 'blockquote', 'dd', 'div', 'dl', 'dt',
27+ 'figcaption', 'figure', 'hr', 'li', 'ol', 'p', 'pre', 'ul', 'button', 'datalist', 'fieldset', 'form', 'input',
28+ 'label', 'legend', 'meter', 'optgroup', 'option', 'output', 'progress', 'select', 'textarea', 'applet',
29+ 'frame', 'frameset'
30+]
31+
32+
33+def get_check_no_html():
34+ rep = ['<' + tag + '[^\>]*>' for tag in HTML_TAGS]
35+ rep += ['</' + tag + '>' for tag in HTML_TAGS]
36+ pattern = re.compile('|'.join(rep))
37+
38+ def f(text):
39+ return pattern.search(text.lower()) is None
40+
41+ return f
42+
43+
44+_check_no_html = get_check_no_html()
45+
46+
47+def _is_url(url):
48+ try:
49+ result = urlparse(url)
50+ except ValueError:
51+ return False
52+ if result.scheme not in ('http', 'https'):
53+ return False
54+ return True
55+
56+
57+@functools.lru_cache(maxsize=8192)
58+def _is_url_image(image_url):
59+ if not isinstance(image_url, str):
60+ return False
61+
62+ if image_url.startswith('//'):
63+ image_url = 'https:' + image_url
64+
65+ if image_url.startswith('data:'):
66+ return image_url.startswith('data:image/')
67+
68+ if not _is_url(image_url):
69+ return False
70+
71+ retry = 2
72+
73+ while retry > 0:
74+ a = time()
75+ try:
76+ poolrequests.set_timeout_for_thread(10.0, time())
77+ r = poolrequests.get(image_url, timeout=10.0, allow_redirects=True, headers={
78+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0',
79+ 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
80+ 'Accept-Language': 'en-US;q=0.5,en;q=0.3',
81+ 'Accept-Encoding': 'gzip, deflate, br',
82+ 'DNT': '1',
83+ 'Connection': 'keep-alive',
84+ 'Upgrade-Insecure-Requests': '1',
85+ 'Sec-GPC': '1',
86+ 'Cache-Control': 'max-age=0'
87+ })
88+ if r.headers["content-type"].startswith('image/'):
89+ return True
90+ return False
91+ except requests.exceptions.Timeout:
92+ logger.error('Timeout for %s: %i', image_url, int(time() - a))
93+ retry -= 1
94+ except requests.exceptions.RequestException:
95+ logger.exception('Exception for %s', image_url)
96+ return False
97+
98+
99+def _search_query_to_dict(search_query: SearchQuery) -> typing.Dict[str, typing.Any]:
100+ return {
101+ 'query': search_query.query,
102+ 'lang': search_query.lang,
103+ 'pageno': search_query.pageno,
104+ 'safesearch': search_query.safesearch,
105+ 'time_range': search_query.time_range,
106+ }
107+
108+
109+def _search_query_diff(sq1: SearchQuery, sq2: SearchQuery)\
110+ -> typing.Tuple[typing.Dict[str, typing.Any], typing.Dict[str, typing.Any]]:
111+ param1 = _search_query_to_dict(sq1)
112+ param2 = _search_query_to_dict(sq2)
113+ common = {}
114+ diff = {}
115+ for k, value1 in param1.items():
116+ value2 = param2[k]
117+ if value1 == value2:
118+ common[k] = value1
119+ else:
120+ diff[k] = (value1, value2)
121+ return (common, diff)
122+
123+
124+class TestResults:
125+
126+ __slots__ = 'errors', 'logs', 'languages'
127+
128+ def __init__(self):
129+ self.errors: typing.Dict[str, typing.List[str]] = {}
130+ self.logs: typing.Dict[str, typing.List[typing.Any]] = {}
131+ self.languages: typing.Set[str] = set()
132+
133+ def add_error(self, test, message, *args):
134+ # message to self.errors
135+ errors_for_test = self.errors.setdefault(test, [])
136+ if message not in errors_for_test:
137+ errors_for_test.append(message)
138+ # (message, *args) to self.logs
139+ logs_for_test = self.logs.setdefault(test, [])
140+ if (message, *args) not in logs_for_test:
141+ logs_for_test.append((message, *args))
142+
143+ def add_language(self, language):
144+ self.languages.add(language)
145+
146+ @property
147+ def succesfull(self):
148+ return len(self.errors) == 0
149+
150+ def __iter__(self):
151+ for test_name, errors in self.errors.items():
152+ for error in sorted(errors):
153+ yield (test_name, error)
154+
155+
156+class ResultContainerTests:
157+
158+ __slots__ = 'test_name', 'search_query', 'result_container', 'languages', 'stop_test', 'test_results'
159+
160+ def __init__(self,
161+ test_results: TestResults,
162+ test_name: str,
163+ search_query: SearchQuery,
164+ result_container: ResultContainer):
165+ self.test_name = test_name
166+ self.search_query = search_query
167+ self.result_container = result_container
168+ self.languages: typing.Set[str] = set()
169+ self.test_results = test_results
170+ self.stop_test = False
171+
172+ @property
173+ def result_urls(self):
174+ results = self.result_container.get_ordered_results()
175+ return [result['url'] for result in results]
176+
177+ def _record_error(self, message: str, *args) -> None:
178+ sq = _search_query_to_dict(self.search_query)
179+ sqstr = ' '.join(['{}={!r}'.format(k, v) for k, v in sq.items()])
180+ self.test_results.add_error(self.test_name, message, *args, '(' + sqstr + ')')
181+
182+ def _add_language(self, text: str) -> typing.Optional[str]:
183+ r = cld3.get_language(str(text)) # pylint: disable=E1101
184+ if r is not None and r.probability >= 0.98 and r.is_reliable:
185+ self.languages.add(r.language)
186+ self.test_results.add_language(r.language)
187+ return None
188+
189+ def _check_result(self, result):
190+ if not _check_no_html(result.get('title', '')):
191+ self._record_error('HTML in title', repr(result.get('title', '')))
192+ if not _check_no_html(result.get('content', '')):
193+ self._record_error('HTML in content', repr(result.get('content', '')))
194+
195+ self._add_language(result.get('title', ''))
196+ self._add_language(result.get('content', ''))
197+
198+ template = result.get('template', 'default.html')
199+ if template == 'default.html':
200+ return
201+ if template == 'code.html':
202+ return
203+ if template == 'torrent.html':
204+ return
205+ if template == 'map.html':
206+ return
207+ if template == 'images.html':
208+ thumbnail_src = result.get('thumbnail_src')
209+ if thumbnail_src is not None:
210+ if not _is_url_image(thumbnail_src):
211+ self._record_error('thumbnail_src URL is invalid', thumbnail_src)
212+ elif not _is_url_image(result.get('img_src')):
213+ self._record_error('img_src URL is invalid', result.get('img_src'))
214+ if template == 'videos.html' and not _is_url_image(result.get('thumbnail')):
215+ self._record_error('thumbnail URL is invalid', result.get('img_src'))
216+
217+ def _check_results(self, results: list):
218+ for result in results:
219+ self._check_result(result)
220+
221+ def _check_answers(self, answers):
222+ for answer in answers:
223+ if not _check_no_html(answer):
224+ self._record_error('HTML in answer', answer)
225+
226+ def _check_infoboxes(self, infoboxes):
227+ for infobox in infoboxes:
228+ if not _check_no_html(infobox.get('content', '')):
229+ self._record_error('HTML in infobox content', infobox.get('content', ''))
230+ self._add_language(infobox.get('content', ''))
231+ for attribute in infobox.get('attributes', {}):
232+ if not _check_no_html(attribute.get('value', '')):
233+ self._record_error('HTML in infobox attribute value', attribute.get('value', ''))
234+
235+ def check_basic(self):
236+ if len(self.result_container.unresponsive_engines) > 0:
237+ for message in self.result_container.unresponsive_engines:
238+ self._record_error(message[1] + ' ' + (message[2] or ''))
239+ self.stop_test = True
240+ return
241+
242+ results = self.result_container.get_ordered_results()
243+ if len(results) > 0:
244+ self._check_results(results)
245+
246+ if len(self.result_container.answers) > 0:
247+ self._check_answers(self.result_container.answers)
248+
249+ if len(self.result_container.infoboxes) > 0:
250+ self._check_infoboxes(self.result_container.infoboxes)
251+
252+ def has_infobox(self):
253+ """Check the ResultContainer has at least one infobox"""
254+ if len(self.result_container.infoboxes) == 0:
255+ self._record_error('No infobox')
256+
257+ def has_answer(self):
258+ """Check the ResultContainer has at least one answer"""
259+ if len(self.result_container.answers) == 0:
260+ self._record_error('No answer')
261+
262+ def has_language(self, lang):
263+ """Check at least one title or content of the results is written in the `lang`.
264+
265+ Detected using pycld3, may be not accurate"""
266+ if lang not in self.languages:
267+ self._record_error(lang + ' not found')
268+
269+ def not_empty(self):
270+ """Check the ResultContainer has at least one answer or infobox or result"""
271+ result_types = set()
272+ results = self.result_container.get_ordered_results()
273+ if len(results) > 0:
274+ result_types.add('results')
275+
276+ if len(self.result_container.answers) > 0:
277+ result_types.add('answers')
278+
279+ if len(self.result_container.infoboxes) > 0:
280+ result_types.add('infoboxes')
281+
282+ if len(result_types) == 0:
283+ self._record_error('No result')
284+
285+ def one_title_contains(self, title: str):
286+ """Check one of the title contains `title` (case insensitive comparaison)"""
287+ title = title.lower()
288+ for result in self.result_container.get_ordered_results():
289+ if title in result['title'].lower():
290+ return
291+ self._record_error(('{!r} not found in the title'.format(title)))
292+
293+
294+class CheckerTests:
295+
296+ __slots__ = 'test_results', 'test_name', 'result_container_tests_list'
297+
298+ def __init__(self,
299+ test_results: TestResults,
300+ test_name: str,
301+ result_container_tests_list: typing.List[ResultContainerTests]):
302+ self.test_results = test_results
303+ self.test_name = test_name
304+ self.result_container_tests_list = result_container_tests_list
305+
306+ def unique_results(self):
307+ """Check the results of each ResultContain is unique"""
308+ urls_list = [rct.result_urls for rct in self.result_container_tests_list]
309+ if len(urls_list[0]) > 0:
310+ # results on the first page
311+ for i, urls_i in enumerate(urls_list):
312+ for j, urls_j in enumerate(urls_list):
313+ if i < j and urls_i == urls_j:
314+ common, diff = _search_query_diff(self.result_container_tests_list[i].search_query,
315+ self.result_container_tests_list[j].search_query)
316+ common_str = ' '.join(['{}={!r}'.format(k, v) for k, v in common.items()])
317+ diff1_str = ', ' .join(['{}={!r}'.format(k, v1) for (k, (v1, v2)) in diff.items()])
318+ diff2_str = ', ' .join(['{}={!r}'.format(k, v2) for (k, (v1, v2)) in diff.items()])
319+ self.test_results.add_error(self.test_name,
320+ 'results are identitical for {} and {} ({})'
321+ .format(diff1_str, diff2_str, common_str))
322+
323+
324+class Checker:
325+
326+ __slots__ = 'processor', 'tests', 'test_results'
327+
328+ def __init__(self, processor: EngineProcessor):
329+ self.processor = processor
330+ self.tests = self.processor.get_tests()
331+ self.test_results = TestResults()
332+
333+ @property
334+ def engineref_list(self):
335+ engine_name = self.processor.engine_name
336+ engine_category = self.processor.engine.categories[0]
337+ return [EngineRef(engine_name, engine_category)]
338+
339+ @staticmethod
340+ def search_query_matrix_iterator(engineref_list, matrix):
341+ p = []
342+ for name, values in matrix.items():
343+ if isinstance(values, (tuple, list)):
344+ l = [(name, value) for value in values]
345+ else:
346+ l = [(name, values)]
347+ p.append(l)
348+
349+ for kwargs in itertools.product(*p):
350+ kwargs = {k: v for k, v in kwargs}
351+ query = kwargs['query']
352+ params = dict(kwargs)
353+ del params['query']
354+ yield SearchQuery(query, engineref_list, **params)
355+
356+ def call_test(self, obj, test_description):
357+ if isinstance(test_description, (tuple, list)):
358+ method, args = test_description[0], test_description[1:]
359+ else:
360+ method = test_description
361+ args = ()
362+ if isinstance(method, str) and hasattr(obj, method):
363+ getattr(obj, method)(*args)
364+ elif isinstance(method, types.FunctionType):
365+ method(*args)
366+ else:
367+ self.test_results.add_error(obj.test_name,
368+ 'method {!r} ({}) not found for {}'
369+ .format(method, method.__class__.__name__, obj.__class__.__name__))
370+
371+ def call_tests(self, obj, test_descriptions):
372+ for test_description in test_descriptions:
373+ self.call_test(obj, test_description)
374+
375+ def search(self, search_query: SearchQuery) -> ResultContainer:
376+ result_container = ResultContainer()
377+ engineref_category = search_query.engineref_list[0].category
378+ params = self.processor.get_params(search_query, engineref_category)
379+ if params is not None:
380+ self.processor.search(search_query.query, params, result_container, time(), 5)
381+ return result_container
382+
383+ def get_result_container_tests(self, test_name: str, search_query: SearchQuery) -> ResultContainerTests:
384+ result_container = self.search(search_query)
385+ result_container_check = ResultContainerTests(self.test_results, test_name, search_query, result_container)
386+ result_container_check.check_basic()
387+ return result_container_check
388+
389+ def run_test(self, test_name):
390+ test_parameters = self.tests[test_name]
391+ search_query_list = list(Checker.search_query_matrix_iterator(self.engineref_list, test_parameters['matrix']))
392+ rct_list = [self.get_result_container_tests(test_name, search_query) for search_query in search_query_list]
393+ stop_test = False
394+ if 'result_container' in test_parameters:
395+ for rct in rct_list:
396+ stop_test = stop_test or rct.stop_test
397+ if not rct.stop_test:
398+ self.call_tests(rct, test_parameters['result_container'])
399+ if not stop_test:
400+ if 'test' in test_parameters:
401+ checker_tests = CheckerTests(self.test_results, test_name, rct_list)
402+ self.call_tests(checker_tests, test_parameters['test'])
403+
404+ def run(self):
405+ for test_name in self.tests:
406+ self.run_test(test_name)
searx/search/models.pynew file+69
@@ -0,0 +1,69 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+
3+import typing
4+
5+
6+class EngineRef:
7+
8+ __slots__ = 'name', 'category'
9+
10+ def __init__(self, name: str, category: str):
11+ self.name = name
12+ self.category = category
13+
14+ def __repr__(self):
15+ return "EngineRef({!r}, {!r})".format(self.name, self.category)
16+
17+ def __eq__(self, other):
18+ return self.name == other.name and self.category == other.category
19+
20+ def __hash__(self):
21+ return hash((self.name, self.category))
22+
23+
24+class SearchQuery:
25+ """container for all the search parameters (query, language, etc...)"""
26+
27+ __slots__ = 'query', 'engineref_list', 'lang', 'safesearch', 'pageno', 'time_range',\
28+ 'timeout_limit', 'external_bang'
29+
30+ def __init__(self,
31+ query: str,
32+ engineref_list: typing.List[EngineRef],
33+ lang: str='all',
34+ safesearch: int=0,
35+ pageno: int=1,
36+ time_range: typing.Optional[str]=None,
37+ timeout_limit: typing.Optional[float]=None,
38+ external_bang: typing.Optional[str]=None):
39+ self.query = query
40+ self.engineref_list = engineref_list
41+ self.lang = lang
42+ self.safesearch = safesearch
43+ self.pageno = pageno
44+ self.time_range = time_range
45+ self.timeout_limit = timeout_limit
46+ self.external_bang = external_bang
47+
48+ @property
49+ def categories(self):
50+ return list(set(map(lambda engineref: engineref.category, self.engineref_list)))
51+
52+ def __repr__(self):
53+ return "SearchQuery({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})".\
54+ format(self.query, self.engineref_list, self.lang, self.safesearch,
55+ self.pageno, self.time_range, self.timeout_limit, self.external_bang)
56+
57+ def __eq__(self, other):
58+ return self.query == other.query\
59+ and self.engineref_list == other.engineref_list\
60+ and self.lang == other.lang\
61+ and self.safesearch == other.safesearch\
62+ and self.pageno == other.pageno\
63+ and self.time_range == other.time_range\
64+ and self.timeout_limit == other.timeout_limit\
65+ and self.external_bang == other.external_bang
66+
67+ def __hash__(self):
68+ return hash((self.query, tuple(self.engineref_list), self.lang, self.safesearch, self.pageno, self.time_range,
69+ self.timeout_limit, self.external_bang))
searx/search/processors/abstract.py+12
@@ -37,3 +37,15 @@ class EngineProcessor:
37 @abstractmethod
37 @abstractmethod
38 def search(self, query, params, result_container, start_time, timeout_limit):
38 def search(self, query, params, result_container, start_time, timeout_limit):
39 pass
39 pass
40+
41+ def get_tests(self):
42+ tests = getattr(self.engine, 'tests', None)
43+ if tests is None:
44+ tests = getattr(self.engine, 'additional_tests', {})
45+ tests.update(self.get_default_tests())
46+ return tests
47+ else:
48+ return tests
49+
50+ def get_default_tests(self):
51+ return {}
searx/search/processors/online.py+49−3
@@ -179,15 +179,15 @@ class OnlineProcessor(EngineProcessor):
179 requests_exception = True
179 requests_exception = True
180 elif (issubclass(e.__class__, SearxEngineCaptchaException)):
180 elif (issubclass(e.__class__, SearxEngineCaptchaException)):
181 result_container.add_unresponsive_engine(self.engine_name, 'CAPTCHA required')
181 result_container.add_unresponsive_engine(self.engine_name, 'CAPTCHA required')
182− logger.exception('engine {0} : CAPTCHA')
182+ logger.exception('engine {0} : CAPTCHA'.format(self.engine_name))
183 suspended_time = e.suspended_time # pylint: disable=no-member
183 suspended_time = e.suspended_time # pylint: disable=no-member
184 elif (issubclass(e.__class__, SearxEngineTooManyRequestsException)):
184 elif (issubclass(e.__class__, SearxEngineTooManyRequestsException)):
185 result_container.add_unresponsive_engine(self.engine_name, 'too many requests')
185 result_container.add_unresponsive_engine(self.engine_name, 'too many requests')
186− logger.exception('engine {0} : Too many requests')
186+ logger.exception('engine {0} : Too many requests'.format(self.engine_name))
187 suspended_time = e.suspended_time # pylint: disable=no-member
187 suspended_time = e.suspended_time # pylint: disable=no-member
188 elif (issubclass(e.__class__, SearxEngineAccessDeniedException)):
188 elif (issubclass(e.__class__, SearxEngineAccessDeniedException)):
189 result_container.add_unresponsive_engine(self.engine_name, 'blocked')
189 result_container.add_unresponsive_engine(self.engine_name, 'blocked')
190− logger.exception('engine {0} : Searx is blocked')
190+ logger.exception('engine {0} : Searx is blocked'.format(self.engine_name))
191 suspended_time = e.suspended_time # pylint: disable=no-member
191 suspended_time = e.suspended_time # pylint: disable=no-member
192 else:
192 else:
193 result_container.add_unresponsive_engine(self.engine_name, 'unexpected crash')
193 result_container.add_unresponsive_engine(self.engine_name, 'unexpected crash')
@@ -211,3 +211,49 @@ class OnlineProcessor(EngineProcessor):
211 # reset the suspend variables
211 # reset the suspend variables
212 self.engine.continuous_errors = 0
212 self.engine.continuous_errors = 0
213 self.engine.suspend_end_time = 0
213 self.engine.suspend_end_time = 0
214+
215+ def get_default_tests(self):
216+ tests = {}
217+
218+ tests['simple'] = {
219+ 'matrix': {'query': ('life', 'computer')},
220+ 'result_container': ['not_empty'],
221+ }
222+
223+ if getattr(self.engine, 'paging', False):
224+ tests['paging'] = {
225+ 'matrix': {'query': 'time',
226+ 'pageno': (1, 2, 3)},
227+ 'result_container': ['not_empty'],
228+ 'test': ['unique_results']
229+ }
230+ if 'general' in self.engine.categories:
231+ # avoid documentation about HTML tags (<time> and <input type="time">)
232+ tests['paging']['matrix']['query'] = 'news'
233+
234+ if getattr(self.engine, 'time_range', False):
235+ tests['time_range'] = {
236+ 'matrix': {'query': 'news',
237+ 'time_range': (None, 'day')},
238+ 'result_container': ['not_empty'],
239+ 'test': ['unique_results']
240+ }
241+
242+ if getattr(self.engine, 'lang', False):
243+ tests['lang_fr'] = {
244+ 'matrix': {'query': 'paris', 'lang': 'fr'},
245+ 'result_container': ['not_empty', ('has_lang', 'fr')],
246+ }
247+ tests['lang_en'] = {
248+ 'matrix': {'query': 'paris', 'lang': 'en'},
249+ 'result_container': ['not_empty', ('has_lang', 'en')],
250+ }
251+
252+ if getattr(self.engine, 'safesearch', False):
253+ tests['safesearch'] = {
254+ 'matrix': {'query': 'porn',
255+ 'safesearch': (0, 2)},
256+ 'test': ['unique_results']
257+ }
258+
259+ return tests
searx/search/processors/online_currency.py+10
@@ -55,3 +55,13 @@ class OnlineCurrencyProcessor(OnlineProcessor):
55 params['from_name'] = iso4217_to_name(from_currency, 'en')
55 params['from_name'] = iso4217_to_name(from_currency, 'en')
56 params['to_name'] = iso4217_to_name(to_currency, 'en')
56 params['to_name'] = iso4217_to_name(to_currency, 'en')
57 return params
57 return params
58+
59+ def get_default_tests(self):
60+ tests = {}
61+
62+ tests['currency'] = {
63+ 'matrix': {'query': '1337 usd in rmb'},
64+ 'result_container': ['has_answer'],
65+ }
66+
67+ return tests
searx/search/processors/online_dictionary.py+18
@@ -35,3 +35,21 @@ class OnlineDictionaryProcessor(OnlineProcessor):
35 params['query'] = query
35 params['query'] = query
36
36
37 return params
37 return params
38+
39+ def get_default_tests(self):
40+ tests = {}
41+
42+ if getattr(self.engine, 'paging', False):
43+ tests['translation_paging'] = {
44+ 'matrix': {'query': 'en-es house',
45+ 'pageno': (1, 2, 3)},
46+ 'result_container': ['not_empty', ('one_title_contains', 'house')],
47+ 'test': ['unique_results']
48+ }
49+ else:
50+ tests['translation'] = {
51+ 'matrix': {'query': 'en-es house'},
52+ 'result_container': ['not_empty', ('one_title_contains', 'house')],
53+ }
54+
55+ return tests
searx/settings.yml+40
@@ -102,6 +102,34 @@ outgoing: # communication with search engines
102# - "HTTPS rewrite"
102# - "HTTPS rewrite"
103# - ...
103# - ...
104
104
105+checker:
106+ # disable checker when in debug mode
107+ off_when_debug: True
108+ # scheduling: interval or int
109+ # use "scheduling: False" to disable scheduling
110+ scheduling:
111+ start_after: [300, 1800] # delay to start the first run of the checker
112+ every: [86400, 90000] # how often the checker runs
113+ # additional tests: only for the YAML anchors (see the engines section)
114+ additional_tests:
115+ rosebud: &test_rosebud
116+ matrix:
117+ query: rosebud
118+ lang: en
119+ result_container:
120+ - not_empty
121+ - ['one_title_contains', 'citizen kane']
122+ test:
123+ - unique_results
124+ # tests: only for the YAML anchors (see the engines section)
125+ tests:
126+ infobox: &tests_infobox
127+ infobox:
128+ matrix:
129+ query: ["linux", "new york", "bbc"]
130+ result_container:
131+ - has_infobox
132+
105engines:
133engines:
106 - name: apk mirror
134 - name: apk mirror
107 engine: apkmirror
135 engine: apkmirror
@@ -218,6 +246,7 @@ engines:
218 shortcut : ddd
246 shortcut : ddd
219 weight : 2
247 weight : 2
220 disabled : True
248 disabled : True
249+ tests: *tests_infobox
221
250
222# cloudflare protected
251# cloudflare protected
223# - name : digbt
252# - name : digbt
@@ -262,6 +291,7 @@ engines:
262 shortcut : wd
291 shortcut : wd
263 timeout : 3.0
292 timeout : 3.0
264 weight : 2
293 weight : 2
294+ tests: *tests_infobox
265
295
266 - name : duckduckgo
296 - name : duckduckgo
267 engine : duckduckgo
297 engine : duckduckgo
@@ -278,6 +308,8 @@ engines:
278 engine : etools
308 engine : etools
279 shortcut : eto
309 shortcut : eto
280 disabled : True
310 disabled : True
311+ additional_tests:
312+ rosebud: *test_rosebud
281
313
282 - name : etymonline
314 - name : etymonline
283 engine : xpath
315 engine : xpath
@@ -343,6 +375,8 @@ engines:
343 shortcut : gb
375 shortcut : gb
344 timeout : 3.0
376 timeout : 3.0
345 disabled: True
377 disabled: True
378+ additional_tests:
379+ rosebud: *test_rosebud
346
380
347 - name : gentoo
381 - name : gentoo
348 engine : gentoo
382 engine : gentoo
@@ -646,6 +680,8 @@ engines:
646 shortcut : qw
680 shortcut : qw
647 categories : general
681 categories : general
648 disabled : True
682 disabled : True
683+ additional_tests:
684+ rosebud: *test_rosebud
649
685
650 - name : qwant images
686 - name : qwant images
651 engine : qwant
687 engine : qwant
@@ -745,6 +781,8 @@ engines:
745 shortcut : sp
781 shortcut : sp
746 timeout : 6.0
782 timeout : 6.0
747 disabled : True
783 disabled : True
784+ additional_tests:
785+ rosebud: *test_rosebud
748
786
749 - name : tokyotoshokan
787 - name : tokyotoshokan
750 engine : tokyotoshokan
788 engine : tokyotoshokan
@@ -856,6 +894,8 @@ engines:
856 number_of_results : 5
894 number_of_results : 5
857 search_type : text
895 search_type : text
858 disabled : True
896 disabled : True
897+ additional_tests:
898+ rosebud: *test_rosebud
859
899
860 - name : wikisource
900 - name : wikisource
861 engine : mediawiki
901 engine : mediawiki
searx/shared/__init__.pynew file+31
@@ -0,0 +1,31 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+
3+import logging
4+
5+logger = logging.getLogger('searx.shared')
6+
7+try:
8+ import uwsgi
9+except:
10+ # no uwsgi
11+ from .shared_simple import SimpleSharedDict as SharedDict, schedule
12+ logger.info('Use shared_simple implementation')
13+else:
14+ try:
15+ uwsgi.cache_update('dummy', b'dummy')
16+ if uwsgi.cache_get('dummy') != b'dummy':
17+ raise Exception()
18+ except:
19+ # uwsgi.ini configuration problem: disable all scheduling
20+ logger.error('uwsgi.ini configuration error, add this line to your uwsgi.ini\n'
21+ 'cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1')
22+ from .shared_simple import SimpleSharedDict as SharedDict
23+
24+ def schedule(delay, func, *args):
25+ return False
26+ else:
27+ # uwsgi
28+ from .shared_uwsgi import UwsgiCacheSharedDict as SharedDict, schedule
29+ logger.info('Use shared_uwsgi implementation')
30+
31+storage = SharedDict()
searx/shared/shared_abstract.pynew file+21
@@ -0,0 +1,21 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+from abc import ABC, abstractmethod
3+
4+
5+class SharedDict(ABC):
6+
7+ @abstractmethod
8+ def get_int(self, key):
9+ pass
10+
11+ @abstractmethod
12+ def set_int(self, key, value):
13+ pass
14+
15+ @abstractmethod
16+ def get_str(self, key):
17+ pass
18+
19+ @abstractmethod
20+ def set_str(self, key, value):
21+ pass
searx/shared/shared_simple.pynew file+39
@@ -0,0 +1,39 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+
3+import threading
4+
5+from . import shared_abstract
6+
7+
8+class SimpleSharedDict(shared_abstract.SharedDict):
9+
10+ __slots__ = 'd',
11+
12+ def __init__(self):
13+ self.d = {}
14+
15+ def get_int(self, key):
16+ return self.d.get(key, None)
17+
18+ def set_int(self, key, value):
19+ self.d[key] = value
20+
21+ def get_str(self, key):
22+ return self.d.get(key, None)
23+
24+ def set_str(self, key, value):
25+ self.d[key] = value
26+
27+
28+def schedule(delay, func, *args):
29+ def call_later():
30+ t = threading.Timer(delay, wrapper)
31+ t.daemon = True
32+ t.start()
33+
34+ def wrapper():
35+ call_later()
36+ func(*args)
37+
38+ call_later()
39+ return True
searx/shared/shared_uwsgi.pynew file+64
@@ -0,0 +1,64 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+
3+import time
4+import uwsgi # pylint: disable=E0401
5+from . import shared_abstract
6+
7+
8+_last_signal = 10
9+
10+
11+class UwsgiCacheSharedDict(shared_abstract.SharedDict):
12+
13+ def get_int(self, key):
14+ value = uwsgi.cache_get(key)
15+ if value is None:
16+ return value
17+ else:
18+ return int.from_bytes(value, 'big')
19+
20+ def set_int(self, key, value):
21+ b = value.to_bytes(4, 'big')
22+ uwsgi.cache_update(key, b)
23+
24+ def get_str(self, key):
25+ value = uwsgi.cache_get(key)
26+ if value is None:
27+ return value
28+ else:
29+ return value.decode('utf-8')
30+
31+ def set_str(self, key, value):
32+ b = value.encode('utf-8')
33+ uwsgi.cache_update(key, b)
34+
35+
36+def schedule(delay, func, *args):
37+ """
38+ Can be implemented using a spooler.
39+ https://uwsgi-docs.readthedocs.io/en/latest/PythonDecorators.html
40+
41+ To make the uwsgi configuration simple, use the alternative implementation.
42+ """
43+ global _last_signal
44+
45+ def sighandler(signum):
46+ now = int(time.time())
47+ key = 'scheduler_call_time_signal_' + str(signum)
48+ uwsgi.lock()
49+ try:
50+ updating = uwsgi.cache_get(key)
51+ if updating is not None:
52+ updating = int.from_bytes(updating, 'big')
53+ if now - updating < delay:
54+ return
55+ uwsgi.cache_update(key, now.to_bytes(4, 'big'))
56+ finally:
57+ uwsgi.unlock()
58+ func(*args)
59+
60+ signal_num = _last_signal
61+ _last_signal += 1
62+ uwsgi.register_signal(signal_num, 'worker', sighandler)
63+ uwsgi.add_timer(signal_num, delay)
64+ return True
searx/webapp.py+9−3
@@ -71,7 +71,8 @@ from searx.webadapter import get_search_query_from_webapp, get_selected_categori
71from searx.utils import html_to_text, gen_useragent, dict_subset, match_language
71from searx.utils import html_to_text, gen_useragent, dict_subset, match_language
72from searx.version import VERSION_STRING
72from searx.version import VERSION_STRING
73from searx.languages import language_codes as languages
73from searx.languages import language_codes as languages
74−from searx.search import SearchWithPlugins, initialize
74+from searx.search import SearchWithPlugins, initialize as search_initialize
75+from searx.search.checker import get_result as checker_get_result
75from searx.query import RawTextQuery
76from searx.query import RawTextQuery
76from searx.autocomplete import searx_bang, backends as autocomplete_backends
77from searx.autocomplete import searx_bang, backends as autocomplete_backends
77from searx.plugins import plugins
78from searx.plugins import plugins
@@ -81,7 +82,6 @@ from searx.answerers import answerers
81from searx.poolrequests import get_global_proxies
82from searx.poolrequests import get_global_proxies
82from searx.metrology.error_recorder import errors_per_engines
83from searx.metrology.error_recorder import errors_per_engines
83
84
84−
85# serve pages with HTTP/1.1
85# serve pages with HTTP/1.1
86from werkzeug.serving import WSGIRequestHandler
86from werkzeug.serving import WSGIRequestHandler
87WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server'].get('http_protocol_version', '1.0'))
87WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server'].get('http_protocol_version', '1.0'))
@@ -136,7 +136,7 @@ werkzeug_reloader = flask_run_development or (searx_debug and __name__ == "__mai
136# initialize the engines except on the first run of the werkzeug server.
136# initialize the engines except on the first run of the werkzeug server.
137if not werkzeug_reloader\
137if not werkzeug_reloader\
138 or (werkzeug_reloader and os.environ.get("WERKZEUG_RUN_MAIN") == "true"):
138 or (werkzeug_reloader and os.environ.get("WERKZEUG_RUN_MAIN") == "true"):
139− initialize()
139+ search_initialize(enable_checker=True)
140
140
141babel = Babel(app)
141babel = Babel(app)
142
142
@@ -977,6 +977,12 @@ def stats_errors():
977 return jsonify(result)
977 return jsonify(result)
978
978
979
979
980+@app.route('/stats/checker', methods=['GET'])
981+def stats_checker():
982+ result = checker_get_result()
983+ return jsonify(result)
984+
985+
980@app.route('/robots.txt', methods=['GET'])
986@app.route('/robots.txt', methods=['GET'])
981def robots():
987def robots():
982 return Response("""User-agent: *
988 return Response("""User-agent: *
setup.py+2−1
@@ -49,7 +49,8 @@ setup(
49 },
49 },
50 entry_points={
50 entry_points={
51 'console_scripts': [
51 'console_scripts': [
52− 'searx-run = searx.webapp:run'
52+ 'searx-run = searx.webapp:run',
53+ 'searx-checker = searx.search.checker.__main__:main'
53 ]
54 ]
54 },
55 },
55 package_data={
56 package_data={
utils/searx.sh+4−2
@@ -46,6 +46,7 @@ SEARX_PACKAGES_debian="\
46python3-dev python3-babel python3-venv
46python3-dev python3-babel python3-venv
47uwsgi uwsgi-plugin-python3
47uwsgi uwsgi-plugin-python3
48git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev
48git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev
49+libprotobuf-dev protobuf-compiler
49shellcheck"
50shellcheck"
50
51
51BUILD_PACKAGES_debian="\
52BUILD_PACKAGES_debian="\
@@ -58,6 +59,7 @@ SEARX_PACKAGES_arch="\
58python python-pip python-lxml python-babel
59python python-pip python-lxml python-babel
59uwsgi uwsgi-plugin-python
60uwsgi uwsgi-plugin-python
60git base-devel libxml2
61git base-devel libxml2
62+protobuf
61shellcheck"
63shellcheck"
62
64
63BUILD_PACKAGES_arch="\
65BUILD_PACKAGES_arch="\
@@ -69,7 +71,7 @@ SEARX_PACKAGES_fedora="\
69python python-pip python-lxml python-babel
71python python-pip python-lxml python-babel
70uwsgi uwsgi-plugin-python3
72uwsgi uwsgi-plugin-python3
71git @development-tools libxml2
73git @development-tools libxml2
72−ShellCheck"
74+ShellCheck protobuf-compiler protobuf-devel"
73
75
74BUILD_PACKAGES_fedora="\
76BUILD_PACKAGES_fedora="\
75firefox graphviz graphviz-gd ImageMagick librsvg2-tools
77firefox graphviz graphviz-gd ImageMagick librsvg2-tools
@@ -82,7 +84,7 @@ SEARX_PACKAGES_centos="\
82python36 python36-pip python36-lxml python-babel
84python36 python36-pip python36-lxml python-babel
83uwsgi uwsgi-plugin-python3
85uwsgi uwsgi-plugin-python3
84git @development-tools libxml2
86git @development-tools libxml2
85−ShellCheck"
87+ShellCheck protobuf-compiler protobuf-devel"
86
88
87BUILD_PACKAGES_centos="\
89BUILD_PACKAGES_centos="\
88firefox graphviz graphviz-gd ImageMagick librsvg2-tools
90firefox graphviz graphviz-gd ImageMagick librsvg2-tools
utils/templates/etc/uwsgi/apps-archlinux/searx.ini+4−1
@@ -82,4 +82,7 @@ http = ${SEARX_INTERNAL_HTTP}
82# mkdir -p /run/uwsgi/app/searx
82# mkdir -p /run/uwsgi/app/searx
83# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
83# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
84#
84#
85−# socket = /run/uwsgi/app/searx/socket
85+# socket = /run/uwsgi/app/searx/socket
86+
87+# Cache
88+cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1
utils/templates/etc/uwsgi/apps-available/searx.ini+3
@@ -82,3 +82,6 @@ http = ${SEARX_INTERNAL_HTTP}
82# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
82# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
83#
83#
84# socket = /run/uwsgi/app/searx/socket
84# socket = /run/uwsgi/app/searx/socket
85+
86+# Cache
87+cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1