on master help sign in
[enh] checker: background check

See settings.yml for the options SIGUSR1 signal starts the checker. The result is available at /stats/checker

commit3a9f513521d006a7939538cce368d7b799e32c30 authorAlexandre Flament <alex@al-f.net> dateTue 5 Jan 2021, 11:24 (6 years ago) parent6e2872f
9 files changed+255−97
searx/search/__init__.py+5−67 searx/search/checker/__init__.py+3 searx/search/checker/__main__.py+23−7 searx/search/checker/background.py+106 searx/search/checker/impl.py+11−1 searx/search/models.py+69 searx/settings.yml+27−18 searx/webapp.py+9−3 setup.py+2−1
searx/search/__init__.py+5−67
@@ -28,7 +28,9 @@ from searx.external_bang import get_bang_url
2828from searx.results import ResultContainer
2929from searx import logger
3030from searx.plugins import plugins
31+from searx.search.models import EngineRef, SearchQuery
3132from searx.search.processors import processors, initialize as initialize_processors
33+from searx.search.checker import initialize as initialize_checker
3234
3335
3436logger = logger.getChild('search')
@@ -45,75 +47,11 @@ else:
4547 sys.exit(1)
4648
4749
48−def initialize(settings_engines=None):
50+def initialize(settings_engines=None, enable_checker=False):
4951 settings_engines = settings_engines or settings['engines']
5052 initialize_processors(settings_engines)
51−
52−
53−class EngineRef:
54−
55− __slots__ = 'name', 'category'
56−
57− def __init__(self, name: str, category: str):
58− self.name = name
59− self.category = category
60−
61− def __repr__(self):
62− return "EngineRef({!r}, {!r})".format(self.name, self.category)
63−
64− def __eq__(self, other):
65− return self.name == other.name and self.category == other.category
66−
67− def __hash__(self):
68− return hash((self.name, self.category))
69−
70−
71−class SearchQuery:
72− """container for all the search parameters (query, language, etc...)"""
73−
74− __slots__ = 'query', 'engineref_list', 'lang', 'safesearch', 'pageno', 'time_range',\
75− 'timeout_limit', 'external_bang'
76−
77− def __init__(self,
78− query: str,
79− engineref_list: typing.List[EngineRef],
80− lang: str='all',
81− safesearch: int=0,
82− pageno: int=1,
83− time_range: typing.Optional[str]=None,
84− timeout_limit: typing.Optional[float]=None,
85− external_bang: typing.Optional[str]=None):
86− self.query = query
87− self.engineref_list = engineref_list
88− self.lang = lang
89− self.safesearch = safesearch
90− self.pageno = pageno
91− self.time_range = time_range
92− self.timeout_limit = timeout_limit
93− self.external_bang = external_bang
94−
95− @property
96− def categories(self):
97− return list(set(map(lambda engineref: engineref.category, self.engineref_list)))
98−
99− def __repr__(self):
100− return "SearchQuery({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})".\
101− format(self.query, self.engineref_list, self.lang, self.safesearch,
102− self.pageno, self.time_range, self.timeout_limit, self.external_bang)
103−
104− def __eq__(self, other):
105− return self.query == other.query\
106− and self.engineref_list == other.engineref_list\
107− and self.lang == other.lang\
108− and self.safesearch == other.safesearch\
109− and self.pageno == other.pageno\
110− and self.time_range == other.time_range\
111− and self.timeout_limit == other.timeout_limit\
112− and self.external_bang == other.external_bang
113−
114− def __hash__(self):
115− return hash((self.query, tuple(self.engineref_list), self.lang, self.safesearch, self.pageno, self.time_range,
116− self.timeout_limit, self.external_bang))
53+ if enable_checker:
54+ initialize_checker()
11755
11856
11957class Search:
searx/search/checker/__init__.py+3
@@ -1 +1,4 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+
13from .impl import Checker
4+from .background import initialize, get_result
searx/search/checker/__main__.py+23−7
@@ -1,9 +1,13 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+
13import sys
24import os
5+import argparse
36
47import searx.search
5−import searx.search.processors
68import searx.search.checker
9+from searx.search import processors
10+from searx.engines import engine_shortcuts
711
812
913if sys.stdout.isatty() and os.environ.get('TERM') not in ['dumb', 'unknown']:
@@ -18,20 +22,24 @@ else:
1822 BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = "", "", "", "", "", "", "", ""
1923
2024
21−def iter_processor():
22− if len(sys.argv) > 1:
23− for name, processor in searx.search.processors.items():
24− if name in sys.argv:
25+def iter_processor(engine_name_list):
26+ if len(engine_name_list) > 0:
27+ for name in engine_name_list:
28+ name = engine_shortcuts.get(name, name)
29+ processor = processors.get(name)
30+ if processor is not None:
2531 yield name, processor
32+ else:
33+ print(BOLD_SEQ, 'Engine ', '%-30s' % name, RESET_SEQ, RED, ' Not found ', RESET_SEQ)
2634 else:
2735 for name, processor in searx.search.processors.items():
2836 yield name, processor
2937
3038
31−def main():
39+def run(engine_name_list):
3240 searx.search.initialize()
3341 broken_urls = []
34− for name, processor in iter_processor():
42+ for name, processor in iter_processor(engine_name_list):
3543 if sys.stdout.isatty():
3644 print(BOLD_SEQ, 'Engine ', '%-30s' % name, RESET_SEQ, WHITE, ' Checking', RESET_SEQ)
3745 checker = searx.search.checker.Checker(processor)
@@ -48,5 +56,13 @@ def main():
4856 print('Error fetching', url)
4957
5058
59+def main():
60+ parser = argparse.ArgumentParser(description='Check searx engines.')
61+ parser.add_argument('engine_name_list', metavar='engine name', type=str, nargs='*',
62+ help='engines name or shortcut list. Empty for all engines.')
63+ args = parser.parse_args()
64+ run(args.engine_name_list)
65+
66+
5167if __name__ == '__main__':
5268 main()
searx/search/checker/background.pynew file+106
@@ -0,0 +1,106 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+
3+import json
4+import random
5+import time
6+import threading
7+import os
8+import signal
9+
10+from searx import logger, settings, searx_debug
11+from searx.exceptions import SearxSettingsException
12+from searx.search.processors import processors
13+from searx.search.checker import Checker
14+from searx.shared import schedule, storage
15+
16+
17+CHECKER_RESULT = 'CHECKER_RESULT'
18+running = threading.Lock()
19+
20+
21+def _get_interval(every, error_msg):
22+ if isinstance(every, int):
23+ every = (every, every)
24+ if not isinstance(every, (tuple, list))\
25+ or len(every) != 2\
26+ or not isinstance(every[0], int)\
27+ or not isinstance(every[1], int):
28+ raise SearxSettingsException(error_msg, None)
29+ return every
30+
31+
32+def _get_every():
33+ every = settings.get('checker', {}).get('scheduling', {}).get('every', (300, 1800))
34+ return _get_interval(every, 'checker.scheduling.every is not a int or list')
35+
36+
37+def get_result():
38+ serialized_result = storage.get_str('CHECKER_RESULT')
39+ if serialized_result is not None:
40+ return json.loads(serialized_result)
41+
42+
43+def run():
44+ if not running.acquire(blocking=False):
45+ return
46+ try:
47+ logger.info('Starting checker')
48+ result = {}
49+ for name, processor in processors.items():
50+ logger.debug('Checking %s engine', name)
51+ checker = Checker(processor)
52+ checker.run()
53+ if checker.test_results.succesfull:
54+ result[name] = {'status': True}
55+ else:
56+ result[name] = {'status': False, 'errors': checker.test_results.errors}
57+
58+ storage.set_str('CHECKER_RESULT', json.dumps(result))
59+ logger.info('Check done')
60+ finally:
61+ running.release()
62+
63+
64+def _run_with_delay():
65+ every = _get_every()
66+ delay = random.randint(0, every[1] - every[0])
67+ logger.debug('Start checker in %i seconds', delay)
68+ time.sleep(delay)
69+ run()
70+
71+
72+def _start_scheduling():
73+ every = _get_every()
74+ schedule(every[0], _run_with_delay)
75+ run()
76+
77+
78+def _signal_handler(signum, frame):
79+ t = threading.Thread(target=run)
80+ t.daemon = True
81+ t.start()
82+
83+
84+def initialize():
85+ logger.info('Send SIGUSR1 signal to pid %i to start the checker', os.getpid())
86+ signal.signal(signal.SIGUSR1, _signal_handler)
87+
88+ # special case when debug is activate
89+ if searx_debug and settings.get('checker', {}).get('off_when_debug', True):
90+ logger.info('debug mode: checker is disabled')
91+ return
92+
93+ # check value of checker.scheduling.every now
94+ scheduling = settings.get('checker', {}).get('scheduling', None)
95+ if scheduling is None or not scheduling:
96+ logger.info('Checker scheduler is disabled')
97+ return
98+
99+ #
100+ start_after = scheduling.get('start_after', (300, 1800))
101+ start_after = _get_interval(start_after, 'checker.scheduling.start_after is not a int or list')
102+ delay = random.randint(start_after[0], start_after[1])
103+ logger.info('Start checker in %i seconds', delay)
104+ t = threading.Timer(delay, _start_scheduling)
105+ t.daemon = True
106+ t.start()
searx/search/checker/impl.py+11−1
@@ -1,3 +1,5 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+
13import typing
24import types
35import functools
@@ -11,7 +13,7 @@ import requests.exceptions
1113
1214from searx import poolrequests, logger
1315from searx.results import ResultContainer
14−from searx.search import SearchQuery, EngineRef
16+from searx.search.models import SearchQuery, EngineRef
1517from searx.search.processors import EngineProcessor
1618
1719
@@ -240,18 +242,24 @@ class ResultContainerTests:
240242 self._check_infoboxes(self.result_container.infoboxes)
241243
242244 def has_infobox(self):
245+ """Check the ResultContainer has at least one infobox"""
243246 if len(self.result_container.infoboxes) == 0:
244247 self._record_error('No infobox')
245248
246249 def has_answer(self):
250+ """Check the ResultContainer has at least one answer"""
247251 if len(self.result_container.answers) == 0:
248252 self._record_error('No answer')
249253
250254 def has_language(self, lang):
255+ """Check at least one title or content of the results is written in the `lang`.
256+
257+ Detected using pycld3, may be not accurate"""
251258 if lang not in self.languages:
252259 self._record_error(lang + ' not found')
253260
254261 def not_empty(self):
262+ """Check the ResultContainer has at least one answer or infobox or result"""
255263 result_types = set()
256264 results = self.result_container.get_ordered_results()
257265 if len(results) > 0:
@@ -267,6 +275,7 @@ class ResultContainerTests:
267275 self._record_error('No result')
268276
269277 def one_title_contains(self, title: str):
278+ """Check one of the title contains `title` (case insensitive comparaison)"""
270279 title = title.lower()
271280 for result in self.result_container.get_ordered_results():
272281 if title in result['title'].lower():
@@ -287,6 +296,7 @@ class CheckerTests:
287296 self.result_container_tests_list = result_container_tests_list
288297
289298 def unique_results(self):
299+ """Check the results of each ResultContain is unique"""
290300 urls_list = [rct.result_urls for rct in self.result_container_tests_list]
291301 if len(urls_list[0]) > 0:
292302 # results on the first page
searx/search/models.pynew file+69
@@ -0,0 +1,69 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+
3+import typing
4+
5+
6+class EngineRef:
7+
8+ __slots__ = 'name', 'category'
9+
10+ def __init__(self, name: str, category: str):
11+ self.name = name
12+ self.category = category
13+
14+ def __repr__(self):
15+ return "EngineRef({!r}, {!r})".format(self.name, self.category)
16+
17+ def __eq__(self, other):
18+ return self.name == other.name and self.category == other.category
19+
20+ def __hash__(self):
21+ return hash((self.name, self.category))
22+
23+
24+class SearchQuery:
25+ """container for all the search parameters (query, language, etc...)"""
26+
27+ __slots__ = 'query', 'engineref_list', 'lang', 'safesearch', 'pageno', 'time_range',\
28+ 'timeout_limit', 'external_bang'
29+
30+ def __init__(self,
31+ query: str,
32+ engineref_list: typing.List[EngineRef],
33+ lang: str='all',
34+ safesearch: int=0,
35+ pageno: int=1,
36+ time_range: typing.Optional[str]=None,
37+ timeout_limit: typing.Optional[float]=None,
38+ external_bang: typing.Optional[str]=None):
39+ self.query = query
40+ self.engineref_list = engineref_list
41+ self.lang = lang
42+ self.safesearch = safesearch
43+ self.pageno = pageno
44+ self.time_range = time_range
45+ self.timeout_limit = timeout_limit
46+ self.external_bang = external_bang
47+
48+ @property
49+ def categories(self):
50+ return list(set(map(lambda engineref: engineref.category, self.engineref_list)))
51+
52+ def __repr__(self):
53+ return "SearchQuery({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})".\
54+ format(self.query, self.engineref_list, self.lang, self.safesearch,
55+ self.pageno, self.time_range, self.timeout_limit, self.external_bang)
56+
57+ def __eq__(self, other):
58+ return self.query == other.query\
59+ and self.engineref_list == other.engineref_list\
60+ and self.lang == other.lang\
61+ and self.safesearch == other.safesearch\
62+ and self.pageno == other.pageno\
63+ and self.time_range == other.time_range\
64+ and self.timeout_limit == other.timeout_limit\
65+ and self.external_bang == other.external_bang
66+
67+ def __hash__(self):
68+ return hash((self.query, tuple(self.engineref_list), self.lang, self.safesearch, self.pageno, self.time_range,
69+ self.timeout_limit, self.external_bang))
searx/settings.yml+27−18
@@ -102,24 +102,33 @@ outgoing: # communication with search engines
102102# - "HTTPS rewrite"
103103# - ...
104104
105−additional_tests:
106− rosebud: &test_rosebud
107− matrix:
108− query: rosebud
109− lang: en
110− result_container:
111− - not_empty
112− - [one_title_contains', 'citizen kane']
113− test:
114− - unique_results
115−
116−tests:
117− infobox: &tests_infobox
118− infobox:
119− matrix:
120− query: ["linux", "new york", "bbc"]
121− result_container:
122− - has_infobox
105+checker:
106+ # disable checker when in debug mode
107+ off_when_debug: True
108+ # scheduling: interval or int
109+ # use "scheduling: False" to disable scheduling
110+ scheduling:
111+ start_after: [300, 1800] # delay to start the first run of the checker
112+ every: [86400, 90000] # how often the checker runs
113+ # additional tests: only for the YAML anchors (see the engines section)
114+ additional_tests:
115+ rosebud: &test_rosebud
116+ matrix:
117+ query: rosebud
118+ lang: en
119+ result_container:
120+ - not_empty
121+ - ['one_title_contains', 'citizen kane']
122+ test:
123+ - unique_results
124+ # tests: only for the YAML anchors (see the engines section)
125+ tests:
126+ infobox: &tests_infobox
127+ infobox:
128+ matrix:
129+ query: ["linux", "new york", "bbc"]
130+ result_container:
131+ - has_infobox
123132
124133engines:
125134 - name: apk mirror
searx/webapp.py+9−3
@@ -71,7 +71,8 @@ from searx.webadapter import get_search_query_from_webapp, get_selected_categori
7171from searx.utils import html_to_text, gen_useragent, dict_subset, match_language
7272from searx.version import VERSION_STRING
7373from searx.languages import language_codes as languages
74−from searx.search import SearchWithPlugins, initialize
74+from searx.search import SearchWithPlugins, initialize as search_initialize
75+from searx.search.checker import get_result as checker_get_result
7576from searx.query import RawTextQuery
7677from searx.autocomplete import searx_bang, backends as autocomplete_backends
7778from searx.plugins import plugins
@@ -81,7 +82,6 @@ from searx.answerers import answerers
8182from searx.poolrequests import get_global_proxies
8283from searx.metrology.error_recorder import errors_per_engines
8384
84−
8585# serve pages with HTTP/1.1
8686from werkzeug.serving import WSGIRequestHandler
8787WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server'].get('http_protocol_version', '1.0'))
@@ -136,7 +136,7 @@ werkzeug_reloader = flask_run_development or (searx_debug and __name__ == "__mai
136136# initialize the engines except on the first run of the werkzeug server.
137137if not werkzeug_reloader\
138138 or (werkzeug_reloader and os.environ.get("WERKZEUG_RUN_MAIN") == "true"):
139− initialize()
139+ search_initialize(enable_checker=True)
140140
141141babel = Babel(app)
142142
@@ -977,6 +977,12 @@ def stats_errors():
977977 return jsonify(result)
978978
979979
980+@app.route('/stats/checker', methods=['GET'])
981+def stats_checker():
982+ result = checker_get_result()
983+ return jsonify(result)
984+
985+
980986@app.route('/robots.txt', methods=['GET'])
981987def robots():
982988 return Response("""User-agent: *
setup.py+2−1
@@ -49,7 +49,8 @@ setup(
4949 },
5050 entry_points={
5151 'console_scripts': [
52− 'searx-run = searx.webapp:run'
52+ 'searx-run = searx.webapp:run',
53+ 'searx-checker = searx.search.checker.__main__:main'
5354 ]
5455 },
5556 package_data={