45bfab7 · searx · git.iko.soy

git.iko.soy/searx

|mod] checker: improve searx-checker command line

* output is unbuffered * verbose mode describe more precisly the errrors

commit45bfab77d0154c60f58be0453307cb03b48dca35 authorAlexandre Flament <alex@al-f.net> dateFri 8 Jan 2021, 19:04 (6 years ago) parent3a9f513

2 files changed+67−33

compare with master

unifiedsplit

searx/search/checker/__main__.py+40−14 searx/search/checker/impl.py+27−19

searx/search/checker/__main__.py+40−14

@@ -1,8 +1,10 @@

11# SPDX-License-Identifier: AGPL-3.0-or-later

33import sys

4+import io

45import os

56import argparse

7+import logging

79import searx.search

810import searx.search.checker

@@ -10,6 +12,14 @@ from searx.search import processors

1012from searx.engines import engine_shortcuts

1113

1214

15+# configure logging

16+root = logging.getLogger()

17+handler = logging.StreamHandler(sys.stdout)

18+for h in root.handlers:

19+ root.removeHandler(h)

20+root.addHandler(handler)

21+

22+# color only for a valid terminal

1323if sys.stdout.isatty() and os.environ.get('TERM') not in ['dumb', 'unknown']:

1424 RESET_SEQ = "\033[0m"

1525 COLOR_SEQ = "\033[1;%dm"

@@ -21,7 +31,12 @@ else:

2131 BOLD_SEQ = ""

2232 BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = "", "", "", "", "", "", "", ""

2333

34+# equivalent of 'python -u' (unbuffered stdout, stderr)

35+stdout = io.TextIOWrapper(open(sys.stdout.fileno(), 'wb', 0), write_through=True)

36+stderr = io.TextIOWrapper(open(sys.stderr.fileno(), 'wb', 0), write_through=True)

37+

2438

39+# iterator of processors

2540def iter_processor(engine_name_list):

2641 if len(engine_name_list) > 0:

2742 for name in engine_name_list:

@@ -30,38 +45,49 @@ def iter_processor(engine_name_list):

3045 if processor is not None:

3146 yield name, processor

3247 else:

33− print(BOLD_SEQ, 'Engine ', '%-30s' % name, RESET_SEQ, RED, ' Not found ', RESET_SEQ)

48+ stdout.write(f'{BOLD_SEQ}Engine {name:30}{RESET_SEQ}{RED}Engine does not exist{RESET_SEQ}')

3449 else:

3550 for name, processor in searx.search.processors.items():

3651 yield name, processor

3752

3853

39−def run(engine_name_list):

54+# actual check & display

55+def run(engine_name_list, verbose):

4056 searx.search.initialize()

41− broken_urls = []

4257 for name, processor in iter_processor(engine_name_list):

43− if sys.stdout.isatty():

44− print(BOLD_SEQ, 'Engine ', '%-30s' % name, RESET_SEQ, WHITE, ' Checking', RESET_SEQ)

58+ stdout.write(f'{BOLD_SEQ}Engine {name:30}{RESET_SEQ}Checking\n')

59+ if not sys.stdout.isatty():

60+ stderr.write(f'{BOLD_SEQ}Engine {name:30}{RESET_SEQ}Checking\n')

4561 checker = searx.search.checker.Checker(processor)

4662 checker.run()

4763 if checker.test_results.succesfull:

48− print(BOLD_SEQ, 'Engine ', '%-30s' % name, RESET_SEQ, GREEN, ' OK', RESET_SEQ)

64+ stdout.write(f'{BOLD_SEQ}Engine {name:30}{RESET_SEQ}{GREEN}OK{RESET_SEQ}\n')

65+ if verbose:

66+ stdout.write(f' {"found languages":15}: {" ".join(sorted(list(checker.test_results.languages)))}\n')

4967 else:

50− errors = [test_name + ': ' + error for test_name, error in checker.test_results]

51− print(BOLD_SEQ, 'Engine ', '%-30s' % name, RESET_SEQ, RED, ' Error ', str(errors), RESET_SEQ)

52−

53− broken_urls += checker.test_results.broken_urls

54−

55− for url in broken_urls:

56− print('Error fetching', url)

68+ stdout.write(f'{BOLD_SEQ}Engine {name:30}{RESET_SEQ}{RESET_SEQ}{RED}Error{RESET_SEQ}')

69+ if not verbose:

70+ errors = [test_name + ': ' + error for test_name, error in checker.test_results]

71+ stdout.write(f'{RED}Error {str(errors)}{RESET_SEQ}\n')

72+ else:

73+ stdout.write('\n')

74+ stdout.write(f' {"found languages":15}: {" ".join(sorted(list(checker.test_results.languages)))}\n')

75+ for test_name, logs in checker.test_results.logs.items():

76+ for log in logs:

77+ stdout.write(f' {test_name:15}: {RED}{" ".join(log)}{RESET_SEQ}\n')

5778

5879

80+# call by setup.py

5981def main():

6082 parser = argparse.ArgumentParser(description='Check searx engines.')

6183 parser.add_argument('engine_name_list', metavar='engine name', type=str, nargs='*',

6284 help='engines name or shortcut list. Empty for all engines.')

85+ parser.add_argument('--verbose', '-v',

86+ action='store_true', dest='verbose',

87+ help='Display details about the test results',

88+ default=False)

6389 args = parser.parse_args()

64− run(args.engine_name_list)

90+ run(args.engine_name_list, args.verbose)

6591

6692

6793if __name__ == '__main__':

searx/search/checker/impl.py+27−19

@@ -17,6 +17,8 @@ from searx.search.models import SearchQuery, EngineRef

1717from searx.search.processors import EngineProcessor

1818

1919

20+logger = logger.getChild('searx.search.checker')

21+

2022HTML_TAGS = [

2123 'embed', 'iframe', 'object', 'param', 'picture', 'source', 'svg', 'math', 'canvas', 'noscript', 'script',

2224 'del', 'ins', 'area', 'audio', 'img', 'map', 'track', 'video', 'a', 'abbr', 'b', 'bdi', 'bdo', 'br', 'cite',

@@ -121,20 +123,25 @@ def _search_query_diff(sq1: SearchQuery, sq2: SearchQuery)\

121123

122124class TestResults:

123125

124− __slots__ = 'errors', 'broken_urls'

126+ __slots__ = 'errors', 'logs', 'languages'

125127

126128 def __init__(self):

127129 self.errors: typing.Dict[str, typing.List[str]] = {}

128− self.broken_urls = []

130+ self.logs: typing.Dict[str, typing.List[typing.Any]] = {}

131+ self.languages: typing.Set[str] = set()

129132

130− def add_error(self, test, message):

133+ def add_error(self, test, message, *args):

134+ # message to self.errors

131135 errors_for_test = self.errors.setdefault(test, [])

132136 if message not in errors_for_test:

133137 errors_for_test.append(message)

138+ # (message, *args) to self.logs

139+ logs_for_test = self.logs.setdefault(test, [])

140+ if (message, *args) not in logs_for_test:

141+ logs_for_test.append((message, *args))

134142

135− def add_broken_url(self, url):

136− if url not in self.broken_urls:

137− self.broken_urls.append(url)

143+ def add_language(self, language):

144+ self.languages.add(language)

138145

139146 @property

140147 def succesfull(self):

@@ -167,20 +174,23 @@ class ResultContainerTests:

167174 results = self.result_container.get_ordered_results()

168175 return [result['url'] for result in results]

169176

170− def _record_error(self, message: str) -> None:

171− self.test_results.add_error(self.test_name, message)

177+ def _record_error(self, message: str, *args) -> None:

178+ sq = _search_query_to_dict(self.search_query)

179+ sqstr = ' '.join(['{}={!r}'.format(k, v) for k, v in sq.items()])

180+ self.test_results.add_error(self.test_name, message, *args, '(' + sqstr + ')')

172181

173182 def _add_language(self, text: str) -> typing.Optional[str]:

174183 r = cld3.get_language(str(text)) # pylint: disable=E1101

175− if r is not None and r.probability >= 0.9 and r.is_reliable:

184+ if r is not None and r.probability >= 0.98 and r.is_reliable:

176185 self.languages.add(r.language)

186+ self.test_results.add_language(r.language)

177187 return None

178188

179189 def _check_result(self, result):

180190 if not _check_no_html(result.get('title', '')):

181− self._record_error('HTML in title')

191+ self._record_error('HTML in title', repr(result.get('title', '')))

182192 if not _check_no_html(result.get('content', '')):

183− self._record_error('HTML in content')

193+ self._record_error('HTML in content', repr(result.get('content', '')))

184194

185195 self._add_language(result.get('title', ''))

186196 self._add_language(result.get('content', ''))

@@ -198,13 +208,11 @@ class ResultContainerTests:

198208 thumbnail_src = result.get('thumbnail_src')

199209 if thumbnail_src is not None:

200210 if not _is_url_image(thumbnail_src):

201− self.test_results.add_broken_url(thumbnail_src)

202− self._record_error('thumbnail_src URL is invalid')

211+ self._record_error('thumbnail_src URL is invalid', thumbnail_src)

203212 elif not _is_url_image(result.get('img_src')):

204− self.test_results.add_broken_url(result.get('img_src'))

205− self._record_error('img_src URL is invalid')

213+ self._record_error('img_src URL is invalid', result.get('img_src'))

206214 if template == 'videos.html' and not _is_url_image(result.get('thumbnail')):

207− self._record_error('thumbnail URL is invalid')

215+ self._record_error('thumbnail URL is invalid', result.get('img_src'))

208216

209217 def _check_results(self, results: list):

210218 for result in results:

@@ -213,16 +221,16 @@ class ResultContainerTests:

213221 def _check_answers(self, answers):

214222 for answer in answers:

215223 if not _check_no_html(answer):

216− self._record_error('HTML in answer')

224+ self._record_error('HTML in answer', answer)

217225

218226 def _check_infoboxes(self, infoboxes):

219227 for infobox in infoboxes:

220228 if not _check_no_html(infobox.get('content', '')):

221− self._record_error('HTML in infobox content')

229+ self._record_error('HTML in infobox content', infobox.get('content', ''))

222230 self._add_language(infobox.get('content', ''))

223231 for attribute in infobox.get('attributes', {}):

224232 if not _check_no_html(attribute.get('value', '')):

225− self._record_error('HTML in infobox attribute value')

233+ self._record_error('HTML in infobox attribute value', attribute.get('value', ''))

226234

227235 def check_basic(self):

228236 if len(self.result_container.unresponsive_engines) > 0: