45bfab7 · searx · git.iko.soy

git.iko.soy/searx

on master help sign in

|mod] checker: improve searx-checker command line

45bfab7Alexandre Flament6 years ago

searx/search/checker/__main__.py+40−14

compare with master

unifiedsplit

@@ -1,8 +1,10 @@

1# SPDX-License-Identifier: AGPL-3.0-or-later

3import sys

4+import io

4import os

5import os

5import argparse

6import argparse

7+import logging

7import searx.search

9import searx.search

8import searx.search.checker

10import searx.search.checker

@@ -10,6 +12,14 @@ from searx.search import processors

10from searx.engines import engine_shortcuts

12from searx.engines import engine_shortcuts

15+# configure logging

16+root = logging.getLogger()

17+handler = logging.StreamHandler(sys.stdout)

18+for h in root.handlers:

19+ root.removeHandler(h)

20+root.addHandler(handler)

21+

22+# color only for a valid terminal

13if sys.stdout.isatty() and os.environ.get('TERM') not in ['dumb', 'unknown']:

23if sys.stdout.isatty() and os.environ.get('TERM') not in ['dumb', 'unknown']:

14 RESET_SEQ = "\033[0m"

24 RESET_SEQ = "\033[0m"

15 COLOR_SEQ = "\033[1;%dm"

25 COLOR_SEQ = "\033[1;%dm"

@@ -21,7 +31,12 @@ else:

21 BOLD_SEQ = ""

31 BOLD_SEQ = ""

22 BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = "", "", "", "", "", "", "", ""

32 BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = "", "", "", "", "", "", "", ""

34+# equivalent of 'python -u' (unbuffered stdout, stderr)

35+stdout = io.TextIOWrapper(open(sys.stdout.fileno(), 'wb', 0), write_through=True)

36+stderr = io.TextIOWrapper(open(sys.stderr.fileno(), 'wb', 0), write_through=True)

37+

39+# iterator of processors

25def iter_processor(engine_name_list):

40def iter_processor(engine_name_list):

26 if len(engine_name_list) > 0:

41 if len(engine_name_list) > 0:

27 for name in engine_name_list:

42 for name in engine_name_list:

@@ -30,38 +45,49 @@ def iter_processor(engine_name_list):

30 if processor is not None:

45 if processor is not None:

31 yield name, processor

46 yield name, processor

32 else:

47 else:

33− print(BOLD_SEQ, 'Engine ', '%-30s' % name, RESET_SEQ, RED, ' Not found ', RESET_SEQ)

48+ stdout.write(f'{BOLD_SEQ}Engine {name:30}{RESET_SEQ}{RED}Engine does not exist{RESET_SEQ}')

34 else:

49 else:

35 for name, processor in searx.search.processors.items():

50 for name, processor in searx.search.processors.items():

36 yield name, processor

51 yield name, processor

54+# actual check & display

39−def run(engine_name_list):

55+def run(engine_name_list, verbose):

40 searx.search.initialize()

56 searx.search.initialize()

41− broken_urls = []

42 for name, processor in iter_processor(engine_name_list):

57 for name, processor in iter_processor(engine_name_list):

58+ stdout.write(f'{BOLD_SEQ}Engine {name:30}{RESET_SEQ}Checking\n')

43− if sys.stdout.isatty():

59+ if not sys.stdout.isatty():

44− print(BOLD_SEQ, 'Engine ', '%-30s' % name, RESET_SEQ, WHITE, ' Checking', RESET_SEQ)

60+ stderr.write(f'{BOLD_SEQ}Engine {name:30}{RESET_SEQ}Checking\n')

45 checker = searx.search.checker.Checker(processor)

61 checker = searx.search.checker.Checker(processor)

46 checker.run()

62 checker.run()

47 if checker.test_results.succesfull:

63 if checker.test_results.succesfull:

48− print(BOLD_SEQ, 'Engine ', '%-30s' % name, RESET_SEQ, GREEN, ' OK', RESET_SEQ)

64+ stdout.write(f'{BOLD_SEQ}Engine {name:30}{RESET_SEQ}{GREEN}OK{RESET_SEQ}\n')

65+ if verbose:

66+ stdout.write(f' {"found languages":15}: {" ".join(sorted(list(checker.test_results.languages)))}\n')

49 else:

67 else:

68+ stdout.write(f'{BOLD_SEQ}Engine {name:30}{RESET_SEQ}{RESET_SEQ}{RED}Error{RESET_SEQ}')

69+ if not verbose:

50− errors = [test_name + ': ' + error for test_name, error in checker.test_results]

70+ errors = [test_name + ': ' + error for test_name, error in checker.test_results]

51− print(BOLD_SEQ, 'Engine ', '%-30s' % name, RESET_SEQ, RED, ' Error ', str(errors), RESET_SEQ)

71+ stdout.write(f'{RED}Error {str(errors)}{RESET_SEQ}\n')

52−

72+ else:

53− broken_urls += checker.test_results.broken_urls

73+ stdout.write('\n')

54−

74+ stdout.write(f' {"found languages":15}: {" ".join(sorted(list(checker.test_results.languages)))}\n')

55− for url in broken_urls:

75+ for test_name, logs in checker.test_results.logs.items():

56− print('Error fetching', url)

76+ for log in logs:

77+ stdout.write(f' {test_name:15}: {RED}{" ".join(log)}{RESET_SEQ}\n')

80+# call by setup.py

59def main():

81def main():

60 parser = argparse.ArgumentParser(description='Check searx engines.')

82 parser = argparse.ArgumentParser(description='Check searx engines.')

61 parser.add_argument('engine_name_list', metavar='engine name', type=str, nargs='*',

83 parser.add_argument('engine_name_list', metavar='engine name', type=str, nargs='*',

62 help='engines name or shortcut list. Empty for all engines.')

84 help='engines name or shortcut list. Empty for all engines.')

85+ parser.add_argument('--verbose', '-v',

86+ action='store_true', dest='verbose',

87+ help='Display details about the test results',

88+ default=False)

63 args = parser.parse_args()

89 args = parser.parse_args()

64− run(args.engine_name_list)

90+ run(args.engine_name_list, args.verbose)

67if __name__ == '__main__':

93if __name__ == '__main__':

searx/search/checker/impl.py+27−19

@@ -17,6 +17,8 @@ from searx.search.models import SearchQuery, EngineRef

17from searx.search.processors import EngineProcessor

20+logger = logger.getChild('searx.search.checker')

21+

20HTML_TAGS = [

22HTML_TAGS = [

21 'embed', 'iframe', 'object', 'param', 'picture', 'source', 'svg', 'math', 'canvas', 'noscript', 'script',

23 'embed', 'iframe', 'object', 'param', 'picture', 'source', 'svg', 'math', 'canvas', 'noscript', 'script',

22 'del', 'ins', 'area', 'audio', 'img', 'map', 'track', 'video', 'a', 'abbr', 'b', 'bdi', 'bdo', 'br', 'cite',

24 'del', 'ins', 'area', 'audio', 'img', 'map', 'track', 'video', 'a', 'abbr', 'b', 'bdi', 'bdo', 'br', 'cite',

@@ -121,20 +123,25 @@ def _search_query_diff(sq1: SearchQuery, sq2: SearchQuery)\

121

123

122class TestResults:

124class TestResults:

123

125

124− __slots__ = 'errors', 'broken_urls'

126+ __slots__ = 'errors', 'logs', 'languages'

125

127

126 def __init__(self):

128 def __init__(self):

127 self.errors: typing.Dict[str, typing.List[str]] = {}

129 self.errors: typing.Dict[str, typing.List[str]] = {}

128− self.broken_urls = []

130+ self.logs: typing.Dict[str, typing.List[typing.Any]] = {}

131+ self.languages: typing.Set[str] = set()

129

132

130− def add_error(self, test, message):

133+ def add_error(self, test, message, *args):

134+ # message to self.errors

131 errors_for_test = self.errors.setdefault(test, [])

135 errors_for_test = self.errors.setdefault(test, [])

132 if message not in errors_for_test:

136 if message not in errors_for_test:

133 errors_for_test.append(message)

137 errors_for_test.append(message)

138+ # (message, *args) to self.logs

139+ logs_for_test = self.logs.setdefault(test, [])

140+ if (message, *args) not in logs_for_test:

141+ logs_for_test.append((message, *args))

134

142

135− def add_broken_url(self, url):

143+ def add_language(self, language):

136− if url not in self.broken_urls:

144+ self.languages.add(language)

137− self.broken_urls.append(url)

138

145

139 @property

146 @property

140 def succesfull(self):

147 def succesfull(self):

@@ -167,20 +174,23 @@ class ResultContainerTests:

167 results = self.result_container.get_ordered_results()

174 results = self.result_container.get_ordered_results()

168 return [result['url'] for result in results]

175 return [result['url'] for result in results]

169

176

170− def _record_error(self, message: str) -> None:

177+ def _record_error(self, message: str, *args) -> None:

178+ sq = _search_query_to_dict(self.search_query)

179+ sqstr = ' '.join(['{}={!r}'.format(k, v) for k, v in sq.items()])

171− self.test_results.add_error(self.test_name, message)

180+ self.test_results.add_error(self.test_name, message, *args, '(' + sqstr + ')')

172

181

173 def _add_language(self, text: str) -> typing.Optional[str]:

182 def _add_language(self, text: str) -> typing.Optional[str]:

174 r = cld3.get_language(str(text)) # pylint: disable=E1101

183 r = cld3.get_language(str(text)) # pylint: disable=E1101

175− if r is not None and r.probability >= 0.9 and r.is_reliable:

184+ if r is not None and r.probability >= 0.98 and r.is_reliable:

176 self.languages.add(r.language)

185 self.languages.add(r.language)

186+ self.test_results.add_language(r.language)

177 return None

187 return None

178

188

179 def _check_result(self, result):

189 def _check_result(self, result):

180 if not _check_no_html(result.get('title', '')):

190 if not _check_no_html(result.get('title', '')):

181− self._record_error('HTML in title')

191+ self._record_error('HTML in title', repr(result.get('title', '')))

182 if not _check_no_html(result.get('content', '')):

192 if not _check_no_html(result.get('content', '')):

183− self._record_error('HTML in content')

193+ self._record_error('HTML in content', repr(result.get('content', '')))

184

194

185 self._add_language(result.get('title', ''))

195 self._add_language(result.get('title', ''))

186 self._add_language(result.get('content', ''))

196 self._add_language(result.get('content', ''))

@@ -198,13 +208,11 @@ class ResultContainerTests:

198 thumbnail_src = result.get('thumbnail_src')

208 thumbnail_src = result.get('thumbnail_src')

199 if thumbnail_src is not None:

209 if thumbnail_src is not None:

200 if not _is_url_image(thumbnail_src):

210 if not _is_url_image(thumbnail_src):

201− self.test_results.add_broken_url(thumbnail_src)

202− self._record_error('thumbnail_src URL is invalid')

211+ self._record_error('thumbnail_src URL is invalid', thumbnail_src)

203 elif not _is_url_image(result.get('img_src')):

212 elif not _is_url_image(result.get('img_src')):

204− self.test_results.add_broken_url(result.get('img_src'))

205− self._record_error('img_src URL is invalid')

213+ self._record_error('img_src URL is invalid', result.get('img_src'))

206 if template == 'videos.html' and not _is_url_image(result.get('thumbnail')):

214 if template == 'videos.html' and not _is_url_image(result.get('thumbnail')):

207− self._record_error('thumbnail URL is invalid')

215+ self._record_error('thumbnail URL is invalid', result.get('img_src'))

208

216

209 def _check_results(self, results: list):

217 def _check_results(self, results: list):

210 for result in results:

218 for result in results:

@@ -213,16 +221,16 @@ class ResultContainerTests:

213 def _check_answers(self, answers):

221 def _check_answers(self, answers):

214 for answer in answers:

222 for answer in answers:

215 if not _check_no_html(answer):

223 if not _check_no_html(answer):

216− self._record_error('HTML in answer')

224+ self._record_error('HTML in answer', answer)

217

225

218 def _check_infoboxes(self, infoboxes):

226 def _check_infoboxes(self, infoboxes):

219 for infobox in infoboxes:

227 for infobox in infoboxes:

220 if not _check_no_html(infobox.get('content', '')):

228 if not _check_no_html(infobox.get('content', '')):

221− self._record_error('HTML in infobox content')

229+ self._record_error('HTML in infobox content', infobox.get('content', ''))

222 self._add_language(infobox.get('content', ''))

230 self._add_language(infobox.get('content', ''))

223 for attribute in infobox.get('attributes', {}):

231 for attribute in infobox.get('attributes', {}):

224 if not _check_no_html(attribute.get('value', '')):

232 if not _check_no_html(attribute.get('value', '')):

225− self._record_error('HTML in infobox attribute value')

233+ self._record_error('HTML in infobox attribute value', attribute.get('value', ''))

226

234

227 def check_basic(self):

235 def check_basic(self):

228 if len(self.result_container.unresponsive_engines) > 0:

236 if len(self.result_container.unresponsive_engines) > 0: