on master help sign in
[enh] engine exception handling ++ stats
commite13fe033ede605e2d41443e1e6732b93be681558 authorasciimoo <asciimoo@gmail.com> dateSun 27 Oct 2013, 20:45 (13 years ago) parentca08003
1 file changed+20−3
searx/engines/__init__.py+20−3
searx/engines/__init__.py+20−3
@@ -69,7 +69,7 @@ for section in engines_config.sections():
6969 print '[E] Engine config error: Missing attribute "{0}.{1}"'.format(engine.name, engine_attr)
7070 sys.exit(1)
7171 engines[engine.name] = engine
72− engine.stats = {'result_count': 0, 'search_count': 0, 'page_load_time': 0, 'score_count': 0}
72+ engine.stats = {'result_count': 0, 'search_count': 0, 'page_load_time': 0, 'score_count': 0, 'errors': 0}
7373 if hasattr(engine, 'categories'):
7474 for category_name in engine.categories:
7575 categories.setdefault(category_name, []).append(engine)
@@ -84,7 +84,14 @@ def make_callback(engine_name, results, callback, params):
8484 cb_res = []
8585 response.search_params = params
8686 engines[engine_name].stats['page_load_time'] += (datetime.now() - params['started']).total_seconds()
87− for result in callback(response):
87+ try:
88+ search_results = callback(response)
89+ except Exception, e:
90+ engines[engine_name].stats['errors'] += 1
91+ results[engine_name] = cb_res
92+ print '[E] Error with engine "{0}":\n\t{1}'.format(engine_name, str(e))
93+ return
94+ for result in search_results:
8895 result['engine'] = engine_name
8996 cb_res.append(result)
9097 results[engine_name] = cb_res
@@ -169,8 +176,9 @@ def get_engines_stats():
169176 pageloads = []
170177 results = []
171178 scores = []
179+ errors = []
172180
173− max_pageload = max_results = max_score = 0
181+ max_pageload = max_results = max_score = max_errors = 0
174182 for engine in engines.values():
175183 if engine.stats['search_count'] == 0:
176184 continue
@@ -183,9 +191,11 @@ def get_engines_stats():
183191 max_results = max(results_num, max_results)
184192 max_pageload = max(load_times, max_pageload)
185193 max_score = max(score, max_score)
194+ max_errors = max(max_errors, engine.stats['errors'])
186195 pageloads.append({'avg': load_times, 'name': engine.name})
187196 results.append({'avg': results_num, 'name': engine.name})
188197 scores.append({'avg': score, 'name': engine.name})
198+ errors.append({'avg': engine.stats['errors'], 'name': engine.name})
189199
190200 for engine in pageloads:
191201 engine['percentage'] = int(engine['avg']/max_pageload*100)
@@ -196,8 +206,15 @@ def get_engines_stats():
196206 for engine in scores:
197207 engine['percentage'] = int(engine['avg']/max_score*100)
198208
209+ for engine in errors:
210+ if max_errors:
211+ engine['percentage'] = int(engine['avg']/max_errors*100)
212+ else:
213+ engine['percentage'] = 0
214+
199215
200216 return [('Page loads (sec)', sorted(pageloads, key=itemgetter('avg')))
201217 ,('Number of results', sorted(results, key=itemgetter('avg'), reverse=True))
202218 ,('Scores', sorted(scores, key=itemgetter('avg'), reverse=True))
219+ ,('Errors', sorted(errors, key=itemgetter('avg'), reverse=True))
203220 ]