git.iko.soy/searx
on master help sign in
[mod] better search request exception handling
1c96918Adam Tauber
searx/search.py+16−12
compare with master
unifiedsplit
@@ -34,14 +34,27 @@ from searx.query import Query
34number_of_searches = 0
34number_of_searches = 0
35
35
36
36
37+def search_request_wrapper(fn, url, engine_name, **kwargs):
38+ try:
39+ return fn(url, **kwargs)
40+ except Exception, e:
41+ # increase errors stats
42+ engines[engine_name].stats['errors'] += 1
43+
44+ # print engine name and specific error message
45+ print('[E] Error with engine "{0}":\n\t{1}'.format(
46+ engine_name, str(e)))
47+ return
48+
49+
37def threaded_requests(requests):
50def threaded_requests(requests):
38 timeout_limit = max(r[2]['timeout'] for r in requests)
51 timeout_limit = max(r[2]['timeout'] for r in requests)
39 search_start = time()
52 search_start = time()
40 for fn, url, request_args, engine_name in requests:
53 for fn, url, request_args, engine_name in requests:
41 request_args['timeout'] = timeout_limit
54 request_args['timeout'] = timeout_limit
42 th = threading.Thread(
55 th = threading.Thread(
43− target=fn,
56+ target=search_request_wrapper,
44− args=(url,),
57+ args=(fn, url, engine_name),
45 kwargs=request_args,
58 kwargs=request_args,
46 name='search_request',
59 name='search_request',
47 )
60 )
@@ -79,16 +92,7 @@ def make_callback(engine_name, results_queue, callback, params):
79 return
92 return
80
93
81 # callback
94 # callback
82− try:
83− search_results = callback(response)
95+ search_results = callback(response)
84− except Exception, e:
85− # increase errors stats
86− engines[engine_name].stats['errors'] += 1
87−
88− # print engine name and specific error message
89− print '[E] Error with engine "{0}":\n\t{1}'.format(
90− engine_name, str(e))
91− return
92
96
93 # add results
97 # add results
94 for result in search_results:
98 for result in search_results: