git.iko.soy/searx
on master help sign in
[enh] suspend engines after error
1094753Adam Tauber
searx/engines/__init__.py+3−1
compare with master
unifiedsplit
@@ -40,7 +40,9 @@ engine_default_args = {'paging': False,
40 'safesearch': False,
40 'safesearch': False,
41 'timeout': settings['outgoing']['request_timeout'],
41 'timeout': settings['outgoing']['request_timeout'],
42 'shortcut': '-',
42 'shortcut': '-',
43− 'disabled': False}
43+ 'disabled': False,
44+ 'suspend_end_time': 0,
45+ 'continuous_errors': 0}
44
46
45
47
46def load_module(filename):
48def load_module(filename):
searx/search.py+14−3
@@ -34,16 +34,23 @@ number_of_searches = 0
34
34
35
35
36def search_request_wrapper(fn, url, engine_name, **kwargs):
36def search_request_wrapper(fn, url, engine_name, **kwargs):
37+ ret = None
38+ engine = engines[engine_name]
37 try:
39 try:
38− return fn(url, **kwargs)
40+ ret = fn(url, **kwargs)
41+ with threading.RLock():
42+ engine.continuous_errors = 0
43+ engine.suspend_end_time = 0
39 except:
44 except:
40 # increase errors stats
45 # increase errors stats
41 with threading.RLock():
46 with threading.RLock():
42− engines[engine_name].stats['errors'] += 1
47+ engine.stats['errors'] += 1
48+ engine.continuous_errors += 1
49+ engine.suspend_end_time = time() + min(60, engine.continuous_errors)
43
50
44 # print engine name and specific error message
51 # print engine name and specific error message
45 logger.exception('engine crash: {0}'.format(engine_name))
52 logger.exception('engine crash: {0}'.format(engine_name))
46− return
53+ return ret
47
54
48
55
49def threaded_requests(requests):
56def threaded_requests(requests):
@@ -241,6 +248,10 @@ class Search(object):
241 for engine in categories[categ]
248 for engine in categories[categ]
242 if (engine.name, categ) not in self.blocked_engines)
249 if (engine.name, categ) not in self.blocked_engines)
243
250
251+ # remove suspended engines
252+ self.engines = [e for e in self.engines
253+ if engines[e['name']].suspend_end_time <= time()]
254+
244 # do search-request
255 # do search-request
245 def search(self, request):
256 def search(self, request):
246 global number_of_searches
257 global number_of_searches