git.iko.soy/searx
on master help sign in
Merge pull request #798 from dalf/searchpy4
eaa0fb8Adam Tauber
searx/search.py+90−81
compare with master
unifiedsplit
@@ -20,6 +20,7 @@ import threading
20from thread import start_new_thread
20from thread import start_new_thread
21from time import time
21from time import time
22from uuid import uuid4
22from uuid import uuid4
23+import requests.exceptions
23import searx.poolrequests as requests_lib
24import searx.poolrequests as requests_lib
24from searx.engines import (
25from searx.engines import (
25 categories, engines
26 categories, engines
@@ -37,109 +38,117 @@ number_of_searches = 0
37
38
38
39
39def send_http_request(engine, request_params, timeout_limit):
40def send_http_request(engine, request_params, timeout_limit):
40− response = None
41+ # for page_load_time stats
41− try:
42+ time_before_request = time()
43+
42− # create dictionary which contain all
44+ # create dictionary which contain all
43− # informations about the request
45+ # informations about the request
44− request_args = dict(
46+ request_args = dict(
45− headers=request_params['headers'],
47+ headers=request_params['headers'],
46− cookies=request_params['cookies'],
48+ cookies=request_params['cookies'],
47− timeout=timeout_limit,
49+ timeout=timeout_limit,
48− verify=request_params['verify']
50+ verify=request_params['verify']
49− )
51+ )
52+
50− # specific type of request (GET or POST)
53+ # specific type of request (GET or POST)
51− if request_params['method'] == 'GET':
54+ if request_params['method'] == 'GET':
52− req = requests_lib.get
55+ req = requests_lib.get
53− else:
56+ else:
54− req = requests_lib.post
57+ req = requests_lib.post
55− request_args['data'] = request_params['data']
58+ request_args['data'] = request_params['data']
56−
57− # for page_load_time stats
58− time_before_request = time()
59−
60− # send the request
61− response = req(request_params['url'], **request_args)
62−
63− with threading.RLock():
64− # no error : reset the suspend variables
65− engine.continuous_errors = 0
66− engine.suspend_end_time = 0
67− # update stats with current page-load-time
68− # only the HTTP request
69− engine.stats['page_load_time'] += time() - time_before_request
70− engine.stats['page_load_count'] += 1
71−
72− # is there a timeout (no parsing in this case)
73− timeout_overhead = 0.2 # seconds
74− search_duration = time() - request_params['started']
75− if search_duration > timeout_limit + timeout_overhead:
76− logger.exception('engine timeout on HTTP request:'
77− '{0} (search duration : {1} ms, time-out: {2} )'
78− .format(engine.name, search_duration, timeout_limit))
79− with threading.RLock():
80− engine.stats['errors'] += 1
81− return False
82
59
83− # everything is ok : return the response
60+ # send the request
84− return response
61+ response = req(request_params['url'], **request_args)
85
62
86− except:
63+ # is there a timeout (no parsing in this case)
87− # increase errors stats
64+ timeout_overhead = 0.2 # seconds
88− with threading.RLock():
65+ search_duration = time() - request_params['started']
89− engine.stats['errors'] += 1
66+ if search_duration > timeout_limit + timeout_overhead:
90− engine.continuous_errors += 1
67+ raise Timeout(response=response)
91− engine.suspend_end_time = time() + min(60, engine.continuous_errors)
92
68
93− # print engine name and specific error message
69+ with threading.RLock():
94− logger.exception('engine crash: {0}'.format(engine.name))
70+ # no error : reset the suspend variables
95− return False
71+ engine.continuous_errors = 0
72+ engine.suspend_end_time = 0
73+ # update stats with current page-load-time
74+ # only the HTTP request
75+ engine.stats['page_load_time'] += time() - time_before_request
76+ engine.stats['page_load_count'] += 1
96
77
78+ # everything is ok : return the response
79+ return response
97
80
98−def search_one_request(engine_name, query, request_params, result_container, timeout_limit):
99− engine = engines[engine_name]
100
81
82+def search_one_request(engine, query, request_params, timeout_limit):
101 # update request parameters dependent on
83 # update request parameters dependent on
102 # search-engine (contained in engines folder)
84 # search-engine (contained in engines folder)
103 engine.request(query, request_params)
85 engine.request(query, request_params)
104
86
105− # TODO add support of offline engines
87+ # ignoring empty urls
106 if request_params['url'] is None:
88 if request_params['url'] is None:
107− return False
89+ return []
108
90
109− # ignoring empty urls
110 if not request_params['url']:
91 if not request_params['url']:
111− return False
92+ return []
112
93
113 # send request
94 # send request
114 response = send_http_request(engine, request_params, timeout_limit)
95 response = send_http_request(engine, request_params, timeout_limit)
115
96
116− # parse response
117− success = None
118− if response:
119− # parse the response
97+ # parse the response
120− response.search_params = request_params
98+ response.search_params = request_params
121− try:
122− search_results = engine.response(response)
99+ return engine.response(response)
123− except:
100+
124− logger.exception('engine crash: {0}'.format(engine.name))
101+
125− search_results = []
102+def search_one_request_safe(engine_name, query, request_params, result_container, timeout_limit):
103+ start_time = time()
104+ engine = engines[engine_name]
105+
106+ try:
107+ # send requests and parse the results
108+ search_results = search_one_request(engine, query, request_params, timeout_limit)
126
109
127 # add results
110 # add results
128 for result in search_results:
111 for result in search_results:
129− result['engine'] = engine.name
112+ result['engine'] = engine_name
113+ result_container.extend(engine_name, search_results)
130
114
131− result_container.extend(engine.name, search_results)
115+ # update engine time when there is no exception
132−
116+ with threading.RLock():
133− success = True
117+ engine.stats['engine_time'] += time() - start_time
118+ engine.stats['engine_time_count'] += 1
119+
120+ return True
121+
122+ except Exception as e:
123+ engine.stats['errors'] += 1
124+
125+ search_duration = time() - start_time
126+ requests_exception = False
127+
128+ if (issubclass(e.__class__, requests.exceptions.Timeout)):
129+ # requests timeout (connect or read)
130+ logger.error("engine {0} : HTTP requests timeout"
131+ "(search duration : {1} s, timeout: {2} s) : {3}"
132+ .format(engine_name, search_duration, timeout_limit, e.__class__.__name__))
133+ requests_exception = True
134+ if (issubclass(e.__class__, requests.exceptions.RequestException)):
135+ # other requests exception
136+ logger.exception("engine {0} : requests exception"
137+ "(search duration : {1} s, timeout: {2} s) : {3}"
138+ .format(engine_name, search_duration, timeout_limit, e))
139+ requests_exception = True
134− else:
140+ else:
135− success = False
141+ # others errors
142+ logger.exception('engine {0} : exception : {1}'.format(engine_name, e))
136
143
144+ # update continuous_errors / suspend_end_time
145+ if requests_exception:
137− with threading.RLock():
146+ with threading.RLock():
138− # update stats : total time
147+ engine.continuous_errors += 1
139− engine.stats['engine_time'] += time() - request_params['started']
148+ engine.suspend_end_time = time() + min(60, engine.continuous_errors)
140− engine.stats['engine_time_count'] += 1
141
149
150+ #
142− return success
151+ return False
143
152
144
153
145def search_multiple_requests(requests, result_container, timeout_limit):
154def search_multiple_requests(requests, result_container, timeout_limit):
@@ -148,7 +157,7 @@ def search_multiple_requests(requests, result_container, timeout_limit):
148
157
149 for engine_name, query, request_params in requests:
158 for engine_name, query, request_params in requests:
150 th = threading.Thread(
159 th = threading.Thread(
151− target=search_one_request,
160+ target=search_one_request_safe,
152 args=(engine_name, query, request_params, result_container, timeout_limit),
161 args=(engine_name, query, request_params, result_container, timeout_limit),
153 name=search_id,
162 name=search_id,
154 )
163 )