git.iko.soy/searx
on master help sign in
[mod] timeout in log are readable (the timeouts are compare to the start_time of the request).
28d51fdAlexandre Flament
searx/search.py+11−13
compare with master
unifiedsplit
@@ -37,7 +37,7 @@ logger = logger.getChild('search')
37number_of_searches = 0
37number_of_searches = 0
38
38
39
39
40−def send_http_request(engine, request_params, timeout_limit):
40+def send_http_request(engine, request_params, start_time, timeout_limit):
41 # for page_load_time stats
41 # for page_load_time stats
42 time_before_request = time()
42 time_before_request = time()
43
43
@@ -62,7 +62,8 @@ def send_http_request(engine, request_params, timeout_limit):
62
62
63 # is there a timeout (no parsing in this case)
63 # is there a timeout (no parsing in this case)
64 timeout_overhead = 0.2 # seconds
64 timeout_overhead = 0.2 # seconds
65− search_duration = time() - request_params['started']
65+ time_after_request = time()
66+ search_duration = time_after_request - start_time
66 if search_duration > timeout_limit + timeout_overhead:
67 if search_duration > timeout_limit + timeout_overhead:
67 raise Timeout(response=response)
68 raise Timeout(response=response)
68
69
@@ -72,14 +73,14 @@ def send_http_request(engine, request_params, timeout_limit):
72 engine.suspend_end_time = 0
73 engine.suspend_end_time = 0
73 # update stats with current page-load-time
74 # update stats with current page-load-time
74 # only the HTTP request
75 # only the HTTP request
75− engine.stats['page_load_time'] += time() - time_before_request
76+ engine.stats['page_load_time'] += time_after_request - time_before_request
76 engine.stats['page_load_count'] += 1
77 engine.stats['page_load_count'] += 1
77
78
78 # everything is ok : return the response
79 # everything is ok : return the response
79 return response
80 return response
80
81
81
82
82−def search_one_request(engine, query, request_params, timeout_limit):
83+def search_one_request(engine, query, request_params, start_time, timeout_limit):
83 # update request parameters dependent on
84 # update request parameters dependent on
84 # search-engine (contained in engines folder)
85 # search-engine (contained in engines folder)
85 engine.request(query, request_params)
86 engine.request(query, request_params)
@@ -92,20 +93,19 @@ def search_one_request(engine, query, request_params, timeout_limit):
92 return []
93 return []
93
94
94 # send request
95 # send request
95− response = send_http_request(engine, request_params, timeout_limit)
96+ response = send_http_request(engine, request_params, start_time, timeout_limit)
96
97
97 # parse the response
98 # parse the response
98 response.search_params = request_params
99 response.search_params = request_params
99 return engine.response(response)
100 return engine.response(response)
100
101
101
102
102−def search_one_request_safe(engine_name, query, request_params, result_container, timeout_limit):
103+def search_one_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit):
103− start_time = time()
104 engine = engines[engine_name]
104 engine = engines[engine_name]
105
105
106 try:
106 try:
107 # send requests and parse the results
107 # send requests and parse the results
108− search_results = search_one_request(engine, query, request_params, timeout_limit)
108+ search_results = search_one_request(engine, query, request_params, start_time, timeout_limit)
109
109
110 # add results
110 # add results
111 result_container.extend(engine_name, search_results)
111 result_container.extend(engine_name, search_results)
@@ -149,14 +149,13 @@ def search_one_request_safe(engine_name, query, request_params, result_container
149 return False
149 return False
150
150
151
151
152−def search_multiple_requests(requests, result_container, timeout_limit):
152+def search_multiple_requests(requests, result_container, start_time, timeout_limit):
153− start_time = time()
154 search_id = uuid4().__str__()
153 search_id = uuid4().__str__()
155
154
156 for engine_name, query, request_params in requests:
155 for engine_name, query, request_params in requests:
157 th = threading.Thread(
156 th = threading.Thread(
158 target=search_one_request_safe,
157 target=search_one_request_safe,
159− args=(engine_name, query, request_params, result_container, timeout_limit),
158+ args=(engine_name, query, request_params, result_container, start_time, timeout_limit),
160 name=search_id,
159 name=search_id,
161 )
160 )
162 th._engine_name = engine_name
161 th._engine_name = engine_name
@@ -366,7 +365,6 @@ class Search(object):
366 request_params = default_request_params()
365 request_params = default_request_params()
367 request_params['headers']['User-Agent'] = user_agent
366 request_params['headers']['User-Agent'] = user_agent
368 request_params['category'] = selected_engine['category']
367 request_params['category'] = selected_engine['category']
369− request_params['started'] = start_time
370 request_params['pageno'] = search_query.pageno
368 request_params['pageno'] = search_query.pageno
371
369
372 if hasattr(engine, 'language') and engine.language:
370 if hasattr(engine, 'language') and engine.language:
@@ -386,7 +384,7 @@ class Search(object):
386
384
387 if requests:
385 if requests:
388 # send all search-request
386 # send all search-request
389− search_multiple_requests(requests, self.result_container, timeout_limit - (time() - start_time))
387+ search_multiple_requests(requests, self.result_container, start_time, timeout_limit)
390 start_new_thread(gc.collect, tuple())
388 start_new_thread(gc.collect, tuple())
391
389
392 # return results, suggestions, answers and infoboxes
390 # return results, suggestions, answers and infoboxes