@@ -35,14 +35,53 @@ logger = logger.getChild('search')
3535number_of_searches = 0
3636
3737
38−def search_request_wrapper(fn, url, engine_name, **kwargs):
39− ret = None
40− engine = engines[engine_name]
38+def send_http_request(engine, request_params, timeout_limit):
39+ response = None
4140 try:
42− ret = fn(url, **kwargs)
41+ # create dictionary which contain all
42+ # informations about the request
43+ request_args = dict(
44+ headers=request_params['headers'],
45+ cookies=request_params['cookies'],
46+ timeout=timeout_limit,
47+ verify=request_params['verify']
48+ )
49+ # specific type of request (GET or POST)
50+ if request_params['method'] == 'GET':
51+ req = requests_lib.get
52+ else:
53+ req = requests_lib.post
54+ request_args['data'] = request_params['data']
55+
56+ # for page_load_time stats
57+ time_before_request = time()
58+
59+ # send the request
60+ response = req(request_params['url'], **request_args)
61+
4362 with threading.RLock():
63+ # no error : reset the suspend variables
4464 engine.continuous_errors = 0
4565 engine.suspend_end_time = 0
66+ # update stats with current page-load-time
67+ # only the HTTP request
68+ engine.stats['page_load_time'] += time() - time_before_request
69+ engine.stats['page_load_count'] += 1
70+
71+ # is there a timeout (no parsing in this case)
72+ timeout_overhead = 0.2 # seconds
73+ search_duration = time() - request_params['started']
74+ if search_duration > timeout_limit + timeout_overhead:
75+ logger.exception('engine timeout on HTTP request:'
76+ '{0} (search duration : {1} ms, time-out: {2} )'
77+ .format(engine.name, search_duration, timeout_limit))
78+ with threading.RLock():
79+ engine.stats['errors'] += 1
80+ return False
81+
82+ # everything is ok : return the response
83+ return response
84+
4685 except:
4786 # increase errors stats
4887 with threading.RLock():
@@ -51,20 +90,62 @@ def search_request_wrapper(fn, url, engine_name, **kwargs):
5190 engine.suspend_end_time = time() + min(60, engine.continuous_errors)
5291
5392 # print engine name and specific error message
54− logger.exception('engine crash: {0}'.format(engine_name))
55− return ret
93+ logger.exception('engine crash: {0}'.format(engine.name))
94+ return False
95+
96+
97+def search_one_request(engine_name, query, request_params, result_container, timeout_limit):
98+ engine = engines[engine_name]
99+
100+ # update request parameters dependent on
101+ # search-engine (contained in engines folder)
102+ engine.request(query, request_params)
103+
104+ # TODO add support of offline engines
105+ if request_params['url'] is None:
106+ return False
107+
108+ # ignoring empty urls
109+ if not request_params['url']:
110+ return False
111+
112+ # send request
113+ response = send_http_request(engine, request_params, timeout_limit)
114+
115+ # parse response
116+ success = None
117+ if response:
118+ # parse the response
119+ response.search_params = request_params
120+ search_results = engine.response(response)
121+
122+ # add results
123+ for result in search_results:
124+ result['engine'] = engine.name
125+
126+ result_container.extend(engine.name, search_results)
127+
128+ success = True
129+ else:
130+ success = False
56131
132+ with threading.RLock():
133+ # update stats : total time
134+ engine.stats['engine_time'] += time() - request_params['started']
135+ engine.stats['engine_time_count'] += 1
57136
58−def threaded_requests(requests):
59− timeout_limit = max(r[2]['timeout'] for r in requests)
60− search_start = time()
137+ #
138+ return success
139+
140+
141+def search_multiple_requests(requests, result_container, timeout_limit):
142+ start_time = time()
61143 search_id = uuid4().__str__()
62− for fn, url, request_args, engine_name in requests:
63− request_args['timeout'] = timeout_limit
144+
145+ for engine_name, query, request_params in requests:
64146 th = threading.Thread(
65− target=search_request_wrapper,
66− args=(fn, url, engine_name),
67− kwargs=request_args,
147+ target=search_one_request,
148+ args=(engine_name, query, request_params, result_container, timeout_limit),
68149 name=search_id,
69150 )
70151 th._engine_name = engine_name
@@ -72,7 +153,7 @@ def threaded_requests(requests):
72153
73154 for th in threading.enumerate():
74155 if th.name == search_id:
75− remaining_time = max(0.0, timeout_limit - (time() - search_start))
156+ remaining_time = max(0.0, timeout_limit - (time() - start_time))
76157 th.join(remaining_time)
77158 if th.isAlive():
78159 logger.warning('engine timeout: {0}'.format(th._engine_name))
@@ -90,44 +171,6 @@ def default_request_params():
90171 }
91172
92173
93−# create a callback wrapper for the search engine results
94−def make_callback(engine_name, callback, params, result_container):
95−
96− # creating a callback wrapper for the search engine results
97− def process_callback(response, **kwargs):
98− # check if redirect comparing to the True value,
99− # because resp can be a Mock object, and any attribut name returns something.
100− if response.is_redirect is True:
101− logger.debug('{0} redirect on: {1}'.format(engine_name, response))
102− return
103−
104− response.search_params = params
105−
106− search_duration = time() - params['started']
107− # update stats with current page-load-time
108− with threading.RLock():
109− engines[engine_name].stats['page_load_time'] += search_duration
110−
111− timeout_overhead = 0.2 # seconds
112− timeout_limit = engines[engine_name].timeout + timeout_overhead
113−
114− if search_duration > timeout_limit:
115− with threading.RLock():
116− engines[engine_name].stats['errors'] += 1
117− return
118−
119− # callback
120− search_results = callback(response)
121−
122− # add results
123− for result in search_results:
124− result['engine'] = engine_name
125−
126− result_container.extend(engine_name, search_results)
127−
128− return process_callback
129−
130−
131174def get_search_query_from_webapp(preferences, form):
132175 query = None
133176 query_engines = []
@@ -254,6 +297,9 @@ class Search(object):
254297 def search(self):
255298 global number_of_searches
256299
300+ # start time
301+ start_time = time()
302+
257303 # init vars
258304 requests = []
259305
@@ -266,6 +312,9 @@ class Search(object):
266312
267313 search_query = self.search_query
268314
315+ # max of all selected engine timeout
316+ timeout_limit = 0
317+
269318 # start search-reqest for all selected engines
270319 for selected_engine in search_query.engines:
271320 if selected_engine['name'] not in engines:
@@ -294,7 +343,7 @@ class Search(object):
294343 request_params = default_request_params()
295344 request_params['headers']['User-Agent'] = user_agent
296345 request_params['category'] = selected_engine['category']
297− request_params['started'] = time()
346+ request_params['started'] = start_time
298347 request_params['pageno'] = search_query.pageno
299348
300349 if hasattr(engine, 'language') and engine.language:
@@ -306,52 +355,16 @@ class Search(object):
306355 request_params['safesearch'] = search_query.safesearch
307356 request_params['time_range'] = search_query.time_range
308357
309− # update request parameters dependent on
310− # search-engine (contained in engines folder)
311− engine.request(search_query.query.encode('utf-8'), request_params)
312−
313− if request_params['url'] is None:
314− # TODO add support of offline engines
315− pass
316−
317− # create a callback wrapper for the search engine results
318− callback = make_callback(
319− selected_engine['name'],
320− engine.response,
321− request_params,
322− self.result_container)
323−
324− # create dictionary which contain all
325− # informations about the request
326− request_args = dict(
327− headers=request_params['headers'],
328− hooks=dict(response=callback),
329− cookies=request_params['cookies'],
330− timeout=engine.timeout,
331− verify=request_params['verify']
332− )
333−
334− # specific type of request (GET or POST)
335− if request_params['method'] == 'GET':
336− req = requests_lib.get
337− else:
338− req = requests_lib.post
339− request_args['data'] = request_params['data']
358+ # append request to list
359+ requests.append((selected_engine['name'], search_query.query.encode('utf-8'), request_params))
340360
341− # ignoring empty urls
342− if not request_params['url']:
343− continue
361+ # update timeout_limit
362+ timeout_limit = max(timeout_limit, engine.timeout)
344363
345− # append request to list
346− requests.append((req, request_params['url'],
347− request_args,
348− selected_engine['name']))
349−
350− if not requests:
351− return self.result_container
352− # send all search-request
353− threaded_requests(requests)
354− start_new_thread(gc.collect, tuple())
364+ if requests:
365+ # send all search-request
366+ search_multiple_requests(requests, self.result_container, timeout_limit - (time() - start_time))
367+ start_new_thread(gc.collect, tuple())
355368
356369 # return results, suggestions, answers and infoboxes
357370 return self.result_container