@@ -36,14 +36,53 @@ logger = logger.getChild('search')
3636number_of_searches = 0
3737
3838
39−def search_request_wrapper(fn, url, engine_name, **kwargs):
40− ret = None
41− engine = engines[engine_name]
39+def send_http_request(engine, request_params, timeout_limit):
40+ response = None
4241 try:
43− ret = fn(url, **kwargs)
42+ # create dictionary which contain all
43+ # informations about the request
44+ request_args = dict(
45+ headers=request_params['headers'],
46+ cookies=request_params['cookies'],
47+ timeout=timeout_limit,
48+ verify=request_params['verify']
49+ )
50+ # specific type of request (GET or POST)
51+ if request_params['method'] == 'GET':
52+ req = requests_lib.get
53+ else:
54+ req = requests_lib.post
55+ request_args['data'] = request_params['data']
56+
57+ # for page_load_time stats
58+ time_before_request = time()
59+
60+ # send the request
61+ response = req(request_params['url'], **request_args)
62+
4463 with threading.RLock():
64+ # no error : reset the suspend variables
4565 engine.continuous_errors = 0
4666 engine.suspend_end_time = 0
67+ # update stats with current page-load-time
68+ # only the HTTP request
69+ engine.stats['page_load_time'] += time() - time_before_request
70+ engine.stats['page_load_count'] += 1
71+
72+ # is there a timeout (no parsing in this case)
73+ timeout_overhead = 0.2 # seconds
74+ search_duration = time() - request_params['started']
75+ if search_duration > timeout_limit + timeout_overhead:
76+ logger.exception('engine timeout on HTTP request:'
77+ '{0} (search duration : {1} ms, time-out: {2} )'
78+ .format(engine.name, search_duration, timeout_limit))
79+ with threading.RLock():
80+ engine.stats['errors'] += 1
81+ return False
82+
83+ # everything is ok : return the response
84+ return response
85+
4786 except:
4887 # increase errors stats
4988 with threading.RLock():
@@ -52,20 +91,62 @@ def search_request_wrapper(fn, url, engine_name, **kwargs):
5291 engine.suspend_end_time = time() + min(60, engine.continuous_errors)
5392
5493 # print engine name and specific error message
55− logger.exception('engine crash: {0}'.format(engine_name))
56− return ret
94+ logger.exception('engine crash: {0}'.format(engine.name))
95+ return False
96+
97+
98+def search_one_request(engine_name, query, request_params, result_container, timeout_limit):
99+ engine = engines[engine_name]
100+
101+ # update request parameters dependent on
102+ # search-engine (contained in engines folder)
103+ engine.request(query, request_params)
104+
105+ # TODO add support of offline engines
106+ if request_params['url'] is None:
107+ return False
108+
109+ # ignoring empty urls
110+ if not request_params['url']:
111+ return False
112+
113+ # send request
114+ response = send_http_request(engine, request_params, timeout_limit)
115+
116+ # parse response
117+ success = None
118+ if response:
119+ # parse the response
120+ response.search_params = request_params
121+ search_results = engine.response(response)
122+
123+ # add results
124+ for result in search_results:
125+ result['engine'] = engine.name
126+
127+ result_container.extend(engine.name, search_results)
128+
129+ success = True
130+ else:
131+ success = False
132+
133+ with threading.RLock():
134+ # update stats : total time
135+ engine.stats['engine_time'] += time() - request_params['started']
136+ engine.stats['engine_time_count'] += 1
57137
138+ #
139+ return success
58140
59−def threaded_requests(requests):
60− timeout_limit = max(r[2]['timeout'] for r in requests)
61− search_start = time()
141+
142+def search_multiple_requests(requests, result_container, timeout_limit):
143+ start_time = time()
62144 search_id = uuid4().__str__()
63− for fn, url, request_args, engine_name in requests:
64− request_args['timeout'] = timeout_limit
145+
146+ for engine_name, query, request_params in requests:
65147 th = threading.Thread(
66− target=search_request_wrapper,
67− args=(fn, url, engine_name),
68− kwargs=request_args,
148+ target=search_one_request,
149+ args=(engine_name, query, request_params, result_container, timeout_limit),
69150 name=search_id,
70151 )
71152 th._engine_name = engine_name
@@ -73,7 +154,7 @@ def threaded_requests(requests):
73154
74155 for th in threading.enumerate():
75156 if th.name == search_id:
76− remaining_time = max(0.0, timeout_limit - (time() - search_start))
157+ remaining_time = max(0.0, timeout_limit - (time() - start_time))
77158 th.join(remaining_time)
78159 if th.isAlive():
79160 logger.warning('engine timeout: {0}'.format(th._engine_name))
@@ -91,44 +172,6 @@ def default_request_params():
91172 }
92173
93174
94−# create a callback wrapper for the search engine results
95−def make_callback(engine_name, callback, params, result_container):
96−
97− # creating a callback wrapper for the search engine results
98− def process_callback(response, **kwargs):
99− # check if redirect comparing to the True value,
100− # because resp can be a Mock object, and any attribut name returns something.
101− if response.is_redirect is True:
102− logger.debug('{0} redirect on: {1}'.format(engine_name, response))
103− return
104−
105− response.search_params = params
106−
107− search_duration = time() - params['started']
108− # update stats with current page-load-time
109− with threading.RLock():
110− engines[engine_name].stats['page_load_time'] += search_duration
111−
112− timeout_overhead = 0.2 # seconds
113− timeout_limit = engines[engine_name].timeout + timeout_overhead
114−
115− if search_duration > timeout_limit:
116− with threading.RLock():
117− engines[engine_name].stats['errors'] += 1
118− return
119−
120− # callback
121− search_results = callback(response)
122−
123− # add results
124− for result in search_results:
125− result['engine'] = engine_name
126−
127− result_container.extend(engine_name, search_results)
128−
129− return process_callback
130−
131−
132175def get_search_query_from_webapp(preferences, form):
133176 query = None
134177 query_engines = []
@@ -255,6 +298,10 @@ class Search(object):
255298 def search(self):
256299 global number_of_searches
257300
301+ # start time
302+ start_time = time()
303+
304+ # answeres ?
258305 answerers_results = ask(self.search_query)
259306
260307 if answerers_results:
@@ -274,6 +321,9 @@ class Search(object):
274321
275322 search_query = self.search_query
276323
324+ # max of all selected engine timeout
325+ timeout_limit = 0
326+
277327 # start search-reqest for all selected engines
278328 for selected_engine in search_query.engines:
279329 if selected_engine['name'] not in engines:
@@ -303,7 +353,7 @@ class Search(object):
303353 request_params = default_request_params()
304354 request_params['headers']['User-Agent'] = user_agent
305355 request_params['category'] = selected_engine['category']
306− request_params['started'] = time()
356+ request_params['started'] = start_time
307357 request_params['pageno'] = search_query.pageno
308358
309359 if hasattr(engine, 'language') and engine.language:
@@ -315,52 +365,16 @@ class Search(object):
315365 request_params['safesearch'] = search_query.safesearch
316366 request_params['time_range'] = search_query.time_range
317367
318− # update request parameters dependent on
319− # search-engine (contained in engines folder)
320− engine.request(search_query.query.encode('utf-8'), request_params)
321−
322− if request_params['url'] is None:
323− # TODO add support of offline engines
324− pass
325−
326− # create a callback wrapper for the search engine results
327− callback = make_callback(
328− selected_engine['name'],
329− engine.response,
330− request_params,
331− self.result_container)
332−
333− # create dictionary which contain all
334− # informations about the request
335− request_args = dict(
336− headers=request_params['headers'],
337− hooks=dict(response=callback),
338− cookies=request_params['cookies'],
339− timeout=engine.timeout,
340− verify=request_params['verify']
341− )
342−
343− # specific type of request (GET or POST)
344− if request_params['method'] == 'GET':
345− req = requests_lib.get
346− else:
347− req = requests_lib.post
348− request_args['data'] = request_params['data']
349−
350− # ignoring empty urls
351− if not request_params['url']:
352− continue
353−
354368 # append request to list
355− requests.append((req, request_params['url'],
356− request_args,
357− selected_engine['name']))
369+ requests.append((selected_engine['name'], search_query.query.encode('utf-8'), request_params))
358370
359− if not requests:
360− return self.result_container
361− # send all search-request
362− threaded_requests(requests)
363− start_new_thread(gc.collect, tuple())
371+ # update timeout_limit
372+ timeout_limit = max(timeout_limit, engine.timeout)
373+
374+ if requests:
375+ # send all search-request
376+ search_multiple_requests(requests, self.result_container, timeout_limit - (time() - start_time))
377+ start_new_thread(gc.collect, tuple())
364378
365379 # return results, suggestions, answers and infoboxes
366380 return self.result_container