@@ -19,7 +19,6 @@ import requests as requests_lib
1919import threading
2020import re
2121from itertools import izip_longest, chain
22−from datetime import datetime
2322from operator import itemgetter
2423from Queue import Queue
2524from time import time
@@ -39,6 +38,7 @@ def threaded_requests(requests):
3938 timeout_limit = max(r[2]['timeout'] for r in requests)
4039 search_start = time()
4140 for fn, url, request_args in requests:
41+ request_args['timeout'] = timeout_limit
4242 th = threading.Thread(
4343 target=fn,
4444 args=(url,),
@@ -63,13 +63,7 @@ def default_request_params():
6363
6464
6565# create a callback wrapper for the search engine results
66−def make_callback(engine_name,
67− results_queue,
68− suggestions,
69− answers,
70− infoboxes,
71− callback,
72− params):
66+def make_callback(engine_name, results_queue, callback, params):
7367
7468 # creating a callback wrapper for the search engine results
7569 def process_callback(response, **kwargs):
@@ -87,6 +81,14 @@ def make_callback(engine_name,
8781 engine_name, str(e))
8882 return
8983
84+ timeout_overhead = 0.2 # seconds
85+ search_duration = time() - params['started']
86+ timeout_limit = engines[engine_name].timeout + timeout_overhead
87+ if search_duration > timeout_limit:
88+ engines[engine_name].stats['page_load_time'] += timeout_limit
89+ engines[engine_name].stats['errors'] += 1
90+ return
91+
9092 # add results
9193 for result in search_results:
9294 result['engine'] = engine_name
@@ -94,8 +96,7 @@ def make_callback(engine_name,
9496 results_queue.put_nowait((engine_name, search_results))
9597
9698 # update stats with current page-load-time
97− engines[engine_name].stats['page_load_time'] += \
98− (datetime.now() - params['started']).total_seconds()
99+ engines[engine_name].stats['page_load_time'] += search_duration
99100
100101 return process_callback
101102
@@ -439,14 +440,13 @@ class Search(object):
439440 request_params = default_request_params()
440441 request_params['headers']['User-Agent'] = user_agent
441442 request_params['category'] = selected_engine['category']
442− request_params['started'] = datetime.now()
443+ request_params['started'] = time()
443444 request_params['pageno'] = self.pageno
444445 request_params['language'] = self.lang
445446
446447 # update request parameters dependent on
447448 # search-engine (contained in engines folder)
448− request_params = engine.request(self.query.encode('utf-8'),
449− request_params)
449+ engine.request(self.query.encode('utf-8'), request_params)
450450
451451 if request_params['url'] is None:
452452 # TODO add support of offline engines
@@ -456,12 +456,8 @@ class Search(object):
456456 callback = make_callback(
457457 selected_engine['name'],
458458 results_queue,
459− suggestions,
460− answers,
461− infoboxes,
462459 engine.response,
463− request_params
464− )
460+ request_params)
465461
466462 # create dictionary which contain all
467463 # informations about the request