@@ -77,7 +77,7 @@ def send_http_request(engine, request_params):
77 return req(request_params['url'], **request_args)
77 return req(request_params['url'], **request_args)
80−def search_one_request(engine, query, request_params):
80+def search_one_http_request(engine, query, request_params):
81 # update request parameters dependent on
81 # update request parameters dependent on
82 # search-engine (contained in engines folder)
82 # search-engine (contained in engines folder)
83 engine.request(query, request_params)
83 engine.request(query, request_params)
@@ -97,7 +97,53 @@ def search_one_request(engine, query, request_params):
97 return engine.response(response)
97 return engine.response(response)
100+def search_one_offline_request(engine, query, request_params):
101+ return engine.search(query, request_params)
100def search_one_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit):
104def search_one_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit):
105+ if engines[engine_name].offline:
106+ return search_one_offline_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit)
107+ return search_one_http_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit)
110+def search_one_offline_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit):
111+ engine = engines[engine_name]
114+ search_results = search_one_offline_request(engine, query, request_params)
117+ result_container.extend(engine_name, search_results)
119+ engine_time = time() - start_time
120+ result_container.add_timing(engine_name, engine_time, engine_time)
121+ with threading.RLock():
122+ engine.stats['engine_time'] += engine_time
123+ engine.stats['engine_time_count'] += 1
125+ except ValueError as e:
126+ record_offline_engine_stats_on_error(engine, result_container, start_time)
127+ logger.exception('engine {0} : invalid input : {1}'.format(engine_name, e))
128+ except Exception as e:
129+ record_offline_engine_stats_on_error(engine, result_container, start_time)
131+ result_container.add_unresponsive_engine((
133+ u'{0}: {1}'.format(gettext('unexpected crash'), e),
135+ logger.exception('engine {0} : exception : {1}'.format(engine_name, e))
138+def record_offline_engine_stats_on_error(engine, result_container, start_time):
139+ engine_time = time() - start_time
140+ result_container.add_timing(engine.name, engine_time, engine_time)
142+ with threading.RLock():
143+ engine.stats['errors'] += 1
146+def search_one_http_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit):
101 # set timeout for all HTTP requests
147 # set timeout for all HTTP requests
102 requests_lib.set_timeout_for_thread(timeout_limit, start_time=start_time)
148 requests_lib.set_timeout_for_thread(timeout_limit, start_time=start_time)
103 # reset the HTTP total time
149 # reset the HTTP total time
@@ -111,7 +157,7 @@ def search_one_request_safe(engine_name, query, request_params, result_container
113 # send requests and parse the results
159 # send requests and parse the results
114− search_results = search_one_request(engine, query, request_params)
160+ search_results = search_one_http_request(engine, query, request_params)
116 # check if the engine accepted the request
162 # check if the engine accepted the request
117 if search_results is not None:
163 if search_results is not None:
@@ -427,19 +473,21 @@ class Search(object):
429 # set default request parameters
475 # set default request parameters
477+ if not engine.offline:
430− request_params = default_request_params()
478+ request_params = default_request_params()
431− request_params['headers']['User-Agent'] = user_agent
479+ request_params['headers']['User-Agent'] = user_agent
432− request_params['category'] = selected_engine['category']
433− request_params['pageno'] = search_query.pageno
435− if hasattr(engine, 'language') and engine.language:
481+ if hasattr(engine, 'language') and engine.language:
436− request_params['language'] = engine.language
482+ request_params['language'] = engine.language
438− request_params['language'] = search_query.lang
484+ request_params['language'] = search_query.lang
440− # 0 = None, 1 = Moderate, 2 = Strict
486+ request_params['safesearch'] = search_query.safesearch
441− request_params['safesearch'] = search_query.safesearch
487+ request_params['time_range'] = search_query.time_range
489+ request_params['category'] = selected_engine['category']
442− request_params['time_range'] = search_query.time_range
490+ request_params['pageno'] = search_query.pageno
444 # append request to list
492 # append request to list
445 requests.append((selected_engine['name'], search_query.query, request_params))
493 requests.append((selected_engine['name'], search_query.query, request_params))