@@ -21,6 +21,8 @@ import re
21from itertools import izip_longest, chain
21from itertools import izip_longest, chain
22from datetime import datetime
22from datetime import datetime
23from operator import itemgetter
23from operator import itemgetter
24+from Queue import Queue
24from urlparse import urlparse, unquote
26from urlparse import urlparse, unquote
25from searx.engines import (
27from searx.engines import (
26 categories, engines
28 categories, engines
@@ -34,6 +36,8 @@ number_of_searches = 0
36def threaded_requests(requests):
38def threaded_requests(requests):
39+ timeout_limit = max(r[2]['timeout'] for r in requests)
40+ search_start = time()
37 for fn, url, request_args in requests:
41 for fn, url, request_args in requests:
38 th = threading.Thread(
42 th = threading.Thread(
39 target=fn,
43 target=fn,
@@ -45,7 +49,11 @@ def threaded_requests(requests):
46 for th in threading.enumerate():
50 for th in threading.enumerate():
47 if th.name == 'search_request':
51 if th.name == 'search_request':
52+ remaining_time = max(0.0, timeout_limit - (time() - search_start))
48− th.join()
53+ th.join(remaining_time)
55+ print('engine timeout')
51# get default reqest parameter
59# get default reqest parameter
@@ -56,7 +64,7 @@ def default_request_params():
57# create a callback wrapper for the search engine results
65# create a callback wrapper for the search engine results
58def make_callback(engine_name,
66def make_callback(engine_name,
59− results,
67+ results_queue,
60 suggestions,
68 suggestions,
62 infoboxes,
70 infoboxes,
@@ -74,7 +82,7 @@ def make_callback(engine_name,
74 except Exception, e:
82 except Exception, e:
75 # increase errors stats
83 # increase errors stats
76 engines[engine_name].stats['errors'] += 1
84 engines[engine_name].stats['errors'] += 1
77− results[engine_name] = cb_res
85+ results_queue.put_nowait((engine_name, cb_res))
79 # print engine name and specific error message
87 # print engine name and specific error message
80 print '[E] Error with engine "{0}":\n\t{1}'.format(
88 print '[E] Error with engine "{0}":\n\t{1}'.format(
@@ -104,7 +112,7 @@ def make_callback(engine_name,
104 # append result
112 # append result
105 cb_res.append(result)
113 cb_res.append(result)
107− results[engine_name] = cb_res
115+ results_queue.put_nowait((engine_name, cb_res))
109 # update stats with current page-load-time
117 # update stats with current page-load-time
110 engines[engine_name].stats['page_load_time'] += \
118 engines[engine_name].stats['page_load_time'] += \
@@ -420,7 +428,7 @@ class Search(object):
421 # init vars
429 # init vars
422 requests = []
430 requests = []
423− results = {}
431+ results_queue = Queue()
424 suggestions = set()
432 suggestions = set()
425 answers = set()
433 answers = set()
426 infoboxes = []
434 infoboxes = []
@@ -468,7 +476,7 @@ class Search(object):
468 # create a callback wrapper for the search engine results
476 # create a callback wrapper for the search engine results
469 callback = make_callback(
477 callback = make_callback(
470 selected_engine['name'],
478 selected_engine['name'],
471− results,
479+ results_queue,
472 suggestions,
480 suggestions,
474 infoboxes,
482 infoboxes,
@@ -502,6 +510,11 @@ class Search(object):
502 # send all search-request
510 # send all search-request
503 threaded_requests(requests)
511 threaded_requests(requests)
514+ while not results_queue.empty():
515+ engine_name, engine_results = results_queue.get_nowait()
516+ results[engine_name] = engine_results
505 # update engine-specific stats
518 # update engine-specific stats
506 for engine_name, engine_results in results.items():
519 for engine_name, engine_results in results.items():
507 engines[engine_name].stats['search_count'] += 1
520 engines[engine_name].stats['search_count'] += 1