git.iko.soy/searx
on master help sign in
[fix] use queues to answers/suggestions/infoboxes
d93d703Adam Tauber
searx/search.py+18−22
compare with master
unifiedsplit
@@ -73,7 +73,6 @@ def make_callback(engine_name,
73
73
74 # creating a callback wrapper for the search engine results
74 # creating a callback wrapper for the search engine results
75 def process_callback(response, **kwargs):
75 def process_callback(response, **kwargs):
76− cb_res = []
77 response.search_params = params
76 response.search_params = params
78
77
79 # callback
78 # callback
@@ -82,7 +81,6 @@ def make_callback(engine_name,
82 except Exception, e:
81 except Exception, e:
83 # increase errors stats
82 # increase errors stats
84 engines[engine_name].stats['errors'] += 1
83 engines[engine_name].stats['errors'] += 1
85− results_queue.put_nowait((engine_name, cb_res))
86
84
87 # print engine name and specific error message
85 # print engine name and specific error message
88 print '[E] Error with engine "{0}":\n\t{1}'.format(
86 print '[E] Error with engine "{0}":\n\t{1}'.format(
@@ -93,26 +91,7 @@ def make_callback(engine_name,
93 for result in search_results:
91 for result in search_results:
94 result['engine'] = engine_name
92 result['engine'] = engine_name
95
93
96− # if it is a suggestion, add it to list of suggestions
97− if 'suggestion' in result:
98− # TODO type checks
99− suggestions.add(result['suggestion'])
100− continue
101−
102− # if it is an answer, add it to list of answers
103− if 'answer' in result:
104− answers.add(result['answer'])
105− continue
106−
107− # if it is an infobox, add it to list of infoboxes
108− if 'infobox' in result:
109− infoboxes.append(result)
110− continue
111−
112− # append result
113− cb_res.append(result)
114−
115− results_queue.put_nowait((engine_name, cb_res))
94+ results_queue.put_nowait((engine_name, search_results))
116
95
117 # update stats with current page-load-time
96 # update stats with current page-load-time
118 engines[engine_name].stats['page_load_time'] += \
97 engines[engine_name].stats['page_load_time'] += \
@@ -511,8 +490,25 @@ class Search(object):
511 threaded_requests(requests)
490 threaded_requests(requests)
512
491
513 results = {}
492 results = {}
493+
514 while not results_queue.empty():
494 while not results_queue.empty():
515 engine_name, engine_results = results_queue.get_nowait()
495 engine_name, engine_results = results_queue.get_nowait()
496+
497+ # TODO type checks
498+ [suggestions.add(x['suggestion'])
499+ for x in list(engine_results)
500+ if 'suggestion' in x
501+ and engine_results.remove(x) is None]
502+
503+ [answers.add(x['answer'])
504+ for x in list(engine_results)
505+ if 'answer' in x
506+ and engine_results.remove(x) is None]
507+
508+ infoboxes.extend(x for x in list(engine_results)
509+ if 'infobox' in x
510+ and engine_results.remove(x) is None)
511+
516 results[engine_name] = engine_results
512 results[engine_name] = engine_results
517
513
518 # update engine-specific stats
514 # update engine-specific stats