@@ -16,6 +16,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
18import grequests
18import grequests
19from itertools import izip_longest, chain
20from itertools import izip_longest, chain
20from datetime import datetime
21from datetime import datetime
21from operator import itemgetter
22from operator import itemgetter
@@ -38,17 +39,14 @@ def default_request_params():
40# create a callback wrapper for the search engine results
41# create a callback wrapper for the search engine results
41−def make_callback(engine_name, results, suggestions, callback, params):
42+def make_callback(engine_name, results, suggestions, answers, infoboxes, callback, params):
43 # creating a callback wrapper for the search engine results
44 # creating a callback wrapper for the search engine results
44 def process_callback(response, **kwargs):
45 def process_callback(response, **kwargs):
45 cb_res = []
46 cb_res = []
46 response.search_params = params
47 response.search_params = params
48− # update stats with current page-load-time
49+ # callback
49− engines[engine_name].stats['page_load_time'] += \
50− (datetime.now() - params['started']).total_seconds()
53 search_results = callback(response)
51 search_results = callback(response)
54 except Exception, e:
52 except Exception, e:
@@ -61,6 +59,7 @@ def make_callback(engine_name, results, suggestions, callback, params):
61 engine_name, str(e))
59 engine_name, str(e))
64 for result in search_results:
63 for result in search_results:
65 result['engine'] = engine_name
64 result['engine'] = engine_name
@@ -70,14 +69,37 @@ def make_callback(engine_name, results, suggestions, callback, params):
70 suggestions.add(result['suggestion'])
69 suggestions.add(result['suggestion'])
72+ # if it is an answer, add it to list of answers
73+ if 'answer' in result:
74+ answers.add(result['answer'])
77+ # if it is an infobox, add it to list of infoboxes
78+ if 'infobox' in result:
79+ infoboxes.append(result)
73 # append result
82 # append result
74 cb_res.append(result)
83 cb_res.append(result)
76 results[engine_name] = cb_res
85 results[engine_name] = cb_res
87+ # update stats with current page-load-time
88+ engines[engine_name].stats['page_load_time'] += \
89+ (datetime.now() - params['started']).total_seconds()
78 return process_callback
91 return process_callback
94+# return the meaningful length of the content for a result
95+def content_result_len(content):
96+ if isinstance(content, basestring):
97+ content = re.sub('[,;:!?\./\\\\ ()-_]', '', content)
81# score results and remove duplications
103# score results and remove duplications
82def score_results(results):
104def score_results(results):
83 # calculate scoring parameters
105 # calculate scoring parameters
@@ -99,8 +121,13 @@ def score_results(results):
99 res['host'] = res['host'].replace('www.', '', 1)
121 res['host'] = res['host'].replace('www.', '', 1)
101 res['engines'] = [res['engine']]
123 res['engines'] = [res['engine']]
102 weight = 1.0
125 weight = 1.0
127+ # strip multiple spaces and cariage returns from content
128+ if 'content' in res:
129+ res['content'] = re.sub(' +', ' ', res['content'].strip().replace('\n', ''))
104 # get weight of this engine if possible
131 # get weight of this engine if possible
105 if hasattr(engines[res['engine']], 'weight'):
132 if hasattr(engines[res['engine']], 'weight'):
106 weight = float(engines[res['engine']].weight)
133 weight = float(engines[res['engine']].weight)
@@ -108,9 +135,8 @@ def score_results(results):
108 # calculate score for that engine
135 # calculate score for that engine
109 score = int((flat_len - i) / engines_len) * weight + 1
136 score = int((flat_len - i) / engines_len) * weight + 1
113 # check for duplicates
138 # check for duplicates
114 for new_res in results:
140 for new_res in results:
115 # remove / from the end of the url if required
141 # remove / from the end of the url if required
116 p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path # noqa
142 p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path # noqa
@@ -127,7 +153,7 @@ def score_results(results):
127 # merge duplicates together
153 # merge duplicates together
128 if duplicated:
154 if duplicated:
129 # using content with more text
155 # using content with more text
130− if res.get('content') > duplicated.get('content'):
156+ if content_result_len(res.get('content', '')) > content_result_len(duplicated.get('content', '')):
131 duplicated['content'] = res['content']
157 duplicated['content'] = res['content']
133 # increase result-score
159 # increase result-score
@@ -186,6 +212,64 @@ def score_results(results):
186 return gresults
212 return gresults
215+def merge_two_infoboxes(infobox1, infobox2):
216+ if 'urls' in infobox2:
217+ urls1 = infobox1.get('urls', None)
220+ infobox1.set('urls', urls1)
223+ for url in infobox1.get('urls', []):
224+ urlSet.add(url.get('url', None))
226+ for url in infobox2.get('urls', []):
227+ if url.get('url', None) not in urlSet:
230+ if 'attributes' in infobox2:
231+ attributes1 = infobox1.get('attributes', None)
232+ if attributes1 == None:
234+ infobox1.set('attributes', attributes1)
236+ attributeSet = set()
237+ for attribute in infobox1.get('attributes', []):
238+ if attribute.get('label', None) not in attributeSet:
239+ attributeSet.add(attribute.get('label', None))
241+ for attribute in infobox2.get('attributes', []):
242+ attributes1.append(attribute)
244+ if 'content' in infobox2:
245+ content1 = infobox1.get('content', None)
246+ content2 = infobox2.get('content', '')
247+ if content1 != None:
248+ if content_result_len(content2) > content_result_len(content1):
249+ infobox1['content'] = content2
251+ infobox1.set('content', content2)
254+def merge_infoboxes(infoboxes):
257+ for infobox in infoboxes:
259+ infobox_id = infobox.get('id', None)
260+ if infobox_id != None:
261+ existingIndex = infoboxes_id.get(infobox_id, None)
262+ if existingIndex != None:
263+ merge_two_infoboxes(results[existingIndex], infobox)
267+ results.append(infobox)
268+ infoboxes_id[infobox_id] = len(results)-1
189class Search(object):
273class Search(object):
191 """Search information container"""
275 """Search information container"""
@@ -208,6 +292,8 @@ class Search(object):
209 self.results = []
293 self.results = []
210 self.suggestions = []
294 self.suggestions = []
211 self.request_data = {}
297 self.request_data = {}
213 # set specific language if set
299 # set specific language if set
@@ -293,6 +379,8 @@ class Search(object):
293 requests = []
379 requests = []
294 results = {}
380 results = {}
295 suggestions = set()
381 suggestions = set()
297 # increase number of searches
385 # increase number of searches
298 number_of_searches += 1
386 number_of_searches += 1
@@ -337,6 +425,8 @@ class Search(object):
337 selected_engine['name'],
425 selected_engine['name'],
339 suggestions,
427 suggestions,
340 engine.response,
430 engine.response,
341 request_params
431 request_params
@@ -374,11 +464,14 @@ class Search(object):
374 # score results and remove duplications
464 # score results and remove duplications
375 results = score_results(results)
465 results = score_results(results)
467+ # merge infoboxes according to their ids
468+ infoboxes = merge_infoboxes(infoboxes)
377 # update engine stats, using calculated score
470 # update engine stats, using calculated score
378 for result in results:
471 for result in results:
379 for res_engine in result['engines']:
472 for res_engine in result['engines']:
380 engines[result['engine']]\
473 engines[result['engine']]\
381 .stats['score_count'] += result['score']
474 .stats['score_count'] += result['score']
383− # return results and suggestions
476+ # return results, suggestions, answers and infoboxes
384− return results, suggestions
477+ return results, suggestions, answers, infoboxes