@@ -16,6 +16,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
1616'''
1717
1818import grequests
19+import re
1920from itertools import izip_longest, chain
2021from datetime import datetime
2122from operator import itemgetter
@@ -76,6 +77,13 @@ def make_callback(engine_name, results, suggestions, callback, params):
7677
7778 return process_callback
7879
80+# return the meaningful length of the content for a result
81+def content_result_len(result):
82+ if isinstance(result.get('content'), basestring):
83+ content = re.sub('[,;:!?\./\\\\ ()-_]', '', result.get('content'))
84+ return len(content)
85+ else:
86+ return 0
7987
8088# score results and remove duplications
8189def score_results(results):
@@ -110,6 +118,9 @@ def score_results(results):
110118 duplicated = False
111119
112120 # check for duplicates
121+ if 'content' in res:
122+ res['content'] = re.sub(' +', ' ', res['content'].strip().replace('\n', ''))
123+
113124 for new_res in results:
114125 # remove / from the end of the url if required
115126 p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path # noqa
@@ -126,7 +137,7 @@ def score_results(results):
126137 # merge duplicates together
127138 if duplicated:
128139 # using content with more text
129− if res.get('content') > duplicated.get('content'):
140+ if content_result_len(res) > content_result_len(duplicated):
130141 duplicated['content'] = res['content']
131142
132143 # increase result-score