@@ -16,6 +16,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
18import grequests
18import grequests
19from itertools import izip_longest, chain
20from itertools import izip_longest, chain
20from datetime import datetime
21from datetime import datetime
21from operator import itemgetter
22from operator import itemgetter
@@ -76,6 +77,13 @@ def make_callback(engine_name, results, suggestions, callback, params):
77 return process_callback
78 return process_callback
80+# return the meaningful length of the content for a result
81+def content_result_len(result):
82+ if isinstance(result.get('content'), basestring):
83+ content = re.sub('[,;:!?\./\\\\ ()-_]', '', result.get('content'))
80# score results and remove duplications
88# score results and remove duplications
81def score_results(results):
89def score_results(results):
@@ -110,6 +118,9 @@ def score_results(results):
110 duplicated = False
118 duplicated = False
112 # check for duplicates
120 # check for duplicates
121+ if 'content' in res:
122+ res['content'] = re.sub(' +', ' ', res['content'].strip().replace('\n', ''))
113 for new_res in results:
124 for new_res in results:
114 # remove / from the end of the url if required
125 # remove / from the end of the url if required
115 p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path # noqa
126 p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path # noqa
@@ -126,7 +137,7 @@ def score_results(results):
126 # merge duplicates together
137 # merge duplicates together
127 if duplicated:
138 if duplicated:
128 # using content with more text
139 # using content with more text
129− if res.get('content') > duplicated.get('content'):
140+ if content_result_len(res) > content_result_len(duplicated):
130 duplicated['content'] = res['content']
141 duplicated['content'] = res['content']
132 # increase result-score
143 # increase result-score