@@ -22,6 +22,7 @@ from imp import load_source
2222import grequests
2323from itertools import izip_longest, chain
2424from operator import itemgetter
25+from urlparse import urlparse
2526
2627engine_dir = dirname(realpath(__file__))
2728
@@ -87,16 +88,23 @@ def search(query, request, selected_engines):
8788 results = []
8889 # deduplication + scoring
8990 for i,res in enumerate(flat_res):
91+ res['parsed_url'] = urlparse(res['url'])
9092 score = flat_len - i
9193 duplicated = False
9294 for new_res in results:
93− if res['url'] == new_res['url']:
95+ if res['parsed_url'].netloc == new_res['parsed_url'].netloc and\
96+ res['parsed_url'].path == new_res['parsed_url'].path:
9497 duplicated = new_res
9598 break
9699 if duplicated:
97100 if len(res.get('content', '')) > len(duplicated.get('content', '')):
98101 duplicated['content'] = res['content']
99102 duplicated['score'] += score
103+ if duplicated['parsed_url'].scheme == 'https':
104+ continue
105+ elif res['parsed_url'].scheme == 'https':
106+ duplicated['parsed_url'].scheme == 'https'
107+ duplicated['url'] = duplicated['parsed_url'].geturl()
100108 else:
101109 res['score'] = score
102110 results.append(res)