on master help sign in
[fix] www. domain duplications
commitb226e6462b1a8fa18ee670f3f2738145426b6f41 authorAdam Tauber <asciimoo@gmail.com> dateTue 20 May 2014, 01:16 (12 years ago) parent78d42f0
1 file changed+9−1
searx/engines/__init__.py+9−1
searx/engines/__init__.py+9−1
@@ -154,16 +154,24 @@ def score_results(results):
154154 # deduplication + scoring
155155 for i, res in enumerate(flat_res):
156156 res['parsed_url'] = urlparse(res['url'])
157+ res['host'] = res['parsed_url'].netloc
158+
159+ if res['host'].startswith('www.'):
160+ res['host'] = res['host'].replace('www.', '', 1)
161+
157162 res['engines'] = [res['engine']]
158163 weight = 1.0
164+
159165 if hasattr(engines[res['engine']], 'weight'):
160166 weight = float(engines[res['engine']].weight)
167+
161168 score = int((flat_len - i) / engines_len) * weight + 1
162169 duplicated = False
170+
163171 for new_res in results:
164172 p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path # noqa
165173 p2 = new_res['parsed_url'].path[:-1] if new_res['parsed_url'].path.endswith('/') else new_res['parsed_url'].path # noqa
166− if res['parsed_url'].netloc == new_res['parsed_url'].netloc and\
174+ if res['host'] == new_res['host'] and\
167175 p1 == p2 and\
168176 res['parsed_url'].query == new_res['parsed_url'].query and\
169177 res.get('template') == new_res.get('template'):