@@ -21,7 +21,7 @@ import sys
2121from imp import load_source
2222from itertools import izip_longest, chain
2323from operator import itemgetter
24−from urlparse import urlparse
24+from urlparse import urlparse, unquote
2525from datetime import datetime
2626import grequests
2727from flask.ext.babel import gettext
@@ -153,7 +153,9 @@ def score_results(results):
153153 results = []
154154 # deduplication + scoring
155155 for i, res in enumerate(flat_res):
156+
156157 res['parsed_url'] = urlparse(res['url'])
158+
157159 res['host'] = res['parsed_url'].netloc
158160
159161 if res['host'].startswith('www.'):
@@ -172,7 +174,7 @@ def score_results(results):
172174 p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path # noqa
173175 p2 = new_res['parsed_url'].path[:-1] if new_res['parsed_url'].path.endswith('/') else new_res['parsed_url'].path # noqa
174176 if res['host'] == new_res['host'] and\
175− p1 == p2 and\
177+ unquote(p1) == unquote(p2) and\
176178 res['parsed_url'].query == new_res['parsed_url'].query and\
177179 res.get('template') == new_res.get('template'):
178180 duplicated = new_res
@@ -222,6 +224,10 @@ def search(query, request, selected_engines, pageno=1, lang='all'):
222224 request_params['language'] = lang
223225 request_params = engine.request(query.encode('utf-8'), request_params)
224226
227+ if request_params['url'] is None:
228+ # TODO add support of offline engines
229+ pass
230+
225231 callback = make_callback(
226232 selected_engine['name'],
227233 results,