@@ -156,7 +156,45 @@ def extract_text(xpath_results):
156156 return ' '.join(text.split())
157157
158158
159−def normalize_url(url):
159+def normalize_url(url, base_url):
160+ """Normalize URL: add protocol, join URL with base_url, add trailing slash if there is no path
161+
162+ Args:
163+ * url (str): Relative URL
164+ * base_url (str): Base URL, it must be an absolute URL.
165+
166+ Example:
167+ >>> normalize_url('https://example.com', 'http://example.com/')
168+ 'https://example.com/'
169+ >>> normalize_url('//example.com', 'http://example.com/')
170+ 'http://example.com/'
171+ >>> normalize_url('//example.com', 'https://example.com/')
172+ 'https://example.com/'
173+ >>> normalize_url('/path?a=1', 'https://example.com')
174+ 'https://example.com/path?a=1'
175+ >>> normalize_url('', 'https://example.com')
176+ 'https://example.com/'
177+ >>> normalize_url('/test', '/path')
178+ raise Exception
179+
180+ Raises:
181+ * lxml.etree.ParserError
182+
183+ Returns:
184+ * str: normalized URL
185+ """
186+ if url.startswith('//'):
187+ # add http or https to this kind of url //example.com/
188+ parsed_search_url = urlparse(base_url)
189+ url = '{0}:{1}'.format(parsed_search_url.scheme or 'http', url)
190+ elif url.startswith('/'):
191+ # fix relative url to the search engine
192+ url = urljoin(base_url, url)
193+
194+ # fix relative urls that fall through the crack
195+ if '://' not in url:
196+ url = urljoin(base_url, url)
197+
160198 parsed_url = urlparse(url)
161199
162200 # add a / at this end of the url if there is no path
@@ -165,23 +203,15 @@ def normalize_url(url):
165203 if not parsed_url.path:
166204 url += '/'
167205
168− # FIXME : hack for yahoo
169− if parsed_url.hostname == 'search.yahoo.com'\
170− and parsed_url.path.startswith('/r'):
171− p = parsed_url.path
172− mark = p.find('/**')
173− if mark != -1:
174− return unquote(p[mark + 3:]).decode()
175−
176206 return url
177207
178208
179−def extract_url(xpath_results, search_url):
209+def extract_url(xpath_results, base_url):
180210 """Extract and normalize URL from lxml Element
181211
182212 Args:
183213 * xpath_results (Union[List[html.HtmlElement], html.HtmlElement]): lxml Element(s)
184− * search_url (str): Base URL
214+ * base_url (str): Base URL
185215
186216 Example:
187217 >>> def f(s, search_url):
@@ -210,24 +240,9 @@ def extract_url(xpath_results, search_url):
210240 """
211241 if xpath_results == []:
212242 raise Exception('Empty url resultset')
213− url = extract_text(xpath_results)
214−
215− if url.startswith('//'):
216− # add http or https to this kind of url //example.com/
217− parsed_search_url = urlparse(search_url)
218− url = '{0}:{1}'.format(parsed_search_url.scheme or 'http', url)
219− elif url.startswith('/'):
220− # fix relative url to the search engine
221− url = urljoin(search_url, url)
222−
223− # fix relative urls that fall through the crack
224− if '://' not in url:
225− url = urljoin(search_url, url)
226−
227− # normalize url
228− url = normalize_url(url)
229243
230− return url
244+ url = extract_text(xpath_results)
245+ return normalize_url(url, base_url)
231246
232247
233248def dict_subset(d, properties):