8f914a2 · searx · git.iko.soy

git.iko.soy/searx

on master help sign in

[mod] searx.utils.normalize_url: remove Yahoo hack

8f914a2Alexandre Flament6 years ago

searx/utils.py+43−28

compare with master

unifiedsplit

@@ -156,7 +156,45 @@ def extract_text(xpath_results):

156 return ' '.join(text.split())

157

158

159−def normalize_url(url):

159+def normalize_url(url, base_url):

160+ """Normalize URL: add protocol, join URL with base_url, add trailing slash if there is no path

161+

162+ Args:

163+ * url (str): Relative URL

164+ * base_url (str): Base URL, it must be an absolute URL.

165+

166+ Example:

167+ >>> normalize_url('https://example.com', 'http://example.com/')

168+ 'https://example.com/'

169+ >>> normalize_url('//example.com', 'http://example.com/')

170+ 'http://example.com/'

171+ >>> normalize_url('//example.com', 'https://example.com/')

172+ 'https://example.com/'

173+ >>> normalize_url('/path?a=1', 'https://example.com')

174+ 'https://example.com/path?a=1'

175+ >>> normalize_url('', 'https://example.com')

176+ 'https://example.com/'

177+ >>> normalize_url('/test', '/path')

178+ raise Exception

179+

180+ Raises:

181+ * lxml.etree.ParserError

182+

183+ Returns:

184+ * str: normalized URL

185+ """

186+ if url.startswith('//'):

187+ # add http or https to this kind of url //example.com/

188+ parsed_search_url = urlparse(base_url)

189+ url = '{0}:{1}'.format(parsed_search_url.scheme or 'http', url)

190+ elif url.startswith('/'):

191+ # fix relative url to the search engine

192+ url = urljoin(base_url, url)

193+

194+ # fix relative urls that fall through the crack

195+ if '://' not in url:

196+ url = urljoin(base_url, url)

197+

160 parsed_url = urlparse(url)

198 parsed_url = urlparse(url)

161

199

162 # add a / at this end of the url if there is no path

200 # add a / at this end of the url if there is no path

@@ -165,23 +203,15 @@ def normalize_url(url):

165 if not parsed_url.path:

203 if not parsed_url.path:

166 url += '/'

204 url += '/'

167

205

168− # FIXME : hack for yahoo

169− if parsed_url.hostname == 'search.yahoo.com'\

170− and parsed_url.path.startswith('/r'):

171− p = parsed_url.path

172− mark = p.find('/**')

173− if mark != -1:

174− return unquote(p[mark + 3:]).decode()

175−

176 return url

206 return url

177

207

178

208

179−def extract_url(xpath_results, search_url):

209+def extract_url(xpath_results, base_url):

180 """Extract and normalize URL from lxml Element

210 """Extract and normalize URL from lxml Element

181

211

182 Args:

212 Args:

183 * xpath_results (Union[List[html.HtmlElement], html.HtmlElement]): lxml Element(s)

213 * xpath_results (Union[List[html.HtmlElement], html.HtmlElement]): lxml Element(s)

184− * search_url (str): Base URL

214+ * base_url (str): Base URL

185

215

186 Example:

216 Example:

187 >>> def f(s, search_url):

217 >>> def f(s, search_url):

@@ -210,24 +240,9 @@ def extract_url(xpath_results, search_url):

210 """

240 """

211 if xpath_results == []:

241 if xpath_results == []:

212 raise Exception('Empty url resultset')

242 raise Exception('Empty url resultset')

213− url = extract_text(xpath_results)

214−

215− if url.startswith('//'):

216− # add http or https to this kind of url //example.com/

217− parsed_search_url = urlparse(search_url)

218− url = '{0}:{1}'.format(parsed_search_url.scheme or 'http', url)

219− elif url.startswith('/'):

220− # fix relative url to the search engine

221− url = urljoin(search_url, url)

222−

223− # fix relative urls that fall through the crack

224− if '://' not in url:

225− url = urljoin(search_url, url)

226−

227− # normalize url

228− url = normalize_url(url)

229

243

244+ url = extract_text(xpath_results)

230− return url

245+ return normalize_url(url, base_url)

231

246

232

247

233def dict_subset(d, properties):

248def dict_subset(d, properties):