@@ -10,9 +10,13 @@ from os.path import splitext, join
1010from io import open
1111from random import choice
1212from html.parser import HTMLParser
13−from lxml.etree import XPath
13+from urllib.parse import urljoin, urlparse, unquote
14+
15+from lxml import html
16+from lxml.etree import XPath, _ElementStringResult, _ElementUnicodeResult
1417from babel.core import get_global
1518
19+
1620from searx import settings
1721from searx.version import VERSION_STRING
1822from searx.languages import language_codes
@@ -106,6 +110,74 @@ def html_to_text(html):
106110 return s.get_text()
107111
108112
113+def extract_text(xpath_results):
114+ '''
115+ if xpath_results is list, extract the text from each result and concat the list
116+ if xpath_results is a xml element, extract all the text node from it
117+ ( text_content() method from lxml )
118+ if xpath_results is a string element, then it's already done
119+ '''
120+ if type(xpath_results) == list:
121+ # it's list of result : concat everything using recursive call
122+ result = ''
123+ for e in xpath_results:
124+ result = result + extract_text(e)
125+ return result.strip()
126+ elif type(xpath_results) in [_ElementStringResult, _ElementUnicodeResult]:
127+ # it's a string
128+ return ''.join(xpath_results)
129+ else:
130+ # it's a element
131+ text = html.tostring(
132+ xpath_results, encoding='unicode', method='text', with_tail=False
133+ )
134+ text = text.strip().replace('\n', ' ')
135+ return ' '.join(text.split())
136+
137+
138+def extract_url(xpath_results, search_url):
139+ if xpath_results == []:
140+ raise Exception('Empty url resultset')
141+ url = extract_text(xpath_results)
142+
143+ if url.startswith('//'):
144+ # add http or https to this kind of url //example.com/
145+ parsed_search_url = urlparse(search_url)
146+ url = '{0}:{1}'.format(parsed_search_url.scheme or 'http', url)
147+ elif url.startswith('/'):
148+ # fix relative url to the search engine
149+ url = urljoin(search_url, url)
150+
151+ # fix relative urls that fall through the crack
152+ if '://' not in url:
153+ url = urljoin(search_url, url)
154+
155+ # normalize url
156+ url = normalize_url(url)
157+
158+ return url
159+
160+
161+def normalize_url(url):
162+ parsed_url = urlparse(url)
163+
164+ # add a / at this end of the url if there is no path
165+ if not parsed_url.netloc:
166+ raise Exception('Cannot parse url')
167+ if not parsed_url.path:
168+ url += '/'
169+
170+ # FIXME : hack for yahoo
171+ if parsed_url.hostname == 'search.yahoo.com'\
172+ and parsed_url.path.startswith('/r'):
173+ p = parsed_url.path
174+ mark = p.find('/**')
175+ if mark != -1:
176+ return unquote(p[mark + 3:]).decode()
177+
178+ return url
179+
180+
109181def dict_subset(d, properties):
110182 result = {}
111183 for k in properties: