@@ -10,9 +10,13 @@ from os.path import splitext, join
1010from io import open
1111from random import choice
1212from html.parser import HTMLParser
13−from lxml.etree import XPath
13+from urllib.parse import urljoin, urlparse, unquote
14+
15+from lxml import html
16+from lxml.etree import XPath, _ElementStringResult, _ElementUnicodeResult
1417from babel.core import get_global
1518
19+
1620from searx import settings
1721from searx.version import VERSION_STRING
1822from searx.languages import language_codes
@@ -35,12 +39,17 @@ lang_to_lc_cache = dict()
3539
3640
3741def searx_useragent():
42+ """Return the searx User Agent"""
3843 return 'searx/{searx_version} {suffix}'.format(
3944 searx_version=VERSION_STRING,
4045 suffix=settings['outgoing'].get('useragent_suffix', ''))
4146
4247
4348def gen_useragent(os=None):
49+ """Return a random browser User Agent
50+
51+ See searx/data/useragents.json
52+ """
4453 return str(useragents['ua'].format(os=os or choice(useragents['os']), version=choice(useragents['versions'])))
4554
4655
@@ -95,18 +104,156 @@ class HTMLTextExtractor(HTMLParser):
95104 return ''.join(self.result).strip()
96105
97106
98−def html_to_text(html):
99− html = html.replace('\n', ' ')
100− html = ' '.join(html.split())
107+def html_to_text(html_str):
108+ """Extract text from a HTML string
109+
110+ Args:
111+ * html_str (str): string HTML
112+
113+ Returns:
114+ * str: extracted text
115+
116+ Examples:
117+ >>> html_to_text('Example <span id="42">#2</span>')
118+ 'Example #2'
119+
120+ >>> html_to_text('<style>.span { color: red; }</style><span>Example</span>')
121+ 'Example'
122+ """
123+ html_str = html_str.replace('\n', ' ')
124+ html_str = ' '.join(html_str.split())
101125 s = HTMLTextExtractor()
102126 try:
103− s.feed(html)
127+ s.feed(html_str)
104128 except HTMLTextExtractorException:
105− logger.debug("HTMLTextExtractor: invalid HTML\n%s", html)
129+ logger.debug("HTMLTextExtractor: invalid HTML\n%s", html_str)
106130 return s.get_text()
107131
108132
133+def extract_text(xpath_results):
134+ """Extract text from a lxml result
135+
136+ * if xpath_results is list, extract the text from each result and concat the list
137+ * if xpath_results is a xml element, extract all the text node from it
138+ ( text_content() method from lxml )
139+ * if xpath_results is a string element, then it's already done
140+ """
141+ if type(xpath_results) == list:
142+ # it's list of result : concat everything using recursive call
143+ result = ''
144+ for e in xpath_results:
145+ result = result + extract_text(e)
146+ return result.strip()
147+ elif type(xpath_results) in [_ElementStringResult, _ElementUnicodeResult]:
148+ # it's a string
149+ return ''.join(xpath_results)
150+ else:
151+ # it's a element
152+ text = html.tostring(
153+ xpath_results, encoding='unicode', method='text', with_tail=False
154+ )
155+ text = text.strip().replace('\n', ' ')
156+ return ' '.join(text.split())
157+
158+
159+def normalize_url(url, base_url):
160+ """Normalize URL: add protocol, join URL with base_url, add trailing slash if there is no path
161+
162+ Args:
163+ * url (str): Relative URL
164+ * base_url (str): Base URL, it must be an absolute URL.
165+
166+ Example:
167+ >>> normalize_url('https://example.com', 'http://example.com/')
168+ 'https://example.com/'
169+ >>> normalize_url('//example.com', 'http://example.com/')
170+ 'http://example.com/'
171+ >>> normalize_url('//example.com', 'https://example.com/')
172+ 'https://example.com/'
173+ >>> normalize_url('/path?a=1', 'https://example.com')
174+ 'https://example.com/path?a=1'
175+ >>> normalize_url('', 'https://example.com')
176+ 'https://example.com/'
177+ >>> normalize_url('/test', '/path')
178+ raise Exception
179+
180+ Raises:
181+ * lxml.etree.ParserError
182+
183+ Returns:
184+ * str: normalized URL
185+ """
186+ if url.startswith('//'):
187+ # add http or https to this kind of url //example.com/
188+ parsed_search_url = urlparse(base_url)
189+ url = '{0}:{1}'.format(parsed_search_url.scheme or 'http', url)
190+ elif url.startswith('/'):
191+ # fix relative url to the search engine
192+ url = urljoin(base_url, url)
193+
194+ # fix relative urls that fall through the crack
195+ if '://' not in url:
196+ url = urljoin(base_url, url)
197+
198+ parsed_url = urlparse(url)
199+
200+ # add a / at this end of the url if there is no path
201+ if not parsed_url.netloc:
202+ raise Exception('Cannot parse url')
203+ if not parsed_url.path:
204+ url += '/'
205+
206+ return url
207+
208+
209+def extract_url(xpath_results, base_url):
210+ """Extract and normalize URL from lxml Element
211+
212+ Args:
213+ * xpath_results (Union[List[html.HtmlElement], html.HtmlElement]): lxml Element(s)
214+ * base_url (str): Base URL
215+
216+ Example:
217+ >>> def f(s, search_url):
218+ >>> return searx.utils.extract_url(html.fromstring(s), search_url)
219+ >>> f('<span id="42">https://example.com</span>', 'http://example.com/')
220+ 'https://example.com/'
221+ >>> f('https://example.com', 'http://example.com/')
222+ 'https://example.com/'
223+ >>> f('//example.com', 'http://example.com/')
224+ 'http://example.com/'
225+ >>> f('//example.com', 'https://example.com/')
226+ 'https://example.com/'
227+ >>> f('/path?a=1', 'https://example.com')
228+ 'https://example.com/path?a=1'
229+ >>> f('', 'https://example.com')
230+ raise lxml.etree.ParserError
231+ >>> searx.utils.extract_url([], 'https://example.com')
232+ raise Exception
233+
234+ Raises:
235+ * Exception
236+ * lxml.etree.ParserError
237+
238+ Returns:
239+ * str: normalized URL
240+ """
241+ if xpath_results == []:
242+ raise Exception('Empty url resultset')
243+
244+ url = extract_text(xpath_results)
245+ return normalize_url(url, base_url)
246+
247+
109248def dict_subset(d, properties):
249+ """Extract a subset of a dict
250+
251+ Examples:
252+ >>> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'C'])
253+ {'A': 'a', 'C': 'c'}
254+ >>> >> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'D'])
255+ {'A': 'a'}
256+ """
110257 result = {}
111258 for k in properties:
112259 if k in d:
@@ -114,8 +261,19 @@ def dict_subset(d, properties):
114261 return result
115262
116263
117−# get element in list or default value
118264def list_get(a_list, index, default=None):
265+ """Get element in list or default value
266+
267+ Examples:
268+ >>> list_get(['A', 'B', 'C'], 0)
269+ 'A'
270+ >>> list_get(['A', 'B', 'C'], 3)
271+ None
272+ >>> list_get(['A', 'B', 'C'], 3, 'default')
273+ 'default'
274+ >>> list_get(['A', 'B', 'C'], -1)
275+ 'C'
276+ """
119277 if len(a_list) > index:
120278 return a_list[index]
121279 else:
@@ -123,6 +281,21 @@ def list_get(a_list, index, default=None):
123281
124282
125283def get_torrent_size(filesize, filesize_multiplier):
284+ """
285+
286+ Args:
287+ * filesize (str): size
288+ * filesize_multiplier (str): TB, GB, .... TiB, GiB...
289+
290+ Returns:
291+ * int: number of bytes
292+
293+ Example:
294+ >>> get_torrent_size('5', 'GB')
295+ 5368709120
296+ >>> get_torrent_size('3.14', 'MiB')
297+ 3140000
298+ """
126299 try:
127300 filesize = float(filesize)
128301
@@ -149,14 +322,18 @@ def get_torrent_size(filesize, filesize_multiplier):
149322
150323
151324def convert_str_to_int(number_str):
325+ """Convert number_str to int or 0 if number_str is not a number."""
152326 if number_str.isdigit():
153327 return int(number_str)
154328 else:
155329 return 0
156330
157331
158−# convert a variable to integer or return 0 if it's not a number
159332def int_or_zero(num):
333+ """Convert num to int or 0. num can be either a str or a list.
334+ If num is a list, the first element is converted to int (or return 0 if the list is empty).
335+ If num is a str, see convert_str_to_int
336+ """
160337 if isinstance(num, list):
161338 if len(num) < 1:
162339 return 0
@@ -165,6 +342,22 @@ def int_or_zero(num):
165342
166343
167344def is_valid_lang(lang):
345+ """Return language code and name if lang describe a language.
346+
347+ Examples:
348+ >>> is_valid_lang('zz')
349+ False
350+ >>> is_valid_lang('uk')
351+ (True, 'uk', 'ukrainian')
352+ >>> is_valid_lang(b'uk')
353+ (True, 'uk', 'ukrainian')
354+ >>> is_valid_lang('en')
355+ (True, 'en', 'english')
356+ >>> searx.utils.is_valid_lang('Español')
357+ (True, 'es', 'spanish')
358+ >>> searx.utils.is_valid_lang('Spanish')
359+ (True, 'es', 'spanish')
360+ """
168361 if isinstance(lang, bytes):
169362 lang = lang.decode()
170363 is_abbr = (len(lang) == 2)
@@ -192,8 +385,8 @@ def _get_lang_to_lc_dict(lang_list):
192385 return value
193386
194387
195−# auxiliary function to match lang_code in lang_list
196388def _match_language(lang_code, lang_list=[], custom_aliases={}):
389+ """auxiliary function to match lang_code in lang_list"""
197390 # replace language code with a custom alias if necessary
198391 if lang_code in custom_aliases:
199392 lang_code = custom_aliases[lang_code]
@@ -215,8 +408,8 @@ def _match_language(lang_code, lang_list=[], custom_aliases={}):
215408 return _get_lang_to_lc_dict(lang_list).get(lang_code, None)
216409
217410
218−# get the language code from lang_list that best matches locale_code
219411def match_language(locale_code, lang_list=[], custom_aliases={}, fallback='en-US'):
412+ """get the language code from lang_list that best matches locale_code"""
220413 # try to get language from given locale_code
221414 language = _match_language(locale_code, lang_list, custom_aliases)
222415 if language:
@@ -258,6 +451,7 @@ def load_module(filename, module_dir):
258451
259452
260453def to_string(obj):
454+ """Convert obj to its string representation."""
261455 if isinstance(obj, str):
262456 return obj
263457 if isinstance(obj, Number):
@@ -269,13 +463,19 @@ def to_string(obj):
269463
270464
271465def ecma_unescape(s):
272− """
273− python implementation of the unescape javascript function
466+ """Python implementation of the unescape javascript function
274467
275468 https://www.ecma-international.org/ecma-262/6.0/#sec-unescape-string
276469 https://developer.mozilla.org/fr/docs/Web/JavaScript/Reference/Objets_globaux/unescape
470+
471+ Examples:
472+ >>> ecma_unescape('%u5409')
473+ '吉'
474+ >>> ecma_unescape('%20')
475+ ' '
476+ >>> ecma_unescape('%F3')
477+ 'ó'
277478 """
278− # s = unicode(s)
279479 # "%u5409" becomes "吉"
280480 s = ecma_unescape4_re.sub(lambda e: chr(int(e.group(1), 16)), s)
281481 # "%20" becomes " ", "%F3" becomes "ó"
@@ -299,6 +499,11 @@ def get_engine_from_settings(name):
299499
300500
301501def get_xpath(xpath_str):
502+ """Return cached compiled XPath
503+
504+ There is no thread lock.
505+ Worst case scenario, xpath_str is compiled more than one time.
506+ """
302507 result = xpath_cache.get(xpath_str, None)
303508 if result is None:
304509 result = XPath(xpath_str)
@@ -307,5 +512,6 @@ def get_xpath(xpath_str):
307512
308513
309514def eval_xpath(element, xpath_str):
515+ """Equivalent of element.xpath(xpath_str) but compile xpath_str once for all."""
310516 xpath = get_xpath(xpath_str)
311517 return xpath(element)