@@ -10,7 +10,7 @@ from html.parser import HTMLParser
1010from urllib.parse import urljoin, urlparse
1111
1212from lxml import html
13−from lxml.etree import XPath, _ElementStringResult, _ElementUnicodeResult
13+from lxml.etree import ElementBase, XPath, XPathError, XPathSyntaxError, _ElementStringResult, _ElementUnicodeResult
1414from babel.core import get_global
1515
1616
@@ -18,6 +18,7 @@ from searx import settings
1818from searx.data import USER_AGENTS
1919from searx.version import VERSION_STRING
2020from searx.languages import language_codes
21+from searx.exceptions import SearxXPathSyntaxException, SearxEngineXPathException
2122from searx import logger
2223
2324
@@ -33,6 +34,13 @@ xpath_cache = dict()
3334lang_to_lc_cache = dict()
3435
3536
37+class NotSetClass:
38+ pass
39+
40+
41+NOTSET = NotSetClass()
42+
43+
3644def searx_useragent():
3745 """Return the searx User Agent"""
3846 return 'searx/{searx_version} {suffix}'.format(
@@ -125,7 +133,7 @@ def html_to_text(html_str):
125133 return s.get_text()
126134
127135
128−def extract_text(xpath_results):
136+def extract_text(xpath_results, allow_none=False):
129137 """Extract text from a lxml result
130138
131139 * if xpath_results is list, extract the text from each result and concat the list
@@ -133,22 +141,27 @@ def extract_text(xpath_results):
133141 ( text_content() method from lxml )
134142 * if xpath_results is a string element, then it's already done
135143 """
136− if type(xpath_results) == list:
144+ if isinstance(xpath_results, list):
137145 # it's list of result : concat everything using recursive call
138146 result = ''
139147 for e in xpath_results:
140148 result = result + extract_text(e)
141149 return result.strip()
142− elif type(xpath_results) in [_ElementStringResult, _ElementUnicodeResult]:
143− # it's a string
144− return ''.join(xpath_results)
145− else:
150+ elif isinstance(xpath_results, ElementBase):
146151 # it's a element
147152 text = html.tostring(
148153 xpath_results, encoding='unicode', method='text', with_tail=False
149154 )
150155 text = text.strip().replace('\n', ' ')
151156 return ' '.join(text.split())
157+ elif isinstance(xpath_results, (_ElementStringResult, _ElementUnicodeResult, str, Number, bool)):
158+ return str(xpath_results)
159+ elif xpath_results is None and allow_none:
160+ return None
161+ elif xpath_results is None and not allow_none:
162+ raise ValueError('extract_text(None, allow_none=False)')
163+ else:
164+ raise ValueError('unsupported type')
152165
153166
154167def normalize_url(url, base_url):
@@ -170,7 +183,7 @@ def normalize_url(url, base_url):
170183 >>> normalize_url('', 'https://example.com')
171184 'https://example.com/'
172185 >>> normalize_url('/test', '/path')
173− raise Exception
186+ raise ValueError
174187
175188 Raises:
176189 * lxml.etree.ParserError
@@ -194,7 +207,7 @@ def normalize_url(url, base_url):
194207
195208 # add a / at this end of the url if there is no path
196209 if not parsed_url.netloc:
197− raise Exception('Cannot parse url')
210+ raise ValueError('Cannot parse url')
198211 if not parsed_url.path:
199212 url += '/'
200213
@@ -224,17 +237,17 @@ def extract_url(xpath_results, base_url):
224237 >>> f('', 'https://example.com')
225238 raise lxml.etree.ParserError
226239 >>> searx.utils.extract_url([], 'https://example.com')
227− raise Exception
240+ raise ValueError
228241
229242 Raises:
230− * Exception
243+ * ValueError
231244 * lxml.etree.ParserError
232245
233246 Returns:
234247 * str: normalized URL
235248 """
236249 if xpath_results == []:
237− raise Exception('Empty url resultset')
250+ raise ValueError('Empty url resultset')
238251
239252 url = extract_text(xpath_results)
240253 return normalize_url(url, base_url)
@@ -258,7 +271,6 @@ def dict_subset(d, properties):
258271
259272def list_get(a_list, index, default=None):
260273 """Get element in list or default value
261−
262274 Examples:
263275 >>> list_get(['A', 'B', 'C'], 0)
264276 'A'
@@ -310,7 +322,7 @@ def get_torrent_size(filesize, filesize_multiplier):
310322 filesize = int(filesize * 1000 * 1000)
311323 elif filesize_multiplier == 'KiB':
312324 filesize = int(filesize * 1000)
313− except:
325+ except ValueError:
314326 filesize = None
315327
316328 return filesize
@@ -506,20 +518,110 @@ def get_engine_from_settings(name):
506518 return {}
507519
508520
509−def get_xpath(xpath_str):
521+def get_xpath(xpath_spec):
510522 """Return cached compiled XPath
511523
512524 There is no thread lock.
513525 Worst case scenario, xpath_str is compiled more than one time.
526+
527+ Args:
528+ * xpath_spec (str|lxml.etree.XPath): XPath as a str or lxml.etree.XPath
529+
530+ Returns:
531+ * result (bool, float, list, str): Results.
532+
533+ Raises:
534+ * TypeError: Raise when xpath_spec is neither a str nor a lxml.etree.XPath
535+ * SearxXPathSyntaxException: Raise when there is a syntax error in the XPath
536+ """
537+ if isinstance(xpath_spec, str):
538+ result = xpath_cache.get(xpath_spec, None)
539+ if result is None:
540+ try:
541+ result = XPath(xpath_spec)
542+ except XPathSyntaxError as e:
543+ raise SearxXPathSyntaxException(xpath_spec, str(e.msg))
544+ xpath_cache[xpath_spec] = result
545+ return result
546+
547+ if isinstance(xpath_spec, XPath):
548+ return xpath_spec
549+
550+ raise TypeError('xpath_spec must be either a str or a lxml.etree.XPath')
551+
552+
553+def eval_xpath(element, xpath_spec):
554+ """Equivalent of element.xpath(xpath_str) but compile xpath_str once for all.
555+ See https://lxml.de/xpathxslt.html#xpath-return-values
556+
557+ Args:
558+ * element (ElementBase): [description]
559+ * xpath_spec (str|lxml.etree.XPath): XPath as a str or lxml.etree.XPath
560+
561+ Returns:
562+ * result (bool, float, list, str): Results.
563+
564+ Raises:
565+ * TypeError: Raise when xpath_spec is neither a str nor a lxml.etree.XPath
566+ * SearxXPathSyntaxException: Raise when there is a syntax error in the XPath
567+ * SearxEngineXPathException: Raise when the XPath can't be evaluated.
514568 """
515− result = xpath_cache.get(xpath_str, None)
516− if result is None:
517− result = XPath(xpath_str)
518− xpath_cache[xpath_str] = result
569+ xpath = get_xpath(xpath_spec)
570+ try:
571+ return xpath(element)
572+ except XPathError as e:
573+ arg = ' '.join([str(i) for i in e.args])
574+ raise SearxEngineXPathException(xpath_spec, arg)
575+
576+
577+def eval_xpath_list(element, xpath_spec, min_len=None):
578+ """Same as eval_xpath, check if the result is a list
579+
580+ Args:
581+ * element (ElementBase): [description]
582+ * xpath_spec (str|lxml.etree.XPath): XPath as a str or lxml.etree.XPath
583+ * min_len (int, optional): [description]. Defaults to None.
584+
585+ Raises:
586+ * TypeError: Raise when xpath_spec is neither a str nor a lxml.etree.XPath
587+ * SearxXPathSyntaxException: Raise when there is a syntax error in the XPath
588+ * SearxEngineXPathException: raise if the result is not a list
589+
590+ Returns:
591+ * result (bool, float, list, str): Results.
592+ """
593+ result = eval_xpath(element, xpath_spec)
594+ if not isinstance(result, list):
595+ raise SearxEngineXPathException(xpath_spec, 'the result is not a list')
596+ if min_len is not None and min_len > len(result):
597+ raise SearxEngineXPathException(xpath_spec, 'len(xpath_str) < ' + str(min_len))
519598 return result
520599
521600
522−def eval_xpath(element, xpath_str):
523− """Equivalent of element.xpath(xpath_str) but compile xpath_str once for all."""
524− xpath = get_xpath(xpath_str)
525− return xpath(element)
601+def eval_xpath_getindex(elements, xpath_spec, index, default=NOTSET):
602+ """Call eval_xpath_list then get one element using the index parameter.
603+ If the index does not exist, either aise an exception is default is not set,
604+ other return the default value (can be None).
605+
606+ Args:
607+ * elements (ElementBase): lxml element to apply the xpath.
608+ * xpath_spec (str|lxml.etree.XPath): XPath as a str or lxml.etree.XPath.
609+ * index (int): index to get
610+ * default (Object, optional): Defaults if index doesn't exist.
611+
612+ Raises:
613+ * TypeError: Raise when xpath_spec is neither a str nor a lxml.etree.XPath
614+ * SearxXPathSyntaxException: Raise when there is a syntax error in the XPath
615+ * SearxEngineXPathException: if the index is not found. Also see eval_xpath.
616+
617+ Returns:
618+ * result (bool, float, list, str): Results.
619+ """
620+ result = eval_xpath_list(elements, xpath_spec)
621+ if index >= -len(result) and index < len(result):
622+ return result[index]
623+ if default == NOTSET:
624+ # raise an SearxEngineXPathException instead of IndexError
625+ # to record xpath_spec
626+ raise SearxEngineXPathException(xpath_spec, 'index ' + str(index) + ' not found')
627+ return default