89fbb85 · searx · git.iko.soy

docs/dev/engine_overview.rst+12−10

compare with master

unifiedsplit

@@ -134,16 +134,18 @@ The function ``def request(query, params):`` always returns the ``params``

134variable. Inside searx, the following paramters can be used to specify a search

135request:

136

137−============ =========== =========================================================

137+================== =========== ========================================================================

138−argument type information

138+argument type information

139−============ =========== =========================================================

139+================== =========== ========================================================================

140−url string requested url

140+url string requested url

141−method string HTTP request method

141+method string HTTP request method

142−headers set HTTP header information

142+headers set HTTP header information

143−data set HTTP data information (parsed if ``method != 'GET'``)

143+data set HTTP data information (parsed if ``method != 'GET'``)

144−cookies set HTTP cookies

144+cookies set HTTP cookies

145−verify boolean Performing SSL-Validity check

145+verify boolean Performing SSL-Validity check

146−============ =========== =========================================================

146+max_redirects int maximum redirects, hard limit

147+soft_max_redirects int maximum redirects, soft limit. Record an error but don't stop the engine

148+================== =========== ========================================================================

147

149

148

150

149example code

151example code

searx/engines/1337x.py+7−7

@@ -1,6 +1,6 @@

1from urllib.parse import quote, urljoin

2from lxml import html

3−from searx.utils import extract_text, get_torrent_size

3+from searx.utils import extract_text, get_torrent_size, eval_xpath, eval_xpath_list, eval_xpath_getindex

4

5

6url = 'https://1337x.to/'

@@ -20,12 +20,12 @@ def response(resp):

20

21 dom = html.fromstring(resp.text)

22

23− for result in dom.xpath('//table[contains(@class, "table-list")]/tbody//tr'):

23+ for result in eval_xpath_list(dom, '//table[contains(@class, "table-list")]/tbody//tr'):

24− href = urljoin(url, result.xpath('./td[contains(@class, "name")]/a[2]/@href')[0])

24+ href = urljoin(url, eval_xpath_getindex(result, './td[contains(@class, "name")]/a[2]/@href', 0))

25− title = extract_text(result.xpath('./td[contains(@class, "name")]/a[2]'))

25+ title = extract_text(eval_xpath(result, './td[contains(@class, "name")]/a[2]'))

26− seed = extract_text(result.xpath('.//td[contains(@class, "seeds")]'))

26+ seed = extract_text(eval_xpath(result, './/td[contains(@class, "seeds")]'))

27− leech = extract_text(result.xpath('.//td[contains(@class, "leeches")]'))

27+ leech = extract_text(eval_xpath(result, './/td[contains(@class, "leeches")]'))

28− filesize_info = extract_text(result.xpath('.//td[contains(@class, "size")]/text()'))

28+ filesize_info = extract_text(eval_xpath(result, './/td[contains(@class, "size")]/text()'))

29 filesize, filesize_multiplier = filesize_info.split()

30 filesize = get_torrent_size(filesize, filesize_multiplier)

31

searx/engines/__init__.py+2−1

@@ -132,8 +132,9 @@ def load_engine(engine_data):

132 lambda: engine._fetch_supported_languages(get(engine.supported_languages_url)))

133

134 engine.stats = {

135+ 'sent_search_count': 0, # sent search

136+ 'search_count': 0, # succesful search

135 'result_count': 0,

137 'result_count': 0,

136− 'search_count': 0,

137 'engine_time': 0,

138 'engine_time': 0,

138 'engine_time_count': 0,

139 'engine_time_count': 0,

139 'score_count': 0,

140 'score_count': 0,

searx/engines/acgsou.py+12−15

@@ -11,7 +11,7 @@

11

12from urllib.parse import urlencode

13from lxml import html

14−from searx.utils import extract_text, get_torrent_size

14+from searx.utils import extract_text, get_torrent_size, eval_xpath_list, eval_xpath_getindex

15

16# engine dependent config

17categories = ['files', 'images', 'videos', 'music']

@@ -37,29 +37,26 @@ def request(query, params):

37def response(resp):

38 results = []

39 dom = html.fromstring(resp.text)

40− for result in dom.xpath(xpath_results):

40+ for result in eval_xpath_list(dom, xpath_results):

41 # defaults

42 filesize = 0

43 magnet_link = "magnet:?xt=urn:btih:{}&tr=http://tracker.acgsou.com:2710/announce"

44

45− try:

46− category = extract_text(result.xpath(xpath_category)[0])

45+ category = extract_text(eval_xpath_getindex(result, xpath_category, 0, default=[]))

47− except:

48− pass

49−

50− page_a = result.xpath(xpath_title)[0]

46+ page_a = eval_xpath_getindex(result, xpath_title, 0)

51 title = extract_text(page_a)

47 title = extract_text(page_a)

52 href = base_url + page_a.attrib.get('href')

48 href = base_url + page_a.attrib.get('href')

53

49

54 magnet_link = magnet_link.format(page_a.attrib.get('href')[5:-5])

50 magnet_link = magnet_link.format(page_a.attrib.get('href')[5:-5])

55

51

56− try:

57− filesize_info = result.xpath(xpath_filesize)[0]

52+ filesize_info = eval_xpath_getindex(result, xpath_filesize, 0, default=None)

53+ if filesize_info:

54+ try:

58− filesize = filesize_info[:-2]

55+ filesize = filesize_info[:-2]

59− filesize_multiplier = filesize_info[-2:]

56+ filesize_multiplier = filesize_info[-2:]

60− filesize = get_torrent_size(filesize, filesize_multiplier)

57+ filesize = get_torrent_size(filesize, filesize_multiplier)

61− except:

58+ except:

62− pass

59+ pass

63 # I didn't add download/seed/leech count since as I figured out they are generated randomly everytime

60 # I didn't add download/seed/leech count since as I figured out they are generated randomly everytime

64 content = 'Category: "{category}".'

61 content = 'Category: "{category}".'

65 content = content.format(category=category)

62 content = content.format(category=category)

searx/engines/ahmia.py+7−7

@@ -12,7 +12,7 @@

12

13from urllib.parse import urlencode, urlparse, parse_qs

14from lxml.html import fromstring

15−from searx.engines.xpath import extract_url, extract_text

15+from searx.engines.xpath import extract_url, extract_text, eval_xpath_list, eval_xpath

16

17# engine config

18categories = ['onions']

@@ -50,17 +50,17 @@ def response(resp):

50

51 # trim results so there's not way too many at once

52 first_result_index = page_size * (resp.search_params.get('pageno', 1) - 1)

53− all_results = dom.xpath(results_xpath)

53+ all_results = eval_xpath_list(dom, results_xpath)

54 trimmed_results = all_results[first_result_index:first_result_index + page_size]

55

56 # get results

57 for result in trimmed_results:

58 # remove ahmia url and extract the actual url for the result

59− raw_url = extract_url(result.xpath(url_xpath), search_url)

59+ raw_url = extract_url(eval_xpath_list(result, url_xpath, min_len=1), search_url)

60 cleaned_url = parse_qs(urlparse(raw_url).query).get('redirect_url', [''])[0]

61

62− title = extract_text(result.xpath(title_xpath))

62+ title = extract_text(eval_xpath(result, title_xpath))

63− content = extract_text(result.xpath(content_xpath))

63+ content = extract_text(eval_xpath(result, content_xpath))

64

65 results.append({'url': cleaned_url,

66 'title': title,

@@ -68,11 +68,11 @@ def response(resp):

68 'is_onion': True})

69

70 # get spelling corrections

71− for correction in dom.xpath(correction_xpath):

71+ for correction in eval_xpath_list(dom, correction_xpath):

72 results.append({'correction': extract_text(correction)})

73

74 # get number of results

75− number_of_results = dom.xpath(number_of_results_xpath)

75+ number_of_results = eval_xpath(dom, number_of_results_xpath)

76 if number_of_results:

77 try:

78 results.append({'number_of_results': int(extract_text(number_of_results))})

searx/engines/apkmirror.py+5−4

@@ -11,7 +11,7 @@

11

12from urllib.parse import urlencode

13from lxml import html

14−from searx.utils import extract_text

14+from searx.utils import extract_text, eval_xpath_list, eval_xpath_getindex

15

16

17# engine dependent config

@@ -42,12 +42,13 @@ def response(resp):

42 dom = html.fromstring(resp.text)

43

44 # parse results

45− for result in dom.xpath('.//div[@id="content"]/div[@class="listWidget"]/div[@class="appRow"]'):

45+ for result in eval_xpath_list(dom, './/div[@id="content"]/div[@class="listWidget"]/div[@class="appRow"]'):

46

47− link = result.xpath('.//h5/a')[0]

47+ link = eval_xpath_getindex(result, './/h5/a', 0)

48 url = base_url + link.attrib.get('href') + '#downloads'

49 title = extract_text(link)

50+ thumbnail_src = base_url\

50− thumbnail_src = base_url + result.xpath('.//img')[0].attrib.get('src').replace('&w=32&h=32', '&w=64&h=64')

51+ + eval_xpath_getindex(result, './/img', 0).attrib.get('src').replace('&w=32&h=32', '&w=64&h=64')

51

52

52 res = {

53 res = {

53 'url': url,

54 'url': url,

searx/engines/archlinux.py+3−3

@@ -13,7 +13,7 @@

13

14from urllib.parse import urlencode, urljoin

15from lxml import html

16−from searx.utils import extract_text

16+from searx.utils import extract_text, eval_xpath_list, eval_xpath_getindex

17

18# engine dependent config

19categories = ['it']

@@ -131,8 +131,8 @@ def response(resp):

131 dom = html.fromstring(resp.text)

132

133 # parse results

134− for result in dom.xpath(xpath_results):

134+ for result in eval_xpath_list(dom, xpath_results):

135− link = result.xpath(xpath_link)[0]

135+ link = eval_xpath_getindex(result, xpath_link, 0)

136 href = urljoin(base_url, link.attrib.get('href'))

137 title = extract_text(link)

138

searx/engines/arxiv.py+9−11

@@ -13,6 +13,7 @@

13

14from lxml import html

15from datetime import datetime

16+from searx.utils import eval_xpath_list, eval_xpath_getindex

16

17

18

18categories = ['science']

19categories = ['science']

@@ -42,29 +43,26 @@ def response(resp):

42 results = []

43 results = []

43

44

44 dom = html.fromstring(resp.content)

45 dom = html.fromstring(resp.content)

45− search_results = dom.xpath('//entry')

46

47− for entry in search_results:

47+ for entry in eval_xpath_list(dom, '//entry'):

48− title = entry.xpath('.//title')[0].text

48+ title = eval_xpath_getindex(entry, './/title', 0).text

49

50− url = entry.xpath('.//id')[0].text

50+ url = eval_xpath_getindex(entry, './/id', 0).text

51

52 content_string = '{doi_content}{abstract_content}'

53

54− abstract = entry.xpath('.//summary')[0].text

54+ abstract = eval_xpath_getindex(entry, './/summary', 0).text

55

56 # If a doi is available, add it to the snipppet

57− try:

57+ doi_element = eval_xpath_getindex(entry, './/link[@title="doi"]', 0, default=None)

58− doi_content = entry.xpath('.//link[@title="doi"]')[0].text

58+ doi_content = doi_element.text if doi_element is not None else ''

59− content = content_string.format(doi_content=doi_content, abstract_content=abstract)

60− except:

61− content = content_string.format(doi_content="", abstract_content=abstract)

59+ content = content_string.format(doi_content=doi_content, abstract_content=abstract)

62

60

63 if len(content) > 300:

61 if len(content) > 300:

64 content = content[0:300] + "..."

62 content = content[0:300] + "..."

65 # TODO: center snippet on query term

63 # TODO: center snippet on query term

66

64

67− publishedDate = datetime.strptime(entry.xpath('.//published')[0].text, '%Y-%m-%dT%H:%M:%SZ')

65+ publishedDate = datetime.strptime(eval_xpath_getindex(entry, './/published', 0).text, '%Y-%m-%dT%H:%M:%SZ')

68

66

69 res_dict = {'url': url,

67 res_dict = {'url': url,

70 'title': title,

68 'title': title,

searx/engines/bing_news.py+7−6

@@ -15,7 +15,8 @@ from datetime import datetime

15from dateutil import parser

16from urllib.parse import urlencode, urlparse, parse_qsl

17from lxml import etree

18+from lxml.etree import XPath

18−from searx.utils import list_get, match_language

19+from searx.utils import match_language, eval_xpath_getindex

19from searx.engines.bing import language_aliases

20from searx.engines.bing import language_aliases

20from searx.engines.bing import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import

21from searx.engines.bing import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import

21

22

@@ -94,12 +95,12 @@ def response(resp):

94 # parse results

95 # parse results

95 for item in rss.xpath('./channel/item'):

96 for item in rss.xpath('./channel/item'):

96 # url / title / content

97 # url / title / content

97− url = url_cleanup(item.xpath('./link/text()')[0])

98+ url = url_cleanup(eval_xpath_getindex(item, './link/text()', 0, default=None))

98− title = list_get(item.xpath('./title/text()'), 0, url)

99+ title = eval_xpath_getindex(item, './title/text()', 0, default=url)

99− content = list_get(item.xpath('./description/text()'), 0, '')

100+ content = eval_xpath_getindex(item, './description/text()', 0, default='')

100

101

101 # publishedDate

102 # publishedDate

102− publishedDate = list_get(item.xpath('./pubDate/text()'), 0)

103+ publishedDate = eval_xpath_getindex(item, './pubDate/text()', 0, default=None)

103 try:

104 try:

104 publishedDate = parser.parse(publishedDate, dayfirst=False)

105 publishedDate = parser.parse(publishedDate, dayfirst=False)

105 except TypeError:

106 except TypeError:

@@ -108,7 +109,7 @@ def response(resp):

108 publishedDate = datetime.now()

109 publishedDate = datetime.now()

109

110

110 # thumbnail

111 # thumbnail

111− thumbnail = list_get(item.xpath('./News:Image/text()', namespaces=ns), 0)

112+ thumbnail = eval_xpath_getindex(item, XPath('./News:Image/text()', namespaces=ns), 0, default=None)

112 if thumbnail is not None:

113 if thumbnail is not None:

113 thumbnail = image_url_cleanup(thumbnail)

114 thumbnail = image_url_cleanup(thumbnail)

114

115

searx/engines/duckduckgo_images.py+3−5

@@ -15,6 +15,7 @@

15

16from json import loads

17from urllib.parse import urlencode

18+from searx.exceptions import SearxEngineAPIException

18from searx.engines.duckduckgo import get_region_code

19from searx.engines.duckduckgo import get_region_code

19from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import

20from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import

20from searx.poolrequests import get

21from searx.poolrequests import get

@@ -37,7 +38,7 @@ def get_vqd(query, headers):

37 res = get(query_url, headers=headers)

38 res = get(query_url, headers=headers)

38 content = res.text

39 content = res.text

39 if content.find('vqd=\'') == -1:

40 if content.find('vqd=\'') == -1:

40− raise Exception('Request failed')

41+ raise SearxEngineAPIException('Request failed')

41 vqd = content[content.find('vqd=\'') + 5:]

42 vqd = content[content.find('vqd=\'') + 5:]

42 vqd = vqd[:vqd.find('\'')]

43 vqd = vqd[:vqd.find('\'')]

43 return vqd

44 return vqd

@@ -71,10 +72,7 @@ def response(resp):

71 results = []

72 results = []

72

73

73 content = resp.text

74 content = resp.text

74− try:

75− res_json = loads(content)

75+ res_json = loads(content)

76− except:

77− raise Exception('Cannot parse results')

78

76

79 # parse results

77 # parse results

80 for result in res_json['results']:

78 for result in res_json['results']:

searx/engines/elasticsearch.py+2−1

@@ -1,5 +1,6 @@

1from json import loads, dumps

2from requests.auth import HTTPBasicAuth

3+from searx.exceptions import SearxEngineAPIException

3

4

5

5base_url = 'http://localhost:9200'

6base_url = 'http://localhost:9200'

@@ -107,7 +108,7 @@ def response(resp):

107

108

108 resp_json = loads(resp.text)

109 resp_json = loads(resp.text)

109 if 'error' in resp_json:

110 if 'error' in resp_json:

110− raise Exception(resp_json['error'])

111+ raise SearxEngineAPIException(resp_json['error'])

111

112

112 for result in resp_json['hits']['hits']:

113 for result in resp_json['hits']['hits']:

113 r = {key: str(value) if not key.startswith('_') else value for key, value in result['_source'].items()}

114 r = {key: str(value) if not key.startswith('_') else value for key, value in result['_source'].items()}

searx/engines/google.py+22−30

@@ -20,9 +20,10 @@ Definitions`_.

20

21from urllib.parse import urlencode, urlparse

22from lxml import html

23−from flask_babel import gettext

24from searx import logger

23from searx import logger

25−from searx.utils import match_language, extract_text, eval_xpath

24+from searx.utils import match_language, extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex

25+from searx.exceptions import SearxEngineCaptchaException

26+

26

27

27logger = logger.getChild('google engine')

28logger = logger.getChild('google engine')

28

29

@@ -131,14 +132,6 @@ suggestion_xpath = '//div[contains(@class, "card-section")]//a'

131spelling_suggestion_xpath = '//div[@class="med"]/p/a'

132spelling_suggestion_xpath = '//div[@class="med"]/p/a'

132

133

134

134−def extract_text_from_dom(result, xpath):

135− """returns extract_text on the first result selected by the xpath or None"""

136− r = eval_xpath(result, xpath)

137− if len(r) > 0:

138− return extract_text(r[0])

139− return None

140−

141−

142def get_lang_country(params, lang_list, custom_aliases):

135def get_lang_country(params, lang_list, custom_aliases):

143 """Returns a tuple with *langauage* on its first and *country* on its second

136 """Returns a tuple with *langauage* on its first and *country* on its second

144 position."""

137 position."""

@@ -210,10 +203,10 @@ def response(resp):

210 # detect google sorry

203 # detect google sorry

211 resp_url = urlparse(resp.url)

204 resp_url = urlparse(resp.url)

212 if resp_url.netloc == 'sorry.google.com' or resp_url.path == '/sorry/IndexRedirect':

205 if resp_url.netloc == 'sorry.google.com' or resp_url.path == '/sorry/IndexRedirect':

213− raise RuntimeWarning('sorry.google.com')

206+ raise SearxEngineCaptchaException()

214

207

215 if resp_url.path.startswith('/sorry'):

208 if resp_url.path.startswith('/sorry'):

216− raise RuntimeWarning(gettext('CAPTCHA required'))

209+ raise SearxEngineCaptchaException()

217

210

218 # which subdomain ?

211 # which subdomain ?

219 # subdomain = resp.search_params.get('google_subdomain')

212 # subdomain = resp.search_params.get('google_subdomain')

@@ -229,18 +222,17 @@ def response(resp):

229 logger.debug("did not found 'answer'")

222 logger.debug("did not found 'answer'")

230

223

231 # results --> number_of_results

224 # results --> number_of_results

232− try:

225+ try:

233− _txt = eval_xpath(dom, '//div[@id="result-stats"]//text()')[0]

226+ _txt = eval_xpath_getindex(dom, '//div[@id="result-stats"]//text()', 0)

234− _digit = ''.join([n for n in _txt if n.isdigit()])

227+ _digit = ''.join([n for n in _txt if n.isdigit()])

235− number_of_results = int(_digit)

228+ number_of_results = int(_digit)

236− results.append({'number_of_results': number_of_results})

229+ results.append({'number_of_results': number_of_results})

237−

238− except Exception as e: # pylint: disable=broad-except

230+ except Exception as e: # pylint: disable=broad-except

239− logger.debug("did not 'number_of_results'")

231+ logger.debug("did not 'number_of_results'")

240− logger.error(e, exc_info=True)

232+ logger.error(e, exc_info=True)

241

233

242 # parse results

234 # parse results

243− for result in eval_xpath(dom, results_xpath):

235+ for result in eval_xpath_list(dom, results_xpath):

244

236

245 # google *sections*

237 # google *sections*

246 if extract_text(eval_xpath(result, g_section_with_header)):

238 if extract_text(eval_xpath(result, g_section_with_header)):

@@ -248,14 +240,14 @@ def response(resp):

248 continue

240 continue

249

241

250 try:

242 try:

251− title_tag = eval_xpath(result, title_xpath)

243+ title_tag = eval_xpath_getindex(result, title_xpath, 0, default=None)

252− if not title_tag:

244+ if title_tag is None:

253 # this not one of the common google results *section*

245 # this not one of the common google results *section*

254 logger.debug('ingoring <div class="g" ../> section: missing title')

246 logger.debug('ingoring <div class="g" ../> section: missing title')

255 continue

247 continue

256− title = extract_text(title_tag[0])

248+ title = extract_text(title_tag)

257− url = eval_xpath(result, href_xpath)[0]

249+ url = eval_xpath_getindex(result, href_xpath, 0)

258− content = extract_text_from_dom(result, content_xpath)

250+ content = extract_text(eval_xpath_getindex(result, content_xpath, 0, default=None), allow_none=True)

259 results.append({

251 results.append({

260 'url': url,

252 'url': url,

261 'title': title,

253 'title': title,

@@ -270,11 +262,11 @@ def response(resp):

270 continue

262 continue

271

263

272 # parse suggestion

264 # parse suggestion

273− for suggestion in eval_xpath(dom, suggestion_xpath):

265+ for suggestion in eval_xpath_list(dom, suggestion_xpath):

274 # append suggestion

266 # append suggestion

275 results.append({'suggestion': extract_text(suggestion)})

267 results.append({'suggestion': extract_text(suggestion)})

276

268

277− for correction in eval_xpath(dom, spelling_suggestion_xpath):

269+ for correction in eval_xpath_list(dom, spelling_suggestion_xpath):

278 results.append({'correction': extract_text(correction)})

270 results.append({'correction': extract_text(correction)})

279

271

280 # return results

272 # return results

@@ -286,7 +278,7 @@ def _fetch_supported_languages(resp):

286 ret_val = {}

278 ret_val = {}

287 dom = html.fromstring(resp.text)

279 dom = html.fromstring(resp.text)

288

280

289− radio_buttons = eval_xpath(dom, '//*[@id="langSec"]//input[@name="lr"]')

281+ radio_buttons = eval_xpath_list(dom, '//*[@id="langSec"]//input[@name="lr"]')

290

282

291 for x in radio_buttons:

283 for x in radio_buttons:

292 name = x.get("data-name")

284 name = x.get("data-name")

searx/engines/google_images.py+3−3

@@ -26,8 +26,8 @@ Definitions`_.

26

27from urllib.parse import urlencode, urlparse, unquote

28from lxml import html

29−from flask_babel import gettext

30from searx import logger

29from searx import logger

30+from searx.exceptions import SearxEngineCaptchaException

31from searx.utils import extract_text, eval_xpath

32from searx.engines.google import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import

33

@@ -128,10 +128,10 @@ def response(resp):

128 # detect google sorry

129 resp_url = urlparse(resp.url)

130 if resp_url.netloc == 'sorry.google.com' or resp_url.path == '/sorry/IndexRedirect':

131− raise RuntimeWarning('sorry.google.com')

131+ raise SearxEngineCaptchaException()

132

133 if resp_url.path.startswith('/sorry'):

134− raise RuntimeWarning(gettext('CAPTCHA required'))

134+ raise SearxEngineCaptchaException()

135

136 # which subdomain ?

137 # subdomain = resp.search_params.get('google_subdomain')

searx/engines/google_videos.py+5−5

@@ -13,7 +13,7 @@

13from datetime import date, timedelta

14from urllib.parse import urlencode

15from lxml import html

16−from searx.utils import extract_text

16+from searx.utils import extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex

17import re

18

19# engine dependent config

@@ -66,11 +66,11 @@ def response(resp):

66 dom = html.fromstring(resp.text)

67

68 # parse results

69− for result in dom.xpath('//div[@class="g"]'):

69+ for result in eval_xpath_list(dom, '//div[@class="g"]'):

70

71− title = extract_text(result.xpath('.//h3'))

71+ title = extract_text(eval_xpath(result, './/h3'))

72− url = result.xpath('.//div[@class="r"]/a/@href')[0]

72+ url = eval_xpath_getindex(result, './/div[@class="r"]/a/@href', 0)

73− content = extract_text(result.xpath('.//span[@class="st"]'))

73+ content = extract_text(eval_xpath(result, './/span[@class="st"]'))

74

75 # get thumbnails

76 script = str(dom.xpath('//script[contains(., "_setImagesSrc")]')[0].text)

searx/engines/xpath.py+15−14

@@ -1,6 +1,6 @@

1from lxml import html

2from urllib.parse import urlencode

3−from searx.utils import extract_text, extract_url, eval_xpath

3+from searx.utils import extract_text, extract_url, eval_xpath, eval_xpath_list

4

5search_url = None

6url_xpath = None

@@ -42,21 +42,22 @@ def response(resp):

42 is_onion = True if 'onions' in categories else False

43

44 if results_xpath:

45− for result in eval_xpath(dom, results_xpath):

45+ for result in eval_xpath_list(dom, results_xpath):

46− url = extract_url(eval_xpath(result, url_xpath), search_url)

46+ url = extract_url(eval_xpath_list(result, url_xpath, min_len=1), search_url)

47− title = extract_text(eval_xpath(result, title_xpath))

47+ title = extract_text(eval_xpath_list(result, title_xpath, min_len=1))

48− content = extract_text(eval_xpath(result, content_xpath))

48+ content = extract_text(eval_xpath_list(result, content_xpath, min_len=1))

49 tmp_result = {'url': url, 'title': title, 'content': content}

50

51 # add thumbnail if available

52 if thumbnail_xpath:

53− thumbnail_xpath_result = eval_xpath(result, thumbnail_xpath)

53+ thumbnail_xpath_result = eval_xpath_list(result, thumbnail_xpath)

54 if len(thumbnail_xpath_result) > 0:

55 tmp_result['img_src'] = extract_url(thumbnail_xpath_result, search_url)

56

57 # add alternative cached url if available

58 if cached_xpath:

59− tmp_result['cached_url'] = cached_url + extract_text(result.xpath(cached_xpath))

59+ tmp_result['cached_url'] = cached_url\

60+ + extract_text(eval_xpath_list(result, cached_xpath, min_len=1))

60

61

61 if is_onion:

62 if is_onion:

62 tmp_result['is_onion'] = True

63 tmp_result['is_onion'] = True

@@ -66,19 +67,19 @@ def response(resp):

66 if cached_xpath:

67 if cached_xpath:

67 for url, title, content, cached in zip(

68 for url, title, content, cached in zip(

68 (extract_url(x, search_url) for

69 (extract_url(x, search_url) for

69− x in dom.xpath(url_xpath)),

70+ x in eval_xpath_list(dom, url_xpath)),

70− map(extract_text, dom.xpath(title_xpath)),

71+ map(extract_text, eval_xpath_list(dom, title_xpath)),

71− map(extract_text, dom.xpath(content_xpath)),

72+ map(extract_text, eval_xpath_list(dom, content_xpath)),

72− map(extract_text, dom.xpath(cached_xpath))

73+ map(extract_text, eval_xpath_list(dom, cached_xpath))

73 ):

74 ):

74 results.append({'url': url, 'title': title, 'content': content,

75 results.append({'url': url, 'title': title, 'content': content,

75 'cached_url': cached_url + cached, 'is_onion': is_onion})

76 'cached_url': cached_url + cached, 'is_onion': is_onion})

76 else:

77 else:

77 for url, title, content in zip(

78 for url, title, content in zip(

78 (extract_url(x, search_url) for

79 (extract_url(x, search_url) for

79− x in dom.xpath(url_xpath)),

80+ x in eval_xpath_list(dom, url_xpath)),

80− map(extract_text, dom.xpath(title_xpath)),

81+ map(extract_text, eval_xpath_list(dom, title_xpath)),

81− map(extract_text, dom.xpath(content_xpath))

82+ map(extract_text, eval_xpath_list(dom, content_xpath))

82 ):

83 ):

83 results.append({'url': url, 'title': title, 'content': content, 'is_onion': is_onion})

84 results.append({'url': url, 'title': title, 'content': content, 'is_onion': is_onion})

84

85

searx/engines/youtube_api.py+2−1

@@ -11,6 +11,7 @@

11from json import loads

12from dateutil import parser

13from urllib.parse import urlencode

14+from searx.exceptions import SearxEngineAPIException

14

15

15# engine dependent config

16# engine dependent config

16categories = ['videos', 'music']

17categories = ['videos', 'music']

@@ -48,7 +49,7 @@ def response(resp):

48 search_results = loads(resp.text)

49 search_results = loads(resp.text)

49

50

50 if 'error' in search_results and 'message' in search_results['error']:

51 if 'error' in search_results and 'message' in search_results['error']:

51− raise Exception(search_results['error']['message'])

52+ raise SearxEngineAPIException(search_results['error']['message'])

52

53

53 # return empty array if there are no results

54 # return empty array if there are no results

54 if 'items' not in search_results:

55 if 'items' not in search_results:

searx/exceptions.py+37

@@ -34,8 +34,45 @@ class SearxParameterException(SearxException):

34

35

36class SearxSettingsException(SearxException):

37+ """Error while loading the settings"""

37

38

38 def __init__(self, message, filename):

39 def __init__(self, message, filename):

39 super().__init__(message)

40 super().__init__(message)

40 self.message = message

41 self.message = message

41 self.filename = filename

42 self.filename = filename

43+

44+

45+class SearxEngineException(SearxException):

46+ """Error inside an engine"""

47+

48+

49+class SearxXPathSyntaxException(SearxEngineException):

50+ """Syntax error in a XPATH"""

51+

52+ def __init__(self, xpath_spec, message):

53+ super().__init__(str(xpath_spec) + " " + message)

54+ self.message = message

55+ # str(xpath_spec) to deal with str and XPath instance

56+ self.xpath_str = str(xpath_spec)

57+

58+

59+class SearxEngineResponseException(SearxEngineException):

60+ """Impossible to parse the result of an engine"""

61+

62+

63+class SearxEngineAPIException(SearxEngineResponseException):

64+ """The website has returned an application error"""

65+

66+

67+class SearxEngineCaptchaException(SearxEngineResponseException):

68+ """The website has returned a CAPTCHA"""

69+

70+

71+class SearxEngineXPathException(SearxEngineResponseException):

72+ """Error while getting the result of an XPath expression"""

73+

74+ def __init__(self, xpath_spec, message):

75+ super().__init__(str(xpath_spec) + " " + message)

76+ self.message = message

77+ # str(xpath_spec) to deal with str and XPath instance

78+ self.xpath_str = str(xpath_spec)

searx/metrology/__init__.pynew file

No changes to the content.

searx/metrology/error_recorder.pynew file+142

@@ -0,0 +1,142 @@

1+import typing

2+import inspect

3+import logging

4+from json import JSONDecodeError

5+from urllib.parse import urlparse

6+from requests.exceptions import RequestException

7+from searx.exceptions import SearxXPathSyntaxException, SearxEngineXPathException

8+from searx import logger

9+

10+

11+logging.basicConfig(level=logging.INFO)

12+

13+errors_per_engines = {}

14+

15+

16+class ErrorContext:

17+

18+ __slots__ = 'filename', 'function', 'line_no', 'code', 'exception_classname', 'log_message', 'log_parameters'

19+

20+ def __init__(self, filename, function, line_no, code, exception_classname, log_message, log_parameters):

21+ self.filename = filename

22+ self.function = function

23+ self.line_no = line_no

24+ self.code = code

25+ self.exception_classname = exception_classname

26+ self.log_message = log_message

27+ self.log_parameters = log_parameters

28+

29+ def __eq__(self, o) -> bool:

30+ if not isinstance(o, ErrorContext):

31+ return False

32+ return self.filename == o.filename and self.function == o.function and self.line_no == o.line_no\

33+ and self.code == o.code and self.exception_classname == o.exception_classname\

34+ and self.log_message == o.log_message and self.log_parameters == o.log_parameters

35+

36+ def __hash__(self):

37+ return hash((self.filename, self.function, self.line_no, self.code, self.exception_classname, self.log_message,

38+ self.log_parameters))

39+

40+ def __repr__(self):

41+ return "ErrorContext({!r}, {!r}, {!r}, {!r}, {!r}, {!r})".\

42+ format(self.filename, self.line_no, self.code, self.exception_classname, self.log_message,

43+ self.log_parameters)

44+

45+

46+def add_error_context(engine_name: str, error_context: ErrorContext) -> None:

47+ errors_for_engine = errors_per_engines.setdefault(engine_name, {})

48+ errors_for_engine[error_context] = errors_for_engine.get(error_context, 0) + 1

49+ logger.debug('⚠️ %s: %s', engine_name, str(error_context))

50+

51+

52+def get_trace(traces):

53+ previous_trace = traces[-1]

54+ for trace in reversed(traces):

55+ if trace.filename.endswith('searx/search.py'):

56+ if previous_trace.filename.endswith('searx/poolrequests.py'):

57+ return trace

58+ if previous_trace.filename.endswith('requests/models.py'):

59+ return trace

60+ return previous_trace

61+ previous_trace = trace

62+ return traces[-1]

63+

64+

65+def get_hostname(exc: RequestException) -> typing.Optional[None]:

66+ url = exc.request.url

67+ if url is None and exc.response is not None:

68+ url = exc.response.url

69+ return urlparse(url).netloc

70+

71+

72+def get_request_exception_messages(exc: RequestException)\

73+ -> typing.Tuple[typing.Optional[str], typing.Optional[str], typing.Optional[str]]:

74+ url = None

75+ status_code = None

76+ reason = None

77+ hostname = None

78+ if exc.request is not None:

79+ url = exc.request.url

80+ if url is None and exc.response is not None:

81+ url = exc.response.url

82+ if url is not None:

83+ hostname = str(urlparse(url).netloc)

84+ if exc.response is not None:

85+ status_code = str(exc.response.status_code)

86+ reason = exc.response.reason

87+ return (status_code, reason, hostname)

88+

89+

90+def get_messages(exc, filename) -> typing.Tuple:

91+ if isinstance(exc, JSONDecodeError):

92+ return (exc.msg, )

93+ if isinstance(exc, TypeError):

94+ return (str(exc), )

95+ if isinstance(exc, ValueError) and 'lxml' in filename:

96+ return (str(exc), )

97+ if isinstance(exc, RequestException):

98+ return get_request_exception_messages(exc)

99+ if isinstance(exc, SearxXPathSyntaxException):

100+ return (exc.xpath_str, exc.message)

101+ if isinstance(exc, SearxEngineXPathException):

102+ return (exc.xpath_str, exc.message)

103+ return ()

104+

105+

106+def get_exception_classname(exc: Exception) -> str:

107+ exc_class = exc.__class__

108+ exc_name = exc_class.__qualname__

109+ exc_module = exc_class.__module__

110+ if exc_module is None or exc_module == str.__class__.__module__:

111+ return exc_name

112+ return exc_module + '.' + exc_name

113+

114+

115+def get_error_context(framerecords, exception_classname, log_message, log_parameters) -> ErrorContext:

116+ searx_frame = get_trace(framerecords)

117+ filename = searx_frame.filename

118+ function = searx_frame.function

119+ line_no = searx_frame.lineno

120+ code = searx_frame.code_context[0].strip()

121+ del framerecords

122+ return ErrorContext(filename, function, line_no, code, exception_classname, log_message, log_parameters)

123+

124+

125+def record_exception(engine_name: str, exc: Exception) -> None:

126+ framerecords = inspect.trace()

127+ try:

128+ exception_classname = get_exception_classname(exc)

129+ log_parameters = get_messages(exc, framerecords[-1][1])

130+ error_context = get_error_context(framerecords, exception_classname, None, log_parameters)

131+ add_error_context(engine_name, error_context)

132+ finally:

133+ del framerecords

134+

135+

136+def record_error(engine_name: str, log_message: str, log_parameters: typing.Optional[typing.Tuple] = None) -> None:

137+ framerecords = list(reversed(inspect.stack()[1:]))

138+ try:

139+ error_context = get_error_context(framerecords, None, log_message, log_parameters or ())

140+ add_error_context(engine_name, error_context)

141+ finally:

142+ del framerecords

searx/results.py+9

@@ -4,6 +4,7 @@ from threading import RLock

4from urllib.parse import urlparse, unquote

5from searx import logger

6from searx.engines import engines

7+from searx.metrology.error_recorder import record_error

7

8

9

9CONTENT_LEN_IGNORED_CHARS_REGEX = re.compile(r'[,;:!?\./\\\\ ()-_]', re.M | re.U)

10CONTENT_LEN_IGNORED_CHARS_REGEX = re.compile(r'[,;:!?\./\\\\ ()-_]', re.M | re.U)

@@ -161,6 +162,7 @@ class ResultContainer:

161

162

162 def extend(self, engine_name, results):

163 def extend(self, engine_name, results):

163 standard_result_count = 0

164 standard_result_count = 0

165+ error_msgs = set()

164 for result in list(results):

166 for result in list(results):

165 result['engine'] = engine_name

167 result['engine'] = engine_name

166 if 'suggestion' in result:

168 if 'suggestion' in result:

@@ -177,14 +179,21 @@ class ResultContainer:

177 # standard result (url, title, content)

179 # standard result (url, title, content)

178 if 'url' in result and not isinstance(result['url'], str):

180 if 'url' in result and not isinstance(result['url'], str):

179 logger.debug('result: invalid URL: %s', str(result))

181 logger.debug('result: invalid URL: %s', str(result))

182+ error_msgs.add('invalid URL')

180 elif 'title' in result and not isinstance(result['title'], str):

183 elif 'title' in result and not isinstance(result['title'], str):

181 logger.debug('result: invalid title: %s', str(result))

184 logger.debug('result: invalid title: %s', str(result))

185+ error_msgs.add('invalid title')

182 elif 'content' in result and not isinstance(result['content'], str):

186 elif 'content' in result and not isinstance(result['content'], str):

183 logger.debug('result: invalid content: %s', str(result))

187 logger.debug('result: invalid content: %s', str(result))

188+ error_msgs.add('invalid content')

184 else:

189 else:

185 self._merge_result(result, standard_result_count + 1)

190 self._merge_result(result, standard_result_count + 1)

186 standard_result_count += 1

191 standard_result_count += 1

187

192

193+ if len(error_msgs) > 0:

194+ for msg in error_msgs:

195+ record_error(engine_name, 'some results are invalids: ' + msg)

196+

188 if engine_name in engines:

197 if engine_name in engines:

189 with RLock():

198 with RLock():

190 engines[engine_name].stats['search_count'] += 1

199 engines[engine_name].stats['search_count'] += 1

searx/search.py+49−5

@@ -20,6 +20,7 @@ import gc

20import threading

21from time import time

22from uuid import uuid4

23+from urllib.parse import urlparse

23from _thread import start_new_thread

24from _thread import start_new_thread

24

25

25import requests.exceptions

26import requests.exceptions

@@ -31,6 +32,8 @@ from searx.utils import gen_useragent

31from searx.results import ResultContainer

32from searx.results import ResultContainer

32from searx import logger

33from searx import logger

33from searx.plugins import plugins

34from searx.plugins import plugins

35+from searx.exceptions import SearxEngineCaptchaException

36+from searx.metrology.error_recorder import record_exception, record_error

34

37

35

38

36logger = logger.getChild('search')

39logger = logger.getChild('search')

@@ -120,6 +123,14 @@ def send_http_request(engine, request_params):

120 if hasattr(engine, 'proxies'):

123 if hasattr(engine, 'proxies'):

121 request_args['proxies'] = requests_lib.get_proxies(engine.proxies)

124 request_args['proxies'] = requests_lib.get_proxies(engine.proxies)

122

125

126+ # max_redirects

127+ max_redirects = request_params.get('max_redirects')

128+ if max_redirects:

129+ request_args['max_redirects'] = max_redirects

130+

131+ # soft_max_redirects

132+ soft_max_redirects = request_params.get('soft_max_redirects', max_redirects or 0)

133+

123 # specific type of request (GET or POST)

134 # specific type of request (GET or POST)

124 if request_params['method'] == 'GET':

135 if request_params['method'] == 'GET':

125 req = requests_lib.get

136 req = requests_lib.get

@@ -129,7 +140,23 @@ def send_http_request(engine, request_params):

129 request_args['data'] = request_params['data']

140 request_args['data'] = request_params['data']

130

141

131 # send the request

142 # send the request

132− return req(request_params['url'], **request_args)

143+ response = req(request_params['url'], **request_args)

144+

145+ # check HTTP status

146+ response.raise_for_status()

147+

148+ # check soft limit of the redirect count

149+ if len(response.history) > soft_max_redirects:

150+ # unexpected redirect : record an error

151+ # but the engine might still return valid results.

152+ status_code = str(response.status_code or '')

153+ reason = response.reason or ''

154+ hostname = str(urlparse(response.url or '').netloc)

155+ record_error(engine.name,

156+ '{} redirects, maximum: {}'.format(len(response.history), soft_max_redirects),

157+ (status_code, reason, hostname))

158+

159+ return response

133

160

134

161

135def search_one_http_request(engine, query, request_params):

162def search_one_http_request(engine, query, request_params):

@@ -183,8 +210,9 @@ def search_one_http_request_safe(engine_name, query, request_params, result_cont

183 # update stats with the total HTTP time

210 # update stats with the total HTTP time

184 engine.stats['page_load_time'] += page_load_time

211 engine.stats['page_load_time'] += page_load_time

185 engine.stats['page_load_count'] += 1

212 engine.stats['page_load_count'] += 1

186−

187 except Exception as e:

213 except Exception as e:

214+ record_exception(engine_name, e)

215+

188 # Timing

216 # Timing

189 engine_time = time() - start_time

217 engine_time = time() - start_time

190 page_load_time = requests_lib.get_time_for_thread()

218 page_load_time = requests_lib.get_time_for_thread()

@@ -195,23 +223,29 @@ def search_one_http_request_safe(engine_name, query, request_params, result_cont

195 engine.stats['errors'] += 1

223 engine.stats['errors'] += 1

196

224

197 if (issubclass(e.__class__, requests.exceptions.Timeout)):

225 if (issubclass(e.__class__, requests.exceptions.Timeout)):

198− result_container.add_unresponsive_engine(engine_name, 'timeout')

226+ result_container.add_unresponsive_engine(engine_name, 'HTTP timeout')

199 # requests timeout (connect or read)

227 # requests timeout (connect or read)

200 logger.error("engine {0} : HTTP requests timeout"

228 logger.error("engine {0} : HTTP requests timeout"

201 "(search duration : {1} s, timeout: {2} s) : {3}"

229 "(search duration : {1} s, timeout: {2} s) : {3}"

202 .format(engine_name, engine_time, timeout_limit, e.__class__.__name__))

230 .format(engine_name, engine_time, timeout_limit, e.__class__.__name__))

203 requests_exception = True

231 requests_exception = True

204 elif (issubclass(e.__class__, requests.exceptions.RequestException)):

232 elif (issubclass(e.__class__, requests.exceptions.RequestException)):

205− result_container.add_unresponsive_engine(engine_name, 'request exception')

233+ result_container.add_unresponsive_engine(engine_name, 'HTTP error')

206 # other requests exception

234 # other requests exception

207 logger.exception("engine {0} : requests exception"

235 logger.exception("engine {0} : requests exception"

208 "(search duration : {1} s, timeout: {2} s) : {3}"

236 "(search duration : {1} s, timeout: {2} s) : {3}"

209 .format(engine_name, engine_time, timeout_limit, e))

237 .format(engine_name, engine_time, timeout_limit, e))

210 requests_exception = True

238 requests_exception = True

239+ elif (issubclass(e.__class__, SearxEngineCaptchaException)):

240+ result_container.add_unresponsive_engine(engine_name, 'CAPTCHA required')

241+ logger.exception('engine {0} : CAPTCHA')

211 else:

242 else:

212− result_container.add_unresponsive_engine(engine_name, 'unexpected crash', str(e))

243+ result_container.add_unresponsive_engine(engine_name, 'unexpected crash')

213 # others errors

244 # others errors

214 logger.exception('engine {0} : exception : {1}'.format(engine_name, e))

245 logger.exception('engine {0} : exception : {1}'.format(engine_name, e))

246+ else:

247+ if getattr(threading.current_thread(), '_timeout', False):

248+ record_error(engine_name, 'Timeout')

215

249

216 # suspend or not the engine if there are HTTP errors

250 # suspend or not the engine if there are HTTP errors

217 with threading.RLock():

251 with threading.RLock():

@@ -255,12 +289,17 @@ def search_one_offline_request_safe(engine_name, query, request_params, result_c

255 engine.stats['engine_time_count'] += 1

289 engine.stats['engine_time_count'] += 1

256

290

257 except ValueError as e:

291 except ValueError as e:

292+ record_exception(engine_name, e)

258 record_offline_engine_stats_on_error(engine, result_container, start_time)

293 record_offline_engine_stats_on_error(engine, result_container, start_time)

259 logger.exception('engine {0} : invalid input : {1}'.format(engine_name, e))

294 logger.exception('engine {0} : invalid input : {1}'.format(engine_name, e))

260 except Exception as e:

295 except Exception as e:

296+ record_exception(engine_name, e)

261 record_offline_engine_stats_on_error(engine, result_container, start_time)

297 record_offline_engine_stats_on_error(engine, result_container, start_time)

262 result_container.add_unresponsive_engine(engine_name, 'unexpected crash', str(e))

298 result_container.add_unresponsive_engine(engine_name, 'unexpected crash', str(e))

263 logger.exception('engine {0} : exception : {1}'.format(engine_name, e))

299 logger.exception('engine {0} : exception : {1}'.format(engine_name, e))

300+ else:

301+ if getattr(threading.current_thread(), '_timeout', False):

302+ record_error(engine_name, 'Timeout')

264

303

265

304

266def search_one_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit):

305def search_one_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit):

@@ -278,6 +317,7 @@ def search_multiple_requests(requests, result_container, start_time, timeout_lim

278 args=(engine_name, query, request_params, result_container, start_time, timeout_limit),

317 args=(engine_name, query, request_params, result_container, start_time, timeout_limit),

279 name=search_id,

318 name=search_id,

280 )

319 )

320+ th._timeout = False

281 th._engine_name = engine_name

321 th._engine_name = engine_name

282 th.start()

322 th.start()

283

323

@@ -286,6 +326,7 @@ def search_multiple_requests(requests, result_container, start_time, timeout_lim

286 remaining_time = max(0.0, timeout_limit - (time() - start_time))

326 remaining_time = max(0.0, timeout_limit - (time() - start_time))

287 th.join(remaining_time)

327 th.join(remaining_time)

288 if th.is_alive():

328 if th.is_alive():

329+ th._timeout = True

289 result_container.add_unresponsive_engine(th._engine_name, 'timeout')

330 result_container.add_unresponsive_engine(th._engine_name, 'timeout')

290 logger.warning('engine timeout: {0}'.format(th._engine_name))

331 logger.warning('engine timeout: {0}'.format(th._engine_name))

291

332

@@ -385,6 +426,9 @@ class Search:

385 request_params['category'] = engineref.category

426 request_params['category'] = engineref.category

386 request_params['pageno'] = self.search_query.pageno

427 request_params['pageno'] = self.search_query.pageno

387

428

429+ with threading.RLock():

430+ engine.stats['sent_search_count'] += 1

431+

388 return request_params, engine.timeout

432 return request_params, engine.timeout

389

433

390 # do search-request

434 # do search-request

searx/utils.py+125−41

@@ -10,7 +10,7 @@ from html.parser import HTMLParser

10from urllib.parse import urljoin, urlparse

11

12from lxml import html

13−from lxml.etree import XPath, _ElementStringResult, _ElementUnicodeResult

13+from lxml.etree import ElementBase, XPath, XPathError, XPathSyntaxError, _ElementStringResult, _ElementUnicodeResult

14from babel.core import get_global

15

16

@@ -18,6 +18,7 @@ from searx import settings

18from searx.data import USER_AGENTS

19from searx.version import VERSION_STRING

20from searx.languages import language_codes

21+from searx.exceptions import SearxXPathSyntaxException, SearxEngineXPathException

21from searx import logger

22from searx import logger

22

23

24

@@ -33,6 +34,13 @@ xpath_cache = dict()

33lang_to_lc_cache = dict()

34lang_to_lc_cache = dict()

34

35

36

37+class NotSetClass:

38+ pass

39+

40+

41+NOTSET = NotSetClass()

42+

43+

36def searx_useragent():

44def searx_useragent():

37 """Return the searx User Agent"""

45 """Return the searx User Agent"""

38 return 'searx/{searx_version} {suffix}'.format(

46 return 'searx/{searx_version} {suffix}'.format(

@@ -125,7 +133,7 @@ def html_to_text(html_str):

125 return s.get_text()

133 return s.get_text()

126

134

127

135

128−def extract_text(xpath_results):

136+def extract_text(xpath_results, allow_none=False):

129 """Extract text from a lxml result

137 """Extract text from a lxml result

130

138

131 * if xpath_results is list, extract the text from each result and concat the list

139 * if xpath_results is list, extract the text from each result and concat the list

@@ -133,22 +141,27 @@ def extract_text(xpath_results):

133 ( text_content() method from lxml )

141 ( text_content() method from lxml )

134 * if xpath_results is a string element, then it's already done

142 * if xpath_results is a string element, then it's already done

135 """

143 """

136− if type(xpath_results) == list:

144+ if isinstance(xpath_results, list):

137 # it's list of result : concat everything using recursive call

145 # it's list of result : concat everything using recursive call

138 result = ''

146 result = ''

139 for e in xpath_results:

147 for e in xpath_results:

140 result = result + extract_text(e)

148 result = result + extract_text(e)

141 return result.strip()

149 return result.strip()

142− elif type(xpath_results) in [_ElementStringResult, _ElementUnicodeResult]:

150+ elif isinstance(xpath_results, ElementBase):

143− # it's a string

144− return ''.join(xpath_results)

145− else:

146 # it's a element

151 # it's a element

147 text = html.tostring(

152 text = html.tostring(

148 xpath_results, encoding='unicode', method='text', with_tail=False

153 xpath_results, encoding='unicode', method='text', with_tail=False

149 )

154 )

150 text = text.strip().replace('\n', ' ')

155 text = text.strip().replace('\n', ' ')

151 return ' '.join(text.split())

156 return ' '.join(text.split())

157+ elif isinstance(xpath_results, (_ElementStringResult, _ElementUnicodeResult, str, Number, bool)):

158+ return str(xpath_results)

159+ elif xpath_results is None and allow_none:

160+ return None

161+ elif xpath_results is None and not allow_none:

162+ raise ValueError('extract_text(None, allow_none=False)')

163+ else:

164+ raise ValueError('unsupported type')

152

165

153

166

154def normalize_url(url, base_url):

167def normalize_url(url, base_url):

@@ -170,7 +183,7 @@ def normalize_url(url, base_url):

170 >>> normalize_url('', 'https://example.com')

183 >>> normalize_url('', 'https://example.com')

171 'https://example.com/'

184 'https://example.com/'

172 >>> normalize_url('/test', '/path')

185 >>> normalize_url('/test', '/path')

173− raise Exception

186+ raise ValueError

174

187

175 Raises:

188 Raises:

176 * lxml.etree.ParserError

189 * lxml.etree.ParserError

@@ -194,7 +207,7 @@ def normalize_url(url, base_url):

194

207

195 # add a / at this end of the url if there is no path

208 # add a / at this end of the url if there is no path

196 if not parsed_url.netloc:

209 if not parsed_url.netloc:

197− raise Exception('Cannot parse url')

210+ raise ValueError('Cannot parse url')

198 if not parsed_url.path:

211 if not parsed_url.path:

199 url += '/'

212 url += '/'

200

213

@@ -224,17 +237,17 @@ def extract_url(xpath_results, base_url):

224 >>> f('', 'https://example.com')

237 >>> f('', 'https://example.com')

225 raise lxml.etree.ParserError

238 raise lxml.etree.ParserError

226 >>> searx.utils.extract_url([], 'https://example.com')

239 >>> searx.utils.extract_url([], 'https://example.com')

227− raise Exception

240+ raise ValueError

228

241

229 Raises:

242 Raises:

230− * Exception

243+ * ValueError

231 * lxml.etree.ParserError

244 * lxml.etree.ParserError

232

245

233 Returns:

246 Returns:

234 * str: normalized URL

247 * str: normalized URL

235 """

248 """

236 if xpath_results == []:

249 if xpath_results == []:

237− raise Exception('Empty url resultset')

250+ raise ValueError('Empty url resultset')

238

251

239 url = extract_text(xpath_results)

252 url = extract_text(xpath_results)

240 return normalize_url(url, base_url)

253 return normalize_url(url, base_url)

@@ -256,25 +269,6 @@ def dict_subset(d, properties):

256 return result

269 return result

257

270

258

271

259−def list_get(a_list, index, default=None):

260− """Get element in list or default value

261−

262− Examples:

263− >>> list_get(['A', 'B', 'C'], 0)

264− 'A'

265− >>> list_get(['A', 'B', 'C'], 3)

266− None

267− >>> list_get(['A', 'B', 'C'], 3, 'default')

268− 'default'

269− >>> list_get(['A', 'B', 'C'], -1)

270− 'C'

271− """

272− if len(a_list) > index:

273− return a_list[index]

274− else:

275− return default

276−

277−

278def get_torrent_size(filesize, filesize_multiplier):

272def get_torrent_size(filesize, filesize_multiplier):

279 """

273 """

280

274

@@ -310,7 +304,7 @@ def get_torrent_size(filesize, filesize_multiplier):

310 filesize = int(filesize * 1000 * 1000)

304 filesize = int(filesize * 1000 * 1000)

311 elif filesize_multiplier == 'KiB':

305 elif filesize_multiplier == 'KiB':

312 filesize = int(filesize * 1000)

306 filesize = int(filesize * 1000)

313− except:

307+ except ValueError:

314 filesize = None

308 filesize = None

315

309

316 return filesize

310 return filesize

@@ -506,20 +500,110 @@ def get_engine_from_settings(name):

506 return {}

500 return {}

507

501

508

502

509−def get_xpath(xpath_str):

503+def get_xpath(xpath_spec):

510 """Return cached compiled XPath

504 """Return cached compiled XPath

511

505

512 There is no thread lock.

506 There is no thread lock.

513 Worst case scenario, xpath_str is compiled more than one time.

507 Worst case scenario, xpath_str is compiled more than one time.

508+

509+ Args:

510+ * xpath_spec (str|lxml.etree.XPath): XPath as a str or lxml.etree.XPath

511+

512+ Returns:

513+ * result (bool, float, list, str): Results.

514+

515+ Raises:

516+ * TypeError: Raise when xpath_spec is neither a str nor a lxml.etree.XPath

517+ * SearxXPathSyntaxException: Raise when there is a syntax error in the XPath

514 """

518 """

519+ if isinstance(xpath_spec, str):

515− result = xpath_cache.get(xpath_str, None)

520+ result = xpath_cache.get(xpath_spec, None)

516− if result is None:

521+ if result is None:

522+ try:

517− result = XPath(xpath_str)

523+ result = XPath(xpath_spec)

524+ except XPathSyntaxError as e:

525+ raise SearxXPathSyntaxException(xpath_spec, str(e.msg))

518− xpath_cache[xpath_str] = result

526+ xpath_cache[xpath_spec] = result

527+ return result

528+

529+ if isinstance(xpath_spec, XPath):

530+ return xpath_spec

531+

532+ raise TypeError('xpath_spec must be either a str or a lxml.etree.XPath')

533+

534+

535+def eval_xpath(element, xpath_spec):

536+ """Equivalent of element.xpath(xpath_str) but compile xpath_str once for all.

537+ See https://lxml.de/xpathxslt.html#xpath-return-values

538+

539+ Args:

540+ * element (ElementBase): [description]

541+ * xpath_spec (str|lxml.etree.XPath): XPath as a str or lxml.etree.XPath

542+

543+ Returns:

544+ * result (bool, float, list, str): Results.

545+

546+ Raises:

547+ * TypeError: Raise when xpath_spec is neither a str nor a lxml.etree.XPath

548+ * SearxXPathSyntaxException: Raise when there is a syntax error in the XPath

549+ * SearxEngineXPathException: Raise when the XPath can't be evaluated.

550+ """

551+ xpath = get_xpath(xpath_spec)

552+ try:

553+ return xpath(element)

554+ except XPathError as e:

555+ arg = ' '.join([str(i) for i in e.args])

556+ raise SearxEngineXPathException(xpath_spec, arg)

557+

558+

559+def eval_xpath_list(element, xpath_spec, min_len=None):

560+ """Same as eval_xpath, check if the result is a list

561+

562+ Args:

563+ * element (ElementBase): [description]

564+ * xpath_spec (str|lxml.etree.XPath): XPath as a str or lxml.etree.XPath

565+ * min_len (int, optional): [description]. Defaults to None.

566+

567+ Raises:

568+ * TypeError: Raise when xpath_spec is neither a str nor a lxml.etree.XPath

569+ * SearxXPathSyntaxException: Raise when there is a syntax error in the XPath

570+ * SearxEngineXPathException: raise if the result is not a list

571+

572+ Returns:

573+ * result (bool, float, list, str): Results.

574+ """

575+ result = eval_xpath(element, xpath_spec)

576+ if not isinstance(result, list):

577+ raise SearxEngineXPathException(xpath_spec, 'the result is not a list')

578+ if min_len is not None and min_len > len(result):

579+ raise SearxEngineXPathException(xpath_spec, 'len(xpath_str) < ' + str(min_len))

519 return result

580 return result

520

581

521

582

522−def eval_xpath(element, xpath_str):

583+def eval_xpath_getindex(elements, xpath_spec, index, default=NOTSET):

523− """Equivalent of element.xpath(xpath_str) but compile xpath_str once for all."""

584+ """Call eval_xpath_list then get one element using the index parameter.

524− xpath = get_xpath(xpath_str)

585+ If the index does not exist, either aise an exception is default is not set,

525− return xpath(element)

586+ other return the default value (can be None).

587+

588+ Args:

589+ * elements (ElementBase): lxml element to apply the xpath.

590+ * xpath_spec (str|lxml.etree.XPath): XPath as a str or lxml.etree.XPath.

591+ * index (int): index to get

592+ * default (Object, optional): Defaults if index doesn't exist.

593+

594+ Raises:

595+ * TypeError: Raise when xpath_spec is neither a str nor a lxml.etree.XPath

596+ * SearxXPathSyntaxException: Raise when there is a syntax error in the XPath

597+ * SearxEngineXPathException: if the index is not found. Also see eval_xpath.

598+

599+ Returns:

600+ * result (bool, float, list, str): Results.

601+ """

602+ result = eval_xpath_list(elements, xpath_spec)

603+ if index >= -len(result) and index < len(result):

604+ return result[index]

605+ if default == NOTSET:

606+ # raise an SearxEngineXPathException instead of IndexError

607+ # to record xpath_spec

608+ raise SearxEngineXPathException(xpath_spec, 'index ' + str(index) + ' not found')

609+ return default

searx/webapp.py+29

@@ -79,6 +79,7 @@ from searx.plugins.oa_doi_rewrite import get_doi_resolver

79from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES

80from searx.answerers import answerers

81from searx.poolrequests import get_global_proxies

82+from searx.metrology.error_recorder import errors_per_engines

82

83

84

84# serve pages with HTTP/1.1

85# serve pages with HTTP/1.1

@@ -943,6 +944,34 @@ def stats():

943 )

944 )

944

945

946

947+@app.route('/stats/errors', methods=['GET'])

948+def stats_errors():

949+ result = {}

950+ engine_names = list(errors_per_engines.keys())

951+ engine_names.sort()

952+ for engine_name in engine_names:

953+ error_stats = errors_per_engines[engine_name]

954+ sent_search_count = max(engines[engine_name].stats['sent_search_count'], 1)

955+ sorted_context_count_list = sorted(error_stats.items(), key=lambda context_count: context_count[1])

956+ r = []

957+ percentage_sum = 0

958+ for context, count in sorted_context_count_list:

959+ percentage = round(20 * count / sent_search_count) * 5

960+ percentage_sum += percentage

961+ r.append({

962+ 'filename': context.filename,

963+ 'function': context.function,

964+ 'line_no': context.line_no,

965+ 'code': context.code,

966+ 'exception_classname': context.exception_classname,

967+ 'log_message': context.log_message,

968+ 'log_parameters': context.log_parameters,

969+ 'percentage': percentage,

970+ })

971+ result[engine_name] = sorted(r, reverse=True, key=lambda d: d['percentage'])

972+ return jsonify(result)

973+

974+

946@app.route('/robots.txt', methods=['GET'])

975@app.route('/robots.txt', methods=['GET'])

947def robots():

976def robots():

948 return Response("""User-agent: *

977 return Response("""User-agent: *

tests/unit/test_utils.py+90

@@ -3,6 +3,7 @@ import lxml.etree

3from lxml import html

4

5from searx.testing import SearxTestCase

6+from searx.exceptions import SearxXPathSyntaxException, SearxEngineXPathException

6from searx import utils

7from searx import utils

7

8

9

@@ -57,8 +58,16 @@ class TestUtils(SearxTestCase):

57 dom = html.fromstring(html_str)

58 dom = html.fromstring(html_str)

58 self.assertEqual(utils.extract_text(dom), 'Test text')

59 self.assertEqual(utils.extract_text(dom), 'Test text')

59 self.assertEqual(utils.extract_text(dom.xpath('//span')), 'Test text')

60 self.assertEqual(utils.extract_text(dom.xpath('//span')), 'Test text')

61+ self.assertEqual(utils.extract_text(dom.xpath('//span/text()')), 'Test text')

62+ self.assertEqual(utils.extract_text(dom.xpath('count(//span)')), '3.0')

63+ self.assertEqual(utils.extract_text(dom.xpath('boolean(//span)')), 'True')

60 self.assertEqual(utils.extract_text(dom.xpath('//img/@src')), 'test.jpg')

64 self.assertEqual(utils.extract_text(dom.xpath('//img/@src')), 'test.jpg')

61 self.assertEqual(utils.extract_text(dom.xpath('//unexistingtag')), '')

65 self.assertEqual(utils.extract_text(dom.xpath('//unexistingtag')), '')

66+ self.assertEqual(utils.extract_text(None, allow_none=True), None)

67+ with self.assertRaises(ValueError):

68+ utils.extract_text(None)

69+ with self.assertRaises(ValueError):

70+ utils.extract_text({})

62

71

63 def test_extract_url(self):

72 def test_extract_url(self):

64 def f(html_str, search_url):

73 def f(html_str, search_url):

@@ -136,3 +145,84 @@ class TestHTMLTextExtractor(SearxTestCase):

136 text = 'Lorem ipsumdolor sit amet'

145 text = 'Lorem ipsumdolor sit amet'

137 with self.assertRaises(utils.HTMLTextExtractorException):

146 with self.assertRaises(utils.HTMLTextExtractorException):

138 self.html_text_extractor.feed(text)

147 self.html_text_extractor.feed(text)

148+

149+

150+class TestXPathUtils(SearxTestCase):

151+

152+ TEST_DOC = """<ul>

153+ <li>Text in bold and italic </li>

154+ <li>Another text <img src="data:image/gif;base64,R0lGODlhAQABAIAAAAUEBAAAACwAAAAAAQABAAACAkQBADs="></li>

155+ </ul>"""

156+

157+ def test_get_xpath_cache(self):

158+ xp1 = utils.get_xpath('//a')

159+ xp2 = utils.get_xpath('//div')

160+ xp3 = utils.get_xpath('//a')

161+

162+ self.assertEqual(id(xp1), id(xp3))

163+ self.assertNotEqual(id(xp1), id(xp2))

164+

165+ def test_get_xpath_type(self):

166+ utils.get_xpath(lxml.etree.XPath('//a'))

167+

168+ with self.assertRaises(TypeError):

169+ utils.get_xpath([])

170+

171+ def test_get_xpath_invalid(self):

172+ invalid_xpath = '//a[0].text'

173+ with self.assertRaises(SearxXPathSyntaxException) as context:

174+ utils.get_xpath(invalid_xpath)

175+

176+ self.assertEqual(context.exception.message, 'Invalid expression')

177+ self.assertEqual(context.exception.xpath_str, invalid_xpath)

178+

179+ def test_eval_xpath_unregistered_function(self):

180+ doc = html.fromstring(TestXPathUtils.TEST_DOC)

181+

182+ invalid_function_xpath = 'int(//a)'

183+ with self.assertRaises(SearxEngineXPathException) as context:

184+ utils.eval_xpath(doc, invalid_function_xpath)

185+

186+ self.assertEqual(context.exception.message, 'Unregistered function')

187+ self.assertEqual(context.exception.xpath_str, invalid_function_xpath)

188+

189+ def test_eval_xpath(self):

190+ doc = html.fromstring(TestXPathUtils.TEST_DOC)

191+

192+ self.assertEqual(utils.eval_xpath(doc, '//p'), [])

193+ self.assertEqual(utils.eval_xpath(doc, '//i/text()'), ['italic'])

194+ self.assertEqual(utils.eval_xpath(doc, 'count(//i)'), 1.0)

195+

196+ def test_eval_xpath_list(self):

197+ doc = html.fromstring(TestXPathUtils.TEST_DOC)

198+

199+ # check a not empty list

200+ self.assertEqual(utils.eval_xpath_list(doc, '//i/text()'), ['italic'])

201+

202+ # check min_len parameter

203+ with self.assertRaises(SearxEngineXPathException) as context:

204+ utils.eval_xpath_list(doc, '//p', min_len=1)

205+ self.assertEqual(context.exception.message, 'len(xpath_str) < 1')

206+ self.assertEqual(context.exception.xpath_str, '//p')

207+

208+ def test_eval_xpath_getindex(self):

209+ doc = html.fromstring(TestXPathUtils.TEST_DOC)

210+

211+ # check index 0

212+ self.assertEqual(utils.eval_xpath_getindex(doc, '//i/text()', 0), 'italic')

213+

214+ # default is 'something'

215+ self.assertEqual(utils.eval_xpath_getindex(doc, '//i/text()', 1, default='something'), 'something')

216+

217+ # default is None

218+ self.assertEqual(utils.eval_xpath_getindex(doc, '//i/text()', 1, default=None), None)

219+

220+ # index not found

221+ with self.assertRaises(SearxEngineXPathException) as context:

222+ utils.eval_xpath_getindex(doc, '//i/text()', 1)

223+ self.assertEqual(context.exception.message, 'index 1 not found')

224+

225+ # not a list

226+ with self.assertRaises(SearxEngineXPathException) as context:

227+ utils.eval_xpath_getindex(doc, 'count(//i)', 1)

228+ self.assertEqual(context.exception.message, 'the result is not a list')