64cccae · searx · git.iko.soy

git.iko.soy/searx

on master help sign in

[mod] various engines: use eval_xpath* functions and searx.exceptions.*

64cccaeAlexandre Flament6 years ago

searx/engines/ahmia.py+7−7

compare with master

unifiedsplit

@@ -12,7 +12,7 @@

13from urllib.parse import urlencode, urlparse, parse_qs

14from lxml.html import fromstring

15−from searx.engines.xpath import extract_url, extract_text

15+from searx.engines.xpath import extract_url, extract_text, eval_xpath_list, eval_xpath

17# engine config

18categories = ['onions']

@@ -50,17 +50,17 @@ def response(resp):

51 # trim results so there's not way too many at once

52 first_result_index = page_size * (resp.search_params.get('pageno', 1) - 1)

53− all_results = dom.xpath(results_xpath)

53+ all_results = eval_xpath_list(dom, results_xpath)

54 trimmed_results = all_results[first_result_index:first_result_index + page_size]

56 # get results

57 for result in trimmed_results:

58 # remove ahmia url and extract the actual url for the result

59− raw_url = extract_url(result.xpath(url_xpath), search_url)

59+ raw_url = extract_url(eval_xpath_list(result, url_xpath, min_len=1), search_url)

60 cleaned_url = parse_qs(urlparse(raw_url).query).get('redirect_url', [''])[0]

62− title = extract_text(result.xpath(title_xpath))

62+ title = extract_text(eval_xpath(result, title_xpath))

63− content = extract_text(result.xpath(content_xpath))

63+ content = extract_text(eval_xpath(result, content_xpath))

65 results.append({'url': cleaned_url,

66 'title': title,

@@ -68,11 +68,11 @@ def response(resp):

68 'is_onion': True})

70 # get spelling corrections

71− for correction in dom.xpath(correction_xpath):

71+ for correction in eval_xpath_list(dom, correction_xpath):

72 results.append({'correction': extract_text(correction)})

74 # get number of results

75− number_of_results = dom.xpath(number_of_results_xpath)

75+ number_of_results = eval_xpath(dom, number_of_results_xpath)

76 if number_of_results:

77 try:

78 results.append({'number_of_results': int(extract_text(number_of_results))})

searx/engines/duckduckgo_images.py+3−5

@@ -15,6 +15,7 @@

16from json import loads

17from urllib.parse import urlencode

18+from searx.exceptions import SearxEngineAPIException

18from searx.engines.duckduckgo import get_region_code

19from searx.engines.duckduckgo import get_region_code

19from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import

20from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import

20from searx.poolrequests import get

21from searx.poolrequests import get

@@ -37,7 +38,7 @@ def get_vqd(query, headers):

37 res = get(query_url, headers=headers)

38 res = get(query_url, headers=headers)

38 content = res.text

39 content = res.text

39 if content.find('vqd=\'') == -1:

40 if content.find('vqd=\'') == -1:

40− raise Exception('Request failed')

41+ raise SearxEngineAPIException('Request failed')

41 vqd = content[content.find('vqd=\'') + 5:]

42 vqd = content[content.find('vqd=\'') + 5:]

42 vqd = vqd[:vqd.find('\'')]

43 vqd = vqd[:vqd.find('\'')]

43 return vqd

44 return vqd

@@ -71,10 +72,7 @@ def response(resp):

71 results = []

72 results = []

73 content = resp.text

74 content = resp.text

74− try:

75− res_json = loads(content)

75+ res_json = loads(content)

76− except:

77− raise Exception('Cannot parse results')

79 # parse results

77 # parse results

80 for result in res_json['results']:

78 for result in res_json['results']:

searx/engines/elasticsearch.py+2−1

@@ -1,5 +1,6 @@

1from json import loads, dumps

2from requests.auth import HTTPBasicAuth

3+from searx.exceptions import SearxEngineAPIException

5base_url = 'http://localhost:9200'

6base_url = 'http://localhost:9200'

@@ -107,7 +108,7 @@ def response(resp):

107

108

108 resp_json = loads(resp.text)

109 resp_json = loads(resp.text)

109 if 'error' in resp_json:

110 if 'error' in resp_json:

110− raise Exception(resp_json['error'])

111+ raise SearxEngineAPIException(resp_json['error'])

111

112

112 for result in resp_json['hits']['hits']:

113 for result in resp_json['hits']['hits']:

113 r = {key: str(value) if not key.startswith('_') else value for key, value in result['_source'].items()}

114 r = {key: str(value) if not key.startswith('_') else value for key, value in result['_source'].items()}

searx/engines/google.py+22−30

@@ -20,9 +20,10 @@ Definitions`_.

21from urllib.parse import urlencode, urlparse

22from lxml import html

23−from flask_babel import gettext

24from searx import logger

23from searx import logger

25−from searx.utils import match_language, extract_text, eval_xpath

24+from searx.utils import match_language, extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex

25+from searx.exceptions import SearxEngineCaptchaException

26+

27logger = logger.getChild('google engine')

28logger = logger.getChild('google engine')

@@ -131,14 +132,6 @@ suggestion_xpath = '//div[contains(@class, "card-section")]//a'

131spelling_suggestion_xpath = '//div[@class="med"]/p/a'

132spelling_suggestion_xpath = '//div[@class="med"]/p/a'

132

133

134

134−def extract_text_from_dom(result, xpath):

135− """returns extract_text on the first result selected by the xpath or None"""

136− r = eval_xpath(result, xpath)

137− if len(r) > 0:

138− return extract_text(r[0])

139− return None

140−

141−

142def get_lang_country(params, lang_list, custom_aliases):

135def get_lang_country(params, lang_list, custom_aliases):

143 """Returns a tuple with *langauage* on its first and *country* on its second

136 """Returns a tuple with *langauage* on its first and *country* on its second

144 position."""

137 position."""

@@ -210,10 +203,10 @@ def response(resp):

210 # detect google sorry

203 # detect google sorry

211 resp_url = urlparse(resp.url)

204 resp_url = urlparse(resp.url)

212 if resp_url.netloc == 'sorry.google.com' or resp_url.path == '/sorry/IndexRedirect':

205 if resp_url.netloc == 'sorry.google.com' or resp_url.path == '/sorry/IndexRedirect':

213− raise RuntimeWarning('sorry.google.com')

206+ raise SearxEngineCaptchaException()

214

207

215 if resp_url.path.startswith('/sorry'):

208 if resp_url.path.startswith('/sorry'):

216− raise RuntimeWarning(gettext('CAPTCHA required'))

209+ raise SearxEngineCaptchaException()

217

210

218 # which subdomain ?

211 # which subdomain ?

219 # subdomain = resp.search_params.get('google_subdomain')

212 # subdomain = resp.search_params.get('google_subdomain')

@@ -229,18 +222,17 @@ def response(resp):

229 logger.debug("did not found 'answer'")

222 logger.debug("did not found 'answer'")

230

223

231 # results --> number_of_results

224 # results --> number_of_results

232− try:

225+ try:

233− _txt = eval_xpath(dom, '//div[@id="result-stats"]//text()')[0]

226+ _txt = eval_xpath_getindex(dom, '//div[@id="result-stats"]//text()', 0)

234− _digit = ''.join([n for n in _txt if n.isdigit()])

227+ _digit = ''.join([n for n in _txt if n.isdigit()])

235− number_of_results = int(_digit)

228+ number_of_results = int(_digit)

236− results.append({'number_of_results': number_of_results})

229+ results.append({'number_of_results': number_of_results})

237−

238− except Exception as e: # pylint: disable=broad-except

230+ except Exception as e: # pylint: disable=broad-except

239− logger.debug("did not 'number_of_results'")

231+ logger.debug("did not 'number_of_results'")

240− logger.error(e, exc_info=True)

232+ logger.error(e, exc_info=True)

241

233

242 # parse results

234 # parse results

243− for result in eval_xpath(dom, results_xpath):

235+ for result in eval_xpath_list(dom, results_xpath):

244

236

245 # google *sections*

237 # google *sections*

246 if extract_text(eval_xpath(result, g_section_with_header)):

238 if extract_text(eval_xpath(result, g_section_with_header)):

@@ -248,14 +240,14 @@ def response(resp):

248 continue

240 continue

249

241

250 try:

242 try:

251− title_tag = eval_xpath(result, title_xpath)

243+ title_tag = eval_xpath_getindex(result, title_xpath, 0, default=None)

252− if not title_tag:

244+ if title_tag is None:

253 # this not one of the common google results *section*

245 # this not one of the common google results *section*

254 logger.debug('ingoring <div class="g" ../> section: missing title')

246 logger.debug('ingoring <div class="g" ../> section: missing title')

255 continue

247 continue

256− title = extract_text(title_tag[0])

248+ title = extract_text(title_tag)

257− url = eval_xpath(result, href_xpath)[0]

249+ url = eval_xpath_getindex(result, href_xpath, 0)

258− content = extract_text_from_dom(result, content_xpath)

250+ content = extract_text(eval_xpath_getindex(result, content_xpath, 0, default=None), allow_none=True)

259 results.append({

251 results.append({

260 'url': url,

252 'url': url,

261 'title': title,

253 'title': title,

@@ -270,11 +262,11 @@ def response(resp):

270 continue

262 continue

271

263

272 # parse suggestion

264 # parse suggestion

273− for suggestion in eval_xpath(dom, suggestion_xpath):

265+ for suggestion in eval_xpath_list(dom, suggestion_xpath):

274 # append suggestion

266 # append suggestion

275 results.append({'suggestion': extract_text(suggestion)})

267 results.append({'suggestion': extract_text(suggestion)})

276

268

277− for correction in eval_xpath(dom, spelling_suggestion_xpath):

269+ for correction in eval_xpath_list(dom, spelling_suggestion_xpath):

278 results.append({'correction': extract_text(correction)})

270 results.append({'correction': extract_text(correction)})

279

271

280 # return results

272 # return results

@@ -286,7 +278,7 @@ def _fetch_supported_languages(resp):

286 ret_val = {}

278 ret_val = {}

287 dom = html.fromstring(resp.text)

279 dom = html.fromstring(resp.text)

288

280

289− radio_buttons = eval_xpath(dom, '//*[@id="langSec"]//input[@name="lr"]')

281+ radio_buttons = eval_xpath_list(dom, '//*[@id="langSec"]//input[@name="lr"]')

290

282

291 for x in radio_buttons:

283 for x in radio_buttons:

292 name = x.get("data-name")

284 name = x.get("data-name")

searx/engines/google_images.py+3−3

@@ -26,8 +26,8 @@ Definitions`_.

27from urllib.parse import urlencode, urlparse, unquote

28from lxml import html

29−from flask_babel import gettext

30from searx import logger

29from searx import logger

30+from searx.exceptions import SearxEngineCaptchaException

31from searx.utils import extract_text, eval_xpath

32from searx.engines.google import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import

@@ -128,10 +128,10 @@ def response(resp):

128 # detect google sorry

129 resp_url = urlparse(resp.url)

130 if resp_url.netloc == 'sorry.google.com' or resp_url.path == '/sorry/IndexRedirect':

131− raise RuntimeWarning('sorry.google.com')

131+ raise SearxEngineCaptchaException()

132

133 if resp_url.path.startswith('/sorry'):

134− raise RuntimeWarning(gettext('CAPTCHA required'))

134+ raise SearxEngineCaptchaException()

135

136 # which subdomain ?

137 # subdomain = resp.search_params.get('google_subdomain')

searx/engines/google_videos.py+5−5

@@ -13,7 +13,7 @@

13from datetime import date, timedelta

14from urllib.parse import urlencode

15from lxml import html

16−from searx.utils import extract_text

16+from searx.utils import extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex

17import re

19# engine dependent config

@@ -66,11 +66,11 @@ def response(resp):

66 dom = html.fromstring(resp.text)

68 # parse results

69− for result in dom.xpath('//div[@class="g"]'):

69+ for result in eval_xpath_list(dom, '//div[@class="g"]'):

71− title = extract_text(result.xpath('.//h3'))

71+ title = extract_text(eval_xpath(result, './/h3'))

72− url = result.xpath('.//div[@class="r"]/a/@href')[0]

72+ url = eval_xpath_getindex(result, './/div[@class="r"]/a/@href', 0)

73− content = extract_text(result.xpath('.//span[@class="st"]'))

73+ content = extract_text(eval_xpath(result, './/span[@class="st"]'))

75 # get thumbnails

76 script = str(dom.xpath('//script[contains(., "_setImagesSrc")]')[0].text)

searx/engines/youtube_api.py+2−1

@@ -11,6 +11,7 @@

11from json import loads

12from dateutil import parser

13from urllib.parse import urlencode

14+from searx.exceptions import SearxEngineAPIException

15# engine dependent config

16# engine dependent config

16categories = ['videos', 'music']

17categories = ['videos', 'music']

@@ -48,7 +49,7 @@ def response(resp):

48 search_results = loads(resp.text)

49 search_results = loads(resp.text)

50 if 'error' in search_results and 'message' in search_results['error']:

51 if 'error' in search_results and 'message' in search_results['error']:

51− raise Exception(search_results['error']['message'])

52+ raise SearxEngineAPIException(search_results['error']['message'])

53 # return empty array if there are no results

54 # return empty array if there are no results

54 if 'items' not in search_results:

55 if 'items' not in search_results: