9f8ebec · searx · git.iko.soy

docs/dev/engine_overview.rst+13−13

compare with master

unifiedsplit

@@ -134,19 +134,19 @@ The function ``def request(query, params):`` always returns the ``params``

134variable. Inside searx, the following paramters can be used to specify a search

135request:

136

137−================== =========== ==========================================================================

137+=================== =========== ==========================================================================

138−argument type information

138+argument type information

139−================== =========== ==========================================================================

139+=================== =========== ==========================================================================

140−url string requested url

140+url string requested url

141−method string HTTP request method

141+method string HTTP request method

142−headers set HTTP header information

142+headers set HTTP header information

143−data set HTTP data information (parsed if ``method != 'GET'``)

143+data set HTTP data information (parsed if ``method != 'GET'``)

144−cookies set HTTP cookies

144+cookies set HTTP cookies

145−verify boolean Performing SSL-Validity check

145+verify boolean Performing SSL-Validity check

146−max_redirects int maximum redirects, hard limit

146+max_redirects int maximum redirects, hard limit

147−soft_max_redirects int maximum redirects, soft limit. Record an error but don't stop the engine

147+soft_max_redirects int maximum redirects, soft limit. Record an error but don't stop the engine

148−raise_for_status bool True by default: raise an exception if the HTTP code of response is >= 300

148+raise_for_httperror bool True by default: raise an exception if the HTTP code of response is >= 300

149−================== =========== ==========================================================================

149+=================== =========== ==========================================================================

150

151

152example code

searx/engines/__init__.py+6−2

@@ -281,8 +281,12 @@ def initialize_engines(engine_list):

281 load_engines(engine_list)

282

283 def engine_init(engine_name, init_fn):

284+ try:

284− init_fn(get_engine_from_settings(engine_name))

285+ init_fn(get_engine_from_settings(engine_name))

286+ except Exception:

287+ logger.exception('%s engine: Fail to initialize', engine_name)

288+ else:

285− logger.debug('%s engine: Initialized', engine_name)

289+ logger.debug('%s engine: Initialized', engine_name)

286

290

287 for engine_name, engine in engines.items():

291 for engine_name, engine in engines.items():

288 if hasattr(engine, 'init'):

292 if hasattr(engine, 'init'):

searx/engines/qwant.py+16−11

@@ -14,6 +14,8 @@ from datetime import datetime

14from json import loads

15from urllib.parse import urlencode

16from searx.utils import html_to_text, match_language

17+from searx.exceptions import SearxEngineAPIException, SearxEngineCaptchaException

18+from searx.raise_for_httperror import raise_for_httperror

17

19

18

20

19# engine dependent config

21# engine dependent config

@@ -24,8 +26,7 @@ supported_languages_url = 'https://qwant.com/region'

24

26

25category_to_keyword = {'general': 'web',

27category_to_keyword = {'general': 'web',

26 'images': 'images',

28 'images': 'images',

27− 'news': 'news',

29+ 'news': 'news'}

28− 'social media': 'social'}

29

30

30# search-url

31# search-url

31url = 'https://api.qwant.com/api/search/{keyword}?count=10&offset={offset}&f=&{query}&t={keyword}&uiv=4'

32url = 'https://api.qwant.com/api/search/{keyword}?count=10&offset={offset}&f=&{query}&t={keyword}&uiv=4'

@@ -51,6 +52,7 @@ def request(query, params):

51 params['url'] += '&locale=' + language.replace('-', '_').lower()

52 params['url'] += '&locale=' + language.replace('-', '_').lower()

52

53

53 params['headers']['User-Agent'] = 'Mozilla/5.0 (X11; Linux x86_64; rv:69.0) Gecko/20100101 Firefox/69.0'

54 params['headers']['User-Agent'] = 'Mozilla/5.0 (X11; Linux x86_64; rv:69.0) Gecko/20100101 Firefox/69.0'

55+ params['raise_for_httperror'] = False

54 return params

56 return params

55

57

56

58

@@ -58,8 +60,20 @@ def request(query, params):

58def response(resp):

60def response(resp):

59 results = []

61 results = []

60

62

63+ # According to https://www.qwant.com/js/app.js

64+ if resp.status_code == 429:

65+ raise SearxEngineCaptchaException()

66+

67+ # raise for other errors

68+ raise_for_httperror(resp)

69+

70+ # load JSON result

61 search_results = loads(resp.text)

71 search_results = loads(resp.text)

62

72

73+ # check for an API error

74+ if search_results.get('status') != 'success':

75+ raise SearxEngineAPIException('API error ' + str(search_results.get('error', '')))

76+

63 # return empty array if there are no results

77 # return empty array if there are no results

64 if 'data' not in search_results:

78 if 'data' not in search_results:

65 return []

79 return []

@@ -90,15 +104,6 @@ def response(resp):

90 'thumbnail_src': thumbnail_src,

104 'thumbnail_src': thumbnail_src,

91 'img_src': img_src})

105 'img_src': img_src})

92

106

93− elif category_to_keyword.get(categories[0], '') == 'social':

94− published_date = datetime.fromtimestamp(result['date'], None)

95− img_src = result.get('img', None)

96− results.append({'url': res_url,

97− 'title': title,

98− 'publishedDate': published_date,

99− 'content': content,

100− 'img_src': img_src})

101−

102 elif category_to_keyword.get(categories[0], '') == 'news':

107 elif category_to_keyword.get(categories[0], '') == 'news':

103 published_date = datetime.fromtimestamp(result['date'], None)

108 published_date = datetime.fromtimestamp(result['date'], None)

104 media = result.get('media', [])

109 media = result.get('media', [])

searx/engines/wikidata.py−3

@@ -161,9 +161,6 @@ def request(query, params):

161

162def response(resp):

163 results = []

164− if resp.status_code != 200:

165− logger.debug('SPARQL endpoint error %s', resp.content.decode())

166− resp.raise_for_status()

167 jsonresponse = loads(resp.content.decode())

164 jsonresponse = loads(resp.content.decode())

168

165

169 language = resp.search_params['language'].lower()

166 language = resp.search_params['language'].lower()

searx/engines/wikipedia.py+3−1

@@ -14,6 +14,7 @@ from urllib.parse import quote

14from json import loads

15from lxml.html import fromstring

16from searx.utils import match_language, searx_useragent

17+from searx.raise_for_httperror import raise_for_httperror

17

18

18# search-url

19# search-url

19search_url = 'https://{language}.wikipedia.org/api/rest_v1/page/summary/{title}'

20search_url = 'https://{language}.wikipedia.org/api/rest_v1/page/summary/{title}'

@@ -37,7 +38,7 @@ def request(query, params):

37 language=url_lang(params['language']))

38 language=url_lang(params['language']))

38

39

39 params['headers']['User-Agent'] = searx_useragent()

40 params['headers']['User-Agent'] = searx_useragent()

40− params['raise_for_status'] = False

41+ params['raise_for_httperror'] = False

41 params['soft_max_redirects'] = 2

42 params['soft_max_redirects'] = 2

42

43

43 return params

44 return params

@@ -47,6 +48,7 @@ def request(query, params):

47def response(resp):

48def response(resp):

48 if resp.status_code == 404:

49 if resp.status_code == 404:

49 return []

50 return []

51+ raise_for_httperror(resp)

50

52

51 results = []

53 results = []

52 api_result = loads(resp.text)

54 api_result = loads(resp.text)

searx/exceptions.py+27−2

@@ -64,8 +64,33 @@ class SearxEngineAPIException(SearxEngineResponseException):

64 """The website has returned an application error"""

65

66

67+class SearxEngineAccessDeniedException(SearxEngineResponseException):

68+ """The website is blocking the access"""

69+

70+ def __init__(self, suspended_time=24 * 3600, message='Access denied'):

71+ super().__init__(message + ', suspended_time=' + str(suspended_time))

72+ self.suspended_time = suspended_time

73+ self.message = message

74+

75+

67−class SearxEngineCaptchaException(SearxEngineResponseException):

76+class SearxEngineCaptchaException(SearxEngineAccessDeniedException):

68− """The website has returned a CAPTCHA"""

77+ """The website has returned a CAPTCHA

78+

79+ By default, searx stops sending requests to this engine for 1 day.

80+ """

81+

82+ def __init__(self, suspended_time=24 * 3600, message='CAPTCHA'):

83+ super().__init__(message=message, suspended_time=suspended_time)

84+

85+

86+class SearxEngineTooManyRequestsException(SearxEngineAccessDeniedException):

87+ """The website has returned a Too Many Request status code

88+

89+ By default, searx stops sending requests to this engine for 1 hour.

90+ """

91+

92+ def __init__(self, suspended_time=3600, message='Too many request'):

93+ super().__init__(message=message, suspended_time=suspended_time)

69

94

70

95

71class SearxEngineXPathException(SearxEngineResponseException):

96class SearxEngineXPathException(SearxEngineResponseException):

searx/metrology/error_recorder.py+6−1

@@ -4,7 +4,8 @@ import logging

4from json import JSONDecodeError

5from urllib.parse import urlparse

6from requests.exceptions import RequestException

7−from searx.exceptions import SearxXPathSyntaxException, SearxEngineXPathException

7+from searx.exceptions import (SearxXPathSyntaxException, SearxEngineXPathException, SearxEngineAPIException,

8+ SearxEngineAccessDeniedException)

8from searx import logger

9from searx import logger

9

10

11

@@ -100,6 +101,10 @@ def get_messages(exc, filename) -> typing.Tuple:

100 return (exc.xpath_str, exc.message)

101 return (exc.xpath_str, exc.message)

101 if isinstance(exc, SearxEngineXPathException):

102 if isinstance(exc, SearxEngineXPathException):

102 return (exc.xpath_str, exc.message)

103 return (exc.xpath_str, exc.message)

104+ if isinstance(exc, SearxEngineAPIException):

105+ return (str(exc.args[0]), )

106+ if isinstance(exc, SearxEngineAccessDeniedException):

107+ return (exc.message, )

103 return ()

108 return ()

104

109

105

110

searx/poolrequests.py+11

@@ -7,6 +7,7 @@ import requests

7

8from searx import settings

9from searx import logger

10+from searx.raise_for_httperror import raise_for_httperror

10

11

12

12logger = logger.getChild('poolrequests')

13logger = logger.getChild('poolrequests')

@@ -156,6 +157,12 @@ def request(method, url, **kwargs):

156 if timeout is not None:

157 if timeout is not None:

157 kwargs['timeout'] = timeout

158 kwargs['timeout'] = timeout

158

159

160+ # raise_for_error

161+ check_for_httperror = True

162+ if 'raise_for_httperror' in kwargs:

163+ check_for_httperror = kwargs['raise_for_httperror']

164+ del kwargs['raise_for_httperror']

165+

159 # do request

166 # do request

160 response = session.request(method=method, url=url, **kwargs)

167 response = session.request(method=method, url=url, **kwargs)

161

168

@@ -176,6 +183,10 @@ def request(method, url, **kwargs):

176 if hasattr(threadLocal, 'total_time'):

183 if hasattr(threadLocal, 'total_time'):

177 threadLocal.total_time += time_after_request - time_before_request

184 threadLocal.total_time += time_after_request - time_before_request

178

185

186+ # raise an exception

187+ if check_for_httperror:

188+ raise_for_httperror(response)

189+

179 return response

190 return response

180

191

181

192

searx/raise_for_httperror.pynew file+66

@@ -0,0 +1,66 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+"""

3+Raise exception for an HTTP response is an error.

4+"""

5+from searx.exceptions import (SearxEngineCaptchaException, SearxEngineTooManyRequestsException,

6+ SearxEngineAccessDeniedException)

7+

8+

9+def is_cloudflare_challenge(resp):

10+ if resp.status_code in [429, 503]:

11+ if ('__cf_chl_jschl_tk__=' in resp.text)\

12+ or ('/cdn-cgi/challenge-platform/' in resp.text

13+ and 'orchestrate/jsch/v1' in resp.text

14+ and 'window._cf_chl_enter(' in resp.text):

15+ return True

16+ if resp.status_code == 403 and '__cf_chl_captcha_tk__=' in resp.text:

17+ return True

18+ return False

19+

20+

21+def is_cloudflare_firewall(resp):

22+ return resp.status_code == 403 and '<span class="cf-error-code">1020</span>' in resp.text

23+

24+

25+def raise_for_cloudflare_captcha(resp):

26+ if resp.headers.get('Server', '').startswith('cloudflare'):

27+ if is_cloudflare_challenge(resp):

28+ # https://support.cloudflare.com/hc/en-us/articles/200170136-Understanding-Cloudflare-Challenge-Passage-Captcha-

29+ # suspend for 2 weeks

30+ raise SearxEngineCaptchaException(message='Cloudflare CAPTCHA', suspended_time=3600 * 24 * 15)

31+

32+ if is_cloudflare_firewall(resp):

33+ raise SearxEngineAccessDeniedException(message='Cloudflare Firewall', suspended_time=3600 * 24)

34+

35+

36+def raise_for_recaptcha(resp):

37+ if resp.status_code == 503 \

38+ and '"https://www.google.com/recaptcha/' in resp.text:

39+ raise SearxEngineCaptchaException(message='ReCAPTCHA', suspended_time=3600 * 24 * 7)

40+

41+

42+def raise_for_captcha(resp):

43+ raise_for_cloudflare_captcha(resp)

44+ raise_for_recaptcha(resp)

45+

46+

47+def raise_for_httperror(resp):

48+ """Raise exception for an HTTP response is an error.

49+

50+ Args:

51+ resp (requests.Response): Response to check

52+

53+ Raises:

54+ requests.HTTPError: raise by resp.raise_for_status()

55+ searx.exceptions.SearxEngineAccessDeniedException: raise when the HTTP status code is 402 or 403.

56+ searx.exceptions.SearxEngineTooManyRequestsException: raise when the HTTP status code is 429.

57+ searx.exceptions.SearxEngineCaptchaException: raise when if CATPCHA challenge is detected.

58+ """

59+ if resp.status_code and resp.status_code >= 400:

60+ raise_for_captcha(resp)

61+ if resp.status_code in (402, 403):

62+ raise SearxEngineAccessDeniedException(message='HTTP error ' + str(resp.status_code),

63+ suspended_time=3600 * 24)

64+ if resp.status_code == 429:

65+ raise SearxEngineTooManyRequestsException()

66+ resp.raise_for_status()

searx/search.py+24−12

@@ -32,7 +32,8 @@ from searx.utils import gen_useragent

32from searx.results import ResultContainer

33from searx import logger

34from searx.plugins import plugins

35−from searx.exceptions import SearxEngineCaptchaException

35+from searx.exceptions import (SearxEngineAccessDeniedException, SearxEngineCaptchaException,

36+ SearxEngineTooManyRequestsException,)

36from searx.metrology.error_recorder import record_exception, record_error

37from searx.metrology.error_recorder import record_exception, record_error

37

38

39

@@ -131,6 +132,9 @@ def send_http_request(engine, request_params):

131 # soft_max_redirects

132 # soft_max_redirects

132 soft_max_redirects = request_params.get('soft_max_redirects', max_redirects or 0)

133 soft_max_redirects = request_params.get('soft_max_redirects', max_redirects or 0)

133

134

135+ # raise_for_status

136+ request_args['raise_for_httperror'] = request_params.get('raise_for_httperror', False)

137+

134 # specific type of request (GET or POST)

138 # specific type of request (GET or POST)

135 if request_params['method'] == 'GET':

139 if request_params['method'] == 'GET':

136 req = requests_lib.get

140 req = requests_lib.get

@@ -142,10 +146,6 @@ def send_http_request(engine, request_params):

142 # send the request

146 # send the request

143 response = req(request_params['url'], **request_args)

147 response = req(request_params['url'], **request_args)

144

148

145− # check HTTP status

146− if request_params.get('raise_for_status'):

147− response.raise_for_status()

148−

149 # check soft limit of the redirect count

150 if len(response.history) > soft_max_redirects:

151 # unexpected redirect : record an error

@@ -191,6 +191,7 @@ def search_one_http_request_safe(engine_name, query, request_params, result_cont

191

192 # suppose everything will be alright

193 requests_exception = False

194+ suspended_time = None

194

195

195 try:

196 try:

196 # send requests and parse the results

197 # send requests and parse the results

@@ -240,6 +241,15 @@ def search_one_http_request_safe(engine_name, query, request_params, result_cont

240 elif (issubclass(e.__class__, SearxEngineCaptchaException)):

241 elif (issubclass(e.__class__, SearxEngineCaptchaException)):

241 result_container.add_unresponsive_engine(engine_name, 'CAPTCHA required')

242 result_container.add_unresponsive_engine(engine_name, 'CAPTCHA required')

242 logger.exception('engine {0} : CAPTCHA')

243 logger.exception('engine {0} : CAPTCHA')

244+ suspended_time = e.suspended_time # pylint: disable=no-member

245+ elif (issubclass(e.__class__, SearxEngineTooManyRequestsException)):

246+ result_container.add_unresponsive_engine(engine_name, 'too many requests')

247+ logger.exception('engine {0} : Too many requests')

248+ suspended_time = e.suspended_time # pylint: disable=no-member

249+ elif (issubclass(e.__class__, SearxEngineAccessDeniedException)):

250+ result_container.add_unresponsive_engine(engine_name, 'blocked')

251+ logger.exception('engine {0} : Searx is blocked')

252+ suspended_time = e.suspended_time # pylint: disable=no-member

243 else:

253 else:

244 result_container.add_unresponsive_engine(engine_name, 'unexpected crash')

254 result_container.add_unresponsive_engine(engine_name, 'unexpected crash')

245 # others errors

255 # others errors

@@ -248,16 +258,18 @@ def search_one_http_request_safe(engine_name, query, request_params, result_cont

248 if getattr(threading.current_thread(), '_timeout', False):

258 if getattr(threading.current_thread(), '_timeout', False):

249 record_error(engine_name, 'Timeout')

259 record_error(engine_name, 'Timeout')

250

260

251− # suspend or not the engine if there are HTTP errors

261+ # suspend the engine if there is an HTTP error

262+ # or suspended_time is defined

252 with threading.RLock():

263 with threading.RLock():

253− if requests_exception:

264+ if requests_exception or suspended_time:

254 # update continuous_errors / suspend_end_time

265 # update continuous_errors / suspend_end_time

255 engine.continuous_errors += 1

266 engine.continuous_errors += 1

267+ if suspended_time is None:

256− engine.suspend_end_time = time() + min(settings['search']['max_ban_time_on_fail'],

268+ suspended_time = min(settings['search']['max_ban_time_on_fail'],

257− engine.continuous_errors * settings['search']['ban_time_on_fail'])

269+ engine.continuous_errors * settings['search']['ban_time_on_fail'])

270+ engine.suspend_end_time = time() + suspended_time

258 else:

271 else:

259− # no HTTP error (perhaps an engine error)

260− # anyway, reset the suspend variables

272+ # reset the suspend variables

261 engine.continuous_errors = 0

273 engine.continuous_errors = 0

262 engine.suspend_end_time = 0

274 engine.suspend_end_time = 0

263

275

@@ -342,7 +354,7 @@ def default_request_params():

342 'cookies': {},

354 'cookies': {},

343 'verify': True,

355 'verify': True,

344 'auth': None,

356 'auth': None,

345− 'raise_for_status': True

357+ 'raise_for_httperror': True

346 }

358 }

347

359

348

360

searx/settings.yml+7−11

@@ -647,11 +647,6 @@ engines:

647 shortcut : qwn

648 categories : news

649

650− - name : qwant social

651− engine : qwant

652− shortcut : qws

653− categories : social media

654−

655# - name: library

650# - name: library

656# engine: recoll

651# engine: recoll

657# shortcut: lib

652# shortcut: lib

@@ -817,12 +812,13 @@ engines:

817 # Or you can use the html non-stable engine, activated by default

812 # Or you can use the html non-stable engine, activated by default

818 engine : youtube_noapi

813 engine : youtube_noapi

819

814

815+ # tmp suspended: Cloudflare CAPTCHA

820− - name : yggtorrent

816+ #- name : yggtorrent

821− engine : yggtorrent

817+ # engine : yggtorrent

822− shortcut : ygg

818+ # shortcut : ygg

823− url: https://www2.yggtorrent.si/

819+ # url: https://www2.yggtorrent.si/

824− disabled : True

820+ # disabled : True

825− timeout : 4.0

821+ # timeout : 4.0

826

822

827 - name : dailymotion

823 - name : dailymotion

828 engine : dailymotion

824 engine : dailymotion