d703119 · searx · git.iko.soy

git.iko.soy/searx

[enh] add raise_for_httperror

check HTTP response: * detect some comme CAPTCHA challenge (no solving). In this case the engine is suspended for long a time. * otherwise raise HTTPError as before the check is done in poolrequests.py (was before in search.py). update qwant, wikipedia, wikidata to use raise_for_httperror instead of raise_for_status

commitd703119d3a313a406482b121ee94c6afee3bc307 authorAlexandre Flament <alex@al-f.net> dateWed 9 Dec 2020, 21:23 (6 years ago) parent033f39b

11 files changed+179−56

compare with master

unifiedsplit

docs/dev/engine_overview.rst+13−13 searx/engines/__init__.py+6−2 searx/engines/qwant.py+16−11 searx/engines/wikidata.py−3 searx/engines/wikipedia.py+3−1 searx/exceptions.py+27−2 searx/metrology/error_recorder.py+6−1 searx/poolrequests.py+11 searx/raise_for_httperror.py+66 searx/search.py+24−12 searx/settings.yml+7−11

docs/dev/engine_overview.rst+13−13

@@ -134,19 +134,19 @@ The function ``def request(query, params):`` always returns the ``params``

134134variable. Inside searx, the following paramters can be used to specify a search

135135request:

136136

137−================== =========== ==========================================================================

138−argument type information

139−================== =========== ==========================================================================

140−url string requested url

141−method string HTTP request method

142−headers set HTTP header information

143−data set HTTP data information (parsed if ``method != 'GET'``)

144−cookies set HTTP cookies

145−verify boolean Performing SSL-Validity check

146−max_redirects int maximum redirects, hard limit

147−soft_max_redirects int maximum redirects, soft limit. Record an error but don't stop the engine

148−raise_for_status bool True by default: raise an exception if the HTTP code of response is >= 300

149−================== =========== ==========================================================================

137+=================== =========== ==========================================================================

138+argument type information

139+=================== =========== ==========================================================================

140+url string requested url

141+method string HTTP request method

142+headers set HTTP header information

143+data set HTTP data information (parsed if ``method != 'GET'``)

144+cookies set HTTP cookies

145+verify boolean Performing SSL-Validity check

146+max_redirects int maximum redirects, hard limit

147+soft_max_redirects int maximum redirects, soft limit. Record an error but don't stop the engine

148+raise_for_httperror bool True by default: raise an exception if the HTTP code of response is >= 300

149+=================== =========== ==========================================================================

150150

151151

152152example code

searx/engines/__init__.py+6−2

@@ -281,8 +281,12 @@ def initialize_engines(engine_list):

281281 load_engines(engine_list)

282282

283283 def engine_init(engine_name, init_fn):

284− init_fn(get_engine_from_settings(engine_name))

285− logger.debug('%s engine: Initialized', engine_name)

284+ try:

285+ init_fn(get_engine_from_settings(engine_name))

286+ except Exception:

287+ logger.exception('%s engine: Fail to initialize', engine_name)

288+ else:

289+ logger.debug('%s engine: Initialized', engine_name)

286290

287291 for engine_name, engine in engines.items():

288292 if hasattr(engine, 'init'):

searx/engines/qwant.py+16−11

@@ -14,6 +14,8 @@ from datetime import datetime

1414from json import loads

1515from urllib.parse import urlencode

1616from searx.utils import html_to_text, match_language

17+from searx.exceptions import SearxEngineAPIException, SearxEngineCaptchaException

18+from searx.raise_for_httperror import raise_for_httperror

1719

1820

1921# engine dependent config

@@ -24,8 +26,7 @@ supported_languages_url = 'https://qwant.com/region'

2426

2527category_to_keyword = {'general': 'web',

2628 'images': 'images',

27− 'news': 'news',

28− 'social media': 'social'}

29+ 'news': 'news'}

2930

3031# search-url

3132url = 'https://api.qwant.com/api/search/{keyword}?count=10&offset={offset}&f=&{query}&t={keyword}&uiv=4'

@@ -51,6 +52,7 @@ def request(query, params):

5152 params['url'] += '&locale=' + language.replace('-', '_').lower()

5253

5354 params['headers']['User-Agent'] = 'Mozilla/5.0 (X11; Linux x86_64; rv:69.0) Gecko/20100101 Firefox/69.0'

55+ params['raise_for_httperror'] = False

5456 return params

5557

5658

@@ -58,8 +60,20 @@ def request(query, params):

5860def response(resp):

5961 results = []

6062

63+ # According to https://www.qwant.com/js/app.js

64+ if resp.status_code == 429:

65+ raise SearxEngineCaptchaException()

66+

67+ # raise for other errors

68+ raise_for_httperror(resp)

69+

70+ # load JSON result

6171 search_results = loads(resp.text)

6272

73+ # check for an API error

74+ if search_results.get('status') != 'success':

75+ raise SearxEngineAPIException('API error ' + str(search_results.get('error', '')))

76+

6377 # return empty array if there are no results

6478 if 'data' not in search_results:

6579 return []

@@ -90,15 +104,6 @@ def response(resp):

90104 'thumbnail_src': thumbnail_src,

91105 'img_src': img_src})

92106

93− elif category_to_keyword.get(categories[0], '') == 'social':

94− published_date = datetime.fromtimestamp(result['date'], None)

95− img_src = result.get('img', None)

96− results.append({'url': res_url,

97− 'title': title,

98− 'publishedDate': published_date,

99− 'content': content,

100− 'img_src': img_src})

101−

102107 elif category_to_keyword.get(categories[0], '') == 'news':

103108 published_date = datetime.fromtimestamp(result['date'], None)

104109 media = result.get('media', [])

searx/engines/wikidata.py−3

@@ -161,9 +161,6 @@ def request(query, params):

161161

162162def response(resp):

163163 results = []

164− if resp.status_code != 200:

165− logger.debug('SPARQL endpoint error %s', resp.content.decode())

166− resp.raise_for_status()

167164 jsonresponse = loads(resp.content.decode())

168165

169166 language = resp.search_params['language'].lower()

searx/engines/wikipedia.py+3−1

@@ -14,6 +14,7 @@ from urllib.parse import quote

1414from json import loads

1515from lxml.html import fromstring

1616from searx.utils import match_language, searx_useragent

17+from searx.raise_for_httperror import raise_for_httperror

1718

1819# search-url

1920search_url = 'https://{language}.wikipedia.org/api/rest_v1/page/summary/{title}'

@@ -37,7 +38,7 @@ def request(query, params):

3738 language=url_lang(params['language']))

3839

3940 params['headers']['User-Agent'] = searx_useragent()

40− params['raise_for_status'] = False

41+ params['raise_for_httperror'] = False

4142 params['soft_max_redirects'] = 2

4243

4344 return params

@@ -47,6 +48,7 @@ def request(query, params):

4748def response(resp):

4849 if resp.status_code == 404:

4950 return []

51+ raise_for_httperror(resp)

5052

5153 results = []

5254 api_result = loads(resp.text)

searx/exceptions.py+27−2

@@ -64,8 +64,33 @@ class SearxEngineAPIException(SearxEngineResponseException):

6464 """The website has returned an application error"""

6565

6666

67−class SearxEngineCaptchaException(SearxEngineResponseException):

68− """The website has returned a CAPTCHA"""

67+class SearxEngineAccessDeniedException(SearxEngineResponseException):

68+ """The website is blocking the access"""

69+

70+ def __init__(self, suspended_time=24 * 3600, message='Access denied'):

71+ super().__init__(message + ', suspended_time=' + str(suspended_time))

72+ self.suspended_time = suspended_time

73+ self.message = message

74+

75+

76+class SearxEngineCaptchaException(SearxEngineAccessDeniedException):

77+ """The website has returned a CAPTCHA

78+

79+ By default, searx stops sending requests to this engine for 1 day.

80+ """

81+

82+ def __init__(self, suspended_time=24 * 3600, message='CAPTCHA'):

83+ super().__init__(message=message, suspended_time=suspended_time)

84+

85+

86+class SearxEngineTooManyRequestsException(SearxEngineAccessDeniedException):

87+ """The website has returned a Too Many Request status code

88+

89+ By default, searx stops sending requests to this engine for 1 hour.

90+ """

91+

92+ def __init__(self, suspended_time=3600, message='Too many request'):

93+ super().__init__(message=message, suspended_time=suspended_time)

6994

7095

7196class SearxEngineXPathException(SearxEngineResponseException):

searx/metrology/error_recorder.py+6−1

@@ -4,7 +4,8 @@ import logging

44from json import JSONDecodeError

55from urllib.parse import urlparse

66from requests.exceptions import RequestException

7−from searx.exceptions import SearxXPathSyntaxException, SearxEngineXPathException

7+from searx.exceptions import (SearxXPathSyntaxException, SearxEngineXPathException, SearxEngineAPIException,

8+ SearxEngineAccessDeniedException)

89from searx import logger

910

1011

@@ -100,6 +101,10 @@ def get_messages(exc, filename) -> typing.Tuple:

100101 return (exc.xpath_str, exc.message)

101102 if isinstance(exc, SearxEngineXPathException):

102103 return (exc.xpath_str, exc.message)

104+ if isinstance(exc, SearxEngineAPIException):

105+ return (str(exc.args[0]), )

106+ if isinstance(exc, SearxEngineAccessDeniedException):

107+ return (exc.message, )

103108 return ()

104109

105110

searx/poolrequests.py+11

@@ -7,6 +7,7 @@ import requests

88from searx import settings

99from searx import logger

10+from searx.raise_for_httperror import raise_for_httperror

1011

1112

1213logger = logger.getChild('poolrequests')

@@ -156,6 +157,12 @@ def request(method, url, **kwargs):

156157 if timeout is not None:

157158 kwargs['timeout'] = timeout

158159

160+ # raise_for_error

161+ check_for_httperror = True

162+ if 'raise_for_httperror' in kwargs:

163+ check_for_httperror = kwargs['raise_for_httperror']

164+ del kwargs['raise_for_httperror']

165+

159166 # do request

160167 response = session.request(method=method, url=url, **kwargs)

161168

@@ -176,6 +183,10 @@ def request(method, url, **kwargs):

176183 if hasattr(threadLocal, 'total_time'):

177184 threadLocal.total_time += time_after_request - time_before_request

178185

186+ # raise an exception

187+ if check_for_httperror:

188+ raise_for_httperror(response)

189+

179190 return response

180191

181192

searx/raise_for_httperror.pynew file+66

@@ -0,0 +1,66 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+"""

3+Raise exception for an HTTP response is an error.

4+"""

5+from searx.exceptions import (SearxEngineCaptchaException, SearxEngineTooManyRequestsException,

6+ SearxEngineAccessDeniedException)

9+def is_cloudflare_challenge(resp):

10+ if resp.status_code in [429, 503]:

11+ if ('__cf_chl_jschl_tk__=' in resp.text)\

12+ or ('/cdn-cgi/challenge-platform/' in resp.text

13+ and 'orchestrate/jsch/v1' in resp.text

14+ and 'window._cf_chl_enter(' in resp.text):

15+ return True

16+ if resp.status_code == 403 and '__cf_chl_captcha_tk__=' in resp.text:

17+ return True

18+ return False

19+

20+

21+def is_cloudflare_firewall(resp):

22+ return resp.status_code == 403 and '<span class="cf-error-code">1020</span>' in resp.text

23+

24+

25+def raise_for_cloudflare_captcha(resp):

26+ if resp.headers.get('Server', '').startswith('cloudflare'):

27+ if is_cloudflare_challenge(resp):

28+ # https://support.cloudflare.com/hc/en-us/articles/200170136-Understanding-Cloudflare-Challenge-Passage-Captcha-

29+ # suspend for 2 weeks

30+ raise SearxEngineCaptchaException(message='Cloudflare CAPTCHA', suspended_time=3600 * 24 * 15)

31+

32+ if is_cloudflare_firewall(resp):

33+ raise SearxEngineAccessDeniedException(message='Cloudflare Firewall', suspended_time=3600 * 24)

34+

35+

36+def raise_for_recaptcha(resp):

37+ if resp.status_code == 503 \

38+ and '"https://www.google.com/recaptcha/' in resp.text:

39+ raise SearxEngineCaptchaException(message='ReCAPTCHA', suspended_time=3600 * 24 * 7)

40+

41+

42+def raise_for_captcha(resp):

43+ raise_for_cloudflare_captcha(resp)

44+ raise_for_recaptcha(resp)

45+

46+

47+def raise_for_httperror(resp):

48+ """Raise exception for an HTTP response is an error.

49+

50+ Args:

51+ resp (requests.Response): Response to check

52+

53+ Raises:

54+ requests.HTTPError: raise by resp.raise_for_status()

55+ searx.exceptions.SearxEngineAccessDeniedException: raise when the HTTP status code is 402 or 403.

56+ searx.exceptions.SearxEngineTooManyRequestsException: raise when the HTTP status code is 429.

57+ searx.exceptions.SearxEngineCaptchaException: raise when if CATPCHA challenge is detected.

58+ """

59+ if resp.status_code and resp.status_code >= 400:

60+ raise_for_captcha(resp)

61+ if resp.status_code in (402, 403):

62+ raise SearxEngineAccessDeniedException(message='HTTP error ' + str(resp.status_code),

63+ suspended_time=3600 * 24)

64+ if resp.status_code == 429:

65+ raise SearxEngineTooManyRequestsException()

66+ resp.raise_for_status()

searx/search.py+24−12

@@ -32,7 +32,8 @@ from searx.utils import gen_useragent

3232from searx.results import ResultContainer

3333from searx import logger

3434from searx.plugins import plugins

35−from searx.exceptions import SearxEngineCaptchaException

35+from searx.exceptions import (SearxEngineAccessDeniedException, SearxEngineCaptchaException,

36+ SearxEngineTooManyRequestsException,)

3637from searx.metrology.error_recorder import record_exception, record_error

3738

3839

@@ -131,6 +132,9 @@ def send_http_request(engine, request_params):

131132 # soft_max_redirects

132133 soft_max_redirects = request_params.get('soft_max_redirects', max_redirects or 0)

133134

135+ # raise_for_status

136+ request_args['raise_for_httperror'] = request_params.get('raise_for_httperror', False)

137+

134138 # specific type of request (GET or POST)

135139 if request_params['method'] == 'GET':

136140 req = requests_lib.get

@@ -142,10 +146,6 @@ def send_http_request(engine, request_params):

142146 # send the request

143147 response = req(request_params['url'], **request_args)

144148

145− # check HTTP status

146− if request_params.get('raise_for_status'):

147− response.raise_for_status()

148−

149149 # check soft limit of the redirect count

150150 if len(response.history) > soft_max_redirects:

151151 # unexpected redirect : record an error

@@ -191,6 +191,7 @@ def search_one_http_request_safe(engine_name, query, request_params, result_cont

191191

192192 # suppose everything will be alright

193193 requests_exception = False

194+ suspended_time = None

194195

195196 try:

196197 # send requests and parse the results

@@ -240,6 +241,15 @@ def search_one_http_request_safe(engine_name, query, request_params, result_cont

240241 elif (issubclass(e.__class__, SearxEngineCaptchaException)):

241242 result_container.add_unresponsive_engine(engine_name, 'CAPTCHA required')

242243 logger.exception('engine {0} : CAPTCHA')

244+ suspended_time = e.suspended_time # pylint: disable=no-member

245+ elif (issubclass(e.__class__, SearxEngineTooManyRequestsException)):

246+ result_container.add_unresponsive_engine(engine_name, 'too many requests')

247+ logger.exception('engine {0} : Too many requests')

248+ suspended_time = e.suspended_time # pylint: disable=no-member

249+ elif (issubclass(e.__class__, SearxEngineAccessDeniedException)):

250+ result_container.add_unresponsive_engine(engine_name, 'blocked')

251+ logger.exception('engine {0} : Searx is blocked')

252+ suspended_time = e.suspended_time # pylint: disable=no-member

243253 else:

244254 result_container.add_unresponsive_engine(engine_name, 'unexpected crash')

245255 # others errors

@@ -248,16 +258,18 @@ def search_one_http_request_safe(engine_name, query, request_params, result_cont

248258 if getattr(threading.current_thread(), '_timeout', False):

249259 record_error(engine_name, 'Timeout')

250260

251− # suspend or not the engine if there are HTTP errors

261+ # suspend the engine if there is an HTTP error

262+ # or suspended_time is defined

252263 with threading.RLock():

253− if requests_exception:

264+ if requests_exception or suspended_time:

254265 # update continuous_errors / suspend_end_time

255266 engine.continuous_errors += 1

256− engine.suspend_end_time = time() + min(settings['search']['max_ban_time_on_fail'],

257− engine.continuous_errors * settings['search']['ban_time_on_fail'])

267+ if suspended_time is None:

268+ suspended_time = min(settings['search']['max_ban_time_on_fail'],

269+ engine.continuous_errors * settings['search']['ban_time_on_fail'])

270+ engine.suspend_end_time = time() + suspended_time

258271 else:

259− # no HTTP error (perhaps an engine error)

260− # anyway, reset the suspend variables

272+ # reset the suspend variables

261273 engine.continuous_errors = 0

262274 engine.suspend_end_time = 0

263275

@@ -342,7 +354,7 @@ def default_request_params():

342354 'cookies': {},

343355 'verify': True,

344356 'auth': None,

345− 'raise_for_status': True

357+ 'raise_for_httperror': True

346358 }

347359

348360

searx/settings.yml+7−11

@@ -647,11 +647,6 @@ engines:

647647 shortcut : qwn

648648 categories : news

649649

650− - name : qwant social

651− engine : qwant

652− shortcut : qws

653− categories : social media

654−

655650# - name: library

656651# engine: recoll

657652# shortcut: lib

@@ -817,12 +812,13 @@ engines:

817812 # Or you can use the html non-stable engine, activated by default

818813 engine : youtube_noapi

819814

820− - name : yggtorrent

821− engine : yggtorrent

822− shortcut : ygg

823− url: https://www2.yggtorrent.si/

824− disabled : True

825− timeout : 4.0

815+ # tmp suspended: Cloudflare CAPTCHA

816+ #- name : yggtorrent

817+ # engine : yggtorrent

818+ # shortcut : ygg

819+ # url: https://www2.yggtorrent.si/

820+ # disabled : True

821+ # timeout : 4.0

826822

827823 - name : dailymotion

828824 engine : dailymotion