git.iko.soy/searx
on master help sign in
[fix] startpage engine: fetch CAPTCHA & issues related to PR-695
a4bc089Markus Heiser
searx/engines/startpage.py+21−2
compare with master
unifiedsplit
@@ -18,6 +18,11 @@ from babel.localedata import locale_identifiers
18
18
19from searx import network
19from searx import network
20from searx.utils import extract_text, eval_xpath, match_language
20from searx.utils import extract_text, eval_xpath, match_language
21+from searx.exceptions import (
22+ SearxEngineResponseException,
23+ SearxEngineCaptchaException,
24+)
25+
21
26
22# about
27# about
23about = {
28about = {
@@ -54,6 +59,13 @@ sc_code_ts = 0
54sc_code = ''
59sc_code = ''
55
60
56
61
62+def raise_captcha(resp):
63+
64+ if str(resp.url).startswith('https://www.startpage.com/sp/captcha'):
65+ # suspend CAPTCHA for 7 days
66+ raise SearxEngineCaptchaException(suspended_time=7 * 24 * 3600)
67+
68+
57def get_sc_code(headers):
69def get_sc_code(headers):
58 """Get an actual `sc` argument from startpage's home page.
70 """Get an actual `sc` argument from startpage's home page.
59
71
@@ -73,10 +85,17 @@ def get_sc_code(headers):
73 logger.debug("query new sc time-stamp ...")
85 logger.debug("query new sc time-stamp ...")
74
86
75 resp = network.get(base_url, headers=headers)
87 resp = network.get(base_url, headers=headers)
88+ raise_captcha(resp)
76 dom = html.fromstring(resp.text)
89 dom = html.fromstring(resp.text)
77
90
91+ try:
78− # href --> '/?sc=adrKJMgF8xwp20'
92+ # href --> '/?sc=adrKJMgF8xwp20'
79− href = eval_xpath(dom, '//a[@class="footer-home__logo"]')[0].get('href')
93+ href = eval_xpath(dom, '//a[@class="footer-home__logo"]')[0].get('href')
94+ except IndexError as exc:
95+ # suspend startpage API --> https://github.com/searxng/searxng/pull/695
96+ raise SearxEngineResponseException(
97+ suspended_time=7 * 24 * 3600, message="PR-695: query new sc time-stamp failed!"
98+ )
80
99
81 sc_code = href[5:]
100 sc_code = href[5:]
82 sc_code_ts = time()
101 sc_code_ts = time()