@@ -1,16 +1,30 @@
11# SPDX-License-Identifier: AGPL-3.0-or-later
2−"""
3− Startpage (Web)
2+# lint: pylint
3+"""Startpage (Web)
4+
45"""
56
6−from lxml import html
7−from dateutil import parser
8−from datetime import datetime, timedelta
97import re
8+from time import time
9+
10+from urllib.parse import urlencode
1011from unicodedata import normalize, combining
12+from datetime import datetime, timedelta
13+
14+from dateutil import parser
15+from lxml import html
1116from babel import Locale
1217from babel.localedata import locale_identifiers
18+
19+from searx import logger
20+from searx.network import get
1321from searx.utils import extract_text, eval_xpath, match_language
22+from searx.exceptions import (
23+ SearxEngineResponseException,
24+ SearxEngineCaptchaException,
25+)
26+
27+logger = logger.getChild('startpage')
1428
1529# about
1630about = {
@@ -33,7 +47,7 @@ supported_languages_url = 'https://www.startpage.com/do/settings'
3347
3448# search-url
3549base_url = 'https://startpage.com/'
36−search_url = base_url + 'do/search'
50+search_url = base_url + 'sp/search?'
3751
3852# specific xpath variables
3953# ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"]
@@ -42,18 +56,74 @@ results_xpath = '//div[@class="w-gl__result__main"]'
4256link_xpath = './/a[@class="w-gl__result-title result-link"]'
4357content_xpath = './/p[@class="w-gl__description"]'
4458
59+# timestamp of the last fetch of 'sc' code
60+sc_code_ts = 0
61+sc_code = ''
62+
63+
64+def raise_captcha(resp):
65+
66+ if str(resp.url).startswith('https://www.startpage.com/sp/captcha'):
67+ # suspend CAPTCHA for 7 days
68+ raise SearxEngineCaptchaException(suspended_time=7 * 24 * 3600)
69+
70+
71+def get_sc_code(headers):
72+ """Get an actual `sc` argument from startpage's home page.
73+
74+ Startpage puts a `sc` argument on every link. Without this argument
75+ startpage considers the request is from a bot. We do not know what is
76+ encoded in the value of the `sc` argument, but it seems to be a kind of a
77+ *time-stamp*. This *time-stamp* is valid for a few hours.
78+
79+ This function scrap a new *time-stamp* from startpage's home page every hour
80+ (3000 sec).
81+
82+ """
83+
84+ global sc_code_ts, sc_code # pylint: disable=global-statement
85+
86+ if time() > (sc_code_ts + 3000):
87+ logger.debug("query new sc time-stamp ...")
88+
89+ resp = get(base_url, headers=headers)
90+ raise_captcha(resp)
91+ dom = html.fromstring(resp.text)
92+
93+ try:
94+ # href --> '/?sc=adrKJMgF8xwp20'
95+ href = eval_xpath(dom, '//a[@class="footer-home__logo"]')[0].get('href')
96+ except IndexError as exc:
97+ # suspend startpage API --> https://github.com/searxng/searxng/pull/695
98+ raise SearxEngineResponseException(
99+ suspended_time=7 * 24 * 3600, message="PR-695: query new sc time-stamp failed!"
100+ ) from exc
101+
102+ sc_code = href[5:]
103+ sc_code_ts = time()
104+ logger.debug("new value is: %s", sc_code)
105+
106+ return sc_code
107+
45108
46109# do search-request
47110def request(query, params):
48111
49− params['url'] = search_url
50− params['method'] = 'POST'
51− params['data'] = {
112+ # pylint: disable=line-too-long
113+ # The format string from Startpage's FFox add-on [1]::
114+ #
115+ # https://www.startpage.com/do/dsearch?query={searchTerms}&cat=web&pl=ext-ff&language=__MSG_extensionUrlLanguage__&extVersion=1.3.0
116+ #
117+ # [1] https://addons.mozilla.org/en-US/firefox/addon/startpage-private-search/
118+
119+ args = {
52120 'query': query,
53121 'page': params['pageno'],
54122 'cat': 'web',
55− 'cmd': 'process_search',
56− 'engine0': 'v1all',
123+ # 'pl': 'ext-ff',
124+ # 'extVersion': '1.3.0',
125+ # 'abp': "-1",
126+ 'sc': get_sc_code(params['headers']),
57127 }
58128
59129 # set language if specified
@@ -61,9 +131,10 @@ def request(query, params):
61131 lang_code = match_language(params['language'], supported_languages, fallback=None)
62132 if lang_code:
63133 language_name = supported_languages[lang_code]['alias']
64− params['data']['language'] = language_name
65− params['data']['lui'] = language_name
134+ args['language'] = language_name
135+ args['lui'] = language_name
66136
137+ params['url'] = search_url + urlencode(args)
67138 return params
68139
69140
@@ -139,10 +210,11 @@ def response(resp):
139210
140211# get supported languages from their site
141212def _fetch_supported_languages(resp):
142− # startpage's language selector is a mess
143− # each option has a displayed name and a value, either of which may represent the language name
144− # in the native script, the language name in English, an English transliteration of the native name,
145− # the English name of the writing script used by the language, or occasionally something else entirely.
213+ # startpage's language selector is a mess each option has a displayed name
214+ # and a value, either of which may represent the language name in the native
215+ # script, the language name in English, an English transliteration of the
216+ # native name, the English name of the writing script used by the language,
217+ # or occasionally something else entirely.
146218
147219 # this cases are so special they need to be hardcoded, a couple of them are mispellings
148220 language_names = {
@@ -156,7 +228,15 @@ def _fetch_supported_languages(resp):
156228 }
157229
158230 # get the English name of every language known by babel
159− language_names.update({name.lower(): lang_code for lang_code, name in Locale('en')._data['languages'].items()})
231+ language_names.update(
232+ {
233+ # fmt: off
234+ name.lower(): lang_code
235+ # pylint: disable=protected-access
236+ for lang_code, name in Locale('en')._data['languages'].items()
237+ # fmt: on
238+ }
239+ )
160240
161241 # get the native name of every language known by babel
162242 for lang_code in filter(lambda lang_code: lang_code.find('_') == -1, locale_identifiers()):
@@ -181,8 +261,8 @@ def _fetch_supported_languages(resp):
181261 if isinstance(lang_code, str):
182262 supported_languages[lang_code] = {'alias': sp_option_value}
183263 elif isinstance(lang_code, list):
184− for lc in lang_code:
185− supported_languages[lc] = {'alias': sp_option_value}
264+ for _lc in lang_code:
265+ supported_languages[_lc] = {'alias': sp_option_value}
186266 else:
187267 print('Unknown language option in Startpage: {} ({})'.format(sp_option_value, sp_option_text))
188268