@@ -1,30 +1,16 @@
11# SPDX-License-Identifier: AGPL-3.0-or-later
2−# lint: pylint
3−"""Startpage (Web)
4−
2+"""
3+ Startpage (Web)
54"""
65
6+from lxml import html
7+from dateutil import parser
8+from datetime import datetime, timedelta
79import re
8−from time import time
9−
10−from urllib.parse import urlencode
1110from unicodedata import normalize, combining
12−from datetime import datetime, timedelta
13−
14−from dateutil import parser
15−from lxml import html
1611from babel import Locale
1712from babel.localedata import locale_identifiers
18−
19−from searx import logger
20−from searx.network import get
2113from searx.utils import extract_text, eval_xpath, match_language
22−from searx.exceptions import (
23− SearxEngineResponseException,
24− SearxEngineCaptchaException,
25−)
26−
27−logger = logger.getChild('startpage')
2814
2915# about
3016about = {
@@ -47,7 +33,7 @@ supported_languages_url = 'https://www.startpage.com/do/settings'
4733
4834# search-url
4935base_url = 'https://startpage.com/'
50−search_url = base_url + 'sp/search?'
36+search_url = base_url + 'do/search'
5137
5238# specific xpath variables
5339# ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"]
@@ -56,74 +42,18 @@ results_xpath = '//div[@class="w-gl__result__main"]'
5642link_xpath = './/a[@class="w-gl__result-title result-link"]'
5743content_xpath = './/p[@class="w-gl__description"]'
5844
59−# timestamp of the last fetch of 'sc' code
60−sc_code_ts = 0
61−sc_code = ''
62−
63−
64−def raise_captcha(resp):
65−
66− if str(resp.url).startswith('https://www.startpage.com/sp/captcha'):
67− # suspend CAPTCHA for 7 days
68− raise SearxEngineCaptchaException(suspended_time=7 * 24 * 3600)
69−
70−
71−def get_sc_code(headers):
72− """Get an actual `sc` argument from startpage's home page.
73−
74− Startpage puts a `sc` argument on every link. Without this argument
75− startpage considers the request is from a bot. We do not know what is
76− encoded in the value of the `sc` argument, but it seems to be a kind of a
77− *time-stamp*. This *time-stamp* is valid for a few hours.
78−
79− This function scrap a new *time-stamp* from startpage's home page every hour
80− (3000 sec).
81−
82− """
83−
84− global sc_code_ts, sc_code # pylint: disable=global-statement
85−
86− if time() > (sc_code_ts + 3000):
87− logger.debug("query new sc time-stamp ...")
88−
89− resp = get(base_url, headers=headers)
90− raise_captcha(resp)
91− dom = html.fromstring(resp.text)
92−
93− try:
94− # href --> '/?sc=adrKJMgF8xwp20'
95− href = eval_xpath(dom, '//a[@class="footer-home__logo"]')[0].get('href')
96− except IndexError as exc:
97− # suspend startpage API --> https://github.com/searxng/searxng/pull/695
98− raise SearxEngineResponseException(
99− suspended_time=7 * 24 * 3600, message="PR-695: query new sc time-stamp failed!"
100− ) from exc
101−
102− sc_code = href[5:]
103− sc_code_ts = time()
104− logger.debug("new value is: %s", sc_code)
105−
106− return sc_code
107−
10845
10946# do search-request
11047def request(query, params):
11148
112− # pylint: disable=line-too-long
113− # The format string from Startpage's FFox add-on [1]::
114− #
115− # https://www.startpage.com/do/dsearch?query={searchTerms}&cat=web&pl=ext-ff&language=__MSG_extensionUrlLanguage__&extVersion=1.3.0
116− #
117− # [1] https://addons.mozilla.org/en-US/firefox/addon/startpage-private-search/
118−
119− args = {
49+ params['url'] = search_url
50+ params['method'] = 'POST'
51+ params['data'] = {
12052 'query': query,
12153 'page': params['pageno'],
12254 'cat': 'web',
123− # 'pl': 'ext-ff',
124− # 'extVersion': '1.3.0',
125− # 'abp': "-1",
126− 'sc': get_sc_code(params['headers']),
55+ 'cmd': 'process_search',
56+ 'engine0': 'v1all',
12757 }
12858
12959 # set language if specified
@@ -131,10 +61,9 @@ def request(query, params):
13161 lang_code = match_language(params['language'], supported_languages, fallback=None)
13262 if lang_code:
13363 language_name = supported_languages[lang_code]['alias']
134− args['language'] = language_name
135− args['lui'] = language_name
64+ params['data']['language'] = language_name
65+ params['data']['lui'] = language_name
13666
137− params['url'] = search_url + urlencode(args)
13867 return params
13968
14069
@@ -210,11 +139,10 @@ def response(resp):
210139
211140# get supported languages from their site
212141def _fetch_supported_languages(resp):
213− # startpage's language selector is a mess each option has a displayed name
214− # and a value, either of which may represent the language name in the native
215− # script, the language name in English, an English transliteration of the
216− # native name, the English name of the writing script used by the language,
217− # or occasionally something else entirely.
142+ # startpage's language selector is a mess
143+ # each option has a displayed name and a value, either of which may represent the language name
144+ # in the native script, the language name in English, an English transliteration of the native name,
145+ # the English name of the writing script used by the language, or occasionally something else entirely.
218146
219147 # this cases are so special they need to be hardcoded, a couple of them are mispellings
220148 language_names = {
@@ -228,15 +156,7 @@ def _fetch_supported_languages(resp):
228156 }
229157
230158 # get the English name of every language known by babel
231− language_names.update(
232− {
233− # fmt: off
234− name.lower(): lang_code
235− # pylint: disable=protected-access
236− for lang_code, name in Locale('en')._data['languages'].items()
237− # fmt: on
238− }
239− )
159+ language_names.update({name.lower(): lang_code for lang_code, name in Locale('en')._data['languages'].items()})
240160
241161 # get the native name of every language known by babel
242162 for lang_code in filter(lambda lang_code: lang_code.find('_') == -1, locale_identifiers()):
@@ -261,8 +181,8 @@ def _fetch_supported_languages(resp):
261181 if isinstance(lang_code, str):
262182 supported_languages[lang_code] = {'alias': sp_option_value}
263183 elif isinstance(lang_code, list):
264− for _lc in lang_code:
265− supported_languages[_lc] = {'alias': sp_option_value}
184+ for lc in lang_code:
185+ supported_languages[lc] = {'alias': sp_option_value}
266186 else:
267187 print('Unknown language option in Startpage: {} ({})'.format(sp_option_value, sp_option_text))
268188