@@ -5,6 +5,7 @@
55"""
66
77import re
8+from time import time
89
910from urllib.parse import urlencode
1011from unicodedata import normalize, combining
@@ -15,6 +16,7 @@ from lxml import html
1516from babel import Locale
1617from babel.localedata import locale_identifiers
1718
19+from searx import network
1820from searx.utils import extract_text, eval_xpath, match_language
1921
2022# about
@@ -47,6 +49,41 @@ results_xpath = '//div[@class="w-gl__result__main"]'
4749link_xpath = './/a[@class="w-gl__result-title result-link"]'
4850content_xpath = './/p[@class="w-gl__description"]'
4951
52+# timestamp of the last fetch of 'sc' code
53+sc_code_ts = 0
54+sc_code = ''
55+
56+
57+def get_sc_code(headers):
58+ """Get an actual `sc` argument from startpage's home page.
59+
60+ Startpage puts a `sc` argument on every link. Without this argument
61+ startpage considers the request is from a bot. We do not know what is
62+ encoded in the value of the `sc` argument, but it seems to be a kind of a
63+ *time-stamp*. This *time-stamp* is valid for a few hours.
64+
65+ This function scrap a new *time-stamp* from startpage's home page every hour
66+ (3000 sec).
67+
68+ """
69+
70+ global sc_code_ts, sc_code # pylint: disable=global-statement
71+
72+ if time() > (sc_code_ts + 3000):
73+ logger.debug("query new sc time-stamp ...")
74+
75+ resp = network.get(base_url, headers=headers)
76+ dom = html.fromstring(resp.text)
77+
78+ # href --> '/?sc=adrKJMgF8xwp20'
79+ href = eval_xpath(dom, '//a[@class="footer-home__logo"]')[0].get('href')
80+
81+ sc_code = href[5:]
82+ sc_code_ts = time()
83+ logger.debug("new value is: %s", sc_code)
84+
85+ return sc_code
86+
5087
5188# do search-request
5289def request(query, params):
@@ -56,7 +93,7 @@ def request(query, params):
5693 'page': params['pageno'],
5794 'cat': 'web',
5895 # 'abp': "-1",
59− 'sc': 'Mj4jZy61QETj20',
96+ 'sc': get_sc_code(params['headers']),
6097 }
6198
6299 # set language if specified