git.iko.soy/searx
on master help sign in
[fix] gigablast requires a random extra parameter
ee5d2b3Markus Heiser
searx/engines/gigablast.py+45−18
compare with master
unifiedsplit
@@ -1,3 +1,4 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Gigablast (Web)
3 Gigablast (Web)
3
4
@@ -11,20 +12,52 @@
11"""
12"""
12# pylint: disable=missing-function-docstring, invalid-name
13# pylint: disable=missing-function-docstring, invalid-name
13
14
14−from time import time
15+import re
15from json import loads
16from json import loads
17+# from searx import logger
16from searx.url_utils import urlencode
18from searx.url_utils import urlencode
19+from searx.poolrequests import get
17
20
18# engine dependent config
21# engine dependent config
19categories = ['general']
22categories = ['general']
20−paging = True
23+# gigablast's pagination is totally damaged, don't use it
21−number_of_results = 10
24+paging = False
22language_support = True
25language_support = True
23safesearch = True
26safesearch = True
24
27
25# search-url
28# search-url
29+base_url = 'https://gigablast.com'
30+
31+# ugly hack: gigablast requires a random extra parameter which can be extracted
32+# from the source code of the gigablast HTTP client
33+extra_param = ''
34+extra_param_path='/search?c=main&qlangcountry=en-us&q=south&s=10'
35+
36+def parse_extra_param(text):
37+
38+ # example:
39+ #
40+ # var uxrl='/search?c=main&qlangcountry=en-us&q=south&s=10&rand=1590740241635&n';
41+ # uxrl=uxrl+'sab=730863287';
42+ #
43+ # extra_param --> "rand=1590740241635&nsab=730863287"
44+
45+ global extra_param # pylint: disable=global-statement
46+ re_var= None
47+ for line in text.splitlines():
48+ if re_var is None and extra_param_path in line:
49+ var = line.split("=")[0].split()[1] # e.g. var --> 'uxrl'
50+ re_var = re.compile(var + "\\s*=\\s*" + var + "\\s*\\+\\s*'" + "(.*)" + "'(.*)")
51+ extra_param = line.split("'")[1][len(extra_param_path):]
52+ continue
53+ if re_var is not None and re_var.search(line):
54+ extra_param += re_var.search(line).group(1)
55+ break
56+ # logger.debug('gigablast extra_param="%s"', extra_param)
57+
58+def init(engine_settings=None): # pylint: disable=unused-argument
59+ parse_extra_param(get(base_url + extra_param_path).text)
26
60
27−base_url = 'https://gigablast.com/'
28
61
29# do search-request
62# do search-request
30def request(query, params): # pylint: disable=unused-argument
63def request(query, params): # pylint: disable=unused-argument
@@ -34,21 +67,12 @@ def request(query, params): # pylint: disable=unused-argument
34
67
35 query_args = dict(
68 query_args = dict(
36 c = 'main'
69 c = 'main'
37− , n = number_of_results
38 , format = 'json'
70 , format = 'json'
39 , q = query
71 , q = query
40− # The gigablast HTTP client sends a random number and a nsga argument
72+ , dr = 1
41− # (the values don't seem to matter)
73+ , showgoodimages = 0
42− , rand = int(time() * 1000)
43− , nsga = int(time() * 1000)
44 )
74 )
45
75
46− page_no = (params['pageno'] - 1)
47− if page_no:
48− # API quirk; adds +2 to the number_of_results
49− offset = (params['pageno'] - 1) * number_of_results
50− query_args['s'] = offset
51−
52 if params['language'] and params['language'] != 'all':
76 if params['language'] and params['language'] != 'all':
53 query_args['qlangcountry'] = params['language']
77 query_args['qlangcountry'] = params['language']
54 query_args['qlang'] = params['language'].split('-')[0]
78 query_args['qlang'] = params['language'].split('-')[0]
@@ -56,8 +80,8 @@ def request(query, params): # pylint: disable=unused-argument
56 if params['safesearch'] >= 1:
80 if params['safesearch'] >= 1:
57 query_args['ff'] = 1
81 query_args['ff'] = 1
58
82
59− search_url = 'search?' + urlencode(query_args)
83+ search_url = '/search?' + urlencode(query_args)
60− params['url'] = base_url + search_url
84+ params['url'] = base_url + search_url + extra_param
61
85
62 return params
86 return params
63
87
@@ -66,6 +90,9 @@ def response(resp):
66 results = []
90 results = []
67
91
68 response_json = loads(resp.text)
92 response_json = loads(resp.text)
93+
94+ # logger.debug('gigablast returns %s results', len(response_json['results']))
95+
69 for result in response_json['results']:
96 for result in response_json['results']:
70 # see "Example JSON Output (&format=json)"
97 # see "Example JSON Output (&format=json)"
71 # at http://www.gigablast.com/api.html#/search
98 # at http://www.gigablast.com/api.html#/search
@@ -87,7 +114,7 @@ def response(resp):
87 # extend fields
114 # extend fields
88
115
89 subtitle = result.get('title')
116 subtitle = result.get('title')
90− if len(subtitle) > 3:
117+ if len(subtitle) > 3 and subtitle != title:
91 title += " - " + subtitle
118 title += " - " + subtitle
92
119
93 results.append(dict(
120 results.append(dict(