@@ -1,3 +1,4 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
12"""
23 Gigablast (Web)
34
@@ -11,20 +12,52 @@
1112"""
1213# pylint: disable=missing-function-docstring, invalid-name
1314
14−from time import time
15+import re
1516from json import loads
17+# from searx import logger
1618from searx.url_utils import urlencode
19+from searx.poolrequests import get
1720
1821# engine dependent config
1922categories = ['general']
20−paging = True
21−number_of_results = 10
23+# gigablast's pagination is totally damaged, don't use it
24+paging = False
2225language_support = True
2326safesearch = True
2427
2528# search-url
29+base_url = 'https://gigablast.com'
30+
31+# ugly hack: gigablast requires a random extra parameter which can be extracted
32+# from the source code of the gigablast HTTP client
33+extra_param = ''
34+extra_param_path='/search?c=main&qlangcountry=en-us&q=south&s=10'
35+
36+def parse_extra_param(text):
37+
38+ # example:
39+ #
40+ # var uxrl='/search?c=main&qlangcountry=en-us&q=south&s=10&rand=1590740241635&n';
41+ # uxrl=uxrl+'sab=730863287';
42+ #
43+ # extra_param --> "rand=1590740241635&nsab=730863287"
44+
45+ global extra_param # pylint: disable=global-statement
46+ re_var= None
47+ for line in text.splitlines():
48+ if re_var is None and extra_param_path in line:
49+ var = line.split("=")[0].split()[1] # e.g. var --> 'uxrl'
50+ re_var = re.compile(var + "\\s*=\\s*" + var + "\\s*\\+\\s*'" + "(.*)" + "'(.*)")
51+ extra_param = line.split("'")[1][len(extra_param_path):]
52+ continue
53+ if re_var is not None and re_var.search(line):
54+ extra_param += re_var.search(line).group(1)
55+ break
56+ # logger.debug('gigablast extra_param="%s"', extra_param)
57+
58+def init(engine_settings=None): # pylint: disable=unused-argument
59+ parse_extra_param(get(base_url + extra_param_path).text)
2660
27−base_url = 'https://gigablast.com/'
2861
2962# do search-request
3063def request(query, params): # pylint: disable=unused-argument
@@ -34,21 +67,12 @@ def request(query, params): # pylint: disable=unused-argument
3467
3568 query_args = dict(
3669 c = 'main'
37− , n = number_of_results
3870 , format = 'json'
3971 , q = query
40− # The gigablast HTTP client sends a random number and a nsga argument
41− # (the values don't seem to matter)
42− , rand = int(time() * 1000)
43− , nsga = int(time() * 1000)
72+ , dr = 1
73+ , showgoodimages = 0
4474 )
4575
46− page_no = (params['pageno'] - 1)
47− if page_no:
48− # API quirk; adds +2 to the number_of_results
49− offset = (params['pageno'] - 1) * number_of_results
50− query_args['s'] = offset
51−
5276 if params['language'] and params['language'] != 'all':
5377 query_args['qlangcountry'] = params['language']
5478 query_args['qlang'] = params['language'].split('-')[0]
@@ -56,8 +80,8 @@ def request(query, params): # pylint: disable=unused-argument
5680 if params['safesearch'] >= 1:
5781 query_args['ff'] = 1
5882
59− search_url = 'search?' + urlencode(query_args)
60− params['url'] = base_url + search_url
83+ search_url = '/search?' + urlencode(query_args)
84+ params['url'] = base_url + search_url + extra_param
6185
6286 return params
6387
@@ -66,6 +90,9 @@ def response(resp):
6690 results = []
6791
6892 response_json = loads(resp.text)
93+
94+ # logger.debug('gigablast returns %s results', len(response_json['results']))
95+
6996 for result in response_json['results']:
7097 # see "Example JSON Output (&format=json)"
7198 # at http://www.gigablast.com/api.html#/search
@@ -87,7 +114,7 @@ def response(resp):
87114 # extend fields
88115
89116 subtitle = result.get('title')
90− if len(subtitle) > 3:
117+ if len(subtitle) > 3 and subtitle != title:
91118 title += " - " + subtitle
92119
93120 results.append(dict(