@@ -1,3 +1,4 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
12"""
23 Gigablast (Web)
34
@@ -9,121 +10,117 @@
910 @stable yes
1011 @parse url, title, content
1112"""
13+# pylint: disable=missing-function-docstring, invalid-name
1214
13−import random
15+import re
1416from json import loads
15−from time import time
16−from lxml.html import fromstring
17−from searx.poolrequests import get
17+# from searx import logger
1818from searx.url_utils import urlencode
19−from searx.utils import eval_xpath
19+from searx.poolrequests import get
2020
2121# engine dependent config
2222categories = ['general']
23−paging = True
24−number_of_results = 10
23+# gigablast's pagination is totally damaged, don't use it
24+paging = False
2525language_support = True
2626safesearch = True
2727
2828# search-url
29−base_url = 'https://gigablast.com/'
30−search_string = 'search?{query}'\
31− '&n={number_of_results}'\
32− '&c=main'\
33− '&s={offset}'\
34− '&format=json'\
35− '&langcountry={lang}'\
36− '&ff={safesearch}'\
37− '&rand={rxikd}'
38−# specific xpath variables
39−results_xpath = '//response//result'
40−url_xpath = './/url'
41−title_xpath = './/title'
42−content_xpath = './/sum'
43−
44−supported_languages_url = 'https://gigablast.com/search?&rxikd=1'
45−
46−extra_param = '' # gigablast requires a random extra parameter
47−# which can be extracted from the source code of the search page
29+base_url = 'https://gigablast.com'
4830
31+# ugly hack: gigablast requires a random extra parameter which can be extracted
32+# from the source code of the gigablast HTTP client
33+extra_param = ''
34+extra_param_path='/search?c=main&qlangcountry=en-us&q=south&s=10'
4935
5036def parse_extra_param(text):
51− global extra_param
52− param_lines = [x for x in text.splitlines() if x.startswith('var url=') or x.startswith('url=url+')]
53− extra_param = ''
54− for l in param_lines:
55− extra_param += l.split("'")[1]
56− extra_param = extra_param.split('&')[-1]
5737
58−
59−def init(engine_settings=None):
60− parse_extra_param(get('http://gigablast.com/search?c=main&qlangcountry=en-us&q=south&s=10').text)
38+ # example:
39+ #
40+ # var uxrl='/search?c=main&qlangcountry=en-us&q=south&s=10&rand=1590740241635&n';
41+ # uxrl=uxrl+'sab=730863287';
42+ #
43+ # extra_param --> "rand=1590740241635&nsab=730863287"
44+
45+ global extra_param # pylint: disable=global-statement
46+ re_var= None
47+ for line in text.splitlines():
48+ if re_var is None and extra_param_path in line:
49+ var = line.split("=")[0].split()[1] # e.g. var --> 'uxrl'
50+ re_var = re.compile(var + "\\s*=\\s*" + var + "\\s*\\+\\s*'" + "(.*)" + "'(.*)")
51+ extra_param = line.split("'")[1][len(extra_param_path):]
52+ continue
53+ if re_var is not None and re_var.search(line):
54+ extra_param += re_var.search(line).group(1)
55+ break
56+ # logger.debug('gigablast extra_param="%s"', extra_param)
57+
58+def init(engine_settings=None): # pylint: disable=unused-argument
59+ parse_extra_param(get(base_url + extra_param_path).text)
6160
6261
6362# do search-request
64−def request(query, params):
65− print("EXTRAPARAM:", extra_param)
66− offset = (params['pageno'] - 1) * number_of_results
63+def request(query, params): # pylint: disable=unused-argument
6764
68− if params['language'] == 'all':
69− language = 'xx'
70− else:
71− language = params['language'].replace('-', '_').lower()
72− if language.split('-')[0] != 'zh':
73− language = language.split('-')[0]
65+ # see API http://www.gigablast.com/api.html#/search
66+ # Take into account, that the API has some quirks ..
7467
75− if params['safesearch'] >= 1:
76− safesearch = 1
77− else:
78− safesearch = 0
68+ query_args = dict(
69+ c = 'main'
70+ , format = 'json'
71+ , q = query
72+ , dr = 1
73+ , showgoodimages = 0
74+ )
7975
80− # rxieu is some kind of hash from the search query, but accepts random atm
81− search_path = search_string.format(query=urlencode({'q': query}),
82− offset=offset,
83− number_of_results=number_of_results,
84− lang=language,
85− rxikd=int(time() * 1000),
86− safesearch=safesearch)
76+ if params['language'] and params['language'] != 'all':
77+ query_args['qlangcountry'] = params['language']
78+ query_args['qlang'] = params['language'].split('-')[0]
8779
88− params['url'] = base_url + search_path + '&' + extra_param
80+ if params['safesearch'] >= 1:
81+ query_args['ff'] = 1
8982
90− return params
83+ search_url = '/search?' + urlencode(query_args)
84+ params['url'] = base_url + search_url + extra_param
9185
86+ return params
9287
9388# get response from search-request
9489def response(resp):
9590 results = []
9691
97− # parse results
98− try:
99− response_json = loads(resp.text)
100− except:
101− parse_extra_param(resp.text)
102− raise Exception('extra param expired, please reload')
92+ response_json = loads(resp.text)
93+
94+ # logger.debug('gigablast returns %s results', len(response_json['results']))
10395
10496 for result in response_json['results']:
105− # append result
106− results.append({'url': result['url'],
107− 'title': result['title'],
108− 'content': result['sum']})
97+ # see "Example JSON Output (&format=json)"
98+ # at http://www.gigablast.com/api.html#/search
10999
110− # return results
111− return results
100+ # sort out meaningless result
101+
102+ title = result.get('title')
103+ if len(title) < 2:
104+ continue
105+
106+ url = result.get('url')
107+ if len(url) < 9:
108+ continue
109+
110+ content = result.get('sum')
111+ if len(content) < 5:
112+ continue
112113
114+ # extend fields
113115
114−# get supported languages from their site
115−def _fetch_supported_languages(resp):
116− supported_languages = []
117− dom = fromstring(resp.text)
118− links = eval_xpath(dom, '//span[@id="menu2"]/a')
119− for link in links:
120− href = eval_xpath(link, './@href')[0].split('lang%3A')
121− if len(href) == 2:
122− code = href[1].split('_')
123− if len(code) == 2:
124− code = code[0] + '-' + code[1].upper()
125− else:
126− code = code[0]
127− supported_languages.append(code)
128−
129− return supported_languages
116+ subtitle = result.get('title')
117+ if len(subtitle) > 3 and subtitle != title:
118+ title += " - " + subtitle
119+
120+ results.append(dict(
121+ url = url
122+ , title = title
123+ , content = content
124+ ))
125+
126+ return results