@@ -9,14 +9,11 @@
99 @stable yes
1010 @parse url, title, content
1111"""
12+# pylint: disable=missing-function-docstring, invalid-name
1213
13−import random
14−from json import loads
1514from time import time
16−from lxml.html import fromstring
17−from searx.poolrequests import get
15+from json import loads
1816from searx.url_utils import urlencode
19−from searx.utils import eval_xpath
2017
2118# engine dependent config
2219categories = ['general']
@@ -26,83 +23,77 @@ language_support = True
2623safesearch = True
2724
2825# search-url
29−base_url = 'https://gigablast.com/'
30−search_string = 'search?{query}'\
31− '&n={number_of_results}'\
32− '&c=main'\
33− '&s={offset}'\
34− '&format=json'\
35− '&langcountry={lang}'\
36− '&ff={safesearch}'\
37− '&rand={rxikd}'
38−# specific xpath variables
39−results_xpath = '//response//result'
40−url_xpath = './/url'
41−title_xpath = './/title'
42−content_xpath = './/sum'
43−
44−extra_param = '' # gigablast requires a random extra parameter
45−# which can be extracted from the source code of the search page
46−
47−
48−def parse_extra_param(text):
49− global extra_param
50− param_lines = [x for x in text.splitlines() if x.startswith('var url=') or x.startswith('url=url+')]
51− extra_param = ''
52− for l in param_lines:
53− extra_param += l.split("'")[1]
54− extra_param = extra_param.split('&')[-1]
55−
56−
57−def init(engine_settings=None):
58− parse_extra_param(get('http://gigablast.com/search?c=main&qlangcountry=en-us&q=south&s=10').text)
5926
27+base_url = 'https://gigablast.com/'
6028
6129# do search-request
62−def request(query, params):
63− offset = (params['pageno'] - 1) * number_of_results
64−
65− if params['language'] == 'all':
66− language = 'xx'
67− else:
68− language = params['language'].replace('-', '_').lower()
69− if language.split('-')[0] != 'zh':
70− language = language.split('-')[0]
30+def request(query, params): # pylint: disable=unused-argument
31+
32+ # see API http://www.gigablast.com/api.html#/search
33+ # Take into account, that the API has some quirks ..
34+
35+ query_args = dict(
36+ c = 'main'
37+ , n = number_of_results
38+ , format = 'json'
39+ , q = query
40+ # The gigablast HTTP client sends a random number and a nsga argument
41+ # (the values don't seem to matter)
42+ , rand = int(time() * 1000)
43+ , nsga = int(time() * 1000)
44+ )
45+
46+ page_no = (params['pageno'] - 1)
47+ if page_no:
48+ # API quirk; adds +2 to the number_of_results
49+ offset = (params['pageno'] - 1) * number_of_results
50+ query_args['s'] = offset
51+
52+ if params['language'] and params['language'] != 'all':
53+ query_args['qlangcountry'] = params['language']
54+ query_args['qlang'] = params['language'].split('-')[0]
7155
7256 if params['safesearch'] >= 1:
73− safesearch = 1
74− else:
75− safesearch = 0
57+ query_args['ff'] = 1
7658
77− # rxieu is some kind of hash from the search query, but accepts random atm
78− search_path = search_string.format(query=urlencode({'q': query}),
79− offset=offset,
80− number_of_results=number_of_results,
81− lang=language,
82− rxikd=int(time() * 1000),
83− safesearch=safesearch)
84−
85− params['url'] = base_url + search_path + '&' + extra_param
59+ search_url = 'search?' + urlencode(query_args)
60+ params['url'] = base_url + search_url
8661
8762 return params
8863
89−
9064# get response from search-request
9165def response(resp):
9266 results = []
9367
94− # parse results
95− try:
96− response_json = loads(resp.text)
97− except:
98− parse_extra_param(resp.text)
99− raise Exception('extra param expired, please reload')
100−
68+ response_json = loads(resp.text)
10169 for result in response_json['results']:
102− # append result
103− results.append({'url': result['url'],
104− 'title': result['title'],
105− 'content': result['sum']})
70+ # see "Example JSON Output (&format=json)"
71+ # at http://www.gigablast.com/api.html#/search
72+
73+ # sort out meaningless result
74+
75+ title = result.get('title')
76+ if len(title) < 2:
77+ continue
78+
79+ url = result.get('url')
80+ if len(url) < 9:
81+ continue
82+
83+ content = result.get('sum')
84+ if len(content) < 5:
85+ continue
86+
87+ # extend fields
88+
89+ subtitle = result.get('title')
90+ if len(subtitle) > 3:
91+ title += " - " + subtitle
92+
93+ results.append(dict(
94+ url = url
95+ , title = title
96+ , content = content
97+ ))
10698
107− # return results
10899 return results