git.iko.soy/searx
on master help sign in
[fix] rewrite the google engine since Google Web Search API is about to expire
5dc3eb3Dalf
searx/engines/google.py+82−24
compare with master
unifiedsplit
@@ -1,15 +1,17 @@
1## Google (Web)
1## Google (Web)
2#
2#
3# @website https://www.google.com
3# @website https://www.google.com
4−# @provide-api yes (https://developers.google.com/web-search/docs/), deprecated!
4+# @provide-api yes (https://developers.google.com/custom-search/)
5#
5#
6−# @using-api yes
6+# @using-api no
7−# @results JSON
7+# @results HTML
8−# @stable yes (but deprecated)
8+# @stable no (HTML can change)
9−# @parse url, title, content
9+# @parse url, title, content, suggestion
10
10
11from urllib import urlencode
11from urllib import urlencode
12−from json import loads
12+from urlparse import unquote,urlparse,parse_qsl
13+from lxml import html
14+from searx.engines.xpath import extract_text, extract_url
13
15
14# engine dependent config
16# engine dependent config
15categories = ['general']
17categories = ['general']
@@ -17,21 +19,45 @@ paging = True
17language_support = True
19language_support = True
18
20
19# search-url
21# search-url
20−url = 'https://ajax.googleapis.com/'
22+google_hostname = 'www.google.com'
21−search_url = url + 'ajax/services/search/web?v=2.0&start={offset}&rsz=large&safe=off&filter=off&{query}&hl={language}' # noqa
23+search_path = '/search'
24+redirect_path = '/url'
25+images_path = '/images'
26+search_url = 'https://' + google_hostname + search_path + '?{query}&start={offset}&gbv=1'
22
27
28+# specific xpath variables
29+results_xpath= '//li[@class="g"]'
30+url_xpath = './/h3/a/@href'
31+title_xpath = './/h3'
32+content_xpath = './/span[@class="st"]'
33+suggestion_xpath = '//p[@class="_Bmc"]'
34+
35+images_xpath = './/div/a'
36+image_url_xpath = './@href'
37+image_img_src_xpath = './img/@src'
38+
39+# remove google-specific tracking-url
40+def parse_url(url_string):
41+ parsed_url = urlparse(url_string)
42+ if parsed_url.netloc in [google_hostname, ''] and parsed_url.path==redirect_path:
43+ query = dict(parse_qsl(parsed_url.query))
44+ return query['q']
45+ else:
46+ return url_string
23
47
24# do search-request
48# do search-request
25def request(query, params):
49def request(query, params):
26− offset = (params['pageno'] - 1) * 8
50+ offset = (params['pageno'] - 1) * 10
27
51
28− language = 'en-US'
29− if params['language'] != 'all':
52+ if params['language'] == 'all':
53+ language = 'en'
54+ else:
30− language = params['language'].replace('_', '-')
55+ language = params['language'].replace('_','-').lower()
31
56
32 params['url'] = search_url.format(offset=offset,
57 params['url'] = search_url.format(offset=offset,
33− query=urlencode({'q': query}),
58+ query=urlencode({'q': query}))
34− language=language)
59+
60+ params['headers']['Accept-Language'] = language
35
61
36 return params
62 return params
37
63
@@ -40,18 +66,50 @@ def request(query, params):
40def response(resp):
66def response(resp):
41 results = []
67 results = []
42
68
43− search_res = loads(resp.text)
69+ dom = html.fromstring(resp.text)
44−
45− # return empty array if there are no results
46− if not search_res.get('responseData', {}).get('results'):
47− return []
48
70
49 # parse results
71 # parse results
50− for result in search_res['responseData']['results']:
72+ for result in dom.xpath(results_xpath):
73+ title = extract_text(result.xpath(title_xpath)[0])
74+ try:
75+ url = parse_url(extract_url(result.xpath(url_xpath), search_url))
76+ parsed_url = urlparse(url)
77+ if parsed_url.netloc==google_hostname and parsed_url.path==search_path:
78+ # remove the link to google news
79+ continue
80+
81+ if parsed_url.netloc==google_hostname and parsed_url.path==images_path:
82+ # images result
83+ results = results + parse_images(result)
84+ else:
85+ # normal result
86+ content = extract_text(result.xpath(content_xpath)[0])
51− # append result
87+ # append result
52− results.append({'url': result['unescapedUrl'],
88+ results.append({'url': url,
53− 'title': result['titleNoFormatting'],
89+ 'title': title,
54− 'content': result['content']})
90+ 'content': content})
91+ except:
92+ continue
93+
94+ # parse suggestion
95+ for suggestion in dom.xpath(suggestion_xpath):
96+ # append suggestion
97+ results.append({'suggestion': extract_text(suggestion)})
55
98
56 # return results
99 # return results
57 return results
100 return results
101+
102+def parse_images(result):
103+ results = []
104+ for image in result.xpath(images_xpath):
105+ url = parse_url(extract_text(image.xpath(image_url_xpath)[0]))
106+ img_src = extract_text(image.xpath(image_img_src_xpath)[0])
107+
108+ # append result
109+ results.append({'url': url,
110+ 'title': '',
111+ 'content': '',
112+ 'img_src': img_src,
113+ 'template': 'images.html'})
114+
115+ return results