@@ -1,15 +1,17 @@
11## Google (Web)
22#
33# @website https://www.google.com
4−# @provide-api yes (https://developers.google.com/web-search/docs/), deprecated!
4+# @provide-api yes (https://developers.google.com/custom-search/)
55#
6−# @using-api yes
7−# @results JSON
8−# @stable yes (but deprecated)
9−# @parse url, title, content
6+# @using-api no
7+# @results HTML
8+# @stable no (HTML can change)
9+# @parse url, title, content, suggestion
1010
1111from urllib import urlencode
12−from json import loads
12+from urlparse import unquote,urlparse,parse_qsl
13+from lxml import html
14+from searx.engines.xpath import extract_text, extract_url
1315
1416# engine dependent config
1517categories = ['general']
@@ -17,21 +19,45 @@ paging = True
1719language_support = True
1820
1921# search-url
20−url = 'https://ajax.googleapis.com/'
21−search_url = url + 'ajax/services/search/web?v=2.0&start={offset}&rsz=large&safe=off&filter=off&{query}&hl={language}' # noqa
22+google_hostname = 'www.google.com'
23+search_path = '/search'
24+redirect_path = '/url'
25+images_path = '/images'
26+search_url = 'https://' + google_hostname + search_path + '?{query}&start={offset}&gbv=1'
2227
28+# specific xpath variables
29+results_xpath= '//li[@class="g"]'
30+url_xpath = './/h3/a/@href'
31+title_xpath = './/h3'
32+content_xpath = './/span[@class="st"]'
33+suggestion_xpath = '//p[@class="_Bmc"]'
34+
35+images_xpath = './/div/a'
36+image_url_xpath = './@href'
37+image_img_src_xpath = './img/@src'
38+
39+# remove google-specific tracking-url
40+def parse_url(url_string):
41+ parsed_url = urlparse(url_string)
42+ if parsed_url.netloc in [google_hostname, ''] and parsed_url.path==redirect_path:
43+ query = dict(parse_qsl(parsed_url.query))
44+ return query['q']
45+ else:
46+ return url_string
2347
2448# do search-request
2549def request(query, params):
26− offset = (params['pageno'] - 1) * 8
50+ offset = (params['pageno'] - 1) * 10
2751
28− language = 'en-US'
29− if params['language'] != 'all':
30− language = params['language'].replace('_', '-')
52+ if params['language'] == 'all':
53+ language = 'en'
54+ else:
55+ language = params['language'].replace('_','-').lower()
3156
3257 params['url'] = search_url.format(offset=offset,
33− query=urlencode({'q': query}),
34− language=language)
58+ query=urlencode({'q': query}))
59+
60+ params['headers']['Accept-Language'] = language
3561
3662 return params
3763
@@ -40,18 +66,50 @@ def request(query, params):
4066def response(resp):
4167 results = []
4268
43− search_res = loads(resp.text)
44−
45− # return empty array if there are no results
46− if not search_res.get('responseData', {}).get('results'):
47− return []
69+ dom = html.fromstring(resp.text)
4870
4971 # parse results
50− for result in search_res['responseData']['results']:
51− # append result
52− results.append({'url': result['unescapedUrl'],
53− 'title': result['titleNoFormatting'],
54− 'content': result['content']})
72+ for result in dom.xpath(results_xpath):
73+ title = extract_text(result.xpath(title_xpath)[0])
74+ try:
75+ url = parse_url(extract_url(result.xpath(url_xpath), search_url))
76+ parsed_url = urlparse(url)
77+ if parsed_url.netloc==google_hostname and parsed_url.path==search_path:
78+ # remove the link to google news
79+ continue
80+
81+ if parsed_url.netloc==google_hostname and parsed_url.path==images_path:
82+ # images result
83+ results = results + parse_images(result)
84+ else:
85+ # normal result
86+ content = extract_text(result.xpath(content_xpath)[0])
87+ # append result
88+ results.append({'url': url,
89+ 'title': title,
90+ 'content': content})
91+ except:
92+ continue
93+
94+ # parse suggestion
95+ for suggestion in dom.xpath(suggestion_xpath):
96+ # append suggestion
97+ results.append({'suggestion': extract_text(suggestion)})
5598
5699 # return results
57100 return results
101+
102+def parse_images(result):
103+ results = []
104+ for image in result.xpath(images_xpath):
105+ url = parse_url(extract_text(image.xpath(image_url_xpath)[0]))
106+ img_src = extract_text(image.xpath(image_img_src_xpath)[0])
107+
108+ # append result
109+ results.append({'url': url,
110+ 'title': '',
111+ 'content': '',
112+ 'img_src': img_src,
113+ 'template': 'images.html'})
114+
115+ return results