@@ -1,15 +1,15 @@
1−## Google (Web)
2−#
1+# Google (Web)
2+#
33# @website https://www.google.com
44# @provide-api yes (https://developers.google.com/custom-search/)
5−#
5+#
66# @using-api no
77# @results HTML
88# @stable no (HTML can change)
99# @parse url, title, content, suggestion
1010
1111from urllib import urlencode
12−from urlparse import unquote,urlparse,parse_qsl
12+from urlparse import urlparse, parse_qsl
1313from lxml import html
1414from searx.engines.xpath import extract_text, extract_url
1515
@@ -23,10 +23,13 @@ google_hostname = 'www.google.com'
2323search_path = '/search'
2424redirect_path = '/url'
2525images_path = '/images'
26−search_url = 'https://' + google_hostname + search_path + '?{query}&start={offset}&gbv=1'
26+search_url = ('https://' +
27+ google_hostname +
28+ search_path +
29+ '?{query}&start={offset}&gbv=1')
2730
2831# specific xpath variables
29−results_xpath= '//li[@class="g"]'
32+results_xpath = '//li[@class="g"]'
3033url_xpath = './/h3/a/@href'
3134title_xpath = './/h3'
3235content_xpath = './/span[@class="st"]'
@@ -36,15 +39,18 @@ images_xpath = './/div/a'
3639image_url_xpath = './@href'
3740image_img_src_xpath = './img/@src'
3841
42+
3943# remove google-specific tracking-url
4044def parse_url(url_string):
4145 parsed_url = urlparse(url_string)
42− if parsed_url.netloc in [google_hostname, ''] and parsed_url.path==redirect_path:
46+ if (parsed_url.netloc in [google_hostname, '']
47+ and parsed_url.path == redirect_path):
4348 query = dict(parse_qsl(parsed_url.query))
4449 return query['q']
4550 else:
4651 return url_string
4752
53+
4854# do search-request
4955def request(query, params):
5056 offset = (params['pageno'] - 1) * 10
@@ -52,7 +58,7 @@ def request(query, params):
5258 if params['language'] == 'all':
5359 language = 'en'
5460 else:
55− language = params['language'].replace('_','-').lower()
61+ language = params['language'].replace('_', '-').lower()
5662
5763 params['url'] = search_url.format(offset=offset,
5864 query=urlencode({'q': query}))
@@ -74,19 +80,21 @@ def response(resp):
7480 try:
7581 url = parse_url(extract_url(result.xpath(url_xpath), search_url))
7682 parsed_url = urlparse(url)
77− if parsed_url.netloc==google_hostname and parsed_url.path==search_path:
83+ if (parsed_url.netloc == google_hostname
84+ and parsed_url.path == search_path):
7885 # remove the link to google news
7986 continue
8087
81− if parsed_url.netloc==google_hostname and parsed_url.path==images_path:
88+ if (parsed_url.netloc == google_hostname
89+ and parsed_url.path == images_path):
8290 # images result
8391 results = results + parse_images(result)
8492 else:
8593 # normal result
8694 content = extract_text(result.xpath(content_xpath)[0])
8795 # append result
88− results.append({'url': url,
89− 'title': title,
96+ results.append({'url': url,
97+ 'title': title,
9098 'content': content})
9199 except:
92100 continue
@@ -99,12 +107,13 @@ def response(resp):
99107 # return results
100108 return results
101109
110+
102111def parse_images(result):
103112 results = []
104113 for image in result.xpath(images_xpath):
105114 url = parse_url(extract_text(image.xpath(image_url_xpath)[0]))
106115 img_src = extract_text(image.xpath(image_img_src_xpath)[0])
107−
116+
108117 # append result
109118 results.append({'url': url,
110119 'title': '',