git.iko.soy/searx
on master help sign in
[fix] prevent google engine to redirect

nid/pref cookies are also removed

commit5cea4f9445e2b08fe91f9b964cc768920aac7dc8 authorAdam Tauber <asciimoo@gmail.com> dateTue 22 Dec 2015, 20:05 (11 years ago) parent029291e
2 files changed+2−34
compare with master
unifiedsplit
searx/engines/google.py+1−29 searx/tests/engines/test_google.py+1−5
searx/engines/google.py+1−29
@@ -13,7 +13,6 @@ from cgi import escape
1313from urllib import urlencode
1414from urlparse import urlparse, parse_qsl
1515from lxml import html, etree
16−from searx.poolrequests import get
1716from searx.engines.xpath import extract_text, extract_url
1817from searx.search import logger
1918
@@ -91,7 +90,7 @@ url_map = 'https://www.openstreetmap.org/'\
9190search_path = '/search'
9291search_url = ('https://{hostname}' +
9392 search_path +
94− '?{query}&start={offset}&gbv=1')
93+ '?{query}&start={offset}&gbv=1&gws_rd=cr')
9594
9695# other URLs
9796map_hostname_start = 'maps.google.'
@@ -129,27 +128,6 @@ image_img_src_xpath = './img/@src'
129128property_address = "Address"
130129property_phone = "Phone number"
131130
132−# cookies
133−pref_cookie = ''
134−nid_cookie = {}
135−
136−
137−# see https://support.google.com/websearch/answer/873?hl=en
138−def get_google_pref_cookie():
139− global pref_cookie
140− if pref_cookie == '':
141− resp = get('https://www.google.com/ncr', allow_redirects=False)
142− pref_cookie = resp.cookies["PREF"]
143− return pref_cookie
144−
145−
146−def get_google_nid_cookie(google_hostname):
147− global nid_cookie
148− if google_hostname not in nid_cookie:
149− resp = get('https://' + google_hostname)
150− nid_cookie[google_hostname] = resp.cookies.get("NID", None)
151− return nid_cookie[google_hostname]
152−
153131
154132# remove google-specific tracking-url
155133def parse_url(url_string, google_hostname):
@@ -201,12 +179,6 @@ def request(query, params):
201179
202180 params['headers']['Accept-Language'] = language
203181 params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
204− if google_hostname == default_hostname:
205− try:
206− params['cookies']['PREF'] = get_google_pref_cookie()
207− except:
208− logger.warning('cannot fetch PREF cookie')
209− params['cookies']['NID'] = get_google_nid_cookie(google_hostname)
210182
211183 params['google_hostname'] = google_hostname
212184
searx/tests/engines/test_google.py+1−5
@@ -9,7 +9,7 @@ from searx.testing import SearxTestCase
99class TestGoogleEngine(SearxTestCase):
1010
1111 def mock_response(self, text):
12− response = mock.Mock(text=text, url='https://www.google.com/search?q=test&start=0&gbv=1')
12+ response = mock.Mock(text=text, url='https://www.google.com/search?q=test&start=0&gbv=1&gws_rd=cr')
1313 response.search_params = mock.Mock()
1414 response.search_params.get = mock.Mock(return_value='www.google.com')
1515 return response
@@ -23,16 +23,12 @@ class TestGoogleEngine(SearxTestCase):
2323 self.assertIn('url', params)
2424 self.assertIn(query, params['url'])
2525 self.assertIn('google.fr', params['url'])
26− self.assertNotIn('PREF', params['cookies'])
27− self.assertIn('NID', params['cookies'])
2826 self.assertIn('fr', params['headers']['Accept-Language'])
2927
3028 dicto['language'] = 'all'
3129 params = google.request(query, dicto)
3230 self.assertIn('google.com', params['url'])
3331 self.assertIn('en', params['headers']['Accept-Language'])
34− # self.assertIn('PREF', params['cookies'])
35− self.assertIn('NID', params['cookies'])
3632
3733 def test_response(self):
3834 self.assertRaises(AttributeError, google.response, None)