@@ -99,9 +99,9 @@ time_range_dict = {
9999
100100# Filter results. 0: None, 1: Moderate, 2: Strict
101101filter_mapping = {
102− 0 : 'off',
103− 1 : 'medium',
104− 2 : 'high'
102+ 0: 'off',
103+ 1: 'medium',
104+ 2: 'high'
105105}
106106
107107# specific xpath variables
@@ -111,7 +111,7 @@ filter_mapping = {
111111results_xpath = '//div[@class="g"]'
112112
113113# google *sections* are no usual *results*, we ignore them
114−g_section_with_header='./g-section-with-header'
114+g_section_with_header = './g-section-with-header'
115115
116116# the title is a h3 tag relative to the result group
117117title_xpath = './/h3[1]'
@@ -131,6 +131,7 @@ suggestion_xpath = '//div[contains(@class, "card-section")]//a'
131131# *spelling suggestions*, we use them anyway.
132132spelling_suggestion_xpath = '//div[@class="med"]/p/a'
133133
134+
134135def extract_text_from_dom(result, xpath):
135136 """returns extract_text on the first result selected by the xpath or None"""
136137 r = eval_xpath(result, xpath)
@@ -138,6 +139,7 @@ def extract_text_from_dom(result, xpath):
138139 return extract_text(r[0])
139140 return None
140141
142+
141143def get_lang_country(params, lang_list, custom_aliases):
142144 """Returns a tuple with *langauage* on its first and *country* on its second
143145 position."""
@@ -159,6 +161,7 @@ def get_lang_country(params, lang_list, custom_aliases):
159161
160162 return language, country, lang_country
161163
164+
162165def request(query, params):
163166 """Google search request"""
164167
@@ -170,7 +173,7 @@ def request(query, params):
170173 subdomain = 'www.' + google_domains.get(country.upper(), 'google.com')
171174
172175 # https://www.google.de/search?q=corona&hl=de-DE&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium
173− query_url = 'https://'+ subdomain + '/search' + "?" + urlencode({
176+ query_url = 'https://' + subdomain + '/search' + "?" + urlencode({
174177 'q': query,
175178 'hl': lang_country,
176179 'lr': "lang_" + language,
@@ -190,16 +193,17 @@ def request(query, params):
190193 # en-US,en;q=0.8,en;q=0.5
191194 params['headers']['Accept-Language'] = (
192195 lang_country + ',' + language + ';q=0.8,' + language + ';q=0.5'
193− )
196+ )
194197 logger.debug("HTTP header Accept-Language --> %s",
195198 params['headers']['Accept-Language'])
196199 params['headers']['Accept'] = (
197200 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
198− )
199− #params['google_subdomain'] = subdomain
201+ )
202+ # params['google_subdomain'] = subdomain
200203
201204 return params
202205
206+
203207def response(resp):
204208 """Get response from google's search request"""
205209 results = []
@@ -249,16 +253,16 @@ def response(resp):
249253 url = eval_xpath(result, href_xpath)[0]
250254 content = extract_text_from_dom(result, content_xpath)
251255 results.append({
252− 'url': url,
253− 'title': title,
254− 'content': content
255− })
256+ 'url': url,
257+ 'title': title,
258+ 'content': content
259+ })
256260 except Exception as e: # pylint: disable=broad-except
257261 logger.error(e, exc_info=True)
258− #from lxml import etree
259− #logger.debug(etree.tostring(result, pretty_print=True))
260− #import pdb
261− #pdb.set_trace()
262+ # from lxml import etree
263+ # logger.debug(etree.tostring(result, pretty_print=True))
264+ # import pdb
265+ # pdb.set_trace()
262266 continue
263267
264268 # parse suggestion
@@ -272,6 +276,7 @@ def response(resp):
272276 # return results
273277 return results
274278
279+
275280# get supported languages from their site
276281def _fetch_supported_languages(resp):
277282 ret_val = {}