@@ -132,11 +132,12 @@ suggestion_xpath = '//div[contains(@class, "card-section")]//a'
132132spelling_suggestion_xpath = '//div[@class="med"]/p/a'
133133
134134
135−def get_lang_info(params, lang_list, custom_aliases):
135+def get_lang_info(params, lang_list, custom_aliases, supported_any_language):
136136 ret_val = {}
137137
138138 _lang = params['language']
139− if _lang.lower() == 'all':
139+ _any_language = _lang.lower() == 'all'
140+ if _any_language:
140141 _lang = 'en-US'
141142
142143 language = match_language(_lang, lang_list, custom_aliases)
@@ -158,31 +159,36 @@ def get_lang_info(params, lang_list, custom_aliases):
158159 # the combination (en-US, en-EN, de-DE, de-AU, fr-FR, fr-FR)
159160 lang_country = '%s-%s' % (language, country)
160161
161− # Accept-Language: fr-CH, fr;q=0.8, en;q=0.6, *;q=0.5
162− ret_val['Accept-Language'] = ','.join([
163− lang_country,
164− language + ';q=0.8,',
165− 'en;q=0.6',
166− '*;q=0.5',
167− ])
168−
169162 # subdomain
170163 ret_val['subdomain'] = 'www.' + google_domains.get(country.upper(), 'google.com')
171164
165+ ret_val['params'] = {}
166+ ret_val['headers'] = {}
167+
168+ if _any_language and supported_any_language:
169+ # based on whoogle
170+ ret_val['params']['source'] = 'lnt'
171+ else:
172+ # Accept-Language: fr-CH, fr;q=0.8, en;q=0.6, *;q=0.5
173+ ret_val['headers']['Accept-Language'] = ','.join([
174+ lang_country,
175+ language + ';q=0.8,',
176+ 'en;q=0.6',
177+ '*;q=0.5',
178+ ])
179+
180+ # lr parameter:
181+ # https://developers.google.com/custom-search/docs/xml_results#lrsp
182+ # Language Collection Values:
183+ # https://developers.google.com/custom-search/docs/xml_results_appendices#languageCollections
184+ ret_val['params']['lr'] = "lang_" + lang_list.get(lang_country, language)
185+
186+ ret_val['params']['hl'] = lang_list.get(lang_country, language)
187+
172188 # hl parameter:
173189 # https://developers.google.com/custom-search/docs/xml_results#hlsp The
174190 # Interface Language:
175191 # https://developers.google.com/custom-search/docs/xml_results_appendices#interfaceLanguages
176−
177− ret_val['hl'] = lang_list.get(lang_country, language)
178−
179− # lr parameter:
180− # https://developers.google.com/custom-search/docs/xml_results#lrsp
181− # Language Collection Values:
182− # https://developers.google.com/custom-search/docs/xml_results_appendices#languageCollections
183−
184− ret_val['lr'] = "lang_" + lang_list.get(lang_country, language)
185−
186192 return ret_val
187193
188194def detect_google_sorry(resp):
@@ -197,14 +203,13 @@ def request(query, params):
197203
198204 lang_info = get_lang_info(
199205 # pylint: disable=undefined-variable
200− params, supported_languages, language_aliases
206+ params, supported_languages, language_aliases, True
201207 )
202208
203209 # https://www.google.de/search?q=corona&hl=de&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium
204210 query_url = 'https://' + lang_info['subdomain'] + '/search' + "?" + urlencode({
205211 'q': query,
206− 'hl': lang_info['hl'],
207− 'lr': lang_info['lr'],
212+ **lang_info['params'],
208213 'ie': "utf8",
209214 'oe': "utf8",
210215 'start': offset,
@@ -218,8 +223,8 @@ def request(query, params):
218223 logger.debug("query_url --> %s", query_url)
219224 params['url'] = query_url
220225
221− logger.debug("HTTP header Accept-Language --> %s", lang_info['Accept-Language'])
222− params['headers']['Accept-Language'] = lang_info['Accept-Language']
226+ logger.debug("HTTP header Accept-Language --> %s", lang_info.get('Accept-Language'))
227+ params['headers'].update(lang_info['headers'])
223228 params['headers']['Accept'] = (
224229 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
225230 )