git.iko.soy/searx
on master help sign in
[fix] normalize the language & region aspects of all google engines

BTW: make the engines ready for search.checker: - replace eval_xpath by eval_xpath_getindex and eval_xpath_list - google_images: remove outer try/except block Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

commitb1fefec40d6fb874cd8b248f78cb83441d1c4c5f authorMarkus Heiser <markus.heiser@darmarit.de> dateTue 26 Jan 2021, 11:49 (6 years ago) parent923b490
4 files changed+193−185
compare with master
unifiedsplit
searx/engines/google.py+61−37 searx/engines/google_images.py+72−71 searx/engines/google_news.py+35−34 searx/engines/google_videos.py+25−43
searx/engines/google.py+61−37
@@ -1,11 +1,11 @@
11# SPDX-License-Identifier: AGPL-3.0-or-later
22"""Google (Web)
33
4− For detailed description of the *REST-full* API see: `Query Parameter
5− Definitions`_.
4+For detailed description of the *REST-full* API see: `Query Parameter
5+Definitions`_.
66
7− .. _Query Parameter Definitions:
8− https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions
7+.. _Query Parameter Definitions:
8+ https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions
99"""
1010
1111# pylint: disable=invalid-name, missing-function-docstring
@@ -16,7 +16,6 @@ from searx import logger
1616from searx.utils import match_language, extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex
1717from searx.exceptions import SearxEngineCaptchaException
1818
19−
2019logger = logger.getChild('google engine')
2120
2221# about
@@ -56,7 +55,7 @@ google_domains = {
5655 'NZ': 'google.co.nz', # New Zealand
5756 'PH': 'google.com.ph', # Philippines
5857 'SG': 'google.com.sg', # Singapore
59− # 'US': 'google.us', # United States, redirect to .com
58+ 'US': 'google.com', # United States (google.us) redirects to .com
6059 'ZA': 'google.co.za', # South Africa
6160 'AR': 'google.com.ar', # Argentina
6261 'CL': 'google.cl', # Chile
@@ -87,7 +86,7 @@ google_domains = {
8786 'TH': 'google.co.th', # Thailand
8887 'TR': 'google.com.tr', # Turkey
8988 'UA': 'google.com.ua', # Ukraine
90− # 'CN': 'google.cn', # China, only from China ?
89+ 'CN': 'google.com.hk', # There is no google.cn, we use .com.hk for zh-CN
9190 'HK': 'google.com.hk', # Hong Kong
9291 'TW': 'google.com.tw' # Taiwan
9392}
@@ -134,26 +133,58 @@ suggestion_xpath = '//div[contains(@class, "card-section")]//a'
134133spelling_suggestion_xpath = '//div[@class="med"]/p/a'
135134
136135
137−def get_lang_country(params, lang_list, custom_aliases):
138− """Returns a tuple with *langauage* on its first and *country* on its second
139− position."""
140− language = params['language']
141− if language == 'all':
142− language = 'en-US'
136+def get_lang_info(params, lang_list, custom_aliases):
137+ ret_val = {}
138+
139+ _lang = params['language']
140+ if _lang.lower() == 'all':
141+ _lang = 'en-US'
142+
143+ language = match_language(_lang, lang_list, custom_aliases)
144+ ret_val['language'] = language
143145
144− language_array = language.split('-')
146+ # the requested language from params (en, en-US, de, de-AT, fr, fr-CA, ...)
147+ _l = _lang.split('-')
145148
146− if len(language_array) == 2:
147− country = language_array[1]
149+ # the country code (US, AT, CA)
150+ if len(_l) == 2:
151+ country = _l[1]
148152 else:
149− country = language_array[0].upper()
153+ country = _l[0].upper()
154+ if country == 'EN':
155+ country = 'US'
150156
151− language = match_language(language, lang_list, custom_aliases)
157+ ret_val['country'] = country
158+
159+ # the combination (en-US, en-EN, de-DE, de-AU, fr-FR, fr-FR)
152160 lang_country = '%s-%s' % (language, country)
153− if lang_country == 'en-EN':
154− lang_country = 'en'
155161
156− return language, country, lang_country
162+ # Accept-Language: fr-CH, fr;q=0.8, en;q=0.6, *;q=0.5
163+ ret_val['Accept-Language'] = ','.join([
164+ lang_country,
165+ language + ';q=0.8,',
166+ 'en;q=0.6',
167+ '*;q=0.5',
168+ ])
169+
170+ # subdomain
171+ ret_val['subdomain'] = 'www.' + google_domains.get(country.upper(), 'google.com')
172+
173+ # hl parameter:
174+ # https://developers.google.com/custom-search/docs/xml_results#hlsp The
175+ # Interface Language:
176+ # https://developers.google.com/custom-search/docs/xml_results_appendices#interfaceLanguages
177+
178+ ret_val['hl'] = lang_list.get(lang_country, language)
179+
180+ # lr parameter:
181+ # https://developers.google.com/custom-search/docs/xml_results#lrsp
182+ # Language Collection Values:
183+ # https://developers.google.com/custom-search/docs/xml_results_appendices#languageCollections
184+
185+ ret_val['lr'] = "lang_" + lang_list.get(lang_country, language)
186+
187+ return ret_val
157188
158189def detect_google_sorry(resp):
159190 resp_url = urlparse(resp.url)
@@ -165,17 +196,17 @@ def request(query, params):
165196 """Google search request"""
166197
167198 offset = (params['pageno'] - 1) * 10
168− language, country, lang_country = get_lang_country(
199+
200+ lang_info = get_lang_info(
169201 # pylint: disable=undefined-variable
170202 params, supported_languages, language_aliases
171203 )
172− subdomain = 'www.' + google_domains.get(country.upper(), 'google.com')
173204
174− # https://www.google.de/search?q=corona&hl=de-DE&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium
175− query_url = 'https://' + subdomain + '/search' + "?" + urlencode({
205+ # https://www.google.de/search?q=corona&hl=de&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium
206+ query_url = 'https://' + lang_info['subdomain'] + '/search' + "?" + urlencode({
176207 'q': query,
177− 'hl': lang_country,
178− 'lr': "lang_" + language,
208+ 'hl': lang_info['hl'],
209+ 'lr': lang_info['lr'],
179210 'ie': "utf8",
180211 'oe': "utf8",
181212 'start': offset,
@@ -186,19 +217,14 @@ def request(query, params):
186217 if params['safesearch']:
187218 query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
188219
189− params['url'] = query_url
190220 logger.debug("query_url --> %s", query_url)
221+ params['url'] = query_url
191222
192− # en-US,en;q=0.8,en;q=0.5
193− params['headers']['Accept-Language'] = (
194− lang_country + ',' + language + ';q=0.8,' + language + ';q=0.5'
195− )
196− logger.debug("HTTP header Accept-Language --> %s",
197− params['headers']['Accept-Language'])
223+ logger.debug("HTTP header Accept-Language --> %s", lang_info['Accept-Language'])
224+ params['headers']['Accept-Language'] = lang_info['Accept-Language']
198225 params['headers']['Accept'] = (
199226 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
200227 )
201− # params['google_subdomain'] = subdomain
202228
203229 return params
204230
@@ -209,8 +235,6 @@ def response(resp):
209235 detect_google_sorry(resp)
210236
211237 results = []
212− # which subdomain ?
213− # subdomain = resp.search_params.get('google_subdomain')
214238
215239 # convert the text to dom
216240 dom = html.fromstring(resp.text)
searx/engines/google_images.py+72−71
@@ -10,35 +10,50 @@ Definitions`_.
1010 ``data:` scheme).::
1111
1212 Header set Content-Security-Policy "img-src 'self' data: ;"
13+
14+.. _Query Parameter Definitions:
15+ https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions
16+.. _data URLs:
17+ https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs
1318"""
1419
1520from urllib.parse import urlencode, unquote
1621from lxml import html
22+
1723from searx import logger
18−from searx.utils import extract_text, eval_xpath
19−from searx.engines.google import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import
24+from searx.utils import (
25+ eval_xpath,
26+ eval_xpath_list,
27+ eval_xpath_getindex,
28+ extract_text,
29+)
2030
2131from searx.engines.google import (
22− get_lang_country,
23− google_domains,
32+ get_lang_info,
2433 time_range_dict,
2534 detect_google_sorry,
2635)
2736
37+# pylint: disable=unused-import
38+from searx.engines.google import (
39+ supported_languages_url
40+ , _fetch_supported_languages
41+)
42+# pylint: enable=unused-import
43+
2844logger = logger.getChild('google images')
2945
3046# about
3147about = {
32− "website": 'https://images.google.com/',
48+ "website": 'https://images.google.com',
3349 "wikidata_id": 'Q521550',
34− "official_api_documentation": 'https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions', # NOQA
50+ "official_api_documentation": 'https://developers.google.com/custom-search',
3551 "use_official_api": False,
3652 "require_api_key": False,
3753 "results": 'HTML',
3854}
3955
4056# engine dependent config
41−
4257categories = ['images']
4358paging = False
4459language_support = True
@@ -84,17 +99,16 @@ def scrap_img_by_id(script, data_id):
8499def request(query, params):
85100 """Google-Video search request"""
86101
87− language, country, lang_country = get_lang_country(
102+ lang_info = get_lang_info(
88103 # pylint: disable=undefined-variable
89104 params, supported_languages, language_aliases
90105 )
91− subdomain = 'www.' + google_domains.get(country.upper(), 'google.com')
92106
93− query_url = 'https://' + subdomain + '/search' + "?" + urlencode({
107+ query_url = 'https://' + lang_info['subdomain'] + '/search' + "?" + urlencode({
94108 'q': query,
95109 'tbm': "isch",
96− 'hl': lang_country,
97− 'lr': "lang_" + language,
110+ 'hl': lang_info['hl'],
111+ 'lr': lang_info['lr'],
98112 'ie': "utf8",
99113 'oe': "utf8",
100114 'num': 30,
@@ -105,17 +119,14 @@ def request(query, params):
105119 if params['safesearch']:
106120 query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
107121
108− params['url'] = query_url
109122 logger.debug("query_url --> %s", query_url)
123+ params['url'] = query_url
110124
111− params['headers']['Accept-Language'] = (
112− "%s,%s;q=0.8,%s;q=0.5" % (lang_country, language, language))
113− logger.debug(
114− "HTTP Accept-Language --> %s", params['headers']['Accept-Language'])
125+ logger.debug("HTTP header Accept-Language --> %s", lang_info['Accept-Language'])
126+ params['headers']['Accept-Language'] = lang_info['Accept-Language']
115127 params['headers']['Accept'] = (
116128 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
117129 )
118− # params['google_subdomain'] = subdomain
119130 return params
120131
121132
@@ -125,13 +136,11 @@ def response(resp):
125136
126137 detect_google_sorry(resp)
127138
128− # which subdomain ?
129− # subdomain = resp.search_params.get('google_subdomain')
130−
131139 # convert the text to dom
132140 dom = html.fromstring(resp.text)
133141 img_bas64_map = scrap_out_thumbs(dom)
134− img_src_script = eval_xpath(dom, '//script[contains(., "AF_initDataCallback({key: ")]')[1].text
142+ img_src_script = eval_xpath_getindex(
143+ dom, '//script[contains(., "AF_initDataCallback({key: ")]', 1).text
135144
136145 # parse results
137146 #
@@ -156,55 +165,47 @@ def response(resp):
156165 return results
157166
158167 root = root[0]
159− for img_node in eval_xpath(root, './/img[contains(@class, "rg_i")]'):
160−
161− try:
162− img_alt = eval_xpath(img_node, '@alt')[0]
163−
164− img_base64_id = eval_xpath(img_node, '@data-iid')
165− if img_base64_id:
166− img_base64_id = img_base64_id[0]
167− thumbnail_src = img_bas64_map[img_base64_id]
168+ for img_node in eval_xpath_list(root, './/img[contains(@class, "rg_i")]'):
169+
170+ img_alt = eval_xpath_getindex(img_node, '@alt', 0)
171+
172+ img_base64_id = eval_xpath(img_node, '@data-iid')
173+ if img_base64_id:
174+ img_base64_id = img_base64_id[0]
175+ thumbnail_src = img_bas64_map[img_base64_id]
176+ else:
177+ thumbnail_src = eval_xpath(img_node, '@src')
178+ if not thumbnail_src:
179+ thumbnail_src = eval_xpath(img_node, '@data-src')
180+ if thumbnail_src:
181+ thumbnail_src = thumbnail_src[0]
168182 else:
169− thumbnail_src = eval_xpath(img_node, '@src')
170− if not thumbnail_src:
171− thumbnail_src = eval_xpath(img_node, '@data-src')
172− if thumbnail_src:
173− thumbnail_src = thumbnail_src[0]
174− else:
175− thumbnail_src = ''
176−
177− link_node = eval_xpath(img_node, '../../../a[2]')[0]
178− url = eval_xpath(link_node, '@href')[0]
179−
180− pub_nodes = eval_xpath(link_node, './div/div')
181− pub_descr = img_alt
182− pub_source = ''
183− if pub_nodes:
184− pub_descr = extract_text(pub_nodes[0])
185− pub_source = extract_text(pub_nodes[1])
186−
187− img_src_id = eval_xpath(img_node, '../../../@data-id')[0]
188− src_url = scrap_img_by_id(img_src_script, img_src_id)
189− if not src_url:
190− src_url = thumbnail_src
191−
192− results.append({
193− 'url': url,
194− 'title': img_alt,
195− 'content': pub_descr,
196− 'source': pub_source,
197− 'img_src': src_url,
198− # 'img_format': img_format,
199− 'thumbnail_src': thumbnail_src,
200− 'template': 'images.html'
201− })
202− except Exception as e: # pylint: disable=broad-except
203− logger.error(e, exc_info=True)
204− # from lxml import etree
205− # logger.debug(etree.tostring(img_node, pretty_print=True))
206− # import pdb
207− # pdb.set_trace()
208− continue
183+ thumbnail_src = ''
184+
185+ link_node = eval_xpath_getindex(img_node, '../../../a[2]', 0)
186+ url = eval_xpath_getindex(link_node, '@href', 0)
187+
188+ pub_nodes = eval_xpath(link_node, './div/div')
189+ pub_descr = img_alt
190+ pub_source = ''
191+ if pub_nodes:
192+ pub_descr = extract_text(pub_nodes[0])
193+ pub_source = extract_text(pub_nodes[1])
194+
195+ img_src_id = eval_xpath_getindex(img_node, '../../../@data-id', 0)
196+ src_url = scrap_img_by_id(img_src_script, img_src_id)
197+ if not src_url:
198+ src_url = thumbnail_src
199+
200+ results.append({
201+ 'url': url,
202+ 'title': img_alt,
203+ 'content': pub_descr,
204+ 'source': pub_source,
205+ 'img_src': src_url,
206+ # 'img_format': img_format,
207+ 'thumbnail_src': thumbnail_src,
208+ 'template': 'images.html'
209+ })
209210
210211 return results
searx/engines/google_news.py+35−34
@@ -2,13 +2,16 @@
22"""Google (News)
33
44For detailed description of the *REST-full* API see: `Query Parameter
5−Definitions`_. Not all parameters can be appied, e.g. num_ (the number of
6−search results to return) is ignored.
5+Definitions`_. Not all parameters can be appied:
6+
7+- num_ : the number of search results is ignored
8+- save_ : is ignored / Google-News results are always *SafeSearch*
79
810.. _Query Parameter Definitions:
911 https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions
1012
1113.. _num: https://developers.google.com/custom-search/docs/xml_results#numsp
14+.. _save: https://developers.google.com/custom-search/docs/xml_results#safesp
1215
1316"""
1417
@@ -32,20 +35,19 @@ from searx.utils import (
3235from searx.engines.google import (
3336 supported_languages_url,
3437 _fetch_supported_languages,
35− detect_google_sorry,
3638)
3739# pylint: enable=unused-import
3840
3941from searx.engines.google import (
40− get_lang_country,
41− filter_mapping,
42+ get_lang_info,
43+ detect_google_sorry,
4244)
4345
4446# about
4547about = {
4648 "website": 'https://news.google.com',
4749 "wikidata_id": 'Q12020',
48− "official_api_documentation": None,
50+ "official_api_documentation": 'https://developers.google.com/custom-search',
4951 "use_official_api": False,
5052 "require_api_key": False,
5153 "results": 'HTML',
@@ -69,51 +71,53 @@ paging = False
6971language_support = True
7072use_locale_domain = True
7173time_range_support = True
72−safesearch = True # not really, but it is not generated by google
74+
75+# Google-News results are always *SafeSearch*. Option 'safesearch' is set to
76+# False here, otherwise checker will report safesearch-errors::
77+#
78+# safesearch : results are identitical for safesearch=0 and safesearch=2
79+safesearch = False
7380
7481def request(query, params):
7582 """Google-News search request"""
7683
77− language, country, lang_country = get_lang_country(
84+ lang_info = get_lang_info(
7885 # pylint: disable=undefined-variable
7986 params, supported_languages, language_aliases
8087 )
81− subdomain = 'news.google.com'
8288
83− if params['time_range']: # in time_range_dict:
89+ # google news has only one domain
90+ lang_info['subdomain'] = 'news.google.com'
91+
92+ ceid = "%s:%s" % (lang_info['country'], lang_info['language'])
93+
94+ # google news redirects en to en-US
95+ if lang_info['hl'] == 'en':
96+ lang_info['hl'] = 'en-US'
97+
98+ # Very special to google-news compared to other google engines, the time
99+ # range is included in the search term.
100+ if params['time_range']:
84101 query += ' ' + time_range_dict[params['time_range']]
85102
86− query_url = 'https://'+ subdomain + '/search' + "?" + urlencode({
103+ query_url = 'https://' + lang_info['subdomain'] + '/search' + "?" + urlencode({
87104 'q': query,
88− 'hl': language,
89− 'lr': "lang_" + language,
105+ 'hl': lang_info['hl'],
106+ 'lr': lang_info['lr'],
90107 'ie': "utf8",
91108 'oe': "utf8",
92− 'ceid' : "%s:%s" % (country, language),
93− 'gl' : country,
94− })
109+ 'gl': lang_info['country'],
110+ }) + ('&ceid=%s' % ceid) # ceid includes a ':' character which must not be urlencoded
95111
96− if params['safesearch']:
97− query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
98−
99− params['url'] = query_url
100112 logger.debug("query_url --> %s", query_url)
113+ params['url'] = query_url
101114
102− # en-US,en;q=0.8,en;q=0.5
103− params['headers']['Accept-Language'] = (
104− lang_country + ',' + language + ';q=0.8,' + language + ';q=0.5'
105− )
106− logger.debug("HTTP header Accept-Language --> %s",
107− params['headers']['Accept-Language'])
115+ logger.debug("HTTP header Accept-Language --> %s", lang_info['Accept-Language'])
116+ params['headers']['Accept-Language'] = lang_info['Accept-Language']
108117 params['headers']['Accept'] = (
109118 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
110119 )
111120
112− # hl=en redirect to hl=en-US / en-CA ...
113− params['soft_max_redirects'] = 1
114−
115− #params['google_subdomain'] = subdomain
116−
117121 return params
118122
119123
@@ -123,9 +127,6 @@ def response(resp):
123127
124128 detect_google_sorry(resp)
125129
126− # which subdomain ?
127− # subdomain = resp.search_params.get('google_subdomain')
128−
129130 # convert the text to dom
130131 dom = html.fromstring(resp.text)
131132
searx/engines/google_videos.py+25−43
@@ -1,6 +1,5 @@
11# SPDX-License-Identifier: AGPL-3.0-or-later
2−"""
3−Google (Viedo)
2+"""Google (Video)
43
54For detailed description of the *REST-full* API see: `Query Parameter
65Definitions`_. Not all parameters can be appied.
@@ -22,20 +21,19 @@ Definitions`_. Not all parameters can be appied.
2221# pylint: disable=invalid-name, missing-function-docstring
2322
2423import re
25−from urllib.parse import urlencode, urlparse
24+from urllib.parse import urlencode
2625from lxml import html
2726
2827from searx import logger
29−from searx.exceptions import SearxEngineCaptchaException
3028from searx.utils import (
3129 eval_xpath,
3230 eval_xpath_list,
31+ eval_xpath_getindex,
3332 extract_text,
3433)
3534
3635from searx.engines.google import (
37− get_lang_country,
38− google_domains,
36+ get_lang_info,
3937 time_range_dict,
4038 filter_mapping,
4139 results_xpath,
@@ -44,7 +42,8 @@ from searx.engines.google import (
4442 href_xpath,
4543 content_xpath,
4644 suggestion_xpath,
47− spelling_suggestion_xpath
45+ spelling_suggestion_xpath,
46+ detect_google_sorry,
4847)
4948
5049# pylint: disable=unused-import
@@ -58,12 +57,10 @@ from searx.engines.google import (
5857about = {
5958 "website": 'https://www.google.com',
6059 "wikidata_id": 'Q219885',
61− "official_api_documentation": 'https://developers.google.com/custom-search/',
60+ "official_api_documentation": 'https://developers.google.com/custom-search',
6261 "use_official_api": False,
6362 "require_api_key": False,
6463 "results": 'HTML',
65− "template": 'video.html',
66− "parse": ('url', 'title', 'content', 'thumbnail')
6764}
6865
6966logger = logger.getChild('google video')
@@ -90,7 +87,7 @@ def scrap_out_thumbs(dom):
9087 ret_val = dict()
9188 thumb_name = 'vidthumb'
9289
93− for script in eval_xpath(dom, '//script[contains(., "_setImagesSrc")]'):
90+ for script in eval_xpath_list(dom, '//script[contains(., "_setImagesSrc")]'):
9491 _script = script.text
9592
9693 # var s='data:image/jpeg;base64, ...'
@@ -104,7 +101,7 @@ def scrap_out_thumbs(dom):
104101 ret_val[_vidthumb] = _imgdata[0].replace(r"\x3d", "=")
105102
106103 # {google.ldidly=-1;google.ldi={"vidthumb8":"https://...
107− for script in eval_xpath(dom, '//script[contains(., "google.ldi={")]'):
104+ for script in eval_xpath_list(dom, '//script[contains(., "google.ldi={")]'):
108105 _script = script.text
109106 for key_val in _re(r'"%s\d+\":\"[^\"]*"' % thumb_name).findall( _script) :
110107 match = _re(r'"(%s\d+)":"(.*)"' % thumb_name).search(key_val)
@@ -119,17 +116,16 @@ def scrap_out_thumbs(dom):
119116def request(query, params):
120117 """Google-Video search request"""
121118
122− language, country, lang_country = get_lang_country(
119+ lang_info = get_lang_info(
123120 # pylint: disable=undefined-variable
124121 params, supported_languages, language_aliases
125122 )
126− subdomain = 'www.' + google_domains.get(country.upper(), 'google.com')
127123
128− query_url = 'https://'+ subdomain + '/search' + "?" + urlencode({
124+ query_url = 'https://' + lang_info['subdomain'] + '/search' + "?" + urlencode({
129125 'q': query,
130126 'tbm': "vid",
131− 'hl': lang_country,
132− 'lr': "lang_" + language,
127+ 'hl': lang_info['hl'],
128+ 'lr': lang_info['lr'],
133129 'ie': "utf8",
134130 'oe': "utf8",
135131 })
@@ -139,18 +135,14 @@ def request(query, params):
139135 if params['safesearch']:
140136 query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
141137
142− params['url'] = query_url
143138 logger.debug("query_url --> %s", query_url)
139+ params['url'] = query_url
144140
145− # en-US,en;q=0.8,en;q=0.5
146− params['headers']['Accept-Language'] = (
147− "%s,%s;q=0.8,%s;q=0.5" % (lang_country, language, language))
148− logger.debug(
149− "HTTP Accept-Language --> %s", params['headers']['Accept-Language'])
141+ logger.debug("HTTP header Accept-Language --> %s", lang_info['Accept-Language'])
142+ params['headers']['Accept-Language'] = lang_info['Accept-Language']
150143 params['headers']['Accept'] = (
151144 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
152145 )
153− #params['google_subdomain'] = subdomain
154146 return params
155147
156148
@@ -158,16 +150,7 @@ def response(resp):
158150 """Get response from google's search request"""
159151 results = []
160152
161− # detect google sorry
162− resp_url = urlparse(resp.url)
163− if resp_url.netloc == 'sorry.google.com' or resp_url.path == '/sorry/IndexRedirect':
164− raise SearxEngineCaptchaException()
165−
166− if resp_url.path.startswith('/sorry'):
167− raise SearxEngineCaptchaException()
168−
169− # which subdomain ?
170− # subdomain = resp.search_params.get('google_subdomain')
153+ detect_google_sorry(resp)
171154
172155 # convert the text to dom
173156 dom = html.fromstring(resp.text)
@@ -181,19 +164,18 @@ def response(resp):
181164 logger.debug("ingoring <g-section-with-header>")
182165 continue
183166
184− title = extract_text(eval_xpath(result, title_xpath)[0])
185− url = eval_xpath(result, href_xpath)[0]
186− c_node = eval_xpath(result, content_xpath)[0]
167+ title = extract_text(eval_xpath_getindex(result, title_xpath, 0))
168+ url = eval_xpath_getindex(result, href_xpath, 0)
169+ c_node = eval_xpath_getindex(result, content_xpath, 0)
187170
188171 # <img id="vidthumb1" ...>
189− img_id = eval_xpath(c_node, './div[1]//a/g-img/img/@id')
190− if not img_id:
172+ img_id = eval_xpath_getindex(c_node, './div[1]//a/g-img/img/@id', 0, default=None)
173+ if img_id is None:
191174 continue
192− img_id = img_id[0]
193175 img_src = vidthumb_imgdata.get(img_id, None)
194176 if not img_src:
195177 logger.error("no vidthumb imgdata for: %s" % img_id)
196− img_src = eval_xpath(c_node, './div[1]//a/g-img/img/@src')[0]
178+ img_src = eval_xpath_getindex(c_node, './div[1]//a/g-img/img/@src', 0)
197179
198180 length = extract_text(eval_xpath(c_node, './/div[1]//a/div[3]'))
199181 content = extract_text(eval_xpath(c_node, './/div[2]/span'))
@@ -210,11 +192,11 @@ def response(resp):
210192 })
211193
212194 # parse suggestion
213− for suggestion in eval_xpath(dom, suggestion_xpath):
195+ for suggestion in eval_xpath_list(dom, suggestion_xpath):
214196 # append suggestion
215197 results.append({'suggestion': extract_text(suggestion)})
216198
217− for correction in eval_xpath(dom, spelling_suggestion_xpath):
199+ for correction in eval_xpath_list(dom, spelling_suggestion_xpath):
218200 results.append({'correction': extract_text(correction)})
219201
220202 return results