git.iko.soy/searx
on master help sign in
[fix] normalize the language & region aspects of all google engines
b1fefecMarkus Heiser
searx/engines/google.py+61−37
compare with master
unifiedsplit
@@ -1,11 +1,11 @@
1# SPDX-License-Identifier: AGPL-3.0-or-later
1# SPDX-License-Identifier: AGPL-3.0-or-later
2"""Google (Web)
2"""Google (Web)
3
3
4− For detailed description of the *REST-full* API see: `Query Parameter
4+For detailed description of the *REST-full* API see: `Query Parameter
5− Definitions`_.
5+Definitions`_.
6
6
7− .. _Query Parameter Definitions:
7+.. _Query Parameter Definitions:
8− https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions
8+ https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions
9"""
9"""
10
10
11# pylint: disable=invalid-name, missing-function-docstring
11# pylint: disable=invalid-name, missing-function-docstring
@@ -16,7 +16,6 @@ from searx import logger
16from searx.utils import match_language, extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex
16from searx.utils import match_language, extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex
17from searx.exceptions import SearxEngineCaptchaException
17from searx.exceptions import SearxEngineCaptchaException
18
18
19−
20logger = logger.getChild('google engine')
19logger = logger.getChild('google engine')
21
20
22# about
21# about
@@ -56,7 +55,7 @@ google_domains = {
56 'NZ': 'google.co.nz', # New Zealand
55 'NZ': 'google.co.nz', # New Zealand
57 'PH': 'google.com.ph', # Philippines
56 'PH': 'google.com.ph', # Philippines
58 'SG': 'google.com.sg', # Singapore
57 'SG': 'google.com.sg', # Singapore
59− # 'US': 'google.us', # United States, redirect to .com
58+ 'US': 'google.com', # United States (google.us) redirects to .com
60 'ZA': 'google.co.za', # South Africa
59 'ZA': 'google.co.za', # South Africa
61 'AR': 'google.com.ar', # Argentina
60 'AR': 'google.com.ar', # Argentina
62 'CL': 'google.cl', # Chile
61 'CL': 'google.cl', # Chile
@@ -87,7 +86,7 @@ google_domains = {
87 'TH': 'google.co.th', # Thailand
86 'TH': 'google.co.th', # Thailand
88 'TR': 'google.com.tr', # Turkey
87 'TR': 'google.com.tr', # Turkey
89 'UA': 'google.com.ua', # Ukraine
88 'UA': 'google.com.ua', # Ukraine
90− # 'CN': 'google.cn', # China, only from China ?
89+ 'CN': 'google.com.hk', # There is no google.cn, we use .com.hk for zh-CN
91 'HK': 'google.com.hk', # Hong Kong
90 'HK': 'google.com.hk', # Hong Kong
92 'TW': 'google.com.tw' # Taiwan
91 'TW': 'google.com.tw' # Taiwan
93}
92}
@@ -134,26 +133,58 @@ suggestion_xpath = '//div[contains(@class, "card-section")]//a'
134spelling_suggestion_xpath = '//div[@class="med"]/p/a'
133spelling_suggestion_xpath = '//div[@class="med"]/p/a'
135
134
136
135
137−def get_lang_country(params, lang_list, custom_aliases):
136+def get_lang_info(params, lang_list, custom_aliases):
138− """Returns a tuple with *langauage* on its first and *country* on its second
137+ ret_val = {}
139− position."""
138+
140− language = params['language']
139+ _lang = params['language']
141− if language == 'all':
140+ if _lang.lower() == 'all':
142− language = 'en-US'
141+ _lang = 'en-US'
142+
143+ language = match_language(_lang, lang_list, custom_aliases)
144+ ret_val['language'] = language
143
145
144− language_array = language.split('-')
146+ # the requested language from params (en, en-US, de, de-AT, fr, fr-CA, ...)
147+ _l = _lang.split('-')
145
148
149+ # the country code (US, AT, CA)
146− if len(language_array) == 2:
150+ if len(_l) == 2:
147− country = language_array[1]
151+ country = _l[1]
148 else:
152 else:
149− country = language_array[0].upper()
153+ country = _l[0].upper()
154+ if country == 'EN':
155+ country = 'US'
150
156
151− language = match_language(language, lang_list, custom_aliases)
157+ ret_val['country'] = country
158+
159+ # the combination (en-US, en-EN, de-DE, de-AU, fr-FR, fr-FR)
152 lang_country = '%s-%s' % (language, country)
160 lang_country = '%s-%s' % (language, country)
153− if lang_country == 'en-EN':
154− lang_country = 'en'
155
161
156− return language, country, lang_country
162+ # Accept-Language: fr-CH, fr;q=0.8, en;q=0.6, *;q=0.5
163+ ret_val['Accept-Language'] = ','.join([
164+ lang_country,
165+ language + ';q=0.8,',
166+ 'en;q=0.6',
167+ '*;q=0.5',
168+ ])
169+
170+ # subdomain
171+ ret_val['subdomain'] = 'www.' + google_domains.get(country.upper(), 'google.com')
172+
173+ # hl parameter:
174+ # https://developers.google.com/custom-search/docs/xml_results#hlsp The
175+ # Interface Language:
176+ # https://developers.google.com/custom-search/docs/xml_results_appendices#interfaceLanguages
177+
178+ ret_val['hl'] = lang_list.get(lang_country, language)
179+
180+ # lr parameter:
181+ # https://developers.google.com/custom-search/docs/xml_results#lrsp
182+ # Language Collection Values:
183+ # https://developers.google.com/custom-search/docs/xml_results_appendices#languageCollections
184+
185+ ret_val['lr'] = "lang_" + lang_list.get(lang_country, language)
186+
187+ return ret_val
157
188
158def detect_google_sorry(resp):
189def detect_google_sorry(resp):
159 resp_url = urlparse(resp.url)
190 resp_url = urlparse(resp.url)
@@ -165,17 +196,17 @@ def request(query, params):
165 """Google search request"""
196 """Google search request"""
166
197
167 offset = (params['pageno'] - 1) * 10
198 offset = (params['pageno'] - 1) * 10
168− language, country, lang_country = get_lang_country(
199+
200+ lang_info = get_lang_info(
169 # pylint: disable=undefined-variable
201 # pylint: disable=undefined-variable
170 params, supported_languages, language_aliases
202 params, supported_languages, language_aliases
171 )
203 )
172− subdomain = 'www.' + google_domains.get(country.upper(), 'google.com')
173
204
174− # https://www.google.de/search?q=corona&hl=de-DE&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium
205+ # https://www.google.de/search?q=corona&hl=de&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium
175− query_url = 'https://' + subdomain + '/search' + "?" + urlencode({
206+ query_url = 'https://' + lang_info['subdomain'] + '/search' + "?" + urlencode({
176 'q': query,
207 'q': query,
177− 'hl': lang_country,
208+ 'hl': lang_info['hl'],
178− 'lr': "lang_" + language,
209+ 'lr': lang_info['lr'],
179 'ie': "utf8",
210 'ie': "utf8",
180 'oe': "utf8",
211 'oe': "utf8",
181 'start': offset,
212 'start': offset,
@@ -186,19 +217,14 @@ def request(query, params):
186 if params['safesearch']:
217 if params['safesearch']:
187 query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
218 query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
188
219
189− params['url'] = query_url
190 logger.debug("query_url --> %s", query_url)
220 logger.debug("query_url --> %s", query_url)
221+ params['url'] = query_url
191
222
192− # en-US,en;q=0.8,en;q=0.5
193− params['headers']['Accept-Language'] = (
194− lang_country + ',' + language + ';q=0.8,' + language + ';q=0.5'
195− )
196− logger.debug("HTTP header Accept-Language --> %s",
223+ logger.debug("HTTP header Accept-Language --> %s", lang_info['Accept-Language'])
197− params['headers']['Accept-Language'])
224+ params['headers']['Accept-Language'] = lang_info['Accept-Language']
198 params['headers']['Accept'] = (
225 params['headers']['Accept'] = (
199 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
226 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
200 )
227 )
201− # params['google_subdomain'] = subdomain
202
228
203 return params
229 return params
204
230
@@ -209,8 +235,6 @@ def response(resp):
209 detect_google_sorry(resp)
235 detect_google_sorry(resp)
210
236
211 results = []
237 results = []
212− # which subdomain ?
213− # subdomain = resp.search_params.get('google_subdomain')
214
238
215 # convert the text to dom
239 # convert the text to dom
216 dom = html.fromstring(resp.text)
240 dom = html.fromstring(resp.text)
searx/engines/google_images.py+72−71
@@ -10,35 +10,50 @@ Definitions`_.
10 ``data:` scheme).::
10 ``data:` scheme).::
11
11
12 Header set Content-Security-Policy "img-src 'self' data: ;"
12 Header set Content-Security-Policy "img-src 'self' data: ;"
13+
14+.. _Query Parameter Definitions:
15+ https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions
16+.. _data URLs:
17+ https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs
13"""
18"""
14
19
15from urllib.parse import urlencode, unquote
20from urllib.parse import urlencode, unquote
16from lxml import html
21from lxml import html
22+
17from searx import logger
23from searx import logger
18−from searx.utils import extract_text, eval_xpath
24+from searx.utils import (
19−from searx.engines.google import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import
25+ eval_xpath,
26+ eval_xpath_list,
27+ eval_xpath_getindex,
28+ extract_text,
29+)
20
30
21from searx.engines.google import (
31from searx.engines.google import (
22− get_lang_country,
32+ get_lang_info,
23− google_domains,
24 time_range_dict,
33 time_range_dict,
25 detect_google_sorry,
34 detect_google_sorry,
26)
35)
27
36
37+# pylint: disable=unused-import
38+from searx.engines.google import (
39+ supported_languages_url
40+ , _fetch_supported_languages
41+)
42+# pylint: enable=unused-import
43+
28logger = logger.getChild('google images')
44logger = logger.getChild('google images')
29
45
30# about
46# about
31about = {
47about = {
32− "website": 'https://images.google.com/',
48+ "website": 'https://images.google.com',
33 "wikidata_id": 'Q521550',
49 "wikidata_id": 'Q521550',
34− "official_api_documentation": 'https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions', # NOQA
50+ "official_api_documentation": 'https://developers.google.com/custom-search',
35 "use_official_api": False,
51 "use_official_api": False,
36 "require_api_key": False,
52 "require_api_key": False,
37 "results": 'HTML',
53 "results": 'HTML',
38}
54}
39
55
40# engine dependent config
56# engine dependent config
41−
42categories = ['images']
57categories = ['images']
43paging = False
58paging = False
44language_support = True
59language_support = True
@@ -84,17 +99,16 @@ def scrap_img_by_id(script, data_id):
84def request(query, params):
99def request(query, params):
85 """Google-Video search request"""
100 """Google-Video search request"""
86
101
87− language, country, lang_country = get_lang_country(
102+ lang_info = get_lang_info(
88 # pylint: disable=undefined-variable
103 # pylint: disable=undefined-variable
89 params, supported_languages, language_aliases
104 params, supported_languages, language_aliases
90 )
105 )
91− subdomain = 'www.' + google_domains.get(country.upper(), 'google.com')
92
106
93− query_url = 'https://' + subdomain + '/search' + "?" + urlencode({
107+ query_url = 'https://' + lang_info['subdomain'] + '/search' + "?" + urlencode({
94 'q': query,
108 'q': query,
95 'tbm': "isch",
109 'tbm': "isch",
96− 'hl': lang_country,
110+ 'hl': lang_info['hl'],
97− 'lr': "lang_" + language,
111+ 'lr': lang_info['lr'],
98 'ie': "utf8",
112 'ie': "utf8",
99 'oe': "utf8",
113 'oe': "utf8",
100 'num': 30,
114 'num': 30,
@@ -105,17 +119,14 @@ def request(query, params):
105 if params['safesearch']:
119 if params['safesearch']:
106 query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
120 query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
107
121
108− params['url'] = query_url
109 logger.debug("query_url --> %s", query_url)
122 logger.debug("query_url --> %s", query_url)
123+ params['url'] = query_url
110
124
111− params['headers']['Accept-Language'] = (
125+ logger.debug("HTTP header Accept-Language --> %s", lang_info['Accept-Language'])
112− "%s,%s;q=0.8,%s;q=0.5" % (lang_country, language, language))
113− logger.debug(
114− "HTTP Accept-Language --> %s", params['headers']['Accept-Language'])
126+ params['headers']['Accept-Language'] = lang_info['Accept-Language']
115 params['headers']['Accept'] = (
127 params['headers']['Accept'] = (
116 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
128 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
117 )
129 )
118− # params['google_subdomain'] = subdomain
119 return params
130 return params
120
131
121
132
@@ -125,13 +136,11 @@ def response(resp):
125
136
126 detect_google_sorry(resp)
137 detect_google_sorry(resp)
127
138
128− # which subdomain ?
129− # subdomain = resp.search_params.get('google_subdomain')
130−
131 # convert the text to dom
139 # convert the text to dom
132 dom = html.fromstring(resp.text)
140 dom = html.fromstring(resp.text)
133 img_bas64_map = scrap_out_thumbs(dom)
141 img_bas64_map = scrap_out_thumbs(dom)
142+ img_src_script = eval_xpath_getindex(
134− img_src_script = eval_xpath(dom, '//script[contains(., "AF_initDataCallback({key: ")]')[1].text
143+ dom, '//script[contains(., "AF_initDataCallback({key: ")]', 1).text
135
144
136 # parse results
145 # parse results
137 #
146 #
@@ -156,55 +165,47 @@ def response(resp):
156 return results
165 return results
157
166
158 root = root[0]
167 root = root[0]
159− for img_node in eval_xpath(root, './/img[contains(@class, "rg_i")]'):
168+ for img_node in eval_xpath_list(root, './/img[contains(@class, "rg_i")]'):
160−
169+
161− try:
162− img_alt = eval_xpath(img_node, '@alt')[0]
170+ img_alt = eval_xpath_getindex(img_node, '@alt', 0)
163−
171+
164− img_base64_id = eval_xpath(img_node, '@data-iid')
172+ img_base64_id = eval_xpath(img_node, '@data-iid')
165− if img_base64_id:
173+ if img_base64_id:
166− img_base64_id = img_base64_id[0]
174+ img_base64_id = img_base64_id[0]
167− thumbnail_src = img_bas64_map[img_base64_id]
175+ thumbnail_src = img_bas64_map[img_base64_id]
176+ else:
177+ thumbnail_src = eval_xpath(img_node, '@src')
178+ if not thumbnail_src:
179+ thumbnail_src = eval_xpath(img_node, '@data-src')
180+ if thumbnail_src:
181+ thumbnail_src = thumbnail_src[0]
168 else:
182 else:
169− thumbnail_src = eval_xpath(img_node, '@src')
183+ thumbnail_src = ''
170− if not thumbnail_src:
184+
171− thumbnail_src = eval_xpath(img_node, '@data-src')
172− if thumbnail_src:
173− thumbnail_src = thumbnail_src[0]
174− else:
175− thumbnail_src = ''
176−
177− link_node = eval_xpath(img_node, '../../../a[2]')[0]
185+ link_node = eval_xpath_getindex(img_node, '../../../a[2]', 0)
178− url = eval_xpath(link_node, '@href')[0]
186+ url = eval_xpath_getindex(link_node, '@href', 0)
179−
187+
180− pub_nodes = eval_xpath(link_node, './div/div')
188+ pub_nodes = eval_xpath(link_node, './div/div')
181− pub_descr = img_alt
189+ pub_descr = img_alt
182− pub_source = ''
190+ pub_source = ''
183− if pub_nodes:
191+ if pub_nodes:
184− pub_descr = extract_text(pub_nodes[0])
192+ pub_descr = extract_text(pub_nodes[0])
185− pub_source = extract_text(pub_nodes[1])
193+ pub_source = extract_text(pub_nodes[1])
186−
194+
187− img_src_id = eval_xpath(img_node, '../../../@data-id')[0]
195+ img_src_id = eval_xpath_getindex(img_node, '../../../@data-id', 0)
188− src_url = scrap_img_by_id(img_src_script, img_src_id)
196+ src_url = scrap_img_by_id(img_src_script, img_src_id)
189− if not src_url:
197+ if not src_url:
190− src_url = thumbnail_src
198+ src_url = thumbnail_src
191−
199+
192− results.append({
200+ results.append({
193− 'url': url,
201+ 'url': url,
194− 'title': img_alt,
202+ 'title': img_alt,
195− 'content': pub_descr,
203+ 'content': pub_descr,
196− 'source': pub_source,
204+ 'source': pub_source,
197− 'img_src': src_url,
205+ 'img_src': src_url,
198− # 'img_format': img_format,
206+ # 'img_format': img_format,
199− 'thumbnail_src': thumbnail_src,
207+ 'thumbnail_src': thumbnail_src,
200− 'template': 'images.html'
208+ 'template': 'images.html'
201− })
209+ })
202− except Exception as e: # pylint: disable=broad-except
203− logger.error(e, exc_info=True)
204− # from lxml import etree
205− # logger.debug(etree.tostring(img_node, pretty_print=True))
206− # import pdb
207− # pdb.set_trace()
208− continue
209
210
210 return results
211 return results
searx/engines/google_news.py+35−34
@@ -2,13 +2,16 @@
2"""Google (News)
2"""Google (News)
3
3
4For detailed description of the *REST-full* API see: `Query Parameter
4For detailed description of the *REST-full* API see: `Query Parameter
5−Definitions`_. Not all parameters can be appied, e.g. num_ (the number of
5+Definitions`_. Not all parameters can be appied:
6−search results to return) is ignored.
6+
7+- num_ : the number of search results is ignored
8+- save_ : is ignored / Google-News results are always *SafeSearch*
7
9
8.. _Query Parameter Definitions:
10.. _Query Parameter Definitions:
9 https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions
11 https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions
10
12
11.. _num: https://developers.google.com/custom-search/docs/xml_results#numsp
13.. _num: https://developers.google.com/custom-search/docs/xml_results#numsp
14+.. _save: https://developers.google.com/custom-search/docs/xml_results#safesp
12
15
13"""
16"""
14
17
@@ -32,20 +35,19 @@ from searx.utils import (
32from searx.engines.google import (
35from searx.engines.google import (
33 supported_languages_url,
36 supported_languages_url,
34 _fetch_supported_languages,
37 _fetch_supported_languages,
35− detect_google_sorry,
36)
38)
37# pylint: enable=unused-import
39# pylint: enable=unused-import
38
40
39from searx.engines.google import (
41from searx.engines.google import (
40− get_lang_country,
42+ get_lang_info,
41− filter_mapping,
43+ detect_google_sorry,
42)
44)
43
45
44# about
46# about
45about = {
47about = {
46 "website": 'https://news.google.com',
48 "website": 'https://news.google.com',
47 "wikidata_id": 'Q12020',
49 "wikidata_id": 'Q12020',
48− "official_api_documentation": None,
50+ "official_api_documentation": 'https://developers.google.com/custom-search',
49 "use_official_api": False,
51 "use_official_api": False,
50 "require_api_key": False,
52 "require_api_key": False,
51 "results": 'HTML',
53 "results": 'HTML',
@@ -69,51 +71,53 @@ paging = False
69language_support = True
71language_support = True
70use_locale_domain = True
72use_locale_domain = True
71time_range_support = True
73time_range_support = True
72−safesearch = True # not really, but it is not generated by google
74+
75+# Google-News results are always *SafeSearch*. Option 'safesearch' is set to
76+# False here, otherwise checker will report safesearch-errors::
77+#
78+# safesearch : results are identitical for safesearch=0 and safesearch=2
79+safesearch = False
73
80
74def request(query, params):
81def request(query, params):
75 """Google-News search request"""
82 """Google-News search request"""
76
83
77− language, country, lang_country = get_lang_country(
84+ lang_info = get_lang_info(
78 # pylint: disable=undefined-variable
85 # pylint: disable=undefined-variable
79 params, supported_languages, language_aliases
86 params, supported_languages, language_aliases
80 )
87 )
81− subdomain = 'news.google.com'
82
88
89+ # google news has only one domain
90+ lang_info['subdomain'] = 'news.google.com'
91+
92+ ceid = "%s:%s" % (lang_info['country'], lang_info['language'])
93+
94+ # google news redirects en to en-US
95+ if lang_info['hl'] == 'en':
96+ lang_info['hl'] = 'en-US'
97+
98+ # Very special to google-news compared to other google engines, the time
99+ # range is included in the search term.
83− if params['time_range']: # in time_range_dict:
100+ if params['time_range']:
84 query += ' ' + time_range_dict[params['time_range']]
101 query += ' ' + time_range_dict[params['time_range']]
85
102
86− query_url = 'https://'+ subdomain + '/search' + "?" + urlencode({
103+ query_url = 'https://' + lang_info['subdomain'] + '/search' + "?" + urlencode({
87 'q': query,
104 'q': query,
88− 'hl': language,
105+ 'hl': lang_info['hl'],
89− 'lr': "lang_" + language,
106+ 'lr': lang_info['lr'],
90 'ie': "utf8",
107 'ie': "utf8",
91 'oe': "utf8",
108 'oe': "utf8",
92− 'ceid' : "%s:%s" % (country, language),
93− 'gl' : country,
109+ 'gl': lang_info['country'],
94− })
110+ }) + ('&ceid=%s' % ceid) # ceid includes a ':' character which must not be urlencoded
95
111
96− if params['safesearch']:
97− query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
98−
99− params['url'] = query_url
100 logger.debug("query_url --> %s", query_url)
112 logger.debug("query_url --> %s", query_url)
113+ params['url'] = query_url
101
114
102− # en-US,en;q=0.8,en;q=0.5
103− params['headers']['Accept-Language'] = (
104− lang_country + ',' + language + ';q=0.8,' + language + ';q=0.5'
105− )
106− logger.debug("HTTP header Accept-Language --> %s",
115+ logger.debug("HTTP header Accept-Language --> %s", lang_info['Accept-Language'])
107− params['headers']['Accept-Language'])
116+ params['headers']['Accept-Language'] = lang_info['Accept-Language']
108 params['headers']['Accept'] = (
117 params['headers']['Accept'] = (
109 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
118 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
110 )
119 )
111
120
112− # hl=en redirect to hl=en-US / en-CA ...
113− params['soft_max_redirects'] = 1
114−
115− #params['google_subdomain'] = subdomain
116−
117 return params
121 return params
118
122
119
123
@@ -123,9 +127,6 @@ def response(resp):
123
127
124 detect_google_sorry(resp)
128 detect_google_sorry(resp)
125
129
126− # which subdomain ?
127− # subdomain = resp.search_params.get('google_subdomain')
128−
129 # convert the text to dom
130 # convert the text to dom
130 dom = html.fromstring(resp.text)
131 dom = html.fromstring(resp.text)
131
132
searx/engines/google_videos.py+25−43
@@ -1,6 +1,5 @@
1# SPDX-License-Identifier: AGPL-3.0-or-later
1# SPDX-License-Identifier: AGPL-3.0-or-later
2−"""
3−Google (Viedo)
2+"""Google (Video)
4
3
5For detailed description of the *REST-full* API see: `Query Parameter
4For detailed description of the *REST-full* API see: `Query Parameter
6Definitions`_. Not all parameters can be appied.
5Definitions`_. Not all parameters can be appied.
@@ -22,20 +21,19 @@ Definitions`_. Not all parameters can be appied.
22# pylint: disable=invalid-name, missing-function-docstring
21# pylint: disable=invalid-name, missing-function-docstring
23
22
24import re
23import re
25−from urllib.parse import urlencode, urlparse
24+from urllib.parse import urlencode
26from lxml import html
25from lxml import html
27
26
28from searx import logger
27from searx import logger
29−from searx.exceptions import SearxEngineCaptchaException
30from searx.utils import (
28from searx.utils import (
31 eval_xpath,
29 eval_xpath,
32 eval_xpath_list,
30 eval_xpath_list,
31+ eval_xpath_getindex,
33 extract_text,
32 extract_text,
34)
33)
35
34
36from searx.engines.google import (
35from searx.engines.google import (
37− get_lang_country,
36+ get_lang_info,
38− google_domains,
39 time_range_dict,
37 time_range_dict,
40 filter_mapping,
38 filter_mapping,
41 results_xpath,
39 results_xpath,
@@ -44,7 +42,8 @@ from searx.engines.google import (
44 href_xpath,
42 href_xpath,
45 content_xpath,
43 content_xpath,
46 suggestion_xpath,
44 suggestion_xpath,
47− spelling_suggestion_xpath
45+ spelling_suggestion_xpath,
46+ detect_google_sorry,
48)
47)
49
48
50# pylint: disable=unused-import
49# pylint: disable=unused-import
@@ -58,12 +57,10 @@ from searx.engines.google import (
58about = {
57about = {
59 "website": 'https://www.google.com',
58 "website": 'https://www.google.com',
60 "wikidata_id": 'Q219885',
59 "wikidata_id": 'Q219885',
61− "official_api_documentation": 'https://developers.google.com/custom-search/',
60+ "official_api_documentation": 'https://developers.google.com/custom-search',
62 "use_official_api": False,
61 "use_official_api": False,
63 "require_api_key": False,
62 "require_api_key": False,
64 "results": 'HTML',
63 "results": 'HTML',
65− "template": 'video.html',
66− "parse": ('url', 'title', 'content', 'thumbnail')
67}
64}
68
65
69logger = logger.getChild('google video')
66logger = logger.getChild('google video')
@@ -90,7 +87,7 @@ def scrap_out_thumbs(dom):
90 ret_val = dict()
87 ret_val = dict()
91 thumb_name = 'vidthumb'
88 thumb_name = 'vidthumb'
92
89
93− for script in eval_xpath(dom, '//script[contains(., "_setImagesSrc")]'):
90+ for script in eval_xpath_list(dom, '//script[contains(., "_setImagesSrc")]'):
94 _script = script.text
91 _script = script.text
95
92
96 # var s='data:image/jpeg;base64, ...'
93 # var s='data:image/jpeg;base64, ...'
@@ -104,7 +101,7 @@ def scrap_out_thumbs(dom):
104 ret_val[_vidthumb] = _imgdata[0].replace(r"\x3d", "=")
101 ret_val[_vidthumb] = _imgdata[0].replace(r"\x3d", "=")
105
102
106 # {google.ldidly=-1;google.ldi={"vidthumb8":"https://...
103 # {google.ldidly=-1;google.ldi={"vidthumb8":"https://...
107− for script in eval_xpath(dom, '//script[contains(., "google.ldi={")]'):
104+ for script in eval_xpath_list(dom, '//script[contains(., "google.ldi={")]'):
108 _script = script.text
105 _script = script.text
109 for key_val in _re(r'"%s\d+\":\"[^\"]*"' % thumb_name).findall( _script) :
106 for key_val in _re(r'"%s\d+\":\"[^\"]*"' % thumb_name).findall( _script) :
110 match = _re(r'"(%s\d+)":"(.*)"' % thumb_name).search(key_val)
107 match = _re(r'"(%s\d+)":"(.*)"' % thumb_name).search(key_val)
@@ -119,17 +116,16 @@ def scrap_out_thumbs(dom):
119def request(query, params):
116def request(query, params):
120 """Google-Video search request"""
117 """Google-Video search request"""
121
118
122− language, country, lang_country = get_lang_country(
119+ lang_info = get_lang_info(
123 # pylint: disable=undefined-variable
120 # pylint: disable=undefined-variable
124 params, supported_languages, language_aliases
121 params, supported_languages, language_aliases
125 )
122 )
126− subdomain = 'www.' + google_domains.get(country.upper(), 'google.com')
127
123
128− query_url = 'https://'+ subdomain + '/search' + "?" + urlencode({
124+ query_url = 'https://' + lang_info['subdomain'] + '/search' + "?" + urlencode({
129 'q': query,
125 'q': query,
130 'tbm': "vid",
126 'tbm': "vid",
131− 'hl': lang_country,
127+ 'hl': lang_info['hl'],
132− 'lr': "lang_" + language,
128+ 'lr': lang_info['lr'],
133 'ie': "utf8",
129 'ie': "utf8",
134 'oe': "utf8",
130 'oe': "utf8",
135 })
131 })
@@ -139,18 +135,14 @@ def request(query, params):
139 if params['safesearch']:
135 if params['safesearch']:
140 query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
136 query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
141
137
142− params['url'] = query_url
143 logger.debug("query_url --> %s", query_url)
138 logger.debug("query_url --> %s", query_url)
139+ params['url'] = query_url
144
140
145− # en-US,en;q=0.8,en;q=0.5
141+ logger.debug("HTTP header Accept-Language --> %s", lang_info['Accept-Language'])
146− params['headers']['Accept-Language'] = (
147− "%s,%s;q=0.8,%s;q=0.5" % (lang_country, language, language))
148− logger.debug(
149− "HTTP Accept-Language --> %s", params['headers']['Accept-Language'])
142+ params['headers']['Accept-Language'] = lang_info['Accept-Language']
150 params['headers']['Accept'] = (
143 params['headers']['Accept'] = (
151 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
144 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
152 )
145 )
153− #params['google_subdomain'] = subdomain
154 return params
146 return params
155
147
156
148
@@ -158,16 +150,7 @@ def response(resp):
158 """Get response from google's search request"""
150 """Get response from google's search request"""
159 results = []
151 results = []
160
152
161− # detect google sorry
153+ detect_google_sorry(resp)
162− resp_url = urlparse(resp.url)
163− if resp_url.netloc == 'sorry.google.com' or resp_url.path == '/sorry/IndexRedirect':
164− raise SearxEngineCaptchaException()
165−
166− if resp_url.path.startswith('/sorry'):
167− raise SearxEngineCaptchaException()
168−
169− # which subdomain ?
170− # subdomain = resp.search_params.get('google_subdomain')
171
154
172 # convert the text to dom
155 # convert the text to dom
173 dom = html.fromstring(resp.text)
156 dom = html.fromstring(resp.text)
@@ -181,19 +164,18 @@ def response(resp):
181 logger.debug("ingoring <g-section-with-header>")
164 logger.debug("ingoring <g-section-with-header>")
182 continue
165 continue
183
166
184− title = extract_text(eval_xpath(result, title_xpath)[0])
167+ title = extract_text(eval_xpath_getindex(result, title_xpath, 0))
185− url = eval_xpath(result, href_xpath)[0]
168+ url = eval_xpath_getindex(result, href_xpath, 0)
186− c_node = eval_xpath(result, content_xpath)[0]
169+ c_node = eval_xpath_getindex(result, content_xpath, 0)
187
170
188 # <img id="vidthumb1" ...>
171 # <img id="vidthumb1" ...>
189− img_id = eval_xpath(c_node, './div[1]//a/g-img/img/@id')
172+ img_id = eval_xpath_getindex(c_node, './div[1]//a/g-img/img/@id', 0, default=None)
190− if not img_id:
173+ if img_id is None:
191 continue
174 continue
192− img_id = img_id[0]
193 img_src = vidthumb_imgdata.get(img_id, None)
175 img_src = vidthumb_imgdata.get(img_id, None)
194 if not img_src:
176 if not img_src:
195 logger.error("no vidthumb imgdata for: %s" % img_id)
177 logger.error("no vidthumb imgdata for: %s" % img_id)
196− img_src = eval_xpath(c_node, './div[1]//a/g-img/img/@src')[0]
178+ img_src = eval_xpath_getindex(c_node, './div[1]//a/g-img/img/@src', 0)
197
179
198 length = extract_text(eval_xpath(c_node, './/div[1]//a/div[3]'))
180 length = extract_text(eval_xpath(c_node, './/div[1]//a/div[3]'))
199 content = extract_text(eval_xpath(c_node, './/div[2]/span'))
181 content = extract_text(eval_xpath(c_node, './/div[2]/span'))
@@ -210,11 +192,11 @@ def response(resp):
210 })
192 })
211
193
212 # parse suggestion
194 # parse suggestion
213− for suggestion in eval_xpath(dom, suggestion_xpath):
195+ for suggestion in eval_xpath_list(dom, suggestion_xpath):
214 # append suggestion
196 # append suggestion
215 results.append({'suggestion': extract_text(suggestion)})
197 results.append({'suggestion': extract_text(suggestion)})
216
198
217− for correction in eval_xpath(dom, spelling_suggestion_xpath):
199+ for correction in eval_xpath_list(dom, spelling_suggestion_xpath):
218 results.append({'correction': extract_text(correction)})
200 results.append({'correction': extract_text(correction)})
219
201
220 return results
202 return results