@@ -1,12 +1,45 @@
11# SPDX-License-Identifier: AGPL-3.0-or-later
2−"""
3− Google (News)
2+"""Google (News)
3+
4+For detailed description of the *REST-full* API see: `Query Parameter
5+Definitions`_. Not all parameters can be appied, e.g. num_ (the number of
6+search results to return) is ignored.
7+
8+.. _Query Parameter Definitions:
9+ https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions
10+
11+.. _num: https://developers.google.com/custom-search/docs/xml_results#numsp
12+
413"""
514
15+# pylint: disable=invalid-name, missing-function-docstring
16+
17+import binascii
18+import re
619from urllib.parse import urlencode
20+from base64 import b64decode
721from lxml import html
8−from searx.utils import match_language
9−from searx.engines.google import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import
22+
23+from searx import logger
24+from searx.utils import (
25+ eval_xpath,
26+ eval_xpath_list,
27+ eval_xpath_getindex,
28+ extract_text,
29+)
30+
31+# pylint: disable=unused-import
32+from searx.engines.google import (
33+ supported_languages_url,
34+ _fetch_supported_languages,
35+ detect_google_sorry,
36+)
37+# pylint: enable=unused-import
38+
39+from searx.engines.google import (
40+ get_lang_country,
41+ filter_mapping,
42+)
1043
1144# about
1245about = {
@@ -18,72 +51,143 @@ about = {
1851 "results": 'HTML',
1952}
2053
21−# search-url
54+logger = logger.getChild('google news')
55+
56+# compared to other google engines google-news has a different time range
57+# support. The time range is included in the search term.
58+time_range_dict = {
59+ 'day': 'when:1d',
60+ 'week': 'when:7d',
61+ 'month': 'when:1m',
62+ 'year': 'when:1y',
63+}
64+
65+# engine dependent config
66+
2267categories = ['news']
23−paging = True
68+paging = False
2469language_support = True
25−safesearch = True
70+use_locale_domain = True
2671time_range_support = True
27−number_of_results = 10
28−
29−search_url = 'https://www.google.com/search'\
30− '?{query}'\
31− '&tbm=nws'\
32− '&gws_rd=cr'\
33− '&{search_options}'
34−time_range_attr = "qdr:{range}"
35−time_range_dict = {'day': 'd',
36− 'week': 'w',
37− 'month': 'm',
38− 'year': 'y'}
39−
72+safesearch = True # not really, but it is not generated by google
4073
41−# do search-request
4274def request(query, params):
43−
44− search_options = {
45− 'start': (params['pageno'] - 1) * number_of_results
46− }
47−
48− if params['time_range'] in time_range_dict:
49− search_options['tbs'] = time_range_attr.format(range=time_range_dict[params['time_range']])
50−
51− if safesearch and params['safesearch']:
52− search_options['safe'] = 'on'
53−
54− params['url'] = search_url.format(query=urlencode({'q': query}),
55− search_options=urlencode(search_options))
56−
57− if params['language'] != 'all':
58− language = match_language(params['language'], supported_languages, language_aliases).split('-')[0]
59− if language:
60− params['url'] += '&hl=' + language
75+ """Google-News search request"""
76+
77+ language, country, lang_country = get_lang_country(
78+ # pylint: disable=undefined-variable
79+ params, supported_languages, language_aliases
80+ )
81+ subdomain = 'news.google.com'
82+
83+ if params['time_range']: # in time_range_dict:
84+ query += ' ' + time_range_dict[params['time_range']]
85+
86+ query_url = 'https://'+ subdomain + '/search' + "?" + urlencode({
87+ 'q': query,
88+ 'hl': language,
89+ 'lr': "lang_" + language,
90+ 'ie': "utf8",
91+ 'oe': "utf8",
92+ 'ceid' : "%s:%s" % (country, language),
93+ 'gl' : country,
94+ })
95+
96+ if params['safesearch']:
97+ query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
98+
99+ params['url'] = query_url
100+ logger.debug("query_url --> %s", query_url)
101+
102+ # en-US,en;q=0.8,en;q=0.5
103+ params['headers']['Accept-Language'] = (
104+ lang_country + ',' + language + ';q=0.8,' + language + ';q=0.5'
105+ )
106+ logger.debug("HTTP header Accept-Language --> %s",
107+ params['headers']['Accept-Language'])
108+ params['headers']['Accept'] = (
109+ 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
110+ )
111+ #params['google_subdomain'] = subdomain
61112
62113 return params
63114
64115
65−# get response from search-request
66116def response(resp):
117+ """Get response from google's search request"""
67118 results = []
68119
120+ detect_google_sorry(resp)
121+
122+ # which subdomain ?
123+ # subdomain = resp.search_params.get('google_subdomain')
124+
125+ # convert the text to dom
69126 dom = html.fromstring(resp.text)
70127
71− # parse results
72− for result in dom.xpath('//div[@class="g"]|//div[@class="g _cy"]'):
73− try:
74− r = {
75− 'url': result.xpath('.//a[@class="l lLrAF"]')[0].attrib.get("href"),
76− 'title': ''.join(result.xpath('.//a[@class="l lLrAF"]//text()')),
77− 'content': ''.join(result.xpath('.//div[@class="st"]//text()')),
78− }
79− except:
80− continue
81−
82− imgs = result.xpath('.//img/@src')
83− if len(imgs) and not imgs[0].startswith('data'):
84− r['img_src'] = imgs[0]
85−
86− results.append(r)
128+ for result in eval_xpath_list(dom, '//div[@class="xrnccd"]'):
129+
130+ # The first <a> tag in the <article> contains the link to the
131+ # article The href attribute of the <a> is a google internal link,
132+ # we can't use. The real link is hidden in the jslog attribute:
133+ #
134+ # <a ...
135+ # jslog="95014; 4:https://www.cnn.com/.../index.html; track:click"
136+ # href="./articles/CAIiENu3nGS...?hl=en-US&gl=US&ceid=US%3Aen"
137+ # ... />
138+
139+ jslog = eval_xpath_getindex(result, './article/a/@jslog', 0)
140+ url = re.findall('http[^;]*', jslog)
141+ if url:
142+ url = url[0]
143+ else:
144+ # The real URL is base64 encoded in the json attribute:
145+ # jslog="95014; 5:W251bGwsbnVsbCxudW...giXQ==; track:click"
146+ jslog = jslog.split(";")[1].split(':')[1].strip()
147+ try:
148+ padding = (4 -(len(jslog) % 4)) * "="
149+ jslog = b64decode(jslog + padding)
150+ except binascii.Error:
151+ # URL cant be read, skip this result
152+ continue
153+
154+ # now we have : b'[null, ... null,"https://www.cnn.com/.../index.html"]'
155+ url = re.findall('http[^;"]*', str(jslog))[0]
156+
157+ # the first <h3> tag in the <article> contains the title of the link
158+ title = extract_text(eval_xpath(result, './article/h3[1]'))
159+
160+ # the first <div> tag in the <article> contains the content of the link
161+ content = extract_text(eval_xpath(result, './article/div[1]'))
162+
163+ # the second <div> tag contains origin publisher and the publishing date
164+
165+ pub_date = extract_text(eval_xpath(result, './article/div[2]//time'))
166+ pub_origin = extract_text(eval_xpath(result, './article/div[2]//a'))
167+
168+ pub_info = []
169+ if pub_origin:
170+ pub_info.append(pub_origin)
171+ if pub_date:
172+ # The pub_date is mostly a string like 'yesertday', not a real
173+ # timezone date or time. Therefore we can't use publishedDate.
174+ pub_info.append(pub_date)
175+ pub_info = ', '.join(pub_info)
176+ if pub_info:
177+ content = pub_info + ': ' + content
178+
179+ # The image URL is located in a preceding sibling <img> tag, e.g.:
180+ # "https://lh3.googleusercontent.com/DjhQh7DMszk.....z=-p-h100-w100"
181+ # These URL are long but not personalized (double checked via tor).
182+
183+ img_src = extract_text(result.xpath('preceding-sibling::a/figure/img/@src'))
184+
185+ results.append({
186+ 'url': url,
187+ 'title': title,
188+ 'content': content,
189+ 'img_src': img_src,
190+ })
87191
88192 # return results
89193 return results