@@ -1,13 +1,58 @@
11# SPDX-License-Identifier: AGPL-3.0-or-later
22"""
3− Google (Videos)
3+Google (Viedo)
4+
5+For detailed description of the *REST-full* API see: `Query Parameter
6+Definitions`_. Not all parameters can be appied.
7+
8+.. _admonition:: Content-Security-Policy (CSP)
9+
10+ This engine needs to allow images from the `data URLs`_ (prefixed with the
11+ ``data:` scheme).::
12+
13+ Header set Content-Security-Policy "img-src 'self' data: ;"
14+
15+.. _Query Parameter Definitions:
16+ https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions
17+.. _data URLs:
18+ https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs
19+
420"""
521
6−from datetime import date, timedelta
7−from urllib.parse import urlencode
8−from lxml import html
9−from searx.utils import extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex
22+# pylint: disable=invalid-name, missing-function-docstring
23+
1024import re
25+from urllib.parse import urlencode, urlparse
26+from lxml import html
27+
28+from searx import logger
29+from searx.exceptions import SearxEngineCaptchaException
30+from searx.utils import (
31+ eval_xpath,
32+ eval_xpath_list,
33+ extract_text,
34+)
35+
36+from searx.engines.google import (
37+ get_lang_country,
38+ google_domains,
39+ time_range_dict,
40+ filter_mapping,
41+ results_xpath,
42+ g_section_with_header,
43+ title_xpath,
44+ href_xpath,
45+ content_xpath,
46+ suggestion_xpath,
47+ spelling_suggestion_xpath
48+)
49+
50+# pylint: disable=unused-import
51+from searx.engines.google import (
52+ supported_languages_url
53+ , _fetch_supported_languages
54+)
55+# pylint: enable=unused-import
1156
1257# about
1358about = {
@@ -17,83 +62,162 @@ about = {
1762 "use_official_api": False,
1863 "require_api_key": False,
1964 "results": 'HTML',
65+ "template": 'video.html',
66+ "parse": ('url', 'title', 'content', 'thumbnail')
2067}
2168
69+logger = logger.getChild('google video')
70+
2271# engine dependent config
72+
2373categories = ['videos']
24−paging = True
25−safesearch = True
74+paging = False
75+language_support = True
76+use_locale_domain = True
2677time_range_support = True
27−number_of_results = 10
78+safesearch = True
2879
29−search_url = 'https://www.google.com/search'\
30− '?q={query}'\
31− '&tbm=vid'\
32− '&{search_options}'
33−time_range_attr = "qdr:{range}"
34−time_range_custom_attr = "cdr:1,cd_min:{start},cd_max{end}"
35−time_range_dict = {'day': 'd',
36− 'week': 'w',
37− 'month': 'm'}
80+RE_CACHE = {}
3881
82+def _re(regexpr):
83+ """returns compiled regular expression"""
84+ RE_CACHE[regexpr] = RE_CACHE.get(regexpr, re.compile(regexpr))
85+ return RE_CACHE[regexpr]
3986
40−# do search-request
41−def request(query, params):
42− search_options = {
43− 'ijn': params['pageno'] - 1,
44− 'start': (params['pageno'] - 1) * number_of_results
45− }
87+def scrap_out_thumbs(dom):
88+ """Scrap out thumbnail data from <script> tags.
89+ """
90+ ret_val = dict()
91+ thumb_name = 'vidthumb'
4692
47− if params['time_range'] in time_range_dict:
48− search_options['tbs'] = time_range_attr.format(range=time_range_dict[params['time_range']])
49− elif params['time_range'] == 'year':
50− now = date.today()
51− then = now - timedelta(days=365)
52− start = then.strftime('%m/%d/%Y')
53− end = now.strftime('%m/%d/%Y')
54− search_options['tbs'] = time_range_custom_attr.format(start=start, end=end)
93+ for script in eval_xpath(dom, '//script[contains(., "_setImagesSrc")]'):
94+ _script = script.text
95+
96+ # var s='data:image/jpeg;base64, ...'
97+ _imgdata = _re("s='([^']*)").findall( _script)
98+ if not _imgdata:
99+ continue
100+
101+ # var ii=['vidthumb4','vidthumb7']
102+ for _vidthumb in _re(r"(%s\d+)" % thumb_name).findall(_script):
103+ # At least the equal sign in the URL needs to be decoded
104+ ret_val[_vidthumb] = _imgdata[0].replace(r"\x3d", "=")
105+
106+ # {google.ldidly=-1;google.ldi={"vidthumb8":"https://...
107+ for script in eval_xpath(dom, '//script[contains(., "google.ldi={")]'):
108+ _script = script.text
109+ for key_val in _re(r'"%s\d+\":\"[^\"]*"' % thumb_name).findall( _script) :
110+ match = _re(r'"(%s\d+)":"(.*)"' % thumb_name).search(key_val)
111+ if match:
112+ # At least the equal sign in the URL needs to be decoded
113+ ret_val[match.group(1)] = match.group(2).replace(r"\u003d", "=")
55114
56− if safesearch and params['safesearch']:
57− search_options['safe'] = 'on'
115+ logger.debug("found %s imgdata for: %s", thumb_name, ret_val.keys())
116+ return ret_val
58117
59− params['url'] = search_url.format(query=urlencode({'q': query}),
60− search_options=urlencode(search_options))
61118
119+def request(query, params):
120+ """Google-Video search request"""
121+
122+ language, country, lang_country = get_lang_country(
123+ # pylint: disable=undefined-variable
124+ params, supported_languages, language_aliases
125+ )
126+ subdomain = 'www.' + google_domains.get(country.upper(), 'google.com')
127+
128+ query_url = 'https://'+ subdomain + '/search' + "?" + urlencode({
129+ 'q': query,
130+ 'tbm': "vid",
131+ 'hl': lang_country,
132+ 'lr': "lang_" + language,
133+ 'ie': "utf8",
134+ 'oe': "utf8",
135+ })
136+
137+ if params['time_range'] in time_range_dict:
138+ query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
139+ if params['safesearch']:
140+ query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
141+
142+ params['url'] = query_url
143+ logger.debug("query_url --> %s", query_url)
144+
145+ # en-US,en;q=0.8,en;q=0.5
146+ params['headers']['Accept-Language'] = (
147+ "%s,%s;q=0.8,%s;q=0.5" % (lang_country, language, language))
148+ logger.debug(
149+ "HTTP Accept-Language --> %s", params['headers']['Accept-Language'])
150+ params['headers']['Accept'] = (
151+ 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
152+ )
153+ #params['google_subdomain'] = subdomain
62154 return params
63155
64156
65−# get response from search-request
66157def response(resp):
158+ """Get response from google's search request"""
67159 results = []
68160
161+ # detect google sorry
162+ resp_url = urlparse(resp.url)
163+ if resp_url.netloc == 'sorry.google.com' or resp_url.path == '/sorry/IndexRedirect':
164+ raise SearxEngineCaptchaException()
165+
166+ if resp_url.path.startswith('/sorry'):
167+ raise SearxEngineCaptchaException()
168+
169+ # which subdomain ?
170+ # subdomain = resp.search_params.get('google_subdomain')
171+
172+ # convert the text to dom
69173 dom = html.fromstring(resp.text)
174+ vidthumb_imgdata = scrap_out_thumbs(dom)
70175
71176 # parse results
72− for result in eval_xpath_list(dom, '//div[@class="g"]'):
73−
74− title = extract_text(eval_xpath(result, './/h3'))
75− url = eval_xpath_getindex(result, './/div[@class="r"]/a/@href', 0)
76− content = extract_text(eval_xpath(result, './/span[@class="st"]'))
77−
78− # get thumbnails
79− script = str(dom.xpath('//script[contains(., "_setImagesSrc")]')[0].text)
80− ids = result.xpath('.//div[@class="s"]//img/@id')
81− if len(ids) > 0:
82− thumbnails_data = \
83− re.findall('s=\'(.*?)(?:\\\\[a-z,1-9,\\\\]+\'|\')\;var ii=\[(?:|[\'vidthumb\d+\',]+)\'' + ids[0],
84− script)
85− tmp = []
86− if len(thumbnails_data) != 0:
87− tmp = re.findall('(data:image/jpeg;base64,[a-z,A-Z,0-9,/,\+]+)', thumbnails_data[0])
88− thumbnail = ''
89− if len(tmp) != 0:
90− thumbnail = tmp[-1]
91−
92− # append result
93− results.append({'url': url,
94− 'title': title,
95− 'content': content,
96− 'thumbnail': thumbnail,
97− 'template': 'videos.html'})
177+ for result in eval_xpath_list(dom, results_xpath):
178+
179+ # google *sections*
180+ if extract_text(eval_xpath(result, g_section_with_header)):
181+ logger.debug("ingoring <g-section-with-header>")
182+ continue
183+
184+ title = extract_text(eval_xpath(result, title_xpath)[0])
185+ url = eval_xpath(result, href_xpath)[0]
186+ c_node = eval_xpath(result, content_xpath)[0]
187+
188+ # <img id="vidthumb1" ...>
189+ img_id = eval_xpath(c_node, './div[1]//a/g-img/img/@id')
190+ if not img_id:
191+ continue
192+ img_id = img_id[0]
193+ img_src = vidthumb_imgdata.get(img_id, None)
194+ if not img_src:
195+ logger.error("no vidthumb imgdata for: %s" % img_id)
196+ img_src = eval_xpath(c_node, './div[1]//a/g-img/img/@src')[0]
197+
198+ duration = extract_text(eval_xpath(c_node, './div[1]//a/span'))
199+ content = extract_text(eval_xpath(c_node, './div[2]/span'))
200+ pub_info = extract_text(eval_xpath(c_node, './div[2]/div'))
201+
202+ if len(duration) > 3:
203+ content = duration + " - " + content
204+ if pub_info:
205+ content = content + " (%s)" % pub_info
206+
207+ results.append({
208+ 'url': url,
209+ 'title': title,
210+ 'content': content,
211+ 'thumbnail': img_src,
212+ 'template': 'videos.html',
213+ })
214+
215+ # parse suggestion
216+ for suggestion in eval_xpath(dom, suggestion_xpath):
217+ # append suggestion
218+ results.append({'suggestion': extract_text(suggestion)})
219+
220+ for correction in eval_xpath(dom, spelling_suggestion_xpath):
221+ results.append({'correction': extract_text(correction)})
98222
99223 return results