@@ -1,97 +1,205 @@
1−"""
2− Google (Images)
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+"""Google (Images)
3+
4+:website: https://images.google.com (redirected to subdomain www.)
5+:provide-api: yes (https://developers.google.com/custom-search/)
6+:using-api: not the offical, since it needs registration to another service
7+:results: HTML
8+:stable: no
9+:template: images.html
10+:parse: url, title, content, source, thumbnail_src, img_src
11+
12+For detailed description of the *REST-full* API see: `Query Parameter
13+Definitions`_.
14+
15+.. _admonition:: Content-Security-Policy (CSP)
316
4− @website https://www.google.com
5− @provide-api yes (https://developers.google.com/custom-search/)
17+ This engine needs to allow images from the `data URLs`_ (prefixed with the
18+ ``data:` scheme).::
19+
20+ Header set Content-Security-Policy "img-src 'self' data: ;"
21+
22+.. _Query Parameter Definitions:
23+ https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions
624
7− @using-api no
8− @results HTML chunks with JSON inside
9− @stable no
10− @parse url, title, img_src
1125"""
1226
13−from datetime import date, timedelta
14−from json import loads
1527from lxml import html
16−from searx.url_utils import urlencode
28+from flask_babel import gettext
29+from searx import logger
30+from searx.url_utils import urlencode, urlparse
31+from searx.utils import eval_xpath
32+from searx.engines.xpath import extract_text
33+
34+# pylint: disable=unused-import
35+from searx.engines.google import (
36+ supported_languages_url
37+ , _fetch_supported_languages
38+)
39+# pylint: enable=unused-import
40+
41+from searx.engines.google import (
42+ get_lang_country
43+ , google_domains
44+ , time_range_dict
45+)
46+
47+logger = logger.getChild('google images')
1748
1849# engine dependent config
50+
1951categories = ['images']
20−paging = True
21−safesearch = True
52+paging = False
53+language_support = True
54+use_locale_domain = True
2255time_range_support = True
23−number_of_results = 100
24−
25−search_url = 'https://www.google.com/search'\
26− '?{query}'\
27− '&tbm=isch'\
28− '&yv=2'\
29− '&{search_options}'
30−time_range_attr = "qdr:{range}"
31−time_range_custom_attr = "cdr:1,cd_min:{start},cd_max{end}"
32−time_range_dict = {'day': 'd',
33− 'week': 'w',
34− 'month': 'm'}
56+safesearch = True
3557
58+filter_mapping = {
59+ 0 : 'images',
60+ 1 : 'active',
61+ 2 : 'active'
62+}
63+
64+def scrap_out_thumbs(dom):
65+ """Scrap out thumbnail data from <script> tags.
66+ """
67+ ret_val = dict()
68+ for script in eval_xpath(dom, '//script[contains(., "_setImgSrc(")]'):
69+ _script = script.text
70+ # _setImgSrc('0','data:image\/jpeg;base64,\/9j\/4AAQSkZJR ....');
71+ _thumb_no, _img_data = _script[len("_setImgSrc("):-2].split(",",1)
72+ _thumb_no = _thumb_no.replace("'","")
73+ _img_data = _img_data.replace("'","")
74+ _img_data = _img_data.replace(r"\/", r"/")
75+ ret_val[_thumb_no] = _img_data.replace(r"\x3d", "=")
76+ return ret_val
3677
37−# do search-request
3878def request(query, params):
39− search_options = {
40− 'ijn': params['pageno'] - 1,
41− 'start': (params['pageno'] - 1) * number_of_results
42− }
79+ """Google-Video search request"""
80+
81+ language, country, lang_country = get_lang_country(
82+ # pylint: disable=undefined-variable
83+ params, supported_languages, language_aliases
84+ )
85+ subdomain = 'www.' + google_domains.get(country.upper(), 'google.com')
86+
87+ query_url = 'https://'+ subdomain + '/search' + "?" + urlencode({
88+ 'q': query,
89+ 'tbm': "isch",
90+ 'hl': lang_country,
91+ 'lr': "lang_" + language,
92+ 'ie': "utf8",
93+ 'oe': "utf8",
94+ 'num': 30,
95+ })
4396
4497 if params['time_range'] in time_range_dict:
45− search_options['tbs'] = time_range_attr.format(range=time_range_dict[params['time_range']])
46− elif params['time_range'] == 'year':
47− now = date.today()
48− then = now - timedelta(days=365)
49− start = then.strftime('%m/%d/%Y')
50− end = now.strftime('%m/%d/%Y')
51− search_options['tbs'] = time_range_custom_attr.format(start=start, end=end)
52−
53− if safesearch and params['safesearch']:
54− search_options['safe'] = 'on'
55−
56− params['url'] = search_url.format(query=urlencode({'q': query}),
57− search_options=urlencode(search_options))
58−
98+ query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
99+ if params['safesearch']:
100+ query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
101+
102+ params['url'] = query_url
103+ logger.debug("query_url --> %s", query_url)
104+
105+ params['headers']['Accept-Language'] = (
106+ "%s,%s;q=0.8,%s;q=0.5" % (lang_country, language, language))
107+ logger.debug(
108+ "HTTP Accept-Language --> %s", params['headers']['Accept-Language'])
109+ params['headers']['Accept'] = (
110+ 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
111+ )
112+ #params['google_subdomain'] = subdomain
59113 return params
60114
61115
62−# get response from search-request
63116def response(resp):
117+ """Get response from google's search request"""
64118 results = []
65119
120+ # detect google sorry
121+ resp_url = urlparse(resp.url)
122+ if resp_url.netloc == 'sorry.google.com' or resp_url.path == '/sorry/IndexRedirect':
123+ raise RuntimeWarning('sorry.google.com')
124+
125+ if resp_url.path.startswith('/sorry'):
126+ raise RuntimeWarning(gettext('CAPTCHA required'))
127+
128+ # which subdomain ?
129+ # subdomain = resp.search_params.get('google_subdomain')
130+
131+ # convert the text to dom
66132 dom = html.fromstring(resp.text)
133+ img_bas64_map = scrap_out_thumbs(dom)
67134
68135 # parse results
69− for result in dom.xpath('//div[contains(@class, "rg_meta")]/text()'):
136+ #
137+ # root element::
138+ # <div id="islmp" ..>
139+ # result div per image::
140+ # <div jsmodel="tTXmib"> / <div jsaction="..." data-id="..."
141+ # The data-id matches to a item in a json-data structure in::
142+ # <script nonce="I+vqelcy/01CKiBJi5Z1Ow">AF_initDataCallback({key: 'ds:1', ... data:function(){return [ ...
143+ # In this structure the ling to the origin PNG, JPG or whatever is given
144+ # (we do not blow out the link there, you could still implement that)
145+ # first link per image-div contains a <img> with the data-iid for bas64 encoded image data::
146+ # <img class="rg_i Q4LuWd" data-iid="0"
147+ # second link per image-div is the target link::
148+ # <a class="VFACy kGQAp" href="https://en.wikipedia.org/wiki/The_Sacrament_of_the_Last_Supper">
149+ # the second link also contains two div tags with the *description* and *publisher*::
150+ # <div class="WGvvNb">The Sacrament of the Last Supper ...</div>
151+ # <div class="fxgdke">en.wikipedia.org</div>
152+
153+ root = eval_xpath(dom, '//div[@id="islmp"]')
154+ if not root:
155+ logger.error("did not find root element id='islmp'")
156+ return results
157+
158+ root = root[0]
159+ for img_node in eval_xpath(root, './/img[contains(@class, "rg_i")]'):
70160
71161 try:
72− metadata = loads(result)
73−
74− img_format = metadata.get('ity', '')
75− img_width = metadata.get('ow', '')
76− img_height = metadata.get('oh', '')
77− if img_width and img_height:
78− img_format += " {0}x{1}".format(img_width, img_height)
79−
80− source = metadata.get('st', '')
81− source_url = metadata.get('isu', '')
82− if source_url:
83− source += " ({0})".format(source_url)
84−
85− results.append({'url': metadata['ru'],
86− 'title': metadata['pt'],
87− 'content': metadata.get('s', ''),
88− 'source': source,
89− 'img_format': img_format,
90− 'thumbnail_src': metadata['tu'],
91− 'img_src': metadata['ou'],
92− 'template': 'images.html'})
93−
94− except:
162+ img_alt = eval_xpath(img_node, '@alt')[0]
163+
164+ img_base64_id = eval_xpath(img_node, '@data-iid')
165+ if img_base64_id:
166+ img_base64_id = img_base64_id[0]
167+ thumbnail_src = img_bas64_map[img_base64_id]
168+ else:
169+ thumbnail_src = eval_xpath(img_node, '@src')
170+ if not thumbnail_src:
171+ thumbnail_src = eval_xpath(img_node, '@data-src')
172+ if thumbnail_src:
173+ thumbnail_src = thumbnail_src[0]
174+ else:
175+ thumbnail_src = ''
176+
177+ link_node = eval_xpath(img_node, '../../../a[2]')[0]
178+ url = eval_xpath(link_node, '@href')[0]
179+
180+ pub_nodes = eval_xpath(link_node, './div/div')
181+ pub_descr = img_alt
182+ pub_source = ''
183+ if pub_nodes:
184+ pub_descr = extract_text(pub_nodes[0])
185+ pub_source = extract_text(pub_nodes[1])
186+
187+ results.append({
188+ 'url': url,
189+ 'title': img_alt,
190+ 'content': pub_descr,
191+ 'source': pub_source,
192+ 'img_src': url,
193+ # 'img_format': img_format,
194+ 'thumbnail_src': thumbnail_src,
195+ 'template': 'images.html'
196+ })
197+ except Exception as e: # pylint: disable=broad-except
198+ logger.error(e, exc_info=True)
199+ #from lxml import etree
200+ #logger.debug(etree.tostring(img_node, pretty_print=True))
201+ #import pdb
202+ #pdb.set_trace()
95203 continue
96204
97205 return results