@@ -0,0 +1,157 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+"""Google (Scholar)
3+
4+For detailed description of the *REST-full* API see: `Query Parameter
5+Definitions`_.
6+
7+.. _Query Parameter Definitions:
8+ https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions
9+"""
10+
11+# pylint: disable=invalid-name, missing-function-docstring
12+
13+from urllib.parse import urlencode
14+from datetime import datetime
15+from lxml import html
16+from searx import logger
17+
18+from searx.utils import (
19+ eval_xpath,
20+ eval_xpath_list,
21+ extract_text,
22+)
23+
24+from searx.engines.google import (
25+ get_lang_info,
26+ time_range_dict,
27+ detect_google_sorry,
28+)
29+
30+# pylint: disable=unused-import
31+from searx.engines.google import (
32+ supported_languages_url,
33+ _fetch_supported_languages,
34+)
35+# pylint: enable=unused-import
36+
37+# about
38+about = {
39+ "website": 'https://scholar.google.com',
40+ "wikidata_id": 'Q494817',
41+ "official_api_documentation": 'https://developers.google.com/custom-search',
42+ "use_official_api": False,
43+ "require_api_key": False,
44+ "results": 'HTML',
45+}
46+
47+# engine dependent config
48+categories = ['science']
49+paging = True
50+language_support = True
51+use_locale_domain = True
52+time_range_support = True
53+safesearch = False
54+
55+logger = logger.getChild('google scholar')
56+
57+def time_range_url(params):
58+ """Returns a URL query component for a google-Scholar time range based on
59+ ``params['time_range']``. Google-Scholar does only support ranges in years.
60+ To have any effect, all the Searx ranges (*day*, *week*, *month*, *year*)
61+ are mapped to *year*. If no range is set, an empty string is returned.
62+ Example::
63+
64+ &as_ylo=2019
65+ """
66+ # as_ylo=2016&as_yhi=2019
67+ ret_val = ''
68+ if params['time_range'] in time_range_dict:
69+ ret_val= urlencode({'as_ylo': datetime.now().year -1 })
70+ return '&' + ret_val
71+
72+
73+def request(query, params):
74+ """Google-Scholar search request"""
75+
76+ offset = (params['pageno'] - 1) * 10
77+ lang_info = get_lang_info(
78+ # pylint: disable=undefined-variable
79+
80+
81+ # params, {}, language_aliases
82+
83+ params, supported_languages, language_aliases
84+ )
85+ # subdomain is: scholar.google.xy
86+ lang_info['subdomain'] = lang_info['subdomain'].replace("www.", "scholar.")
87+
88+ query_url = 'https://'+ lang_info['subdomain'] + '/scholar' + "?" + urlencode({
89+ 'q': query,
90+ 'hl': lang_info['hl'],
91+ 'lr': lang_info['lr'],
92+ 'ie': "utf8",
93+ 'oe': "utf8",
94+ 'start' : offset,
95+ })
96+
97+ query_url += time_range_url(params)
98+
99+ logger.debug("query_url --> %s", query_url)
100+ params['url'] = query_url
101+
102+ logger.debug("HTTP header Accept-Language --> %s", lang_info['Accept-Language'])
103+ params['headers']['Accept-Language'] = lang_info['Accept-Language']
104+ params['headers']['Accept'] = (
105+ 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
106+ )
107+
108+ #params['google_subdomain'] = subdomain
109+ return params
110+
111+def response(resp):
112+ """Get response from google's search request"""
113+ results = []
114+
115+ detect_google_sorry(resp)
116+
117+ # which subdomain ?
118+ # subdomain = resp.search_params.get('google_subdomain')
119+
120+ # convert the text to dom
121+ dom = html.fromstring(resp.text)
122+
123+ # parse results
124+ for result in eval_xpath_list(dom, '//div[@class="gs_ri"]'):
125+
126+ title = extract_text(eval_xpath(result, './h3[1]//a'))
127+
128+ if not title:
129+ # this is a [ZITATION] block
130+ continue
131+
132+ url = eval_xpath(result, './h3[1]//a/@href')[0]
133+ content = extract_text(eval_xpath(result, './div[@class="gs_rs"]')) or ''
134+
135+ pub_info = extract_text(eval_xpath(result, './div[@class="gs_a"]'))
136+ if pub_info:
137+ content += "[%s]" % pub_info
138+
139+ pub_type = extract_text(eval_xpath(result, './/span[@class="gs_ct1"]'))
140+ if pub_type:
141+ title = title + " " + pub_type
142+
143+ results.append({
144+ 'url': url,
145+ 'title': title,
146+ 'content': content,
147+ })
148+
149+ # parse suggestion
150+ for suggestion in eval_xpath(dom, '//div[contains(@class, "gs_qsuggest_wrap")]//li//a'):
151+ # append suggestion
152+ results.append({'suggestion': extract_text(suggestion)})
153+
154+ for correction in eval_xpath(dom, '//div[@class="gs_r gs_pda"]/a'):
155+ results.append({'correction': extract_text(correction)})
156+
157+ return results