@@ -1,16 +1,24 @@
11# SPDX-License-Identifier: AGPL-3.0-or-later
2−"""
3− DuckDuckGo (Web)
2+# lint: pylint
3+"""DuckDuckGo Lite
44"""
55
6−from lxml.html import fromstring
76from json import loads
8−from searx.utils import extract_text, match_language, eval_xpath, dict_subset
7+
8+from lxml.html import fromstring
9+
10+from searx.utils import (
11+ dict_subset,
12+ eval_xpath,
13+ eval_xpath_getindex,
14+ extract_text,
15+ match_language,
16+)
917from searx.network import get
1018
1119# about
1220about = {
13− "website": 'https://duckduckgo.com/',
21+ "website": 'https://lite.duckduckgo.com/lite',
1422 "wikidata_id": 'Q12805',
1523 "official_api_documentation": 'https://duckduckgo.com/api',
1624 "use_official_api": False,
@@ -20,7 +28,7 @@ about = {
2028
2129# engine dependent config
2230categories = ['general']
23−paging = False
31+paging = True
2432supported_languages_url = 'https://duckduckgo.com/util/u172.js'
2533time_range_support = True
2634
@@ -34,21 +42,16 @@ language_aliases = {
3442 'zh-HK': 'tzh-HK'
3543}
3644
37−# search-url
38−url = 'https://html.duckduckgo.com/html'
39−url_ping = 'https://duckduckgo.com/t/sl_h'
40−time_range_dict = {'day': 'd',
41− 'week': 'w',
42− 'month': 'm',
43− 'year': 'y'}
44−
45−# specific xpath variables
46−result_xpath = '//div[@class="result results_links results_links_deep web-result "]' # noqa
47−url_xpath = './/a[@class="result__a"]/@href'
48−title_xpath = './/a[@class="result__a"]'
49−content_xpath = './/a[@class="result__snippet"]'
50−correction_xpath = '//div[@id="did_you_mean"]//a'
45+time_range_dict = {
46+ 'day': 'd',
47+ 'week': 'w',
48+ 'month': 'm',
49+ 'year': 'y'
50+}
5151
52+# search-url
53+url = 'https://lite.duckduckgo.com/lite'
54+url_ping = 'https://duckduckgo.com/t/sl_l'
5255
5356# match query's language to a region code that duckduckgo will accept
5457def get_region_code(lang, lang_list=None):
@@ -63,66 +66,111 @@ def get_region_code(lang, lang_list=None):
6366
6467
6568def request(query, params):
66− if params['time_range'] is not None and params['time_range'] not in time_range_dict:
67− return params
6869
6970 params['url'] = url
7071 params['method'] = 'POST'
72+
7173 params['data']['q'] = query
72− params['data']['b'] = ''
74+
75+ # The API is not documented, so we do some reverse engineering and emulate
76+ # what https://lite.duckduckgo.com/lite/ does when you press "next Page"
77+ # link again and again ..
78+
79+ params['headers']['Content-Type'] = 'application/x-www-form-urlencoded'
80+
81+ # initial page does not have an offset
82+ if params['pageno'] == 2:
83+ # second page does have an offset of 30
84+ offset = (params['pageno'] - 1) * 30
85+ params['data']['s'] = offset
86+ params['data']['dc'] = offset + 1
87+
88+ elif params['pageno'] > 2:
89+ # third and following pages do have an offset of 30 + n*50
90+ offset = 30 + (params['pageno'] - 2) * 50
91+ params['data']['s'] = offset
92+ params['data']['dc'] = offset + 1
93+
94+ # initial page does not have additional data in the input form
95+ if params['pageno'] > 1:
96+ # request the second page (and more pages) needs 'o' and 'api' arguments
97+ params['data']['o'] = 'json'
98+ params['data']['api'] = 'd.js'
99+
100+ # initial page does not have additional data in the input form
101+ if params['pageno'] > 2:
102+ # request the third page (and more pages) some more arguments
103+ params['data']['nextParams'] = ''
104+ params['data']['v'] = ''
105+ params['data']['vqd'] = ''
73106
74107 region_code = get_region_code(params['language'], supported_languages)
75108 if region_code:
76109 params['data']['kl'] = region_code
77110 params['cookies']['kl'] = region_code
78111
112+ params['data']['df'] = ''
79113 if params['time_range'] in time_range_dict:
80114 params['data']['df'] = time_range_dict[params['time_range']]
115+ params['cookies']['df'] = time_range_dict[params['time_range']]
81116
82− params['allow_redirects'] = False
117+ logger.debug("param data: %s", params['data'])
118+ logger.debug("param cookies: %s", params['cookies'])
83119 return params
84120
85−
86121# get response from search-request
87122def response(resp):
88− if resp.status_code == 303:
89− return []
90123
91− # ping
92124 headers_ping = dict_subset(resp.request.headers, ['User-Agent', 'Accept-Encoding', 'Accept', 'Cookie'])
93125 get(url_ping, headers=headers_ping)
94126
95− # parse the response
127+ if resp.status_code == 303:
128+ return []
129+
96130 results = []
97131 doc = fromstring(resp.text)
98− for i, r in enumerate(eval_xpath(doc, result_xpath)):
99− if i >= 30:
100− break
101− try:
102− res_url = eval_xpath(r, url_xpath)[-1]
103− except:
104− continue
105132
106− if not res_url:
133+ result_table = eval_xpath(doc, '//html/body/form/div[@class="filters"]/table')
134+ if not len(result_table) >= 3:
135+ # no more results
136+ return []
137+ result_table = result_table[2]
138+
139+ tr_rows = eval_xpath(result_table, './/tr')
140+
141+ # In the last <tr> is the form of the 'previous/next page' links
142+ tr_rows = tr_rows[:-1]
143+
144+ len_tr_rows = len(tr_rows)
145+ offset = 0
146+
147+ while len_tr_rows >= offset + 4:
148+
149+ # assemble table rows we need to scrap
150+ tr_title = tr_rows[offset]
151+ tr_content = tr_rows[offset + 1]
152+ offset += 4
153+
154+ # ignore sponsored Adds <tr class="result-sponsored">
155+ if tr_content.get('class') == 'result-sponsored':
107156 continue
108157
109− title = extract_text(eval_xpath(r, title_xpath))
110− content = extract_text(eval_xpath(r, content_xpath))
158+ a_tag = eval_xpath_getindex(tr_title, './/td//a[@class="result-link"]', 0, None)
159+ if a_tag is None:
160+ continue
111161
112− # append result
113− results.append({'title': title,
114− 'content': content,
115− 'url': res_url})
162+ td_content = eval_xpath_getindex(tr_content, './/td[@class="result-snippet"]', 0, None)
163+ if td_content is None:
164+ continue
116165
117− # parse correction
118− for correction in eval_xpath(doc, correction_xpath):
119− # append correction
120− results.append({'correction': extract_text(correction)})
166+ results.append({
167+ 'title': a_tag.text_content(),
168+ 'content': extract_text(td_content),
169+ 'url': a_tag.get('href'),
170+ })
121171
122− # return results
123172 return results
124173
125−
126174# get supported languages from their site
127175def _fetch_supported_languages(resp):
128176