@@ -12,28 +12,53 @@ DuckDuckGo (definitions)
1212import json
1313from urllib.parse import urlencode
1414from lxml import html
15−from re import compile
15+
16+from searx import logger
17+from searx.data import WIKIDATA_UNITS
1618from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url, language_aliases
17−from searx.utils import extract_text, html_to_text, match_language
19+from searx.utils import extract_text, html_to_text, match_language, get_string_replaces_function
20+from searx.external_urls import get_external_url, get_earth_coordinates_url, area_to_osm_zoom
21+
22+
23+logger = logger.getChild('duckduckgo_definitions')
1824
19−url = 'https://api.duckduckgo.com/'\
25+URL = 'https://api.duckduckgo.com/'\
2026 + '?{query}&format=json&pretty=0&no_redirect=1&d=1'
2127
22−http_regex = compile(r'^http:')
28+WIKIDATA_PREFIX = [
29+ 'http://www.wikidata.org/entity/',
30+ 'https://www.wikidata.org/entity/'
31+]
32+
33+replace_http_by_https = get_string_replaces_function({'http:': 'https:'})
34+
35+
36+def is_broken_text(text):
37+ """ duckduckgo may return something like "<a href="xxxx">http://somewhere Related website<a/>"
2338
39+ The href URL is broken, the "Related website" may contains some HTML.
2440
25−def result_to_text(url, text, htmlResult):
41+ The best solution seems to ignore these results.
42+ """
43+ return text.startswith('http') and ' ' in text
44+
45+
46+def result_to_text(text, htmlResult):
2647 # TODO : remove result ending with "Meaning" or "Category"
48+ result = None
2749 dom = html.fromstring(htmlResult)
2850 a = dom.xpath('//a')
2951 if len(a) >= 1:
30− return extract_text(a[0])
52+ result = extract_text(a[0])
3153 else:
32− return text
54+ result = text
55+ if not is_broken_text(result):
56+ return result
57+ return None
3358
3459
3560def request(query, params):
36− params['url'] = url.format(query=urlencode({'q': query}))
61+ params['url'] = URL.format(query=urlencode({'q': query}))
3762 language = match_language(params['language'], supported_languages, language_aliases)
3863 language = language.split('-')[0]
3964 params['headers']['Accept-Language'] = language
@@ -45,6 +70,14 @@ def response(resp):
4570
4671 search_res = json.loads(resp.text)
4772
73+ # search_res.get('Entity') possible values (not exhaustive) :
74+ # * continent / country / department / location / waterfall
75+ # * actor / musician / artist
76+ # * book / performing art / film / television / media franchise / concert tour / playwright
77+ # * prepared food
78+ # * website / software / os / programming language / file format / software engineer
79+ # * compagny
80+
4881 content = ''
4982 heading = search_res.get('Heading', '')
5083 attributes = []
@@ -55,7 +88,8 @@ def response(resp):
5588 # add answer if there is one
5689 answer = search_res.get('Answer', '')
5790 if answer:
58− if search_res.get('AnswerType', '') not in ['calc']:
91+ logger.debug('AnswerType="%s" Answer="%s"', search_res.get('AnswerType'), answer)
92+ if search_res.get('AnswerType') not in ['calc', 'ip']:
5993 results.append({'answer': html_to_text(answer)})
6094
6195 # add infobox
@@ -66,42 +100,36 @@ def response(resp):
66100 content = content + search_res.get('Abstract', '')
67101
68102 # image
69− image = search_res.get('Image', '')
103+ image = search_res.get('Image')
70104 image = None if image == '' else image
71105
72− # attributes
73− if 'Infobox' in search_res:
74− infobox = search_res.get('Infobox', None)
75− if 'content' in infobox:
76− for info in infobox.get('content'):
77− attributes.append({'label': info.get('label'),
78− 'value': info.get('value')})
79−
80106 # urls
107+ # Official website, Wikipedia page
81108 for ddg_result in search_res.get('Results', []):
82− if 'FirstURL' in ddg_result:
83− firstURL = ddg_result.get('FirstURL', '')
84− text = ddg_result.get('Text', '')
109+ firstURL = ddg_result.get('FirstURL')
110+ text = ddg_result.get('Text')
111+ if firstURL is not None and text is not None:
85112 urls.append({'title': text, 'url': firstURL})
86113 results.append({'title': heading, 'url': firstURL})
87114
88115 # related topics
89116 for ddg_result in search_res.get('RelatedTopics', []):
90117 if 'FirstURL' in ddg_result:
91− suggestion = result_to_text(ddg_result.get('FirstURL', None),
92− ddg_result.get('Text', None),
93− ddg_result.get('Result', None))
94− if suggestion != heading:
95− results.append({'suggestion': suggestion})
118+ firstURL = ddg_result.get('FirstURL')
119+ text = ddg_result.get('Text')
120+ if not is_broken_text(text):
121+ suggestion = result_to_text(text,
122+ ddg_result.get('Result'))
123+ if suggestion != heading and suggestion is not None:
124+ results.append({'suggestion': suggestion})
96125 elif 'Topics' in ddg_result:
97126 suggestions = []
98127 relatedTopics.append({'name': ddg_result.get('Name', ''),
99− 'suggestions': suggestions})
128+ 'suggestions': suggestions})
100129 for topic_result in ddg_result.get('Topics', []):
101− suggestion = result_to_text(topic_result.get('FirstURL', None),
102− topic_result.get('Text', None),
103− topic_result.get('Result', None))
104− if suggestion != heading:
130+ suggestion = result_to_text(topic_result.get('Text'),
131+ topic_result.get('Result'))
132+ if suggestion != heading and suggestion is not None:
105133 suggestions.append(suggestion)
106134
107135 # abstract
@@ -110,7 +138,10 @@ def response(resp):
110138 # add as result ? problem always in english
111139 infobox_id = abstractURL
112140 urls.append({'title': search_res.get('AbstractSource'),
113− 'url': abstractURL})
141+ 'url': abstractURL,
142+ 'official': True})
143+ results.append({'url': abstractURL,
144+ 'title': heading})
114145
115146 # definition
116147 definitionURL = search_res.get('DefinitionURL', '')
@@ -118,53 +149,107 @@ def response(resp):
118149 # add as result ? as answer ? problem always in english
119150 infobox_id = definitionURL
120151 urls.append({'title': search_res.get('DefinitionSource'),
121− 'url': definitionURL})
152+ 'url': definitionURL})
122153
123154 # to merge with wikidata's infobox
124155 if infobox_id:
125− infobox_id = http_regex.sub('https:', infobox_id)
126−
127− # entity
128− entity = search_res.get('Entity', None)
129− # TODO continent / country / department / location / waterfall /
130− # mountain range :
131− # link to map search, get weather, near by locations
132− # TODO musician : link to music search
133− # TODO concert tour : ??
134− # TODO film / actor / television / media franchise :
135− # links to IMDB / rottentomatoes (or scrap result)
136− # TODO music : link tu musicbrainz / last.fm
137− # TODO book : ??
138− # TODO artist / playwright : ??
139− # TODO compagny : ??
140− # TODO software / os : ??
141− # TODO software engineer : ??
142− # TODO prepared food : ??
143− # TODO website : ??
144− # TODO performing art : ??
145− # TODO prepared food : ??
146− # TODO programming language : ??
147− # TODO file format : ??
156+ infobox_id = replace_http_by_https(infobox_id)
157+
158+ # attributes
159+ # some will be converted to urls
160+ if 'Infobox' in search_res:
161+ infobox = search_res.get('Infobox')
162+ if 'content' in infobox:
163+ osm_zoom = 17
164+ coordinates = None
165+ for info in infobox.get('content'):
166+ data_type = info.get('data_type')
167+ data_label = info.get('label')
168+ data_value = info.get('value')
169+
170+ # Workaround: ddg may return a double quote
171+ if data_value == '""':
172+ continue
173+
174+ # Is it an external URL ?
175+ # * imdb_id / facebook_profile / youtube_channel / youtube_video / twitter_profile
176+ # * instagram_profile / rotten_tomatoes / spotify_artist_id / itunes_artist_id / soundcloud_id
177+ # * netflix_id
178+ external_url = get_external_url(data_type, data_value)
179+ if external_url is not None:
180+ urls.append({'title': data_label,
181+ 'url': external_url})
182+ elif data_type in ['instance', 'wiki_maps_trigger', 'google_play_artist_id']:
183+ # ignore instance: Wikidata value from "Instance Of" (Qxxxx)
184+ # ignore wiki_maps_trigger: reference to a javascript
185+ # ignore google_play_artist_id: service shutdown
186+ pass
187+ elif data_type == 'string' and data_label == 'Website':
188+ # There is already an URL for the website
189+ pass
190+ elif data_type == 'area':
191+ attributes.append({'label': data_label,
192+ 'value': area_to_str(data_value),
193+ 'entity': 'P2046'})
194+ osm_zoom = area_to_osm_zoom(data_value.get('amount'))
195+ elif data_type == 'coordinates':
196+ if data_value.get('globe') == 'http://www.wikidata.org/entity/Q2':
197+ # coordinate on Earth
198+ # get the zoom information from the area
199+ coordinates = info
200+ else:
201+ # coordinate NOT on Earth
202+ attributes.append({'label': data_label,
203+ 'value': data_value,
204+ 'entity': 'P625'})
205+ elif data_type == 'string':
206+ attributes.append({'label': data_label,
207+ 'value': data_value})
208+
209+ if coordinates:
210+ data_label = coordinates.get('label')
211+ data_value = coordinates.get('value')
212+ latitude = data_value.get('latitude')
213+ longitude = data_value.get('longitude')
214+ url = get_earth_coordinates_url(latitude, longitude, osm_zoom)
215+ urls.append({'title': 'OpenStreetMap',
216+ 'url': url,
217+ 'entity': 'P625'})
148218
149219 if len(heading) > 0:
150220 # TODO get infobox.meta.value where .label='article_title'
151221 if image is None and len(attributes) == 0 and len(urls) == 1 and\
152222 len(relatedTopics) == 0 and len(content) == 0:
153− results.append({
154− 'url': urls[0]['url'],
155− 'title': heading,
156− 'content': content
157− })
223+ results.append({'url': urls[0]['url'],
224+ 'title': heading,
225+ 'content': content})
158226 else:
159− results.append({
160− 'infobox': heading,
161− 'id': infobox_id,
162− 'entity': entity,
163− 'content': content,
164− 'img_src': image,
165− 'attributes': attributes,
166− 'urls': urls,
167− 'relatedTopics': relatedTopics
168− })
227+ results.append({'infobox': heading,
228+ 'id': infobox_id,
229+ 'content': content,
230+ 'img_src': image,
231+ 'attributes': attributes,
232+ 'urls': urls,
233+ 'relatedTopics': relatedTopics})
169234
170235 return results
236+
237+
238+def unit_to_str(unit):
239+ for prefix in WIKIDATA_PREFIX:
240+ if unit.startswith(prefix):
241+ wikidata_entity = unit[len(prefix):]
242+ return WIKIDATA_UNITS.get(wikidata_entity, unit)
243+ return unit
244+
245+
246+def area_to_str(area):
247+ """parse {'unit': 'http://www.wikidata.org/entity/Q712226', 'amount': '+20.99'}"""
248+ unit = unit_to_str(area.get('unit'))
249+ if unit is not None:
250+ try:
251+ amount = float(area.get('amount'))
252+ return '{} {}'.format(amount, unit)
253+ except ValueError:
254+ pass
255+ return '{} {}'.format(area.get('amount', ''), area.get('unit', ''))