@@ -3,501 +3,686 @@
33 Wikidata
44
55 @website https://wikidata.org
6− @provide-api yes (https://wikidata.org/w/api.php)
6+ @provide-api yes (https://query.wikidata.org/)
77
8− @using-api partially (most things require scraping)
9− @results JSON, HTML
10− @stable no (html can change)
8+ @using-api yes
9+ @results JSON
10+ @stable yes
1111 @parse url, infobox
1212"""
1313
14−from searx import logger
15−from searx.poolrequests import get
16−from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url
17−from searx.utils import extract_text, match_language, eval_xpath
1814
1915from urllib.parse import urlencode
2016from json import loads
21−from lxml.html import fromstring
22−from lxml import etree
17+
18+from dateutil.parser import isoparse
19+from babel.dates import format_datetime, format_date, format_time, get_datetime_format
20+
21+from searx import logger
22+from searx.data import WIKIDATA_UNITS
23+from searx.poolrequests import post, get
24+from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url
25+from searx.utils import match_language, searx_useragent, get_string_replaces_function
26+from searx.external_urls import get_external_url, get_earth_coordinates_url, area_to_osm_zoom
2327
2428logger = logger.getChild('wikidata')
25−result_count = 1
26−
27−# urls
28−wikidata_host = 'https://www.wikidata.org'
29−url_search = wikidata_host \
30− + '/w/index.php?{query}&ns0=1'
31−
32−wikidata_api = wikidata_host + '/w/api.php'
33−url_detail = wikidata_api\
34− + '?action=parse&format=json&{query}'\
35− + '&redirects=1&prop=text%7Cdisplaytitle%7Cparsewarnings'\
36− + '&disableeditsection=1&preview=1§ionpreview=1&disabletoc=1&utf8=1&formatversion=2'
37−
38−url_map = 'https://www.openstreetmap.org/'\
39− + '?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M'
40−url_image = 'https://commons.wikimedia.org/wiki/Special:FilePath/{filename}?width=500&height=400'
41−
42−# xpaths
43−div_ids_xpath = '//div[@id]'
44−wikidata_ids_xpath = '//ul[@class="mw-search-results"]/li//a/@href'
45−title_xpath = '//*[contains(@class,"wikibase-title-label")]'
46−description_xpath = '//div[contains(@class,"wikibase-entitytermsview-heading-description")]'
47−label_xpath = './/div[contains(@class,"wikibase-statementgroupview-property-label")]/a'
48−url_xpath = './/a[contains(@class,"external free") or contains(@class, "wb-external-id")]'
49−wikilink_xpath = './/ul[contains(@class,"wikibase-sitelinklistview-listview")]'\
50− + '/li[contains(@data-wb-siteid,"{wikiid}")]//a/@href'
51−property_row_xpath = './/div[contains(@class,"wikibase-statementview")]'
52−preferred_rank_xpath = './/span[contains(@class,"wikibase-rankselector-preferred")]'
53−value_xpath = './/div[contains(@class,"wikibase-statementview-mainsnak")]'\
54− + '/*/div[contains(@class,"wikibase-snakview-value")]'
55−language_fallback_xpath = '//sup[contains(@class,"wb-language-fallback-indicator")]'
56−calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'
57−media_xpath = value_xpath + '//div[contains(@class,"commons-media-caption")]//a'
58−
59−
60−def get_id_cache(result):
61− id_cache = {}
62− for e in eval_xpath(result, div_ids_xpath):
63− id = e.get('id')
64− if id.startswith('P'):
65− id_cache[id] = e
66− return id_cache
6729
30+# SPARQL
31+SPARQL_ENDPOINT_URL = 'https://query.wikidata.org/sparql'
32+SPARQL_EXPLAIN_URL = 'https://query.wikidata.org/bigdata/namespace/wdq/sparql?explain'
33+WIKIDATA_PROPERTIES = {
34+ 'P434': 'MusicBrainz',
35+ 'P435': 'MusicBrainz',
36+ 'P436': 'MusicBrainz',
37+ 'P966': 'MusicBrainz',
38+ 'P345': 'IMDb',
39+ 'P2397': 'YouTube',
40+ 'P1651': 'YouTube',
41+ 'P2002': 'Twitter',
42+ 'P2013': 'Facebook',
43+ 'P2003': 'Instagram',
44+}
45+
46+# SERVICE wikibase:mwapi : https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual/MWAPI
47+# SERVICE wikibase:label: https://en.wikibooks.org/wiki/SPARQL/SERVICE_-_Label#Manual_Label_SERVICE
48+# https://en.wikibooks.org/wiki/SPARQL/WIKIDATA_Precision,_Units_and_Coordinates
49+# https://www.mediawiki.org/wiki/Wikibase/Indexing/RDF_Dump_Format#Data_model
50+# optmization:
51+# * https://www.wikidata.org/wiki/Wikidata:SPARQL_query_service/query_optimization
52+# * https://github.com/blazegraph/database/wiki/QueryHints
53+QUERY_TEMPLATE = """
54+SELECT ?item ?itemLabel ?itemDescription ?lat ?long %SELECT%
55+WHERE
56+{
57+ SERVICE wikibase:mwapi {
58+ bd:serviceParam wikibase:endpoint "www.wikidata.org";
59+ wikibase:api "EntitySearch";
60+ wikibase:limit 1;
61+ mwapi:search "%QUERY%";
62+ mwapi:language "%LANGUAGE%".
63+ ?item wikibase:apiOutputItem mwapi:item.
64+ }
65+
66+ %WHERE%
67+
68+ SERVICE wikibase:label {
69+ bd:serviceParam wikibase:language "%LANGUAGE%,en".
70+ ?item rdfs:label ?itemLabel .
71+ ?item schema:description ?itemDescription .
72+ %WIKIBASE_LABELS%
73+ }
74+
75+}
76+GROUP BY ?item ?itemLabel ?itemDescription ?lat ?long %GROUP_BY%
77+"""
6878
69−def request(query, params):
70− params['url'] = url_search.format(
71− query=urlencode({'search': query}))
72− return params
79+# Get the calendar names and the property names
80+QUERY_PROPERTY_NAMES = """
81+SELECT ?item ?name
82+WHERE {
83+ {
84+ SELECT ?item
85+ WHERE { ?item wdt:P279* wd:Q12132 }
86+ } UNION {
87+ VALUES ?item { %ATTRIBUTES% }
88+ }
89+ OPTIONAL { ?item rdfs:label ?name. }
90+}
91+"""
7392
7493
75−def response(resp):
76− results = []
77− htmlparser = etree.HTMLParser()
78− html = fromstring(resp.content.decode(), parser=htmlparser)
79− search_results = eval_xpath(html, wikidata_ids_xpath)
94+# https://www.w3.org/TR/sparql11-query/#rSTRING_LITERAL1
95+# https://lists.w3.org/Archives/Public/public-rdf-dawg/2011OctDec/0175.html
96+sparql_string_escape = get_string_replaces_function({'\t': '\\\t',
97+ '\n': '\\\n',
98+ '\r': '\\\r',
99+ '\b': '\\\b',
100+ '\f': '\\\f',
101+ '\"': '\\\"',
102+ '\'': '\\\'',
103+ '\\': '\\\\'})
104+
105+replace_http_by_https = get_string_replaces_function({'http:': 'https:'})
106+
107+
108+def get_headers():
109+ # user agent: https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual#Query_limits
110+ return {
111+ 'Accept': 'application/sparql-results+json',
112+ 'User-Agent': searx_useragent()
113+ }
114+
115+
116+def get_label_for_entity(entity_id, language):
117+ name = WIKIDATA_PROPERTIES.get(entity_id)
118+ if name is None:
119+ name = WIKIDATA_PROPERTIES.get((entity_id, language))
120+ if name is None:
121+ name = WIKIDATA_PROPERTIES.get((entity_id, language.split('-')[0]))
122+ if name is None:
123+ name = WIKIDATA_PROPERTIES.get((entity_id, 'en'))
124+ if name is None:
125+ name = entity_id
126+ return name
127+
128+
129+def send_wikidata_query(query, method='GET'):
130+ if method == 'GET':
131+ # query will be cached by wikidata
132+ http_response = get(SPARQL_ENDPOINT_URL + '?' + urlencode({'query': query}), headers=get_headers())
133+ else:
134+ # query won't be cached by wikidata
135+ http_response = post(SPARQL_ENDPOINT_URL, data={'query': query}, headers=get_headers())
136+ if http_response.status_code != 200:
137+ logger.debug('SPARQL endpoint error %s', http_response.content.decode())
138+ logger.debug('request time %s', str(http_response.elapsed))
139+ http_response.raise_for_status()
140+ return loads(http_response.content.decode())
141+
80142
81− if resp.search_params['language'].split('-')[0] == 'all':
143+def request(query, params):
144+ language = params['language'].split('-')[0]
145+ if language == 'all':
82146 language = 'en'
83147 else:
84− language = match_language(resp.search_params['language'], supported_languages, language_aliases).split('-')[0]
148+ language = match_language(params['language'], supported_languages, language_aliases).split('-')[0]
149+
150+ query, attributes = get_query(query, language)
85151
86− # TODO: make requests asynchronous to avoid timeout when result_count > 1
87− for search_result in search_results[:result_count]:
88− wikidata_id = search_result.split('/')[-1]
89− url = url_detail.format(query=urlencode({'page': wikidata_id, 'uselang': language}))
90− htmlresponse = get(url)
91− jsonresponse = loads(htmlresponse.content.decode())
92− results += getDetail(jsonresponse, wikidata_id, language, resp.search_params['language'], htmlparser)
152+ params['method'] = 'POST'
153+ params['url'] = SPARQL_ENDPOINT_URL
154+ params['data'] = {'query': query}
155+ params['headers'] = get_headers()
156+
157+ params['language'] = language
158+ params['attributes'] = attributes
159+ return params
160+
161+
162+def response(resp):
163+ results = []
164+ if resp.status_code != 200:
165+ logger.debug('SPARQL endpoint error %s', resp.content.decode())
166+ resp.raise_for_status()
167+ jsonresponse = loads(resp.content.decode())
168+
169+ language = resp.search_params['language'].lower()
170+ attributes = resp.search_params['attributes']
171+
172+ seen_entities = set()
173+
174+ for result in jsonresponse.get('results', {}).get('bindings', []):
175+ attribute_result = {key: value['value'] for key, value in result.items()}
176+ entity_url = attribute_result['item']
177+ if entity_url not in seen_entities:
178+ seen_entities.add(entity_url)
179+ results += get_results(attribute_result, attributes, language)
180+ else:
181+ logger.debug('The SPARQL request returns duplicate entities: %s', str(attribute_result))
93182
94183 return results
95184
96185
97−def getDetail(jsonresponse, wikidata_id, language, locale, htmlparser):
186+def get_results(attribute_result, attributes, language):
98187 results = []
99− urls = []
100− attributes = []
188+ infobox_title = attribute_result.get('itemLabel')
189+ infobox_id = attribute_result['item']
190+ infobox_id_lang = None
191+ infobox_urls = []
192+ infobox_attributes = []
193+ infobox_content = attribute_result.get('itemDescription')
194+ img_src = None
195+ img_src_priority = 100
196+
197+ for attribute in attributes:
198+ value = attribute.get_str(attribute_result, language)
199+ if value is not None and value != '':
200+ attribute_type = type(attribute)
201+
202+ if attribute_type in (WDURLAttribute, WDArticle):
203+ # get_select() method : there is group_concat(distinct ...;separator=", ")
204+ # split the value here
205+ for url in value.split(', '):
206+ infobox_urls.append({'title': attribute.get_label(language), 'url': url, **attribute.kwargs})
207+ # "normal" results (not infobox) include official website and Wikipedia links.
208+ if attribute.kwargs.get('official') or attribute_type == WDArticle:
209+ results.append({'title': infobox_title, 'url': url})
210+ # update the infobox_id with the wikipedia URL
211+ # first the local wikipedia URL, and as fallback the english wikipedia URL
212+ if attribute_type == WDArticle\
213+ and ((attribute.language == 'en' and infobox_id_lang is None)
214+ or attribute.language != 'en'):
215+ infobox_id_lang = attribute.language
216+ infobox_id = url
217+ elif attribute_type == WDImageAttribute:
218+ # this attribute is an image.
219+ # replace the current image only the priority is lower
220+ # (the infobox contain only one image).
221+ if attribute.priority < img_src_priority:
222+ img_src = value
223+ img_src_priority = attribute.priority
224+ elif attribute_type == WDGeoAttribute:
225+ # geocoordinate link
226+ # use the area to get the OSM zoom
227+ # Note: ignre the unit (must be km² otherwise the calculation is wrong)
228+ # Should use normalized value p:P2046/psn:P2046/wikibase:quantityAmount
229+ area = attribute_result.get('P2046')
230+ osm_zoom = area_to_osm_zoom(area) if area else 19
231+ url = attribute.get_str(attribute_result, language, osm_zoom=osm_zoom)
232+ if url:
233+ infobox_urls.append({'title': attribute.get_label(language),
234+ 'url': url,
235+ 'entity': attribute.name})
236+ else:
237+ infobox_attributes.append({'label': attribute.get_label(language),
238+ 'value': value,
239+ 'entity': attribute.name})
240+
241+ if infobox_id:
242+ infobox_id = replace_http_by_https(infobox_id)
101243
102− title = jsonresponse.get('parse', {}).get('displaytitle', {})
103− result = jsonresponse.get('parse', {}).get('text', {})
104−
105− if not title or not result:
106− return results
107−
108− title = fromstring(title, parser=htmlparser)
109− for elem in eval_xpath(title, language_fallback_xpath):
110− elem.getparent().remove(elem)
111− title = extract_text(eval_xpath(title, title_xpath))
112−
113− result = fromstring(result, parser=htmlparser)
114− for elem in eval_xpath(result, language_fallback_xpath):
115− elem.getparent().remove(elem)
116−
117− description = extract_text(eval_xpath(result, description_xpath))
118−
119− id_cache = get_id_cache(result)
120−
121− # URLS
122−
123− # official website
124− add_url(urls, result, id_cache, 'P856', results=results)
125−
126− # wikipedia
127− wikipedia_link_count = 0
128− wikipedia_link = get_wikilink(result, language + 'wiki')
129− if wikipedia_link:
130− wikipedia_link_count += 1
131− urls.append({'title': 'Wikipedia (' + language + ')',
132− 'url': wikipedia_link})
133−
134− if language != 'en':
135− wikipedia_en_link = get_wikilink(result, 'enwiki')
136− if wikipedia_en_link:
137− wikipedia_link_count += 1
138− urls.append({'title': 'Wikipedia (en)',
139− 'url': wikipedia_en_link})
140−
141− # TODO: get_wiki_firstlanguage
142− # if wikipedia_link_count == 0:
143−
144− # more wikis
145− add_url(urls, result, id_cache, default_label='Wikivoyage (' + language + ')', link_type=language + 'wikivoyage')
146− add_url(urls, result, id_cache, default_label='Wikiquote (' + language + ')', link_type=language + 'wikiquote')
147− add_url(urls, result, id_cache, default_label='Wikimedia Commons', link_type='commonswiki')
148−
149− add_url(urls, result, id_cache, 'P625', 'OpenStreetMap', link_type='geo')
150−
151− # musicbrainz
152− add_url(urls, result, id_cache, 'P434', 'MusicBrainz', 'http://musicbrainz.org/artist/')
153− add_url(urls, result, id_cache, 'P435', 'MusicBrainz', 'http://musicbrainz.org/work/')
154− add_url(urls, result, id_cache, 'P436', 'MusicBrainz', 'http://musicbrainz.org/release-group/')
155− add_url(urls, result, id_cache, 'P966', 'MusicBrainz', 'http://musicbrainz.org/label/')
156−
157− # IMDb
158− add_url(urls, result, id_cache, 'P345', 'IMDb', 'https://www.imdb.com/', link_type='imdb')
159− # source code repository
160− add_url(urls, result, id_cache, 'P1324')
161− # blog
162− add_url(urls, result, id_cache, 'P1581')
163− # social media links
164− add_url(urls, result, id_cache, 'P2397', 'YouTube', 'https://www.youtube.com/channel/')
165− add_url(urls, result, id_cache, 'P1651', 'YouTube', 'https://www.youtube.com/watch?v=')
166− add_url(urls, result, id_cache, 'P2002', 'Twitter', 'https://twitter.com/')
167− add_url(urls, result, id_cache, 'P2013', 'Facebook', 'https://facebook.com/')
168− add_url(urls, result, id_cache, 'P2003', 'Instagram', 'https://instagram.com/')
169−
170− urls.append({'title': 'Wikidata',
171− 'url': 'https://www.wikidata.org/wiki/'
172− + wikidata_id + '?uselang=' + language})
173−
174− # INFOBOX ATTRIBUTES (ROWS)
175−
176− # DATES
177− # inception date
178− add_attribute(attributes, id_cache, 'P571', date=True)
179− # dissolution date
180− add_attribute(attributes, id_cache, 'P576', date=True)
181− # start date
182− add_attribute(attributes, id_cache, 'P580', date=True)
183− # end date
184− add_attribute(attributes, id_cache, 'P582', date=True)
185− # date of birth
186− add_attribute(attributes, id_cache, 'P569', date=True)
187− # date of death
188− add_attribute(attributes, id_cache, 'P570', date=True)
189− # date of spacecraft launch
190− add_attribute(attributes, id_cache, 'P619', date=True)
191− # date of spacecraft landing
192− add_attribute(attributes, id_cache, 'P620', date=True)
193−
194− # nationality
195− add_attribute(attributes, id_cache, 'P27')
196− # country of origin
197− add_attribute(attributes, id_cache, 'P495')
198− # country
199− add_attribute(attributes, id_cache, 'P17')
200− # headquarters
201− add_attribute(attributes, id_cache, 'Q180')
202−
203− # PLACES
204− # capital
205− add_attribute(attributes, id_cache, 'P36', trim=True)
206− # head of state
207− add_attribute(attributes, id_cache, 'P35', trim=True)
208− # head of government
209− add_attribute(attributes, id_cache, 'P6', trim=True)
210− # type of government
211− add_attribute(attributes, id_cache, 'P122')
212− # official language
213− add_attribute(attributes, id_cache, 'P37')
214− # population
215− add_attribute(attributes, id_cache, 'P1082', trim=True)
216− # area
217− add_attribute(attributes, id_cache, 'P2046')
218− # currency
219− add_attribute(attributes, id_cache, 'P38', trim=True)
220− # heigth (building)
221− add_attribute(attributes, id_cache, 'P2048')
222−
223− # MEDIA
224− # platform (videogames)
225− add_attribute(attributes, id_cache, 'P400')
226− # author
227− add_attribute(attributes, id_cache, 'P50')
228− # creator
229− add_attribute(attributes, id_cache, 'P170')
230− # director
231− add_attribute(attributes, id_cache, 'P57')
232− # performer
233− add_attribute(attributes, id_cache, 'P175')
234− # developer
235− add_attribute(attributes, id_cache, 'P178')
236− # producer
237− add_attribute(attributes, id_cache, 'P162')
238− # manufacturer
239− add_attribute(attributes, id_cache, 'P176')
240− # screenwriter
241− add_attribute(attributes, id_cache, 'P58')
242− # production company
243− add_attribute(attributes, id_cache, 'P272')
244− # record label
245− add_attribute(attributes, id_cache, 'P264')
246− # publisher
247− add_attribute(attributes, id_cache, 'P123')
248− # original network
249− add_attribute(attributes, id_cache, 'P449')
250− # distributor
251− add_attribute(attributes, id_cache, 'P750')
252− # composer
253− add_attribute(attributes, id_cache, 'P86')
254− # publication date
255− add_attribute(attributes, id_cache, 'P577', date=True)
256− # genre
257− add_attribute(attributes, id_cache, 'P136')
258− # original language
259− add_attribute(attributes, id_cache, 'P364')
260− # isbn
261− add_attribute(attributes, id_cache, 'Q33057')
262− # software license
263− add_attribute(attributes, id_cache, 'P275')
264− # programming language
265− add_attribute(attributes, id_cache, 'P277')
266− # version
267− add_attribute(attributes, id_cache, 'P348', trim=True)
268− # narrative location
269− add_attribute(attributes, id_cache, 'P840')
270−
271− # LANGUAGES
272− # number of speakers
273− add_attribute(attributes, id_cache, 'P1098')
274− # writing system
275− add_attribute(attributes, id_cache, 'P282')
276− # regulatory body
277− add_attribute(attributes, id_cache, 'P1018')
278− # language code
279− add_attribute(attributes, id_cache, 'P218')
280−
281− # OTHER
282− # ceo
283− add_attribute(attributes, id_cache, 'P169', trim=True)
284− # founder
285− add_attribute(attributes, id_cache, 'P112')
286− # legal form (company/organization)
287− add_attribute(attributes, id_cache, 'P1454')
288− # operator
289− add_attribute(attributes, id_cache, 'P137')
290− # crew members (tripulation)
291− add_attribute(attributes, id_cache, 'P1029')
292− # taxon
293− add_attribute(attributes, id_cache, 'P225')
294− # chemical formula
295− add_attribute(attributes, id_cache, 'P274')
296− # winner (sports/contests)
297− add_attribute(attributes, id_cache, 'P1346')
298− # number of deaths
299− add_attribute(attributes, id_cache, 'P1120')
300− # currency code
301− add_attribute(attributes, id_cache, 'P498')
302−
303− image = add_image(id_cache)
304−
305− if len(attributes) == 0 and len(urls) == 2 and len(description) == 0:
244+ # add the wikidata URL at the end
245+ infobox_urls.append({'title': 'Wikidata', 'url': attribute_result['item']})
246+
247+ if img_src is None and len(infobox_attributes) == 0 and len(infobox_urls) == 1 and\
248+ len(infobox_content) == 0:
306249 results.append({
307− 'url': urls[0]['url'],
308− 'title': title,
309− 'content': description
310− })
250+ 'url': infobox_urls[0]['url'],
251+ 'title': infobox_title,
252+ 'content': infobox_content
253+ })
311254 else:
312255 results.append({
313− 'infobox': title,
314− 'id': wikipedia_link,
315− 'content': description,
316− 'img_src': image,
317− 'attributes': attributes,
318− 'urls': urls
319− })
320−
256+ 'infobox': infobox_title,
257+ 'id': infobox_id,
258+ 'content': infobox_content,
259+ 'img_src': img_src,
260+ 'urls': infobox_urls,
261+ 'attributes': infobox_attributes
262+ })
321263 return results
322264
323265
324−# only returns first match
325−def add_image(id_cache):
326− # P15: route map, P242: locator map, P154: logo, P18: image, P242: map, P41: flag, P2716: collage, P2910: icon
327− property_ids = ['P15', 'P242', 'P154', 'P18', 'P242', 'P41', 'P2716', 'P2910']
266+def get_query(query, language):
267+ attributes = get_attributes(language)
268+ select = [a.get_select() for a in attributes]
269+ where = list(filter(lambda s: len(s) > 0, [a.get_where() for a in attributes]))
270+ wikibase_label = list(filter(lambda s: len(s) > 0, [a.get_wikibase_label() for a in attributes]))
271+ group_by = list(filter(lambda s: len(s) > 0, [a.get_group_by() for a in attributes]))
272+ query = QUERY_TEMPLATE\
273+ .replace('%QUERY%', sparql_string_escape(query))\
274+ .replace('%SELECT%', ' '.join(select))\
275+ .replace('%WHERE%', '\n '.join(where))\
276+ .replace('%WIKIBASE_LABELS%', '\n '.join(wikibase_label))\
277+ .replace('%GROUP_BY%', ' '.join(group_by))\
278+ .replace('%LANGUAGE%', language)
279+ return query, attributes
328280
329− for property_id in property_ids:
330− image = id_cache.get(property_id, None)
331− if image is not None:
332− image_name = eval_xpath(image, media_xpath)
333− image_src = url_image.replace('{filename}', extract_text(image_name[0]))
334− return image_src
335281
282+def get_attributes(language):
283+ attributes = []
336284
337−# setting trim will only returned high ranked rows OR the first row
338−def add_attribute(attributes, id_cache, property_id, default_label=None, date=False, trim=False):
339− attribute = id_cache.get(property_id, None)
340− if attribute is not None:
285+ def add_value(name):
286+ attributes.append(WDAttribute(name))
287+
288+ def add_amount(name):
289+ attributes.append(WDAmountAttribute(name))
290+
291+ def add_label(name):
292+ attributes.append(WDLabelAttribute(name))
293+
294+ def add_url(name, url_id=None, **kwargs):
295+ attributes.append(WDURLAttribute(name, url_id, kwargs))
296+
297+ def add_image(name, url_id=None, priority=1):
298+ attributes.append(WDImageAttribute(name, url_id, priority))
299+
300+ def add_date(name):
301+ attributes.append(WDDateAttribute(name))
302+
303+ # Dates
304+ for p in ['P571', # inception date
305+ 'P576', # dissolution date
306+ 'P580', # start date
307+ 'P582', # end date
308+ 'P569', # date of birth
309+ 'P570', # date of death
310+ 'P619', # date of spacecraft launch
311+ 'P620']: # date of spacecraft landing
312+ add_date(p)
313+
314+ for p in ['P27', # country of citizenship
315+ 'P495', # country of origin
316+ 'P17', # country
317+ 'P159']: # headquarters location
318+ add_label(p)
319+
320+ # Places
321+ for p in ['P36', # capital
322+ 'P35', # head of state
323+ 'P6', # head of government
324+ 'P122', # basic form of government
325+ 'P37']: # official language
326+ add_label(p)
327+
328+ add_value('P1082') # population
329+ add_amount('P2046') # area
330+ add_amount('P281') # postal code
331+ add_label('P38') # currency
332+ add_amount('P2048') # heigth (building)
333+
334+ # Media
335+ for p in ['P400', # platform (videogames, computing)
336+ 'P50', # author
337+ 'P170', # creator
338+ 'P57', # director
339+ 'P175', # performer
340+ 'P178', # developer
341+ 'P162', # producer
342+ 'P176', # manufacturer
343+ 'P58', # screenwriter
344+ 'P272', # production company
345+ 'P264', # record label
346+ 'P123', # publisher
347+ 'P449', # original network
348+ 'P750', # distributed by
349+ 'P86']: # composer
350+ add_label(p)
351+
352+ add_date('P577') # publication date
353+ add_label('P136') # genre (music, film, artistic...)
354+ add_label('P364') # original language
355+ add_value('P212') # ISBN-13
356+ add_value('P957') # ISBN-10
357+ add_label('P275') # copyright license
358+ add_label('P277') # programming language
359+ add_value('P348') # version
360+ add_label('P840') # narrative location
361+
362+ # Languages
363+ add_value('P1098') # number of speakers
364+ add_label('P282') # writing system
365+ add_label('P1018') # language regulatory body
366+ add_value('P218') # language code (ISO 639-1)
367+
368+ # Other
369+ add_label('P169') # ceo
370+ add_label('P112') # founded by
371+ add_label('P1454') # legal form (company, organization)
372+ add_label('P137') # operator (service, facility, ...)
373+ add_label('P1029') # crew members (tripulation)
374+ add_label('P225') # taxon name
375+ add_value('P274') # chemical formula
376+ add_label('P1346') # winner (sports, contests, ...)
377+ add_value('P1120') # number of deaths
378+ add_value('P498') # currency code (ISO 4217)
379+
380+ # URL
381+ add_url('P856', official=True) # official website
382+ attributes.append(WDArticle(language)) # wikipedia (user language)
383+ if not language.startswith('en'):
384+ attributes.append(WDArticle('en')) # wikipedia (english)
385+
386+ add_url('P1324') # source code repository
387+ add_url('P1581') # blog
388+ add_url('P434', url_id='musicbrainz_artist')
389+ add_url('P435', url_id='musicbrainz_work')
390+ add_url('P436', url_id='musicbrainz_release_group')
391+ add_url('P966', url_id='musicbrainz_label')
392+ add_url('P345', url_id='imdb_id')
393+ add_url('P2397', url_id='youtube_channel')
394+ add_url('P1651', url_id='youtube_video')
395+ add_url('P2002', url_id='twitter_profile')
396+ add_url('P2013', url_id='facebook_profile')
397+ add_url('P2003', url_id='instagram_profile')
398+
399+ # Map
400+ attributes.append(WDGeoAttribute('P625'))
401+
402+ # Image
403+ add_image('P15', priority=1, url_id='wikimedia_image') # route map
404+ add_image('P242', priority=2, url_id='wikimedia_image') # locator map
405+ add_image('P154', priority=3, url_id='wikimedia_image') # logo
406+ add_image('P18', priority=4, url_id='wikimedia_image') # image
407+ add_image('P41', priority=5, url_id='wikimedia_image') # flag
408+ add_image('P2716', priority=6, url_id='wikimedia_image') # collage
409+ add_image('P2910', priority=7, url_id='wikimedia_image') # icon
410+
411+ return attributes
412+
413+
414+class WDAttribute:
415+
416+ __slots__ = 'name',
417+
418+ def __init__(self, name):
419+ self.name = name
420+
421+ def get_select(self):
422+ return '(group_concat(distinct ?{name};separator=", ") as ?{name}s)'.replace('{name}', self.name)
423+
424+ def get_label(self, language):
425+ return get_label_for_entity(self.name, language)
426+
427+ def get_where(self):
428+ return "OPTIONAL { ?item wdt:{name} ?{name} . }".replace('{name}', self.name)
429+
430+ def get_wikibase_label(self):
431+ return ""
432+
433+ def get_group_by(self):
434+ return ""
435+
436+ def get_str(self, result, language):
437+ return result.get(self.name + 's')
341438
342− if default_label:
343− label = default_label
344− else:
345− label = extract_text(eval_xpath(attribute, label_xpath))
346− label = label[0].upper() + label[1:]
347−
348− if date:
349− trim = True
350− # remove calendar name
351− calendar_name = eval_xpath(attribute, calendar_name_xpath)
352− for calendar in calendar_name:
353− calendar.getparent().remove(calendar)
354−
355− concat_values = ""
356− values = []
357− first_value = None
358− for row in eval_xpath(attribute, property_row_xpath):
359− if not first_value or not trim or eval_xpath(row, preferred_rank_xpath):
360− value = eval_xpath(row, value_xpath)
361− if not value:
362− continue
363− value = extract_text(value)
364−
365− # save first value in case no ranked row is found
366− if trim and not first_value:
367− first_value = value
368− else:
369− # to avoid duplicate values
370− if value not in values:
371− concat_values += value + ", "
372− values.append(value)
373−
374− if trim and not values:
375− attributes.append({'label': label,
376− 'value': first_value})
377− else:
378− attributes.append({'label': label,
379− 'value': concat_values[:-2]})
439+ def __repr__(self):
440+ return '<' + str(type(self).__name__) + ':' + self.name + '>'
380441
381442
382−# requires property_id unless it's a wiki link (defined in link_type)
383−def add_url(urls, result, id_cache, property_id=None, default_label=None, url_prefix=None, results=None,
384− link_type=None, only_first=True):
385− links = []
443+class WDAmountAttribute(WDAttribute):
386444
387− # wiki links don't have property in wikidata page
388− if link_type and 'wiki' in link_type:
389− links.append(get_wikilink(result, link_type))
390− else:
391− dom_element = id_cache.get(property_id, None)
392− if dom_element is not None:
393− if not default_label:
394− label = extract_text(eval_xpath(dom_element, label_xpath))
395− label = label[0].upper() + label[1:]
445+ def get_select(self):
446+ return '?{name} ?{name}Unit'.replace('{name}', self.name)
396447
397− if link_type == 'geo':
398− links.append(get_geolink(dom_element))
448+ def get_where(self):
449+ return """ OPTIONAL { ?item p:{name} ?{name}Node .
450+ ?{name}Node rdf:type wikibase:BestRank ; ps:{name} ?{name} .
451+ OPTIONAL { ?{name}Node psv:{name}/wikibase:quantityUnit ?{name}Unit. } }""".replace('{name}', self.name)
399452
400− elif link_type == 'imdb':
401− links.append(get_imdblink(dom_element, url_prefix))
453+ def get_group_by(self):
454+ return self.get_select()
402455
403− else:
404− url_results = eval_xpath(dom_element, url_xpath)
405− for link in url_results:
406− if link is not None:
407− if url_prefix:
408− link = url_prefix + extract_text(link)
409− else:
410− link = extract_text(link)
411− links.append(link)
412−
413− # append urls
414− for url in links:
415− if url is not None:
416− u = {'title': default_label or label, 'url': url}
417− if property_id == 'P856':
418− u['official'] = True
419− u['domain'] = url.split('/')[2]
420− urls.append(u)
421− if results is not None:
422− results.append(u)
423− if only_first:
424− break
425−
426−
427−def get_imdblink(result, url_prefix):
428− imdb_id = eval_xpath(result, value_xpath)
429− if imdb_id:
430− imdb_id = extract_text(imdb_id)
431− id_prefix = imdb_id[:2]
432− if id_prefix == 'tt':
433− url = url_prefix + 'title/' + imdb_id
434− elif id_prefix == 'nm':
435− url = url_prefix + 'name/' + imdb_id
436− elif id_prefix == 'ch':
437− url = url_prefix + 'character/' + imdb_id
438− elif id_prefix == 'co':
439− url = url_prefix + 'company/' + imdb_id
440− elif id_prefix == 'ev':
441− url = url_prefix + 'event/' + imdb_id
442− else:
443− url = None
444− return url
456+ def get_str(self, result, language):
457+ value = result.get(self.name)
458+ unit = result.get(self.name + "Unit")
459+ if unit is not None:
460+ unit = unit.replace('http://www.wikidata.org/entity/', '')
461+ return value + " " + get_label_for_entity(unit, language)
462+ return value
445463
446464
447−def get_geolink(result):
448− coordinates = eval_xpath(result, value_xpath)
449− if not coordinates:
450− return None
451− coordinates = extract_text(coordinates[0])
452− latitude, longitude = coordinates.split(',')
453−
454− # convert to decimal
455− lat = int(latitude[:latitude.find('°')])
456− if latitude.find('\'') >= 0:
457− lat += int(latitude[latitude.find('°') + 1:latitude.find('\'')] or 0) / 60.0
458− if latitude.find('"') >= 0:
459− lat += float(latitude[latitude.find('\'') + 1:latitude.find('"')] or 0) / 3600.0
460− if latitude.find('S') >= 0:
461− lat *= -1
462− lon = int(longitude[:longitude.find('°')])
463− if longitude.find('\'') >= 0:
464− lon += int(longitude[longitude.find('°') + 1:longitude.find('\'')] or 0) / 60.0
465− if longitude.find('"') >= 0:
466− lon += float(longitude[longitude.find('\'') + 1:longitude.find('"')] or 0) / 3600.0
467− if longitude.find('W') >= 0:
468− lon *= -1
469−
470− # TODO: get precision
471− precision = 0.0002
472− # there is no zoom information, deduce from precision (error prone)
473− # samples :
474− # 13 --> 5
475− # 1 --> 6
476− # 0.016666666666667 --> 9
477− # 0.00027777777777778 --> 19
478− # wolframalpha :
479− # quadratic fit { {13, 5}, {1, 6}, {0.0166666, 9}, {0.0002777777,19}}
480− # 14.1186-8.8322 x+0.625447 x^2
481− if precision < 0.0003:
482− zoom = 19
483− else:
484− zoom = int(15 - precision * 8.8322 + precision * precision * 0.625447)
465+class WDArticle(WDAttribute):
466+
467+ __slots__ = 'language', 'kwargs'
468+
469+ def __init__(self, language, kwargs=None):
470+ super().__init__('wikipedia')
471+ self.language = language
472+ self.kwargs = kwargs or {}
473+
474+ def get_label(self, language):
475+ # language parameter is ignored
476+ return "Wikipedia ({language})".replace('{language}', self.language)
477+
478+ def get_select(self):
479+ return "?article{language} ?articleName{language}".replace('{language}', self.language)
480+
481+ def get_where(self):
482+ return """OPTIONAL { ?article{language} schema:about ?item ;
483+ schema:inLanguage "{language}" ;
484+ schema:isPartOf <https://{language}.wikipedia.org/> ;
485+ schema:name ?articleName{language} . }""".replace('{language}', self.language)
486+
487+ def get_group_by(self):
488+ return self.get_select()
489+
490+ def get_str(self, result, language):
491+ key = 'article{language}'.replace('{language}', self.language)
492+ return result.get(key)
493+
494+
495+class WDLabelAttribute(WDAttribute):
496+
497+ def get_select(self):
498+ return '(group_concat(distinct ?{name}Label;separator=", ") as ?{name}Labels)'.replace('{name}', self.name)
485499
486− url = url_map\
487− .replace('{latitude}', str(lat))\
488− .replace('{longitude}', str(lon))\
489− .replace('{zoom}', str(zoom))
500+ def get_where(self):
501+ return "OPTIONAL { ?item wdt:{name} ?{name} . }".replace('{name}', self.name)
490502
491− return url
503+ def get_wikibase_label(self):
504+ return "?{name} rdfs:label ?{name}Label .".replace('{name}', self.name)
492505
506+ def get_str(self, result, language):
507+ return result.get(self.name + 'Labels')
493508
494−def get_wikilink(result, wikiid):
495− url = eval_xpath(result, wikilink_xpath.replace('{wikiid}', wikiid))
496− if not url:
509+
510+class WDURLAttribute(WDAttribute):
511+
512+ HTTP_WIKIMEDIA_IMAGE = 'http://commons.wikimedia.org/wiki/Special:FilePath/'
513+
514+ __slots__ = 'url_id', 'kwargs'
515+
516+ def __init__(self, name, url_id=None, kwargs=None):
517+ super().__init__(name)
518+ self.url_id = url_id
519+ self.kwargs = kwargs
520+
521+ def get_str(self, result, language):
522+ value = result.get(self.name + 's')
523+ if self.url_id and value is not None and value != '':
524+ value = value.split(',')[0]
525+ url_id = self.url_id
526+ if value.startswith(WDURLAttribute.HTTP_WIKIMEDIA_IMAGE):
527+ value = value[len(WDURLAttribute.HTTP_WIKIMEDIA_IMAGE):]
528+ url_id = 'wikimedia_image'
529+ return get_external_url(url_id, value)
530+ return value
531+
532+
533+class WDGeoAttribute(WDAttribute):
534+
535+ def get_label(self, language):
536+ return "OpenStreetMap"
537+
538+ def get_select(self):
539+ return "?{name}Lat ?{name}Long".replace('{name}', self.name)
540+
541+ def get_where(self):
542+ return """OPTIONAL { ?item p:{name}/psv:{name} [
543+ wikibase:geoLatitude ?{name}Lat ;
544+ wikibase:geoLongitude ?{name}Long ] }""".replace('{name}', self.name)
545+
546+ def get_group_by(self):
547+ return self.get_select()
548+
549+ def get_str(self, result, language, osm_zoom=19):
550+ latitude = result.get(self.name + 'Lat')
551+ longitude = result.get(self.name + 'Long')
552+ if latitude and longitude:
553+ return get_earth_coordinates_url(latitude, longitude, osm_zoom)
497554 return None
498− url = url[0]
499− if url.startswith('http://'):
500− url = url.replace('http://', 'https://')
501− elif url.startswith('//'):
502− url = 'https:' + url
503− return url
555+
556+
557+class WDImageAttribute(WDURLAttribute):
558+
559+ __slots__ = 'priority',
560+
561+ def __init__(self, name, url_id=None, priority=100):
562+ super().__init__(name, url_id)
563+ self.priority = priority
564+
565+
566+class WDDateAttribute(WDAttribute):
567+
568+ def get_select(self):
569+ return '?{name} ?{name}timePrecision ?{name}timeZone ?{name}timeCalendar'.replace('{name}', self.name)
570+
571+ def get_where(self):
572+ # To remove duplicate, add
573+ # FILTER NOT EXISTS { ?item p:{name}/psv:{name}/wikibase:timeValue ?{name}bis FILTER (?{name}bis < ?{name}) }
574+ # this filter is too slow, so the response function ignore duplicate results
575+ # (see the seen_entities variable)
576+ return """OPTIONAL { ?item p:{name}/psv:{name} [
577+ wikibase:timeValue ?{name} ;
578+ wikibase:timePrecision ?{name}timePrecision ;
579+ wikibase:timeTimezone ?{name}timeZone ;
580+ wikibase:timeCalendarModel ?{name}timeCalendar ] . }
581+ hint:Prior hint:rangeSafe true;""".replace('{name}', self.name)
582+
583+ def get_group_by(self):
584+ return self.get_select()
585+
586+ def format_8(self, value, locale):
587+ # precision: less than a year
588+ return value
589+
590+ def format_9(self, value, locale):
591+ year = int(value)
592+ # precision: year
593+ if year < 1584:
594+ if year < 0:
595+ return str(year - 1)
596+ return str(year)
597+ timestamp = isoparse(value)
598+ return format_date(timestamp, format='yyyy', locale=locale)
599+
600+ def format_10(self, value, locale):
601+ # precision: month
602+ timestamp = isoparse(value)
603+ return format_date(timestamp, format='MMMM y', locale=locale)
604+
605+ def format_11(self, value, locale):
606+ # precision: day
607+ timestamp = isoparse(value)
608+ return format_date(timestamp, format='full', locale=locale)
609+
610+ def format_13(self, value, locale):
611+ timestamp = isoparse(value)
612+ # precision: minute
613+ return get_datetime_format(format, locale=locale) \
614+ .replace("'", "") \
615+ .replace('{0}', format_time(timestamp, 'full', tzinfo=None,
616+ locale=locale)) \
617+ .replace('{1}', format_date(timestamp, 'short', locale=locale))
618+
619+ def format_14(self, value, locale):
620+ # precision: second.
621+ return format_datetime(isoparse(value), format='full', locale=locale)
622+
623+ DATE_FORMAT = {
624+ '0': ('format_8', 1000000000),
625+ '1': ('format_8', 100000000),
626+ '2': ('format_8', 10000000),
627+ '3': ('format_8', 1000000),
628+ '4': ('format_8', 100000),
629+ '5': ('format_8', 10000),
630+ '6': ('format_8', 1000),
631+ '7': ('format_8', 100),
632+ '8': ('format_8', 10),
633+ '9': ('format_9', 1), # year
634+ '10': ('format_10', 1), # month
635+ '11': ('format_11', 0), # day
636+ '12': ('format_13', 0), # hour (not supported by babel, display minute)
637+ '13': ('format_13', 0), # minute
638+ '14': ('format_14', 0) # second
639+ }
640+
641+ def get_str(self, result, language):
642+ value = result.get(self.name)
643+ if value == '' or value is None:
644+ return None
645+ precision = result.get(self.name + 'timePrecision')
646+ date_format = WDDateAttribute.DATE_FORMAT.get(precision)
647+ if date_format is not None:
648+ format_method = getattr(self, date_format[0])
649+ precision = date_format[1]
650+ try:
651+ if precision >= 1:
652+ t = value.split('-')
653+ if value.startswith('-'):
654+ value = '-' + t[1]
655+ else:
656+ value = t[0]
657+ return format_method(value, language)
658+ except Exception:
659+ return value
660+ return value
661+
662+
663+def debug_explain_wikidata_query(query, method='GET'):
664+ if method == 'GET':
665+ http_response = get(SPARQL_EXPLAIN_URL + '&' + urlencode({'query': query}), headers=get_headers())
666+ else:
667+ http_response = post(SPARQL_EXPLAIN_URL, data={'query': query}, headers=get_headers())
668+ http_response.raise_for_status()
669+ return http_response.content
670+
671+
672+def init(engine_settings=None):
673+ # WIKIDATA_PROPERTIES : add unit symbols
674+ WIKIDATA_PROPERTIES.update(WIKIDATA_UNITS)
675+
676+ # WIKIDATA_PROPERTIES : add property labels
677+ wikidata_property_names = []
678+ for attribute in get_attributes('en'):
679+ if type(attribute) in (WDAttribute, WDAmountAttribute, WDURLAttribute, WDDateAttribute, WDLabelAttribute):
680+ if attribute.name not in WIKIDATA_PROPERTIES:
681+ wikidata_property_names.append("wd:" + attribute.name)
682+ query = QUERY_PROPERTY_NAMES.replace('%ATTRIBUTES%', " ".join(wikidata_property_names))
683+ jsonresponse = send_wikidata_query(query)
684+ for result in jsonresponse.get('results', {}).get('bindings', {}):
685+ name = result['name']['value']
686+ lang = result['name']['xml:lang']
687+ entity_id = result['item']['value'].replace('http://www.wikidata.org/entity/', '')
688+ WIKIDATA_PROPERTIES[(entity_id, lang)] = name.capitalize()