@@ -1,33 +1,57 @@
1−import json
1+# -*- coding: utf-8 -*-
2+"""
3+ Wikidata
4+
5+ @website https://wikidata.org
6+ @provide-api yes (https://wikidata.org/w/api.php)
7+
8+ @using-api partially (most things require scraping)
9+ @results JSON, HTML
10+ @stable no (html can change)
11+ @parse url, infobox
12+"""
213
314from searx import logger
415from searx.poolrequests import get
5−from searx.utils import format_date_by_locale
16+from searx.engines.xpath import extract_text
617
7−from datetime import datetime
8−from dateutil.parser import parse as dateutil_parse
9−from urllib import urlencode
18+from json import loads
1019from lxml.html import fromstring
11−
20+from urllib import urlencode
1221
1322logger = logger.getChild('wikidata')
1423result_count = 1
24+
25+# urls
1526wikidata_host = 'https://www.wikidata.org'
1627url_search = wikidata_host \
1728 + '/wiki/Special:ItemDisambiguation?{query}'
1829
1930wikidata_api = wikidata_host + '/w/api.php'
2031url_detail = wikidata_api\
21− + '?action=wbgetentities&format=json'\
22− + '&props=labels%7Cinfo%7Csitelinks'\
23− + '%7Csitelinks%2Furls%7Cdescriptions%7Cclaims'\
24− + '&{query}'
32+ + '?action=parse&format=json&{query}'\
33+ + '&redirects=1&prop=text%7Cdisplaytitle%7Clanglinks%7Crevid'\
34+ + '&disableeditsection=1&disabletidy=1&preview=1§ionpreview=1&disabletoc=1&utf8=1&formatversion=2'
35+
2536url_map = 'https://www.openstreetmap.org/'\
2637 + '?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M'
27−url_entity_label = wikidata_api\
28− + '?action=wbgetentities&format=json&props=labels&{query}'
38+url_image = 'https://commons.wikimedia.org/wiki/Special:FilePath/{filename}?width=500'
2939
40+# xpaths
3041wikidata_ids_xpath = '//div/ul[@class="wikibase-disambiguation"]/li/a/@title'
42+title_xpath = '//*[contains(@class,"wikibase-title-label")]'
43+description_xpath = '//div[contains(@class,"wikibase-entitytermsview-heading-description")]'
44+property_xpath = '//div[@id="{propertyid}"]'
45+label_xpath = './/div[contains(@class,"wikibase-statementgroupview-property-label")]/a'
46+url_xpath = './/a[contains(@class,"external free") or contains(@class, "wb-external-id")]'
47+wikilink_xpath = './/ul[contains(@class,"wikibase-sitelinklistview-listview")]'\
48+ + '/li[contains(@data-wb-siteid,"{wikiid}")]//a/@href'
49+property_row_xpath = './/div[contains(@class,"wikibase-statementview")]'
50+preferred_rank_xpath = './/span[contains(@class,"wikibase-rankselector-preferred")]'
51+value_xpath = './/div[contains(@class,"wikibase-statementview-mainsnak")]'\
52+ + '/*/div[contains(@class,"wikibase-snakview-value")]'
53+language_fallback_xpath = '//sup[contains(@class,"wb-language-fallback-indicator")]'
54+calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'
3155
3256
3357def request(query, params):
@@ -50,13 +74,13 @@ def response(resp):
5074 if language == 'all':
5175 language = 'en'
5276
53− url = url_detail.format(query=urlencode({'ids': '|'.join(wikidata_ids),
54− 'languages': language + '|en'}))
55−
56− htmlresponse = get(url)
57− jsonresponse = json.loads(htmlresponse.content)
77+ # TODO: make requests asynchronous to avoid timeout when result_count > 1
5878 for wikidata_id in wikidata_ids[:result_count]:
59− results = results + getDetail(jsonresponse, wikidata_id, language, resp.search_params['language'])
79+ url = url_detail.format(query=urlencode({'page': wikidata_id,
80+ 'uselang': language}))
81+ htmlresponse = get(url)
82+ jsonresponse = loads(htmlresponse.content)
83+ results += getDetail(jsonresponse, wikidata_id, language, resp.search_params['language'])
6084
6185 return results
6286
@@ -66,125 +90,194 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
6690 urls = []
6791 attributes = []
6892
69− result = jsonresponse.get('entities', {}).get(wikidata_id, {})
93+ title = jsonresponse.get('parse', {}).get('displaytitle', {})
94+ result = jsonresponse.get('parse', {}).get('text', {})
7095
71− title = result.get('labels', {}).get(language, {}).get('value', None)
72− if title is None:
73− title = result.get('labels', {}).get('en', {}).get('value', None)
74− if title is None:
96+ if not title or not result:
7597 return results
7698
77− description = result\
78− .get('descriptions', {})\
79− .get(language, {})\
80− .get('value', None)
99+ title = fromstring(title)
100+ for elem in title.xpath(language_fallback_xpath):
101+ elem.getparent().remove(elem)
102+ title = extract_text(title.xpath(title_xpath))
81103
82− if description is None:
83− description = result\
84− .get('descriptions', {})\
85− .get('en', {})\
86− .get('value', '')
104+ result = fromstring(result)
105+ for elem in result.xpath(language_fallback_xpath):
106+ elem.getparent().remove(elem)
87107
88− claims = result.get('claims', {})
89− official_website = get_string(claims, 'P856', None)
90− if official_website is not None:
91− urls.append({'title': get_label('P856', language), 'url': official_website})
92− results.append({'title': title, 'url': official_website})
108+ description = extract_text(result.xpath(description_xpath))
93109
94− wikipedia_link_count = 0
95− wikipedia_link = get_wikilink(result, language + 'wiki')
96− wikipedia_link_count += add_url(urls,
97− 'Wikipedia (' + language + ')',
98− wikipedia_link)
99− if language != 'en':
100− wikipedia_en_link = get_wikilink(result, 'enwiki')
101− wikipedia_link_count += add_url(urls,
102− 'Wikipedia (en)',
103− wikipedia_en_link)
104− if wikipedia_link_count == 0:
105− misc_language = get_wiki_firstlanguage(result, 'wiki')
106− if misc_language is not None:
107− add_url(urls,
108− 'Wikipedia (' + misc_language + ')',
109− get_wikilink(result, misc_language + 'wiki'))
110+ # URLS
110111
111− if language != 'en':
112− add_url(urls,
113− 'Wiki voyage (' + language + ')',
114− get_wikilink(result, language + 'wikivoyage'))
112+ # official website
113+ add_url(urls, result, 'P856', results=results)
115114
116− add_url(urls,
117− 'Wiki voyage (en)',
118− get_wikilink(result, 'enwikivoyage'))
115+ # wikipedia
116+ wikipedia_link_count = 0
117+ wikipedia_link = get_wikilink(result, language + 'wiki')
118+ if wikipedia_link:
119+ wikipedia_link_count += 1
120+ urls.append({'title': 'Wikipedia (' + language + ')',
121+ 'url': wikipedia_link})
119122
120123 if language != 'en':
121− add_url(urls,
122− 'Wikiquote (' + language + ')',
123− get_wikilink(result, language + 'wikiquote'))
124−
125− add_url(urls,
126− 'Wikiquote (en)',
127− get_wikilink(result, 'enwikiquote'))
128−
129− add_url(urls,
130− 'Commons wiki',
131− get_wikilink(result, 'commonswiki'))
132−
133− # Location
134− add_url(urls,
135− get_label('P625', language),
136− get_geolink(claims, 'P625', None))
137−
138− add_url(urls,
139− 'Wikidata',
140− 'https://www.wikidata.org/wiki/'
141− + wikidata_id + '?uselang=' + language)
142−
143− musicbrainz_work_id = get_string(claims, 'P435')
144− if musicbrainz_work_id is not None:
145− add_url(urls,
146− 'MusicBrainz',
147− 'http://musicbrainz.org/work/'
148− + musicbrainz_work_id)
149−
150− musicbrainz_artist_id = get_string(claims, 'P434')
151− if musicbrainz_artist_id is not None:
152− add_url(urls,
153− 'MusicBrainz',
154− 'http://musicbrainz.org/artist/'
155− + musicbrainz_artist_id)
156−
157− musicbrainz_release_group_id = get_string(claims, 'P436')
158− if musicbrainz_release_group_id is not None:
159− add_url(urls,
160− 'MusicBrainz',
161− 'http://musicbrainz.org/release-group/'
162− + musicbrainz_release_group_id)
163−
164− musicbrainz_label_id = get_string(claims, 'P966')
165− if musicbrainz_label_id is not None:
166− add_url(urls,
167− 'MusicBrainz',
168− 'http://musicbrainz.org/label/'
169− + musicbrainz_label_id)
170−
171− # musicbrainz_area_id = get_string(claims, 'P982')
172− # P1407 MusicBrainz series ID
173− # P1004 MusicBrainz place ID
174− # P1330 MusicBrainz instrument ID
175− # P1407 MusicBrainz series ID
176−
177− postal_code = get_string(claims, 'P281', None)
178− if postal_code is not None:
179− attributes.append({'label': get_label('P281', language), 'value': postal_code})
180−
181− date_of_birth = get_time(claims, 'P569', locale, None)
182− if date_of_birth is not None:
183− attributes.append({'label': get_label('P569', language), 'value': date_of_birth})
184−
185− date_of_death = get_time(claims, 'P570', locale, None)
186− if date_of_death is not None:
187− attributes.append({'label': get_label('P570', language), 'value': date_of_death})
124+ wikipedia_en_link = get_wikilink(result, 'enwiki')
125+ if wikipedia_en_link:
126+ wikipedia_link_count += 1
127+ urls.append({'title': 'Wikipedia (en)',
128+ 'url': wikipedia_en_link})
129+
130+ # TODO: get_wiki_firstlanguage
131+ # if wikipedia_link_count == 0:
132+
133+ # more wikis
134+ add_url(urls, result, default_label='Wikivoyage (' + language + ')', link_type=language + 'wikivoyage')
135+ add_url(urls, result, default_label='Wikiquote (' + language + ')', link_type=language + 'wikiquote')
136+ add_url(urls, result, default_label='Wikimedia Commons', link_type='commonswiki')
137+
138+ add_url(urls, result, 'P625', 'OpenStreetMap', link_type='geo')
139+
140+ # musicbrainz
141+ add_url(urls, result, 'P434', 'MusicBrainz', 'http://musicbrainz.org/artist/')
142+ add_url(urls, result, 'P435', 'MusicBrainz', 'http://musicbrainz.org/work/')
143+ add_url(urls, result, 'P436', 'MusicBrainz', 'http://musicbrainz.org/release-group/')
144+ add_url(urls, result, 'P966', 'MusicBrainz', 'http://musicbrainz.org/label/')
145+
146+ # IMDb
147+ add_url(urls, result, 'P345', 'IMDb', 'https://www.imdb.com/', link_type='imdb')
148+ # source code repository
149+ add_url(urls, result, 'P1324')
150+ # blog
151+ add_url(urls, result, 'P1581')
152+ # social media links
153+ add_url(urls, result, 'P2397', 'YouTube', 'https://www.youtube.com/channel/')
154+ add_url(urls, result, 'P1651', 'YouTube', 'https://www.youtube.com/watch?v=')
155+ add_url(urls, result, 'P2002', 'Twitter', 'https://twitter.com/')
156+ add_url(urls, result, 'P2013', 'Facebook', 'https://facebook.com/')
157+ add_url(urls, result, 'P2003', 'Instagram', 'https://instagram.com/')
158+
159+ urls.append({'title': 'Wikidata',
160+ 'url': 'https://www.wikidata.org/wiki/'
161+ + wikidata_id + '?uselang=' + language})
162+
163+ # INFOBOX ATTRIBUTES (ROWS)
164+
165+ # inception date
166+ add_attribute(attributes, result, 'P571', date=True)
167+ # dissolution date
168+ add_attribute(attributes, result, 'P576', date=True)
169+ # start date
170+ add_attribute(attributes, result, 'P580', date=True)
171+ # end date
172+ add_attribute(attributes, result, 'P582', date=True)
173+
174+ # date of birth
175+ add_attribute(attributes, result, 'P569', date=True)
176+ # date of death
177+ add_attribute(attributes, result, 'P570', date=True)
178+
179+ # nationality
180+ add_attribute(attributes, result, 'P27')
181+ # country of origin
182+ add_attribute(attributes, result, 'P495')
183+ # country
184+ add_attribute(attributes, result, 'P17')
185+ # headquarters
186+ add_attribute(attributes, result, 'Q180')
187+
188+ # PLACES
189+ # capital
190+ add_attribute(attributes, result, 'P36', trim=True)
191+ # head of state
192+ add_attribute(attributes, result, 'P35', trim=True)
193+ # head of government
194+ add_attribute(attributes, result, 'P6', trim=True)
195+ # type of government
196+ add_attribute(attributes, result, 'P122')
197+ # official language
198+ add_attribute(attributes, result, 'P37')
199+ # population
200+ add_attribute(attributes, result, 'P1082', trim=True)
201+ # area
202+ add_attribute(attributes, result, 'P2046')
203+ # currency
204+ add_attribute(attributes, result, 'P38')
205+ # heigth (building)
206+ add_attribute(attributes, result, 'P2048')
207+
208+ # MEDIA
209+ # platform (videogames)
210+ add_attribute(attributes, result, 'P400')
211+ # author
212+ add_attribute(attributes, result, 'P50')
213+ # creator
214+ add_attribute(attributes, result, 'P170')
215+ # director
216+ add_attribute(attributes, result, 'P57')
217+ # performer
218+ add_attribute(attributes, result, 'P175')
219+ # developer
220+ add_attribute(attributes, result, 'P178')
221+ # producer
222+ add_attribute(attributes, result, 'P162')
223+ # manufacturer
224+ add_attribute(attributes, result, 'P176')
225+ # screenwriter
226+ add_attribute(attributes, result, 'P58')
227+ # production company
228+ add_attribute(attributes, result, 'P272')
229+ # record label
230+ add_attribute(attributes, result, 'P264')
231+ # publisher
232+ add_attribute(attributes, result, 'P123')
233+ # composer
234+ add_attribute(attributes, result, 'P86')
235+ # publication date
236+ add_attribute(attributes, result, 'P577', date=True)
237+ # genre
238+ add_attribute(attributes, result, 'P136')
239+ # original language
240+ add_attribute(attributes, result, 'P364')
241+ # isbn
242+ add_attribute(attributes, result, 'Q33057')
243+ # software license
244+ add_attribute(attributes, result, 'P275')
245+ # programming language
246+ add_attribute(attributes, result, 'P277')
247+ # version
248+ add_attribute(attributes, result, 'P348', trim=True)
249+ # narrative location
250+ add_attribute(attributes, result, 'P840')
251+
252+ # LANGUAGES
253+ # number of speakers
254+ add_attribute(attributes, result, 'P1098')
255+ # writing system
256+ add_attribute(attributes, result, 'P282')
257+ # regulatory body
258+ add_attribute(attributes, result, 'P1018')
259+ # language code
260+ add_attribute(attributes, result, 'P218')
261+
262+ # OTHER
263+ # ceo
264+ add_attribute(attributes, result, 'P169', trim=True)
265+ # founder
266+ add_attribute(attributes, result, 'P112')
267+ # legal form (company/organization)
268+ add_attribute(attributes, result, 'P1454')
269+ # taxon
270+ add_attribute(attributes, result, 'P225')
271+ # chemical formula
272+ add_attribute(attributes, result, 'P274')
273+ # winner (sports/contests)
274+ add_attribute(attributes, result, 'P1346')
275+ # number of deaths
276+ add_attribute(attributes, result, 'P1120')
277+ # currency code
278+ add_attribute(attributes, result, 'P498')
279+
280+ image = add_image(result)
188281
189282 if len(attributes) == 0 and len(urls) == 2 and len(description) == 0:
190283 results.append({
@@ -197,6 +290,7 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
197290 'infobox': title,
198291 'id': wikipedia_link,
199292 'content': description,
293+ 'img_src': image,
200294 'attributes': attributes,
201295 'urls': urls
202296 })
@@ -204,92 +298,149 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
204298 return results
205299
206300
207−def add_url(urls, title, url):
208− if url is not None:
209− urls.append({'title': title, 'url': url})
210− return 1
301+# only returns first match
302+def add_image(result):
303+ # P18: image, P154: logo, P242: map, P41: flag, P2716: collage, P2910: icon
304+ property_ids = ['P18', 'P154', 'P242', 'P41', 'P2716', 'P2910']
305+
306+ for property_id in property_ids:
307+ image = result.xpath(property_xpath.replace('{propertyid}', property_id))
308+ if image:
309+ image_name = image[0].xpath(value_xpath)
310+ image_src = url_image.replace('{filename}', extract_text(image_name[0]))
311+ return image_src
312+
313+
314+# setting trim will only returned high ranked rows OR the first row
315+def add_attribute(attributes, result, property_id, default_label=None, date=False, trim=False):
316+ attribute = result.xpath(property_xpath.replace('{propertyid}', property_id))
317+ if attribute:
318+
319+ if default_label:
320+ label = default_label
321+ else:
322+ label = extract_text(attribute[0].xpath(label_xpath))
323+
324+ if date:
325+ trim = True
326+ # remove calendar name
327+ calendar_name = attribute[0].xpath(calendar_name_xpath)
328+ for calendar in calendar_name:
329+ calendar.getparent().remove(calendar)
330+
331+ concat_values = ""
332+ values = []
333+ first_value = None
334+ for row in attribute[0].xpath(property_row_xpath):
335+ if not first_value or not trim or row.xpath(preferred_rank_xpath):
336+
337+ value = row.xpath(value_xpath)
338+ if not value:
339+ continue
340+ value = extract_text(value)
341+
342+ # save first value in case no ranked row is found
343+ if trim and not first_value:
344+ first_value = value
345+ else:
346+ # to avoid duplicate values
347+ if value not in values:
348+ concat_values += value + ", "
349+ values.append(value)
350+
351+ if trim and not values:
352+ attributes.append({'label': label,
353+ 'value': first_value})
354+ else:
355+ attributes.append({'label': label,
356+ 'value': concat_values[:-2]})
357+
358+
359+# requires property_id unless it's a wiki link (defined in link_type)
360+def add_url(urls, result, property_id=None, default_label=None, url_prefix=None, results=None, link_type=None):
361+ links = []
362+
363+ # wiki links don't have property in wikidata page
364+ if link_type and 'wiki' in link_type:
365+ links.append(get_wikilink(result, link_type))
211366 else:
212− return 0
367+ dom_element = result.xpath(property_xpath.replace('{propertyid}', property_id))
368+ if dom_element:
369+ dom_element = dom_element[0]
370+ if not default_label:
371+ label = extract_text(dom_element.xpath(label_xpath))
372+
373+ if link_type == 'geo':
374+ links.append(get_geolink(dom_element))
375+
376+ elif link_type == 'imdb':
377+ links.append(get_imdblink(dom_element, url_prefix))
378+
379+ else:
380+ url_results = dom_element.xpath(url_xpath)
381+ for link in url_results:
382+ if link is not None:
383+ if url_prefix:
384+ link = url_prefix + extract_text(link)
385+ else:
386+ link = extract_text(link)
387+ links.append(link)
388+
389+ # append urls
390+ for url in links:
391+ if url is not None:
392+ urls.append({'title': default_label or label,
393+ 'url': url})
394+ if results is not None:
395+ results.append({'title': default_label or label,
396+ 'url': url})
397+
398+
399+def get_imdblink(result, url_prefix):
400+ imdb_id = result.xpath(value_xpath)
401+ if imdb_id:
402+ imdb_id = extract_text(imdb_id)
403+ id_prefix = imdb_id[:2]
404+ if id_prefix == 'tt':
405+ url = url_prefix + 'title/' + imdb_id
406+ elif id_prefix == 'nm':
407+ url = url_prefix + 'name/' + imdb_id
408+ elif id_prefix == 'ch':
409+ url = url_prefix + 'character/' + imdb_id
410+ elif id_prefix == 'co':
411+ url = url_prefix + 'company/' + imdb_id
412+ elif id_prefix == 'ev':
413+ url = url_prefix + 'event/' + imdb_id
414+ else:
415+ url = None
416+ return url
213417
214418
215−def get_mainsnak(claims, propertyName):
216− propValue = claims.get(propertyName, {})
217− if len(propValue) == 0:
419+def get_geolink(result):
420+ coordinates = result.xpath(value_xpath)
421+ if not coordinates:
218422 return None
219−
220− propValue = propValue[0].get('mainsnak', None)
221− return propValue
222−
223−
224−def get_string(claims, propertyName, defaultValue=None):
225− propValue = claims.get(propertyName, {})
226− if len(propValue) == 0:
227− return defaultValue
228−
229− result = []
230− for e in propValue:
231− mainsnak = e.get('mainsnak', {})
232−
233− datavalue = mainsnak.get('datavalue', {})
234− if datavalue is not None:
235− result.append(datavalue.get('value', ''))
236−
237− if len(result) == 0:
238− return defaultValue
239− else:
240− # TODO handle multiple urls
241− return result[0]
242−
243−
244−def get_time(claims, propertyName, locale, defaultValue=None):
245− propValue = claims.get(propertyName, {})
246− if len(propValue) == 0:
247− return defaultValue
248−
249− result = []
250− for e in propValue:
251− mainsnak = e.get('mainsnak', {})
252−
253− datavalue = mainsnak.get('datavalue', {})
254− if datavalue is not None:
255− value = datavalue.get('value', '')
256− result.append(value.get('time', ''))
257−
258− if len(result) == 0:
259− date_string = defaultValue
260− else:
261− date_string = ', '.join(result)
262−
263− try:
264− parsed_date = datetime.strptime(date_string, "+%Y-%m-%dT%H:%M:%SZ")
265− except:
266− if date_string.startswith('-'):
267− return date_string.split('T')[0]
268− try:
269− parsed_date = dateutil_parse(date_string, fuzzy=False, default=False)
270− except:
271− logger.debug('could not parse date %s', date_string)
272− return date_string.split('T')[0]
273−
274− return format_date_by_locale(parsed_date, locale)
275−
276−
277−def get_geolink(claims, propertyName, defaultValue=''):
278− mainsnak = get_mainsnak(claims, propertyName)
279−
280− if mainsnak is None:
281− return defaultValue
282−
283− datatype = mainsnak.get('datatype', '')
284− datavalue = mainsnak.get('datavalue', {})
285−
286− if datatype != 'globe-coordinate':
287− return defaultValue
288−
289− value = datavalue.get('value', {})
290−
291− precision = value.get('precision', 0.0002)
292−
423+ coordinates = extract_text(coordinates[0])
424+ latitude, longitude = coordinates.split(',')
425+
426+ # convert to decimal
427+ lat = int(latitude[:latitude.find(u'°')])
428+ if latitude.find('\'') >= 0:
429+ lat += int(latitude[latitude.find(u'°') + 1:latitude.find('\'')] or 0) / 60.0
430+ if latitude.find('"') >= 0:
431+ lat += float(latitude[latitude.find('\'') + 1:latitude.find('"')] or 0) / 3600.0
432+ if latitude.find('S') >= 0:
433+ lat *= -1
434+ lon = int(longitude[:longitude.find(u'°')])
435+ if longitude.find('\'') >= 0:
436+ lon += int(longitude[longitude.find(u'°') + 1:longitude.find('\'')] or 0) / 60.0
437+ if longitude.find('"') >= 0:
438+ lon += float(longitude[longitude.find('\'') + 1:longitude.find('"')] or 0) / 3600.0
439+ if longitude.find('W') >= 0:
440+ lon *= -1
441+
442+ # TODO: get precision
443+ precision = 0.0002
293444 # there is no zoom information, deduce from precision (error prone)
294445 # samples :
295446 # 13 --> 5
@@ -305,39 +456,20 @@ def get_geolink(claims, propertyName, defaultValue=''):
305456 zoom = int(15 - precision * 8.8322 + precision * precision * 0.625447)
306457
307458 url = url_map\
308− .replace('{latitude}', str(value.get('latitude', 0)))\
309− .replace('{longitude}', str(value.get('longitude', 0)))\
459+ .replace('{latitude}', str(lat))\
460+ .replace('{longitude}', str(lon))\
310461 .replace('{zoom}', str(zoom))
311462
312463 return url
313464
314465
315466def get_wikilink(result, wikiid):
316− url = result.get('sitelinks', {}).get(wikiid, {}).get('url', None)
317− if url is None:
318− return url
319− elif url.startswith('http://'):
467+ url = result.xpath(wikilink_xpath.replace('{wikiid}', wikiid))
468+ if not url:
469+ return None
470+ url = url[0]
471+ if url.startswith('http://'):
320472 url = url.replace('http://', 'https://')
321473 elif url.startswith('//'):
322474 url = 'https:' + url
323475 return url
324−
325−
326−def get_wiki_firstlanguage(result, wikipatternid):
327− for k in result.get('sitelinks', {}).keys():
328− if k.endswith(wikipatternid) and len(k) == (2 + len(wikipatternid)):
329− return k[0:2]
330− return None
331−
332−
333−def get_label(entity_id, language):
334− url = url_entity_label.format(query=urlencode({'ids': entity_id,
335− 'languages': language + '|en'}))
336−
337− response = get(url)
338− jsonresponse = json.loads(response.text)
339− label = jsonresponse.get('entities', {}).get(entity_id, {}).get('labels', {}).get(language, {}).get('value', None)
340− if label is None:
341− label = jsonresponse['entities'][entity_id]['labels']['en']['value']
342−
343− return label