@@ -5,17 +5,17 @@ import unicodedata
55import string
66from urllib import urlencode
77from requests import get
8−
8+
99languages = {'de', 'en', 'es', 'fr', 'hu', 'it', 'nl', 'jp'}
10−
10+
1111url_template = 'https://www.wikidata.org/w/api.php?action=wbgetentities&format=json&{query}&props=labels%7Cdatatype%7Cclaims%7Caliases&languages=' + '|'.join(languages)
12−url_wmflabs_template = 'http://wdq.wmflabs.org/api?q='
12+url_wmflabs_template = 'http://wdq.wmflabs.org/api?q='
1313url_wikidata_search_template='http://www.wikidata.org/w/api.php?action=query&list=search&format=json&srnamespace=0&srprop=sectiontitle&{query}'
14−
15−wmflabs_queries = [
14+
15+wmflabs_queries = [
1616 'CLAIM[31:8142]', # all devise
1717]
18−
18+
1919db = {
2020 'iso4217' : {
2121 },
@@ -26,7 +26,7 @@ db = {
2626
2727def remove_accents(data):
2828 return unicodedata.normalize('NFKD', data).lower()
29−
29+
3030
3131def normalize_name(name):
3232 return re.sub(' +',' ', remove_accents(name.lower()).replace('-', ' '))
@@ -66,22 +66,22 @@ def get_property_value(data, name):
6666 prop = data.get('claims', {}).get(name, {})
6767 if len(prop) == 0:
6868 return None
69−
70− value = prop[0].get('mainsnak', {}).get('datavalue', {}).get('value', '')
69+
70+ value = prop[0].get('mainsnak', {}).get('datavalue', {}).get('value', '')
7171 if value == '':
7272 return None
7373
7474 return value
75−
75+
7676
7777def parse_currency(data):
7878 iso4217 = get_property_value(data, 'P498')
79−
79+
8080 if iso4217 is not None:
8181 unit = get_property_value(data, 'P558')
8282 if unit is not None:
8383 add_currency_name(unit, iso4217)
84−
84+
8585 labels = data.get('labels', {})
8686 for language in languages:
8787 name = labels.get(language, {}).get('value', None)
@@ -95,22 +95,22 @@ def parse_currency(data):
9595 alias = aliases[language][i].get('value', None)
9696 add_currency_name(alias, iso4217)
9797
98−
98+
9999def fetch_data(wikidata_ids):
100100 url = url_template.format(query=urlencode({'ids' : '|'.join(wikidata_ids)}))
101101 htmlresponse = get(url)
102102 jsonresponse = json.loads(htmlresponse.content)
103103 entities = jsonresponse.get('entities', {})
104−
104+
105105 for pname in entities:
106106 pvalue = entities.get(pname)
107107 parse_currency(pvalue)
108−
109−
108+
109+
110110def add_q(i):
111111 return "Q" + str(i)
112−
113−
112+
113+
114114def fetch_data_batch(wikidata_ids):
115115 while len(wikidata_ids) > 0:
116116 if len(wikidata_ids) > 50:
@@ -119,8 +119,8 @@ def fetch_data_batch(wikidata_ids):
119119 else:
120120 fetch_data(wikidata_ids)
121121 wikidata_ids = []
122−
123−
122+
123+
124124def wdq_query(query):
125125 url = url_wmflabs_template + query
126126 htmlresponse = get(url)
@@ -131,23 +131,23 @@ def wdq_query(query):
131131 print "error for query '" + query + "' :" + error
132132
133133 fetch_data_batch(qlist)
134−
135−
134+
135+
136136def wd_query(query, offset=0):
137137 qlist = []
138−
138+
139139 url = url_wikidata_search_template.format(query=urlencode({'srsearch': query, 'srlimit': 50, 'sroffset': offset}))
140140 htmlresponse = get(url)
141141 jsonresponse = json.loads(htmlresponse.content)
142142 for r in jsonresponse.get('query', {}).get('search', {}):
143143 qlist.append(r.get('title', ''))
144144 fetch_data_batch(qlist)
145−
145+
146146## fetch ##
147147for q in wmflabs_queries:
148148 wdq_query(q)
149149
150−# static
150+# static
151151add_currency_name(u"euro", 'EUR')
152152add_currency_name(u"euros", 'EUR')
153153add_currency_name(u"dollar", 'USD')