git.iko.soy/searx
on update_data_update_firefox_version.py help sign in
searx/searx_extra/update/update_currencies.py
150 lines 4.7 kB history raw
1#!/usr/bin/env python
3import re
4import unicodedata
5import json
7# set path
8from sys import path
9from os.path import realpath, dirname, join
11from searx import searx_dir, settings
12from searx.engines.wikidata import send_wikidata_query
15# ORDER BY (with all the query fields) is important to keep a deterministic result order
16# so multiple invokation of this script doesn't change currencies.json
17SARQL_REQUEST = """
18SELECT DISTINCT ?iso4217 ?unit ?unicode ?label ?alias WHERE {
19 ?item wdt:P498 ?iso4217; rdfs:label ?label.
20 OPTIONAL { ?item skos:altLabel ?alias FILTER (LANG (?alias) = LANG(?label)). }
21 OPTIONAL { ?item wdt:P5061 ?unit. }
22 OPTIONAL { ?item wdt:P489 ?symbol.
23 ?symbol wdt:P487 ?unicode. }
24 MINUS { ?item wdt:P582 ?end_data . } # Ignore monney with an end date
25 MINUS { ?item wdt:P31/wdt:P279* wd:Q15893266 . } # Ignore "former entity" (obsolete currency)
26 FILTER(LANG(?label) IN (%LANGUAGES_SPARQL%)).
28ORDER BY ?iso4217 ?unit ?unicode ?label ?alias
29"""
31# ORDER BY (with all the query fields) is important to keep a deterministic result order
32# so multiple invokation of this script doesn't change currencies.json
33SPARQL_WIKIPEDIA_NAMES_REQUEST = """
34SELECT DISTINCT ?iso4217 ?article_name WHERE {
35 ?item wdt:P498 ?iso4217 .
36 ?article schema:about ?item ;
37 schema:name ?article_name ;
38 schema:isPartOf [ wikibase:wikiGroup "wikipedia" ]
39 MINUS { ?item wdt:P582 ?end_data . } # Ignore monney with an end date
40 MINUS { ?item wdt:P31/wdt:P279* wd:Q15893266 . } # Ignore "former entity" (obsolete currency)
41 FILTER(LANG(?article_name) IN (%LANGUAGES_SPARQL%)).
43ORDER BY ?iso4217 ?article_name
44"""
47LANGUAGES = settings['locales'].keys()
48LANGUAGES_SPARQL = ', '.join(set(map(lambda l: repr(l.split('_')[0]), LANGUAGES)))
51def remove_accents(name):
52 return unicodedata.normalize('NFKD', name).lower()
55def remove_extra(name):
56 for c in ('(', ':'):
57 if c in name:
58 name = name.split(c)[0].strip()
59 return name
62def _normalize_name(name):
63 name = re.sub(' +', ' ', remove_accents(name.lower()).replace('-', ' '))
64 name = remove_extra(name)
65 return name
68def add_currency_name(db, name, iso4217, normalize_name=True):
69 db_names = db['names']
71 if normalize_name:
72 name = _normalize_name(name)
74 iso4217_set = db_names.setdefault(name, [])
75 if iso4217 not in iso4217_set:
76 iso4217_set.insert(0, iso4217)
79def add_currency_label(db, label, iso4217, language):
80 labels = db['iso4217'].setdefault(iso4217, {})
81 labels[language] = label
84def wikidata_request_result_iterator(request):
85 result = send_wikidata_query(request.replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL))
86 if result is not None:
87 for r in result['results']['bindings']:
88 yield r
91def fetch_db():
92 db = {
93 'names': {},
94 'iso4217': {},
95 }
97 for r in wikidata_request_result_iterator(SPARQL_WIKIPEDIA_NAMES_REQUEST):
98 iso4217 = r['iso4217']['value']
99 article_name = r['article_name']['value']
100 article_lang = r['article_name']['xml:lang']
101 add_currency_name(db, article_name, iso4217)
102 add_currency_label(db, article_name, iso4217, article_lang)
104 for r in wikidata_request_result_iterator(SARQL_REQUEST):
105 iso4217 = r['iso4217']['value']
106 if 'label' in r:
107 label = r['label']['value']
108 label_lang = r['label']['xml:lang']
109 add_currency_name(db, label, iso4217)
110 add_currency_label(db, label, iso4217, label_lang)
112 if 'alias' in r:
113 add_currency_name(db, r['alias']['value'], iso4217)
115 if 'unicode' in r:
116 add_currency_name(db, r['unicode']['value'], iso4217, normalize_name=False)
118 if 'unit' in r:
119 add_currency_name(db, r['unit']['value'], iso4217, normalize_name=False)
121 # reduce memory usage:
122 # replace lists with one item by the item.
123 # see searx.search.processors.online_currency.name_to_iso4217
124 for name in db['names']:
125 if len(db['names'][name]) == 1:
126 db['names'][name] = db['names'][name][0]
128 return db
131def get_filename():
132 return join(join(searx_dir, "data"), "currencies.json")
135def main():
136 #
137 db = fetch_db()
138 # static
139 add_currency_name(db, "euro", 'EUR')
140 add_currency_name(db, "euros", 'EUR')
141 add_currency_name(db, "dollar", 'USD')
142 add_currency_name(db, "dollars", 'USD')
143 add_currency_name(db, "peso", 'MXN')
144 add_currency_name(db, "pesos", 'MXN')
146 with open(get_filename(), 'w', encoding='utf8') as f:
147 json.dump(db, f, ensure_ascii=False, indent=4)
149if __name__ == '__main__':
150 main()