@@ -1,12 +1,43 @@
11#!/usr/bin/env python
2−"""
3−Fetch OSM keys and tags
4−
5−to get the i18n names, the scripts uses query.wikidata.org
6−instead of for example https://taginfo.openstreetmap.org/taginfo/apidoc
2+# lint: pylint
3+# pylint: disable=missing-function-docstring
4+"""Fetch OSM keys and tags.
5+
6+To get the i18n names, the scripts uses `Wikidata Query Service`_ instead of for
7+example `OSM tags API`_ (sidenote: the actual change log from
8+map.atownsend.org.uk_ might be useful to normalize OSM tags)
9+
10+.. _Wikidata Query Service: https://query.wikidata.org/
11+.. _OSM tags API: https://taginfo.openstreetmap.org/taginfo/apidoc
12+.. _map.atownsend.org.uk: https://map.atownsend.org.uk/maps/map/changelog.html
13+
14+:py:obj:`SPARQL_TAGS_REQUEST` :
15+ Wikidata SPARQL query that returns *type-categories* and *types*. The
16+ returned tag is ``Tag:{category}={type}`` (see :py:func:`get_tags`).
17+ Example:
18+
19+ - https://taginfo.openstreetmap.org/tags/building=house#overview
20+ - https://wiki.openstreetmap.org/wiki/Tag:building%3Dhouse
21+ at the bottom of the infobox (right side), there is a link to wikidata:
22+ https://www.wikidata.org/wiki/Q3947
23+ see property "OpenStreetMap tag or key" (P1282)
24+ - https://wiki.openstreetmap.org/wiki/Tag%3Abuilding%3Dbungalow
25+ https://www.wikidata.org/wiki/Q850107
26+
27+:py:obj:`SPARQL_KEYS_REQUEST` :
28+ Wikidata SPARQL query that returns *keys*. Example with "payment":
29+
30+ - https://wiki.openstreetmap.org/wiki/Key%3Apayment
31+ at the bottom of infobox (right side), there is a link to wikidata:
32+ https://www.wikidata.org/wiki/Q1148747
33+ link made using the "OpenStreetMap tag or key" property (P1282)
34+ to be confirm: there is a one wiki page per key ?
35+ - https://taginfo.openstreetmap.org/keys/payment#values
36+ - https://taginfo.openstreetmap.org/keys/payment:cash#values
37+
38+ ``rdfs:label`` get all the labels without language selection
39+ (as opposed to SERVICE ``wikibase:label``).
740
8−https://map.atownsend.org.uk/maps/map/changelog.html (the actual change log)
9−might be useful to normalize OSM tags
1041"""
1142
1243import json
@@ -19,50 +50,28 @@ from searx.engines.wikidata import send_wikidata_query
1950from searx.languages import language_codes
2051from searx.engines.openstreetmap import get_key_rank, VALUE_TO_LINK
2152
22−
23−# nominatim return type category and type
24−# the tag is "Tag:{category}={type}"
25−# Example:
26−# * https://taginfo.openstreetmap.org/tags/building=house#overview
27−# * https://wiki.openstreetmap.org/wiki/Tag:building%3Dhouse
28−# at the bottom of the infobox (right side), there is a link to wikidata:
29−# https://www.wikidata.org/wiki/Q3947
30−# see property "OpenStreetMap tag or key" (P1282)
31−# * https://wiki.openstreetmap.org/wiki/Tag%3Abuilding%3Dbungalow
32−# https://www.wikidata.org/wiki/Q850107
33−SARQL_TAGS_REQUEST = """
53+SPARQL_TAGS_REQUEST = """
3454SELECT ?tag ?item ?itemLabel WHERE {
3555 ?item wdt:P1282 ?tag .
3656 ?item rdfs:label ?itemLabel .
3757 FILTER(STRSTARTS(?tag, 'Tag'))
3858}
39−GROUP BY ?tag ?item ?itemLabel
59+GROUP BY ?tag ?item ?itemLabel
4060ORDER BY ?tag ?item ?itemLabel
4161"""
4262
43−# keys
44−# Example with "payment"":
45−# * https://wiki.openstreetmap.org/wiki/Key%3Apayment
46−# at the bottom of infobox (right side), there is a link to wikidata:
47−# https://www.wikidata.org/wiki/Q1148747
48−# link made using the "OpenStreetMap tag or key" property (P1282)
49−# to be confirm: there is a one wiki page per key ?
50−# * https://taginfo.openstreetmap.org/keys/payment#values
51−# * https://taginfo.openstreetmap.org/keys/payment:cash#values
52−#
53−# rdfs:label get all the labels without language selection
54−# (as opposed to SERVICE wikibase:label)
55−SARQL_KEYS_REQUEST = """
63+SPARQL_KEYS_REQUEST = """
5664SELECT ?key ?item ?itemLabel WHERE {
5765 ?item wdt:P1282 ?key .
5866 ?item rdfs:label ?itemLabel .
5967 FILTER(STRSTARTS(?key, 'Key'))
6068}
61−GROUP BY ?key ?item ?itemLabel
69+GROUP BY ?key ?item ?itemLabel
6270ORDER BY ?key ?item ?itemLabel
6371"""
6472
6573LANGUAGES = [l[0].lower() for l in language_codes]
74+
6675PRESET_KEYS = {
6776 ('wikidata',): {'en': 'Wikidata'},
6877 ('wikipedia',): {'en': 'Wikipedia'},
@@ -71,11 +80,11 @@ PRESET_KEYS = {
7180 ('fax',): {'en': 'Fax'},
7281 ('internet_access', 'ssid'): {'en': 'Wi-Fi'},
7382}
83+
7484INCLUDED_KEYS = {
7585 ('addr', )
7686}
7787
78−
7988def get_preset_keys():
8089 results = collections.OrderedDict()
8190 for keys, value in PRESET_KEYS.items():
@@ -85,10 +94,9 @@ def get_preset_keys():
8594 r.setdefault('*', value)
8695 return results
8796
88−
8997def get_keys():
9098 results = get_preset_keys()
91− response = send_wikidata_query(SARQL_KEYS_REQUEST)
99+ response = send_wikidata_query(SPARQL_KEYS_REQUEST)
92100
93101 for key in response['results']['bindings']:
94102 keys = key['key']['value'].split(':')[1:]
@@ -136,7 +144,7 @@ def get_keys():
136144
137145def get_tags():
138146 results = collections.OrderedDict()
139− response = send_wikidata_query(SARQL_TAGS_REQUEST)
147+ response = send_wikidata_query(SPARQL_TAGS_REQUEST)
140148 for tag in response['results']['bindings']:
141149 tag_names = tag['tag']['value'].split(':')[1].split('=')
142150 if len(tag_names) == 2:
@@ -149,7 +157,6 @@ def get_tags():
149157 results.setdefault(tag_category, {}).setdefault(tag_type, {}).setdefault(lang, label)
150158 return results
151159
152−
153160def optimize_data_lang(translations):
154161 language_to_delete = []
155162 # remove "zh-hk" entry if the value is the same as "zh"
@@ -174,14 +181,12 @@ def optimize_data_lang(translations):
174181 for language in language_to_delete:
175182 del translations[language]
176183
177−
178184def optimize_tags(data):
179185 for v in data.values():
180186 for translations in v.values():
181187 optimize_data_lang(translations)
182188 return data
183189
184−
185190def optimize_keys(data):
186191 for k, v in data.items():
187192 if k == '*':
@@ -190,15 +195,15 @@ def optimize_keys(data):
190195 optimize_keys(v)
191196 return data
192197
193−
194198def get_osm_tags_filename():
195199 return Path(searx_dir) / "data" / "osm_keys_tags.json"
196200
201+if __name__ == '__main__':
197202
198−set_timeout_for_thread(60)
199−result = {
200− 'keys': optimize_keys(get_keys()),
201− 'tags': optimize_tags(get_tags()),
202−}
203−with open(get_osm_tags_filename(), 'w') as f:
204− json.dump(result, f, indent=4, ensure_ascii=False)
203+ set_timeout_for_thread(60)
204+ result = {
205+ 'keys': optimize_keys(get_keys()),
206+ 'tags': optimize_tags(get_tags()),
207+ }
208+ with open(get_osm_tags_filename(), 'w') as f:
209+ json.dump(result, f, indent=4, ensure_ascii=False)