git.iko.soy/searx
on 5fa36eb help sign in
searx/utils/fetch_languages.py
187 lines 6.9 kB history raw
1# -*- coding: utf-8 -*-
3# This script generates languages.py from intersecting each engine's supported languages.
4#
5# The country names are obtained from http://api.geonames.org which requires registering as a user.
6#
7# Output files (engines_languages.json and languages.py)
8# are written in current directory to avoid overwriting in case something goes wrong.
10from requests import get
11from urllib import urlencode
12from lxml.html import fromstring
13from json import loads, dumps
14import io
15from sys import path
16path.append('../searx') # noqa
17from searx.engines import engines
19# Geonames API for country names.
20geonames_user = '' # ADD USER NAME HERE
21country_names_url = 'http://api.geonames.org/countryInfoJSON?{parameters}'
23# Output files.
24engines_languages_file = 'engines_languages.json'
25languages_file = 'languages.py'
27engines_languages = {}
30# To filter out invalid codes and dialects.
31def valid_code(lang_code):
32 # filter invalid codes
33 # sl-SL is technically not invalid, but still a mistake
34 invalid_codes = ['sl-SL', 'wt-WT', 'jw']
35 invalid_countries = ['UK', 'XA', 'XL']
36 if lang_code[:2] == 'xx'\
37 or lang_code in invalid_codes\
38 or lang_code[-2:] in invalid_countries\
39 or is_dialect(lang_code):
40 return False
42 return True
45# Language codes with any additional tags other than language and country.
46def is_dialect(lang_code):
47 lang_code = lang_code.split('-')
48 if len(lang_code) > 2 or len(lang_code[0]) > 3:
49 return True
50 if len(lang_code) == 2 and len(lang_code[1]) > 2:
51 return True
53 return False
56# Get country name in specified language.
57def get_country_name(locale):
58 if geonames_user is '':
59 return ''
61 locale = locale.split('-')
62 if len(locale) != 2:
63 return ''
65 url = country_names_url.format(parameters=urlencode({'lang': locale[0],
66 'country': locale[1],
67 'username': geonames_user}))
68 response = get(url)
69 json = loads(response.text)
70 content = json.get('geonames', None)
71 if content is None or len(content) != 1:
72 print "No country name found for " + locale[0] + "-" + locale[1]
73 return ''
75 return content[0].get('countryName', '')
78# Fetchs supported languages for each engine and writes json file with those.
79def fetch_supported_languages():
80 for engine_name in engines:
81 if hasattr(engines[engine_name], 'fetch_supported_languages'):
82 try:
83 engines_languages[engine_name] = engines[engine_name].fetch_supported_languages()
84 except Exception as e:
85 print e
87 # write json file
88 with io.open(engines_languages_file, "w", encoding="utf-8") as f:
89 f.write(unicode(dumps(engines_languages, ensure_ascii=False, encoding="utf-8")))
92# Join all language lists.
93# Iterate all languages supported by each engine.
94def join_language_lists():
95 global languages
96 # include wikipedia first for more accurate language names
97 languages = {code: lang for code, lang
98 in engines_languages['wikipedia'].iteritems()
99 if valid_code(code)}
101 for engine_name in engines_languages:
102 for locale in engines_languages[engine_name]:
103 if valid_code(locale):
104 # if language is not on list or if it has no name yet
105 if locale not in languages or not languages[locale].get('name'):
106 if isinstance(engines_languages[engine_name], dict):
107 languages[locale] = engines_languages[engine_name][locale]
108 else:
109 languages[locale] = {}
111 # add to counter of engines that support given language
112 lang = locale.split('-')[0]
113 if lang in languages:
114 if 'counter' not in languages[lang]:
115 languages[lang]['counter'] = [engine_name]
116 elif engine_name not in languages[lang]['counter']:
117 languages[lang]['counter'].append(engine_name)
119 # filter list to include only languages supported by most engines
120 min_supported_engines = int(0.75 * len(engines_languages))
121 languages = {code: lang for code, lang
122 in languages.iteritems()
123 if len(lang.get('counter', [])) >= min_supported_engines or
124 len(languages.get(code.split('-')[0], {}).get('counter', [])) >= min_supported_engines}
126 # get locales that have no name or country yet
127 for locale in languages.keys():
128 # try to get language names
129 if not languages[locale].get('name'):
130 name = languages.get(locale.split('-')[0], {}).get('name', None)
131 if name:
132 languages[locale]['name'] = name
133 else:
134 # filter out locales with no name
135 del languages[locale]
136 continue
138 # try to get language name in english
139 if not languages[locale].get('english_name'):
140 languages[locale]['english_name'] = languages.get(locale.split('-')[0], {}).get('english_name', '')
142 # try to get country name
143 if locale.find('-') > 0 and not languages[locale].get('country'):
144 languages[locale]['country'] = get_country_name(locale) or ''
147# Remove countryless language if language is featured in only one country.
148def filter_single_country_languages():
149 prev_lang = None
150 prev_code = None
151 for code in sorted(languages):
152 lang = code.split('-')[0]
153 if lang == prev_lang:
154 countries += 1
155 else:
156 if prev_lang is not None and countries == 1:
157 del languages[prev_lang]
158 languages[prev_code]['country'] = ''
159 countries = 0
160 prev_lang = lang
161 prev_code = code
164# Write languages.py.
165def write_languages_file():
166 new_file = open(languages_file, 'w')
167 file_content = '# -*- coding: utf-8 -*-\n'\
168 + '# list of language codes\n'\
169 + '# this file is generated automatically by utils/update_search_languages.py\n'\
170 + '\nlanguage_codes = ('
171 for code in sorted(languages):
172 file_content += '\n (u"' + code + '"'\
173 + ', u"' + languages[code]['name'].split(' (')[0] + '"'\
174 + ', u"' + languages[code].get('country', '') + '"'\
175 + ', u"' + languages[code].get('english_name', '').split(' (')[0] + '"),'
176 # remove last comma
177 file_content = file_content[:-1]
178 file_content += '\n)\n'
179 new_file.write(file_content.encode('utf8'))
180 new_file.close()
183if __name__ == "__main__":
184 fetch_supported_languages()
185 join_language_lists()
186 filter_single_country_languages()
187 write_languages_file()