git.iko.soy/searx
on eecfff2 help sign in
searx/utils/fetch_languages.py
207 lines 8.6 kB history raw
1# -*- coding: utf-8 -*-
3# This script generates languages.py from intersecting each engine's supported languages.
4#
5# Output files (engines_languages.json and languages.py)
6# are written in current directory to avoid overwriting in case something goes wrong.
8import json
9from pprint import pformat
10from sys import path
11from babel import Locale, UnknownLocaleError
12from babel.languages import get_global
14path.append('../searx') # noqa
15from searx import settings
16from searx.engines import initialize_engines, engines
18# Output files.
19engines_languages_file = 'engines_languages.json'
20languages_file = 'languages.py'
23# Fetchs supported languages for each engine and writes json file with those.
24def fetch_supported_languages():
26 engines_languages = dict()
27 names = list(engines)
28 names.sort()
30 for engine_name in names:
31 if hasattr(engines[engine_name], 'fetch_supported_languages'):
32 engines_languages[engine_name] = engines[engine_name].fetch_supported_languages()
33 print("fetched %s languages from engine %s" % (
34 len(engines_languages[engine_name]), engine_name))
35 if type(engines_languages[engine_name]) == list:
36 engines_languages[engine_name] = sorted(engines_languages[engine_name])
38 # write json file
39 with open(engines_languages_file, 'w', encoding='utf-8') as f:
40 json.dump(engines_languages, f, indent=2, sort_keys=True)
42 return engines_languages
45# Get babel Locale object from lang_code if possible.
46def get_locale(lang_code):
47 try:
48 locale = Locale.parse(lang_code, sep='-')
49 return locale
50 except (UnknownLocaleError, ValueError):
51 return None
54# Join all language lists.
55def join_language_lists(engines_languages):
56 language_list = dict()
57 for engine_name in engines_languages:
58 for lang_code in engines_languages[engine_name]:
60 # apply custom fixes if necessary
61 if lang_code in getattr(engines[engine_name], 'language_aliases', {}).values():
62 lang_code = next(lc for lc, alias in engines[engine_name].language_aliases.items()
63 if lang_code == alias)
65 locale = get_locale(lang_code)
67 # ensure that lang_code uses standard language and country codes
68 if locale and locale.territory:
69 lang_code = "{lang}-{country}".format(lang=locale.language, country=locale.territory)
70 short_code = lang_code.split('-')[0]
72 # add language without country if not in list
73 if short_code not in language_list:
74 if locale:
75 # get language's data from babel's Locale object
76 language_name = locale.get_language_name().title()
77 english_name = locale.english_name.split(' (')[0]
78 elif short_code in engines_languages['wikipedia']:
79 # get language's data from wikipedia if not known by babel
80 language_name = engines_languages['wikipedia'][short_code]['name']
81 english_name = engines_languages['wikipedia'][short_code]['english_name']
82 else:
83 language_name = None
84 english_name = None
86 # add language to list
87 language_list[short_code] = {'name': language_name,
88 'english_name': english_name,
89 'counter': set(),
90 'countries': dict()}
92 # add language with country if not in list
93 if lang_code != short_code and lang_code not in language_list[short_code]['countries']:
94 country_name = ''
95 if locale:
96 # get country name from babel's Locale object
97 country_name = locale.get_territory_name()
99 language_list[short_code]['countries'][lang_code] = {'country_name': country_name,
100 'counter': set()}
102 # count engine for both language_country combination and language alone
103 language_list[short_code]['counter'].add(engine_name)
104 if lang_code != short_code:
105 language_list[short_code]['countries'][lang_code]['counter'].add(engine_name)
107 return language_list
110# Filter language list so it only includes the most supported languages and countries
111def filter_language_list(all_languages):
112 min_engines_per_lang = 15
113 min_engines_per_country = 10
114 main_engines = [engine_name for engine_name in engines.keys()
115 if 'general' in engines[engine_name].categories and
116 engines[engine_name].supported_languages and
117 not engines[engine_name].disabled]
119 # filter list to include only languages supported by most engines or all default general engines
120 filtered_languages = {code: lang for code, lang
121 in all_languages.items()
122 if (len(lang['counter']) >= min_engines_per_lang or
123 all(main_engine in lang['counter']
124 for main_engine in main_engines))}
126 def _copy_lang_data(lang, country_name=None):
127 new_dict = dict()
128 new_dict['name'] = all_languages[lang]['name']
129 new_dict['english_name'] = all_languages[lang]['english_name']
130 if country_name:
131 new_dict['country_name'] = country_name
132 return new_dict
134 def _country_count(i):
135 return len(countries[sorted_countries[i]]['counter'])
137 # for each language get country codes supported by most engines or at least one country code
138 filtered_languages_with_countries = dict()
139 for lang, lang_data in filtered_languages.items():
140 countries = lang_data['countries']
141 filtered_countries = dict()
143 # get language's country codes with enough supported engines
144 for lang_country, country_data in countries.items():
145 if len(country_data['counter']) >= min_engines_per_country:
146 filtered_countries[lang_country] = _copy_lang_data(lang, country_data['country_name'])
148 # add language without countries too if there's more than one country to choose from
149 if len(filtered_countries) > 1:
150 filtered_countries[lang] = _copy_lang_data(lang)
151 elif len(filtered_countries) == 1:
152 # if there's only one country per language, it's not necessary to show country name
153 lang_country = next(iter(filtered_countries))
154 filtered_countries[lang_country]['country_name'] = None
156 # if no country has enough engines try to get most likely country code from babel
157 if not filtered_countries:
158 lang_country = None
159 subtags = get_global('likely_subtags').get(lang)
160 if subtags:
161 country_code = subtags.split('_')[-1]
162 if len(country_code) == 2:
163 lang_country = "{lang}-{country}".format(lang=lang, country=country_code)
165 if lang_country:
166 filtered_countries[lang_country] = _copy_lang_data(lang)
167 else:
168 filtered_countries[lang] = _copy_lang_data(lang)
170 filtered_languages_with_countries.update(filtered_countries)
172 return filtered_languages_with_countries
175# Write languages.py.
176def write_languages_file(languages):
177 file_headers = (
178 "# -*- coding: utf-8 -*-",
179 "# list of language codes",
180 "# this file is generated automatically by utils/fetch_languages.py",
181 "language_codes ="
182 )
184 language_codes = tuple([
185 (
186 code,
187 languages[code]['name'].split(' (')[0],
188 languages[code].get('country_name') or '',
189 languages[code].get('english_name') or ''
190 ) for code in sorted(languages)
191 ])
193 with open(languages_file, 'w') as new_file:
194 file_content = "{file_headers} \\\n{language_codes}".format(
195 file_headers='\n'.join(file_headers),
196 language_codes=pformat(language_codes, indent=4)
197 )
198 new_file.write(file_content)
199 new_file.close()
202if __name__ == "__main__":
203 initialize_engines(settings['engines'])
204 engines_languages = fetch_supported_languages()
205 all_languages = join_language_lists(engines_languages)
206 filtered_languages = filter_language_list(all_languages)
207 write_languages_file(filtered_languages)