git.iko.soy/searx
on 4c0ffa3 help sign in
searx/utils/fetch_languages.py
189 lines 7.9 kB history raw
1# -*- coding: utf-8 -*-
3# This script generates languages.py from intersecting each engine's supported languages.
4#
5# Output files (engines_languages.json and languages.py)
6# are written in current directory to avoid overwriting in case something goes wrong.
8from json import dump
9import io
10from sys import path
11from babel import Locale, UnknownLocaleError
12from babel.languages import get_global
14path.append('../searx') # noqa
15from searx import settings
16from searx.engines import initialize_engines, engines
18# Output files.
19engines_languages_file = 'engines_languages.json'
20languages_file = 'languages.py'
23# Fetchs supported languages for each engine and writes json file with those.
24def fetch_supported_languages():
25 engines_languages = {}
26 for engine_name in engines:
27 if hasattr(engines[engine_name], 'fetch_supported_languages'):
28 try:
29 engines_languages[engine_name] = engines[engine_name].fetch_supported_languages()
30 except Exception as e:
31 print(e)
33 # write json file
34 with io.open(engines_languages_file, "w", encoding="utf-8") as f:
35 dump(engines_languages, f, ensure_ascii=False)
37 return engines_languages
40# Get babel Locale object from lang_code if possible.
41def get_locale(lang_code):
42 try:
43 locale = Locale.parse(lang_code, sep='-')
44 return locale
45 except (UnknownLocaleError, ValueError):
46 return None
49# Append engine_name to list of engines that support locale.
50def add_engine_counter(lang_code, engine_name, languages):
51 if lang_code in languages:
52 if 'counter' not in languages[lang_code]:
53 languages[lang_code]['counter'] = [engine_name]
54 elif engine_name not in languages[lang_code]['counter']:
55 languages[lang_code]['counter'].append(engine_name)
58# Join all language lists.
59# TODO: Add language names from engine's language list if name not known by babel.
60def join_language_lists(engines_languages):
61 language_list = {}
62 for engine_name in engines_languages:
63 for lang_code in engines_languages[engine_name]:
65 # apply custom fixes if necessary
66 if lang_code in getattr(engines[engine_name], 'language_aliases', {}).values():
67 lang_code = next(lc for lc, alias in engines[engine_name].language_aliases.items()
68 if lang_code == alias)
70 locale = get_locale(lang_code)
72 # ensure that lang_code uses standard language and country codes
73 if locale and locale.territory:
74 lang_code = locale.language + '-' + locale.territory
76 # add locale if it's not in list
77 if lang_code not in language_list:
78 if locale:
79 language_list[lang_code] = {'name': locale.get_language_name().title(),
80 'english_name': locale.english_name,
81 'country': locale.get_territory_name() or ''}
83 # also add language without country
84 if locale.language not in language_list:
85 language_list[locale.language] = {'name': locale.get_language_name().title(),
86 'english_name': locale.english_name}
87 else:
88 language_list[lang_code] = {}
90 # count engine for both language_country combination and language alone
91 add_engine_counter(lang_code, engine_name, language_list)
92 add_engine_counter(lang_code.split('-')[0], engine_name, language_list)
94 return language_list
97# Filter language list so it only includes the most supported languages and countries.
98def filter_language_list(all_languages):
99 min_supported_engines = 10
100 main_engines = [engine_name for engine_name in engines.keys()
101 if 'general' in engines[engine_name].categories and
102 engines[engine_name].supported_languages and
103 not engines[engine_name].disabled]
105 # filter list to include only languages supported by most engines or all default general engines
106 filtered_languages = {code: lang for code, lang
107 in all_languages.items()
108 if (len(lang.get('counter', [])) >= min_supported_engines or
109 all(main_engine in lang.get('counter', [])
110 for main_engine in main_engines))}
112 return filtered_languages
115# Add country codes to languages without one and filter out language codes.
116def assign_country_codes(filtered_languages, all_languages):
117 sorted_languages = sorted(all_languages,
118 key=lambda lang: len(all_languages[lang].get('counter', [])),
119 reverse=True)
120 previous_lang = None
121 previous_code = None
122 countries = 0
123 for current_code in sorted(filtered_languages):
124 current_lang = current_code.split('-')[0]
126 # count country codes per language
127 if current_lang == previous_lang:
128 countries += 1
130 else:
131 if previous_lang is not None:
132 # if language has no single country code
133 if countries == 0:
134 # try to get country code with most supported engines
135 for l in sorted_languages:
136 l_parts = l.split('-')
137 if len(l_parts) == 2 and l_parts[0] == previous_lang:
138 filtered_languages[l] = all_languages[l]
139 filtered_languages[l]['country'] = ''
140 countries = 1
141 break
143 if countries == 0:
144 # get most likely country code from babel
145 subtags = get_global('likely_subtags').get(previous_lang)
146 if subtags:
147 subtag_parts = subtags.split('_')
148 new_code = subtag_parts[0] + '-' + subtag_parts[-1]
149 filtered_languages[new_code] = all_languages[previous_lang]
150 countries = 1
152 if countries == 1:
153 # remove countryless version of language if there's only one country
154 del filtered_languages[previous_lang]
155 if previous_code in filtered_languages:
156 filtered_languages[previous_code]['country'] = ''
158 countries = 0
159 previous_lang = current_lang
161 previous_code = current_code
164# Write languages.py.
165def write_languages_file(languages):
166 new_file = open(languages_file, 'wb')
167 file_content = '# -*- coding: utf-8 -*-\n'\
168 + '# list of language codes\n'\
169 + '# this file is generated automatically by utils/update_search_languages.py\n'\
170 + '\nlanguage_codes = ('
171 for code in sorted(languages):
172 file_content += '\n (u"' + code + '"'\
173 + ', u"' + languages[code]['name'].split(' (')[0] + '"'\
174 + ', u"' + languages[code].get('country', '') + '"'\
175 + ', u"' + languages[code].get('english_name', '').split(' (')[0] + '"),'
176 # remove last comma
177 file_content = file_content[:-1]
178 file_content += '\n)\n'
179 new_file.write(file_content.encode('utf8'))
180 new_file.close()
183if __name__ == "__main__":
184 initialize_engines(settings['engines'])
185 engines_languages = fetch_supported_languages()
186 all_languages = join_language_lists(engines_languages)
187 filtered_languages = filter_language_list(all_languages)
188 assign_country_codes(filtered_languages, all_languages)
189 write_languages_file(filtered_languages)