git.iko.soy/searx
on ddee486 help sign in
searx/utils/fetch_languages.py
191 lines 8.1 kB history raw
1# -*- coding: utf-8 -*-
3# This script generates languages.py from intersecting each engine's supported languages.
4#
5# Output files (engines_languages.json and languages.py)
6# are written in current directory to avoid overwriting in case something goes wrong.
8from json import dump
9import io
10from sys import path
11from babel import Locale, UnknownLocaleError
12from babel.languages import get_global
14path.append('../searx') # noqa
15from searx import settings
16from searx.engines import initialize_engines, engines
18# Output files.
19engines_languages_file = 'engines_languages.json'
20languages_file = 'languages.py'
23# Fetchs supported languages for each engine and writes json file with those.
24def fetch_supported_languages():
25 engines_languages = {}
26 for engine_name in engines:
27 if hasattr(engines[engine_name], 'fetch_supported_languages'):
28 try:
29 engines_languages[engine_name] = engines[engine_name].fetch_supported_languages()
30 if type(engines_languages[engine_name]) == list:
31 engines_languages[engine_name] = sorted(engines_languages[engine_name])
32 except Exception as e:
33 print(e)
35 # write json file
36 with io.open(engines_languages_file, "w", encoding="utf-8") as f:
37 dump(engines_languages, f, ensure_ascii=False, indent=4, separators=(',', ': '))
39 return engines_languages
42# Get babel Locale object from lang_code if possible.
43def get_locale(lang_code):
44 try:
45 locale = Locale.parse(lang_code, sep='-')
46 return locale
47 except (UnknownLocaleError, ValueError):
48 return None
51# Append engine_name to list of engines that support locale.
52def add_engine_counter(lang_code, engine_name, languages):
53 if lang_code in languages:
54 if 'counter' not in languages[lang_code]:
55 languages[lang_code]['counter'] = [engine_name]
56 elif engine_name not in languages[lang_code]['counter']:
57 languages[lang_code]['counter'].append(engine_name)
60# Join all language lists.
61# TODO: Add language names from engine's language list if name not known by babel.
62def join_language_lists(engines_languages):
63 language_list = {}
64 for engine_name in engines_languages:
65 for lang_code in engines_languages[engine_name]:
67 # apply custom fixes if necessary
68 if lang_code in getattr(engines[engine_name], 'language_aliases', {}).values():
69 lang_code = next(lc for lc, alias in engines[engine_name].language_aliases.items()
70 if lang_code == alias)
72 locale = get_locale(lang_code)
74 # ensure that lang_code uses standard language and country codes
75 if locale and locale.territory:
76 lang_code = locale.language + '-' + locale.territory
78 # add locale if it's not in list
79 if lang_code not in language_list:
80 if locale:
81 language_list[lang_code] = {'name': locale.get_language_name().title(),
82 'english_name': locale.english_name,
83 'country': locale.get_territory_name() or ''}
85 # also add language without country
86 if locale.language not in language_list:
87 language_list[locale.language] = {'name': locale.get_language_name().title(),
88 'english_name': locale.english_name}
89 else:
90 language_list[lang_code] = {}
92 # count engine for both language_country combination and language alone
93 add_engine_counter(lang_code, engine_name, language_list)
94 add_engine_counter(lang_code.split('-')[0], engine_name, language_list)
96 return language_list
99# Filter language list so it only includes the most supported languages and countries.
100def filter_language_list(all_languages):
101 min_supported_engines = 10
102 main_engines = [engine_name for engine_name in engines.keys()
103 if 'general' in engines[engine_name].categories and
104 engines[engine_name].supported_languages and
105 not engines[engine_name].disabled]
107 # filter list to include only languages supported by most engines or all default general engines
108 filtered_languages = {code: lang for code, lang
109 in all_languages.items()
110 if (len(lang.get('counter', [])) >= min_supported_engines or
111 all(main_engine in lang.get('counter', [])
112 for main_engine in main_engines))}
114 return filtered_languages
117# Add country codes to languages without one and filter out language codes.
118def assign_country_codes(filtered_languages, all_languages):
119 sorted_languages = sorted(all_languages,
120 key=lambda lang: len(all_languages[lang].get('counter', [])),
121 reverse=True)
122 previous_lang = None
123 previous_code = None
124 countries = 0
125 for current_code in sorted(filtered_languages):
126 current_lang = current_code.split('-')[0]
128 # count country codes per language
129 if current_lang == previous_lang:
130 countries += 1
132 else:
133 if previous_lang is not None:
134 # if language has no single country code
135 if countries == 0:
136 # try to get country code with most supported engines
137 for l in sorted_languages:
138 l_parts = l.split('-')
139 if len(l_parts) == 2 and l_parts[0] == previous_lang:
140 filtered_languages[l] = all_languages[l]
141 filtered_languages[l]['country'] = ''
142 countries = 1
143 break
145 if countries == 0:
146 # get most likely country code from babel
147 subtags = get_global('likely_subtags').get(previous_lang)
148 if subtags:
149 subtag_parts = subtags.split('_')
150 new_code = subtag_parts[0] + '-' + subtag_parts[-1]
151 filtered_languages[new_code] = all_languages[previous_lang]
152 countries = 1
154 if countries == 1:
155 # remove countryless version of language if there's only one country
156 del filtered_languages[previous_lang]
157 if previous_code in filtered_languages:
158 filtered_languages[previous_code]['country'] = ''
160 countries = 0
161 previous_lang = current_lang
163 previous_code = current_code
166# Write languages.py.
167def write_languages_file(languages):
168 new_file = open(languages_file, 'wb')
169 file_content = '# -*- coding: utf-8 -*-\n'\
170 + '# list of language codes\n'\
171 + '# this file is generated automatically by utils/update_search_languages.py\n'\
172 + '\nlanguage_codes = ('
173 for code in sorted(languages):
174 file_content += '\n (u"' + code + '"'\
175 + ', u"' + languages[code]['name'].split(' (')[0] + '"'\
176 + ', u"' + languages[code].get('country', '') + '"'\
177 + ', u"' + languages[code].get('english_name', '').split(' (')[0] + '"),'
178 # remove last comma
179 file_content = file_content[:-1]
180 file_content += '\n)\n'
181 new_file.write(file_content.encode('utf8'))
182 new_file.close()
185if __name__ == "__main__":
186 initialize_engines(settings['engines'])
187 engines_languages = fetch_supported_languages()
188 all_languages = join_language_lists(engines_languages)
189 filtered_languages = filter_language_list(all_languages)
190 assign_country_codes(filtered_languages, all_languages)
191 write_languages_file(filtered_languages)