git.iko.soy/searx
on baddb8d help sign in
searx/utils/fetch_languages.py
194 lines 8.1 kB history raw
1# -*- coding: utf-8 -*-
3# This script generates languages.py from intersecting each engine's supported languages.
4#
5# Output files (engines_languages.json and languages.py)
6# are written in current directory to avoid overwriting in case something goes wrong.
8import json
9import io
10from sys import path
11from babel import Locale, UnknownLocaleError
12from babel.languages import get_global
14path.append('../searx') # noqa
15from searx import settings
16from searx.engines import initialize_engines, engines
18# Output files.
19engines_languages_file = 'engines_languages.json'
20languages_file = 'languages.py'
23# Fetchs supported languages for each engine and writes json file with those.
24def fetch_supported_languages():
26 engines_languages = {}
27 names = list(engines)
28 names.sort()
30 for engine_name in names:
31 print("fetching languages of engine %s" % engine_name)
33 if hasattr(engines[engine_name], 'fetch_supported_languages'):
34 engines_languages[engine_name] = engines[engine_name].fetch_supported_languages()
35 if type(engines_languages[engine_name]) == list:
36 engines_languages[engine_name] = sorted(engines_languages[engine_name])
38 # write json file
39 with open(engines_languages_file, 'w', encoding='utf-8') as f:
40 json.dump(engines_languages, f, indent=2, sort_keys=True)
42 return engines_languages
45# Get babel Locale object from lang_code if possible.
46def get_locale(lang_code):
47 try:
48 locale = Locale.parse(lang_code, sep='-')
49 return locale
50 except (UnknownLocaleError, ValueError):
51 return None
54# Append engine_name to list of engines that support locale.
55def add_engine_counter(lang_code, engine_name, languages):
56 if lang_code in languages:
57 if 'counter' not in languages[lang_code]:
58 languages[lang_code]['counter'] = [engine_name]
59 elif engine_name not in languages[lang_code]['counter']:
60 languages[lang_code]['counter'].append(engine_name)
63# Join all language lists.
64# TODO: Add language names from engine's language list if name not known by babel.
65def join_language_lists(engines_languages):
66 language_list = {}
67 for engine_name in engines_languages:
68 for lang_code in engines_languages[engine_name]:
70 # apply custom fixes if necessary
71 if lang_code in getattr(engines[engine_name], 'language_aliases', {}).values():
72 lang_code = next(lc for lc, alias in engines[engine_name].language_aliases.items()
73 if lang_code == alias)
75 locale = get_locale(lang_code)
77 # ensure that lang_code uses standard language and country codes
78 if locale and locale.territory:
79 lang_code = locale.language + '-' + locale.territory
81 # add locale if it's not in list
82 if lang_code not in language_list:
83 if locale:
84 language_list[lang_code] = {'name': locale.get_language_name().title(),
85 'english_name': locale.english_name,
86 'country': locale.get_territory_name() or ''}
88 # also add language without country
89 if locale.language not in language_list:
90 language_list[locale.language] = {'name': locale.get_language_name().title(),
91 'english_name': locale.english_name}
92 else:
93 language_list[lang_code] = {}
95 # count engine for both language_country combination and language alone
96 add_engine_counter(lang_code, engine_name, language_list)
97 add_engine_counter(lang_code.split('-')[0], engine_name, language_list)
99 return language_list
102# Filter language list so it only includes the most supported languages and countries.
103def filter_language_list(all_languages):
104 min_supported_engines = 10
105 main_engines = [engine_name for engine_name in engines.keys()
106 if 'general' in engines[engine_name].categories and
107 engines[engine_name].supported_languages and
108 not engines[engine_name].disabled]
110 # filter list to include only languages supported by most engines or all default general engines
111 filtered_languages = {code: lang for code, lang
112 in all_languages.items()
113 if (len(lang.get('counter', [])) >= min_supported_engines or
114 all(main_engine in lang.get('counter', [])
115 for main_engine in main_engines))}
117 return filtered_languages
120# Add country codes to languages without one and filter out language codes.
121def assign_country_codes(filtered_languages, all_languages):
122 sorted_languages = sorted(all_languages,
123 key=lambda lang: len(all_languages[lang].get('counter', [])),
124 reverse=True)
125 previous_lang = None
126 previous_code = None
127 countries = 0
128 for current_code in sorted(filtered_languages):
129 current_lang = current_code.split('-')[0]
131 # count country codes per language
132 if current_lang == previous_lang:
133 countries += 1
135 else:
136 if previous_lang is not None:
137 # if language has no single country code
138 if countries == 0:
139 # try to get country code with most supported engines
140 for l in sorted_languages:
141 l_parts = l.split('-')
142 if len(l_parts) == 2 and l_parts[0] == previous_lang:
143 filtered_languages[l] = all_languages[l]
144 filtered_languages[l]['country'] = ''
145 countries = 1
146 break
148 if countries == 0:
149 # get most likely country code from babel
150 subtags = get_global('likely_subtags').get(previous_lang)
151 if subtags:
152 subtag_parts = subtags.split('_')
153 new_code = subtag_parts[0] + '-' + subtag_parts[-1]
154 filtered_languages[new_code] = all_languages[previous_lang]
155 countries = 1
157 if countries == 1:
158 # remove countryless version of language if there's only one country
159 del filtered_languages[previous_lang]
160 if previous_code in filtered_languages:
161 filtered_languages[previous_code]['country'] = ''
163 countries = 0
164 previous_lang = current_lang
166 previous_code = current_code
169# Write languages.py.
170def write_languages_file(languages):
171 new_file = open(languages_file, 'wb')
172 file_content = '# -*- coding: utf-8 -*-\n'\
173 + '# list of language codes\n'\
174 + '# this file is generated automatically by utils/update_search_languages.py\n'\
175 + '\nlanguage_codes = ('
176 for code in sorted(languages):
177 file_content += '\n (u"' + code + '"'\
178 + ', u"' + languages[code]['name'].split(' (')[0] + '"'\
179 + ', u"' + languages[code].get('country', '') + '"'\
180 + ', u"' + languages[code].get('english_name', '').split(' (')[0] + '"),'
181 # remove last comma
182 file_content = file_content[:-1]
183 file_content += '\n)\n'
184 new_file.write(file_content.encode('utf8'))
185 new_file.close()
188if __name__ == "__main__":
189 initialize_engines(settings['engines'])
190 engines_languages = fetch_supported_languages()
191 all_languages = join_language_lists(engines_languages)
192 filtered_languages = filter_language_list(all_languages)
193 assign_country_codes(filtered_languages, all_languages)
194 write_languages_file(filtered_languages)