git.iko.soy/searx
on df0d915 help sign in
searx/utils/fetch_languages.py
189 lines 6.9 kB history raw
1# -*- coding: utf-8 -*-
3# This script generates languages.py from intersecting each engine's supported languages.
4#
5# The country names are obtained from http://api.geonames.org which requires registering as a user.
6#
7# Output files (engines_languages.json and languages.py)
8# are written in current directory to avoid overwriting in case something goes wrong.
10from requests import get
11from lxml.html import fromstring
12from json import loads, dump
13import io
14from sys import path
15path.append('../searx') # noqa
16from searx import settings
17from searx.url_utils import urlencode
18from searx.engines import initialize_engines, engines
20# Geonames API for country names.
21geonames_user = '' # ADD USER NAME HERE
22country_names_url = 'http://api.geonames.org/countryInfoJSON?{parameters}'
24# Output files.
25engines_languages_file = 'engines_languages.json'
26languages_file = 'languages.py'
28engines_languages = {}
31# To filter out invalid codes and dialects.
32def valid_code(lang_code):
33 # filter invalid codes
34 # sl-SL is technically not invalid, but still a mistake
35 invalid_codes = ['sl-SL', 'wt-WT', 'jw']
36 invalid_countries = ['UK', 'XA', 'XL']
37 if lang_code[:2] == 'xx'\
38 or lang_code in invalid_codes\
39 or lang_code[-2:] in invalid_countries\
40 or is_dialect(lang_code):
41 return False
43 return True
46# Language codes with any additional tags other than language and country.
47def is_dialect(lang_code):
48 lang_code = lang_code.split('-')
49 if len(lang_code) > 2 or len(lang_code[0]) > 3:
50 return True
51 if len(lang_code) == 2 and len(lang_code[1]) > 2:
52 return True
54 return False
57# Get country name in specified language.
58def get_country_name(locale):
59 if geonames_user is '':
60 return ''
62 locale = locale.split('-')
63 if len(locale) != 2:
64 return ''
66 url = country_names_url.format(parameters=urlencode({'lang': locale[0],
67 'country': locale[1],
68 'username': geonames_user}))
69 response = get(url)
70 json = loads(response.text)
71 content = json.get('geonames', None)
72 if content is None or len(content) != 1:
73 print("No country name found for " + locale[0] + "-" + locale[1])
74 return ''
76 return content[0].get('countryName', '')
79# Fetchs supported languages for each engine and writes json file with those.
80def fetch_supported_languages():
81 initialize_engines(settings['engines'])
82 for engine_name in engines:
83 if hasattr(engines[engine_name], 'fetch_supported_languages'):
84 try:
85 engines_languages[engine_name] = engines[engine_name].fetch_supported_languages()
86 except Exception as e:
87 print(e)
89 # write json file
90 with io.open(engines_languages_file, "w", encoding="utf-8") as f:
91 dump(engines_languages, f, ensure_ascii=False)
94# Join all language lists.
95# Iterate all languages supported by each engine.
96def join_language_lists():
97 global languages
98 # include wikipedia first for more accurate language names
99 languages = {code: lang for code, lang
100 in engines_languages['wikipedia'].items()
101 if valid_code(code)}
103 for engine_name in engines_languages:
104 for locale in engines_languages[engine_name]:
105 if valid_code(locale):
106 # if language is not on list or if it has no name yet
107 if locale not in languages or not languages[locale].get('name'):
108 if isinstance(engines_languages[engine_name], dict):
109 languages[locale] = engines_languages[engine_name][locale]
110 else:
111 languages[locale] = {}
113 # add to counter of engines that support given language
114 lang = locale.split('-')[0]
115 if lang in languages:
116 if 'counter' not in languages[lang]:
117 languages[lang]['counter'] = [engine_name]
118 elif engine_name not in languages[lang]['counter']:
119 languages[lang]['counter'].append(engine_name)
121 # filter list to include only languages supported by most engines
122 min_supported_engines = int(0.70 * len(engines_languages))
123 languages = {code: lang for code, lang
124 in languages.items()
125 if len(lang.get('counter', [])) >= min_supported_engines or
126 len(languages.get(code.split('-')[0], {}).get('counter', [])) >= min_supported_engines}
128 # get locales that have no name or country yet
129 for locale in languages.keys():
130 # try to get language names
131 if not languages[locale].get('name'):
132 name = languages.get(locale.split('-')[0], {}).get('name', None)
133 if name:
134 languages[locale]['name'] = name
135 else:
136 # filter out locales with no name
137 del languages[locale]
138 continue
140 # try to get language name in english
141 if not languages[locale].get('english_name'):
142 languages[locale]['english_name'] = languages.get(locale.split('-')[0], {}).get('english_name', '')
144 # try to get country name
145 if locale.find('-') > 0 and not languages[locale].get('country'):
146 languages[locale]['country'] = get_country_name(locale) or ''
149# Remove countryless language if language is featured in only one country.
150def filter_single_country_languages():
151 prev_lang = None
152 prev_code = None
153 for code in sorted(languages):
154 lang = code.split('-')[0]
155 if lang == prev_lang:
156 countries += 1
157 else:
158 if prev_lang is not None and countries == 1:
159 del languages[prev_lang]
160 languages[prev_code]['country'] = ''
161 countries = 0
162 prev_lang = lang
163 prev_code = code
166# Write languages.py.
167def write_languages_file():
168 new_file = open(languages_file, 'wb')
169 file_content = '# -*- coding: utf-8 -*-\n'\
170 + '# list of language codes\n'\
171 + '# this file is generated automatically by utils/update_search_languages.py\n'\
172 + '\nlanguage_codes = ('
173 for code in sorted(languages):
174 file_content += '\n (u"' + code + '"'\
175 + ', u"' + languages[code]['name'].split(' (')[0] + '"'\
176 + ', u"' + languages[code].get('country', '') + '"'\
177 + ', u"' + languages[code].get('english_name', '').split(' (')[0] + '"),'
178 # remove last comma
179 file_content = file_content[:-1]
180 file_content += '\n)\n'
181 new_file.write(file_content.encode('utf8'))
182 new_file.close()
185if __name__ == "__main__":
186 fetch_supported_languages()
187 join_language_lists()
188 filter_single_country_languages()
189 write_languages_file()