git.iko.soy/searx
on master help sign in
[enh] add supported_languages on engines and auto-generate languages.py
commit149802c56926bf48520c98932c4c36b8152b3d2d authormarc <a01200356@itesm.mx> dateFri 5 Aug 2016, 23:34 (10 years ago) parente58949b
34 files changed+666−128
compare with master
unifiedsplit
searx/autocomplete.py+3−3 searx/engines/__init__.py+1 searx/engines/archlinux.py+3−2 searx/engines/bing.py+1−1 searx/engines/bing_images.py+2−1 searx/engines/bing_news.py+2−1 searx/engines/duckduckgo.py+40−2 searx/engines/duckduckgo_definitions.py+2−1 searx/engines/gigablast.py+1−1 searx/engines/google.py+14 searx/engines/google_news.py+3−1 searx/engines/mediawiki.py+2−1 searx/engines/photon.py+2−2 searx/engines/startpage.py+1−1 searx/engines/subtitleseeker.py+7−2 searx/engines/swisscows.py+2−2 searx/engines/twitter.py+1−1 searx/engines/wikidata.py+2 searx/engines/wikipedia.py+33−2 searx/engines/yacy.py+1−1 searx/engines/yahoo.py+11−1 searx/engines/yahoo_news.py+1−1 searx/engines/yandex.py+1−1 searx/engines/youtube_api.py+1−1 searx/languages.py+388−76 searx/query.py+7−4 searx/templates/courgette/preferences.html+3−3 searx/templates/legacy/preferences.html+3−3 searx/templates/oscar/advanced.html+1 searx/templates/oscar/languages.html+12 searx/templates/oscar/preferences.html+5−6 searx/templates/pix-art/preferences.html+3−3 searx/webapp.py+8−4 utils/update_languages.py+99
searx/autocomplete.py+3−3
@@ -81,17 +81,17 @@ def searx_bang(full_query):
8181 engine_query = full_query.getSearchQuery()[1:]
8282
8383 for lc in language_codes:
84− lang_id, lang_name, country = map(str.lower, lc)
84+ lang_id, lang_name, country, english_name = map(str.lower, lc)
8585
8686 # check if query starts with language-id
8787 if lang_id.startswith(engine_query):
8888 if len(engine_query) <= 2:
89− results.append(':{lang_id}'.format(lang_id=lang_id.split('_')[0]))
89+ results.append(':{lang_id}'.format(lang_id=lang_id.split('-')[0]))
9090 else:
9191 results.append(':{lang_id}'.format(lang_id=lang_id))
9292
9393 # check if query starts with language name
94− if lang_name.startswith(engine_query):
94+ if lang_name.startswith(engine_query) or english_name.startswith(engine_query):
9595 results.append(':{lang_name}'.format(lang_name=lang_name))
9696
9797 # check if query starts with country
searx/engines/__init__.py+1
@@ -38,6 +38,7 @@ engine_shortcuts = {}
3838engine_default_args = {'paging': False,
3939 'categories': ['general'],
4040 'language_support': True,
41+ 'supported_languages': [],
4142 'safesearch': False,
4243 'timeout': settings['outgoing']['request_timeout'],
4344 'shortcut': '-',
searx/engines/archlinux.py+3−2
@@ -29,8 +29,8 @@ xpath_link = './/div[@class="mw-search-result-heading"]/a'
2929
3030# cut 'en' from 'en_US', 'de' from 'de_CH', and so on
3131def locale_to_lang_code(locale):
32− if locale.find('_') >= 0:
33− locale = locale.split('_')[0]
32+ if locale.find('-') >= 0:
33+ locale = locale.split('-')[0]
3434 return locale
3535
3636
@@ -95,6 +95,7 @@ main_langs = {
9595 'uk': 'Українська',
9696 'zh': '简体中文'
9797}
98+supported_languages = dict(lang_urls, **main_langs)
9899
99100
100101# do search-request
searx/engines/bing.py+1−1
@@ -32,7 +32,7 @@ def request(query, params):
3232 offset = (params['pageno'] - 1) * 10 + 1
3333
3434 if params['language'] != 'all':
35− query = u'language:{} {}'.format(params['language'].split('_')[0].upper(),
35+ query = u'language:{} {}'.format(params['language'].split('-')[0].upper(),
3636 query.decode('utf-8')).encode('utf-8')
3737
3838 search_path = search_string.format(
searx/engines/bing_images.py+2−1
@@ -19,6 +19,7 @@ from urllib import urlencode
1919from lxml import html
2020from json import loads
2121import re
22+from searx.engines.bing import supported_languages
2223
2324# engine dependent config
2425categories = ['images']
@@ -53,7 +54,7 @@ def request(query, params):
5354 if params['language'] == 'all':
5455 language = 'en-US'
5556 else:
56− language = params['language'].replace('_', '-')
57+ language = params['language']
5758
5859 search_path = search_string.format(
5960 query=urlencode({'q': query}),
searx/engines/bing_news.py+2−1
@@ -17,6 +17,7 @@ from datetime import datetime
1717from dateutil import parser
1818from lxml import etree
1919from searx.utils import list_get
20+from searx.engines.bing import supported_languages
2021
2122# engine dependent config
2223categories = ['news']
@@ -74,7 +75,7 @@ def request(query, params):
7475 if params['language'] == 'all':
7576 language = 'en-US'
7677 else:
77− language = params['language'].replace('_', '-')
78+ language = params['language']
7879
7980 params['url'] = _get_url(query, language, offset, params['time_range'])
8081
searx/engines/duckduckgo.py+40−2
@@ -22,6 +22,13 @@ from searx.languages import language_codes
2222categories = ['general']
2323paging = True
2424language_support = True
25+supported_languages = ["es-AR", "en-AU", "de-AT", "fr-BE", "nl-BE", "pt-BR", "bg-BG", "en-CA", "fr-CA", "ca-CT",
26+ "es-CL", "zh-CN", "es-CO", "hr-HR", "cs-CZ", "da-DK", "et-EE", "fi-FI", "fr-FR", "de-DE",
27+ "el-GR", "tzh-HK", "hu-HU", "en-IN", "id-ID", "en-ID", "en-IE", "he-IL", "it-IT", "jp-JP",
28+ "kr-KR", "es-XL", "lv-LV", "lt-LT", "ms-MY", "en-MY", "es-MX", "nl-NL", "en-NZ", "no-NO",
29+ "es-PE", "en-PH", "tl-PH", "pl-PL", "pt-PT", "ro-RO", "ru-RU", "ar-XA", "en-XA", "en-SG",
30+ "sk-SK", "sl-SL", "en-ZA", "es-ES", "ca-ES", "sv-SE", "de-CH", "fr-CH", "it-CH", "tzh-TW",
31+ "th-TH", "tr-TR", "uk-UA", "en-UK", "en-US", "es-US", "vi-VN"]
2532time_range_support = True
2633
2734# search-url
@@ -46,10 +53,23 @@ def request(query, params):
4653
4754 offset = (params['pageno'] - 1) * 30
4855
56+ # custom fixes for languages
4957 if params['language'] == 'all':
5058 locale = None
59+ elif params['language'][:2] == 'ja':
60+ locale = 'jp-jp'
61+ elif params['language'] == 'zh-TW':
62+ locale = 'tw-tzh'
63+ elif params['language'] == 'zh-HK':
64+ locale = 'hk-tzh'
65+ elif params['language'][-2:] == 'SA':
66+ locale = 'xa' + params['language'].split('-')[0]
67+ elif params['language'][-2:] == 'GB':
68+ locale = 'uk' + params['language'].split('-')[0]
69+ elif params['language'] == 'es-419':
70+ locale = 'xl-es'
5171 else:
52− locale = params['language'].split('_')
72+ locale = params['language'].split('-')
5373 if len(locale) == 2:
5474 # country code goes first
5575 locale = locale[1].lower() + '-' + locale[0].lower()
@@ -58,7 +78,25 @@ def request(query, params):
5878 locale = locale[0].lower()
5979 lang_codes = [x[0] for x in language_codes]
6080 for lc in lang_codes:
61− lc = lc.split('_')
81+ lc = lc.split('-')
82+ if locale == lc[0] and len(lc) == 2:
83+ locale = lc[1].lower() + '-' + lc[0].lower()
84+ break
85+
86+ if locale:
87+ params['url'] = url.format(
88+ query=urlencode({'q': query, 'kl': locale}), offset=offset)
89+ else:
90+ locale = params['language'].split('-')
91+ if len(locale) == 2:
92+ # country code goes first
93+ locale = locale[1].lower() + '-' + locale[0].lower()
94+ else:
95+ # tries to get a country code from language
96+ locale = locale[0].lower()
97+ lang_codes = [x[0] for x in language_codes]
98+ for lc in lang_codes:
99+ lc = lc.split('-')
62100 if locale == lc[0]:
63101 locale = lc[1].lower() + '-' + lc[0].lower()
64102 break
searx/engines/duckduckgo_definitions.py+2−1
@@ -4,6 +4,7 @@ from re import compile, sub
44from lxml import html
55from searx.utils import html_to_text
66from searx.engines.xpath import extract_text
7+from searx.engines.duckduckgo import supported_languages
78
89url = 'https://api.duckduckgo.com/'\
910 + '?{query}&format=json&pretty=0&no_redirect=1&d=1'
@@ -23,7 +24,7 @@ def result_to_text(url, text, htmlResult):
2324
2425def request(query, params):
2526 params['url'] = url.format(query=urlencode({'q': query}))
26− params['headers']['Accept-Language'] = params['language']
27+ params['headers']['Accept-Language'] = params['language'].split('-')[0]
2728 return params
2829
2930
searx/engines/gigablast.py+1−1
@@ -48,7 +48,7 @@ def request(query, params):
4848 if params['language'] == 'all':
4949 language = 'xx'
5050 else:
51− language = params['language'][0:2]
51+ language = params['language'].split('-')[0]
5252
5353 if params['safesearch'] >= 1:
5454 safesearch = 1
searx/engines/google.py+14
@@ -23,6 +23,20 @@ categories = ['general']
2323paging = True
2424language_support = True
2525use_locale_domain = True
26+supported_languages = ['de', 'en', 'es', 'es_419', 'fr', 'hr', 'it', 'nl', 'pl', 'pt-BR',
27+ 'pt-PT', 'vi', 'tr', 'ru', 'ar', 'th', 'ko', 'zh-CN', 'zh-TW', 'ja',
28+ 'ach', 'af', 'ak', 'az', 'ms', 'ban', 'xx_bork', 'bs', 'br', 'ca',
29+ 'ceb', 'ckb', 'cs', 'sn', 'co', 'cy', 'da', 'yo', 'et', 'xx_elmer',
30+ 'eo', 'eu', 'ee', 'tl', 'fo', 'gaa', 'ga', 'gd', 'gl', 'gn', 'xx_hacker',
31+ 'ht', 'ha', 'haw', 'bem', 'ig', 'rn', 'id', 'ia', 'zu', 'is', 'jw', 'rw',
32+ 'sw', 'tlh', 'kg', 'mfe', 'kri', 'la', 'lv', 'to', 'lt', 'ln', 'loz',
33+ 'lua', 'lg', 'hu', 'mg', 'mt', 'mi', 'pcm', 'no', 'nso', 'ny', 'nn',
34+ 'uz', 'oc', 'om', 'xx_pirate', 'pt', 'ro', 'mo', 'rm', 'qu', 'nyn', 'crs',
35+ 'sq', 'sd', 'sk', 'sl', 'so', 'st', 'sr_ME', 'sr_Latn', 'su', 'fi', 'sv',
36+ 'tg', 'tt', 'tn', 'tum', 'tk', 'tw', 'fy', 'wo', 'xh', 'el', 'be', 'bg',
37+ 'ky', 'kk', 'mk', 'mn', 'sr', 'uk', 'ka', 'hy', 'yi', 'iw', 'ug', 'ur',
38+ 'ps', 'fa', 'ti', 'am', 'ne', 'mr', 'hi', 'bn', 'pa', 'gu', 'or', 'ta',
39+ 'te', 'kn', 'ml', 'si', 'lo', 'my', 'km', 'chr']
2640time_range_support = True
2741
2842# based on https://en.wikipedia.org/wiki/List_of_Google_domains and tests
searx/engines/google_news.py+3−1
@@ -12,6 +12,8 @@
1212
1313from lxml import html
1414from urllib import urlencode
15+from json import loads
16+from searx.engines.google import supported_languages
1517
1618# search-url
1719categories = ['news']
@@ -50,7 +52,7 @@ def request(query, params):
5052 search_options=urlencode(search_options))
5153
5254 if params['language'] != 'all':
53− language_array = params['language'].lower().split('_')
55+ language_array = params['language'].lower().split('-')
5456 params['url'] += '&lr=lang_' + language_array[0]
5557
5658 return params
searx/engines/mediawiki.py+2−1
@@ -15,6 +15,7 @@
1515from json import loads
1616from string import Formatter
1717from urllib import urlencode, quote
18+from searx.engines.wikipedia import supported_engines
1819
1920# engine dependent config
2021categories = ['general']
@@ -46,7 +47,7 @@ def request(query, params):
4647 if params['language'] == 'all':
4748 language = 'en'
4849 else:
49− language = params['language'].split('_')[0]
50+ language = params['language'].split('-')[0]
5051
5152 # format_string [('https://', 'language', '', None), ('.wikipedia.org/', None, None, None)]
5253 if any(x[1] == 'language' for x in format_strings):
searx/engines/photon.py+2−2
@@ -26,7 +26,7 @@ search_string = 'api/?{query}&limit={limit}'
2626result_base_url = 'https://openstreetmap.org/{osm_type}/{osm_id}'
2727
2828# list of supported languages
29−allowed_languages = ['de', 'en', 'fr', 'it']
29+supported_languages = ['de', 'en', 'fr', 'it']
3030
3131
3232# do search-request
@@ -37,7 +37,7 @@ def request(query, params):
3737
3838 if params['language'] != 'all':
3939 language = params['language'].split('_')[0]
40− if language in allowed_languages:
40+ if language in supported_languages:
4141 params['url'] = params['url'] + "&lang=" + language
4242
4343 # using searx User-Agent
searx/engines/startpage.py+1−1
@@ -47,7 +47,7 @@ def request(query, params):
4747
4848 # set language if specified
4949 if params['language'] != 'all':
50− params['data']['with_language'] = ('lang_' + params['language'].split('_')[0])
50+ params['data']['with_language'] = ('lang_' + params['language'].split('-')[0])
5151
5252 return params
5353
searx/engines/subtitleseeker.py+7−2
@@ -43,8 +43,13 @@ def response(resp):
4343
4444 search_lang = ""
4545
46− if resp.search_params['language'] != 'all':
47− search_lang = [lc[1]
46+ # dirty fix for languages named differenly in their site
47+ if resp.search_params['language'][:2] == 'fa':
48+ search_lang = 'Farsi'
49+ elif resp.search_params['language'] == 'pt_BR':
50+ search_lang = 'Brazilian'
51+ elif resp.search_params['language'] != 'all':
52+ search_lang = [lc[3]
4853 for lc in language_codes
4954 if lc[0][:2] == resp.search_params['language'].split('_')[0]][0]
5055
searx/engines/swisscows.py+2−2
@@ -36,8 +36,8 @@ def request(query, params):
3636 ui_language = 'browser'
3737 region = 'browser'
3838 else:
39− region = params['language'].replace('_', '-')
40− ui_language = params['language'].split('_')[0]
39+ region = params['language']
40+ ui_language = params['language'].split('-')[0]
4141
4242 search_path = search_string.format(
4343 query=urlencode({'query': query,
searx/engines/twitter.py+1−1
@@ -40,7 +40,7 @@ def request(query, params):
4040
4141 # set language if specified
4242 if params['language'] != 'all':
43− params['cookies']['lang'] = params['language'].split('_')[0]
43+ params['cookies']['lang'] = params['language'].split('-')[0]
4444 else:
4545 params['cookies']['lang'] = 'en'
4646
searx/engines/wikidata.py+2
@@ -14,6 +14,8 @@
1414from searx import logger
1515from searx.poolrequests import get
1616from searx.engines.xpath import extract_text
17+from searx.utils import format_date_by_locale
18+from searx.engines.wikipedia import supported_languages
1719
1820from json import loads
1921from lxml.html import fromstring
searx/engines/wikipedia.py+33−2
@@ -13,6 +13,36 @@
1313from json import loads
1414from urllib import urlencode, quote
1515
16+supported_languages = ["en", "sv", "ceb", "de", "nl", "fr", "ru", "it", "es", "war",
17+ "pl", "vi", "ja", "pt", "zh", "uk", "ca", "fa", "no", "sh",
18+ "ar", "fi", "hu", "id", "ro", "cs", "ko", "sr", "ms", "tr",
19+ "eu", "eo", "min", "bg", "da", "kk", "sk", "hy", "he", "zh-min-nan",
20+ "lt", "hr", "sl", "et", "ce", "gl", "nn", "uz", "la", "vo",
21+ "el", "simple", "be", "az", "th", "ur", "ka", "hi", "oc", "ta",
22+ "mk", "mg", "new", "lv", "cy", "bs", "tt", "tl", "te", "pms",
23+ "be-tarask", "br", "sq", "ky", "ht", "jv", "tg", "ast", "zh-yue", "lb",
24+ "mr", "ml", "bn", "pnb", "is", "af", "sco", "ga", "ba", "fy",
25+ "cv", "lmo", "sw", "my", "an", "yo", "ne", "io", "gu", "nds",
26+ "scn", "bpy", "pa", "ku", "als", "kn", "bar", "ia", "qu", "su",
27+ "ckb", "bat-smg", "mn", "arz", "nap", "wa", "bug", "gd", "yi", "map-bms",
28+ "am", "mzn", "fo", "si", "nah", "li", "sah", "vec", "hsb", "or",
29+ "os", "mrj", "sa", "hif", "mhr", "roa-tara", "azb", "pam", "ilo",
30+ "sd", "ps", "se", "mi", "bh", "eml", "bcl", "xmf", "diq", "hak",
31+ "gan", "glk", "vls", "nds-nl", "rue", "bo", "fiu-vro", "co", "sc",
32+ "tk", "csb", "lrc", "vep", "wuu", "km", "szl", "gv", "crh", "kv",
33+ "zh-classical", "frr", "zea", "as", "so", "kw", "nso", "ay", "stq",
34+ "udm", "cdo", "nrm", "ie", "koi", "rm", "pcd", "myv", "mt", "fur",
35+ "ace", "lad", "gn", "lij", "dsb", "dv", "cbk-zam", "ext", "gom",
36+ "kab", "ksh", "ang", "mai", "mwl", "lez", "gag", "ln", "ug", "pi",
37+ "pag", "frp", "sn", "nv", "av", "pfl", "haw", "xal", "krc", "kaa",
38+ "rw", "bxr", "pdc", "to", "kl", "nov", "arc", "kbd", "lo", "bjn",
39+ "pap", "ha", "tet", "ki", "tyv", "tpi", "na", "lbe", "ig", "jbo",
40+ "roa-rup", "ty", "jam", "za", "kg", "mdf", "lg", "wo", "srn", "ab",
41+ "ltg", "zu", "sm", "chr", "om", "tn", "chy", "rmy", "cu", "tw", "tum",
42+ "xh", "bi", "rn", "pih", "got", "ss", "pnt", "bm", "ch", "mo", "ts",
43+ "ady", "iu", "st", "ee", "ny", "fj", "ks", "ak", "ik", "sg", "ve",
44+ "dz", "ff", "ti", "cr", "ng", "cho", "kj", "mh", "ho", "ii", "aa", "mus", "hz", "kr"]
45+
1646# search-url
1747base_url = 'https://{language}.wikipedia.org/'
1848search_postfix = 'w/api.php?'\
@@ -28,10 +58,11 @@ search_postfix = 'w/api.php?'\
2858
2959# set language in base_url
3060def url_lang(lang):
31− if lang == 'all':
61+ lang = lang.split('-')[0]
62+ if lang == 'all' or lang not in supported_languages:
3263 language = 'en'
3364 else:
34− language = lang.split('_')[0]
65+ language = lang
3566
3667 return base_url.format(language=language)
3768
searx/engines/yacy.py+1−1
@@ -53,7 +53,7 @@ def request(query, params):
5353
5454 # add language tag if specified
5555 if params['language'] != 'all':
56− params['url'] += '&lr=lang_' + params['language'].split('_')[0]
56+ params['url'] += '&lr=lang_' + params['language'].split('-')[0]
5757
5858 return params
5959
searx/engines/yahoo.py+11−1
@@ -20,6 +20,10 @@ from searx.engines.xpath import extract_text, extract_url
2020categories = ['general']
2121paging = True
2222language_support = True
23+supported_languages = ["ar", "bg", "ca", "szh", "tzh", "hr", "cs", "da", "nl", "en",
24+ "et", "fi", "fr", "de", "el", "he", "hu", "is", "id", "it", "ja",
25+ "ko", "lv", "lt", "no", "fa", "pl", "pt", "ro", "ru", "sk", "sr",
26+ "sl", "es", "sv", "th", "tr"]
2327time_range_support = True
2428
2529# search-url
@@ -72,7 +76,13 @@ def _get_url(query, offset, language, time_range):
7276def _get_language(params):
7377 if params['language'] == 'all':
7478 return 'en'
75− return params['language'].split('_')[0]
79+ elif params['language'][:2] == 'zh':
80+ if params['language'] == 'zh' or params['language'] == 'zh-CH':
81+ return 'szh'
82+ else:
83+ return 'tzh'
84+ else:
85+ return params['language'].split('-')[0]
7686
7787
7888# do search-request
searx/engines/yahoo_news.py+1−1
@@ -12,7 +12,7 @@
1212from urllib import urlencode
1313from lxml import html
1414from searx.engines.xpath import extract_text, extract_url
15−from searx.engines.yahoo import parse_url
15+from searx.engines.yahoo import parse_url, supported_languages
1616from datetime import datetime, timedelta
1717import re
1818from dateutil import parser
searx/engines/yandex.py+1−1
@@ -36,7 +36,7 @@ content_xpath = './/div[@class="text-container typo typo_text_m typo_line_m orga
3636
3737
3838def request(query, params):
39− lang = params['language'].split('_')[0]
39+ lang = params['language'].split('-')[0]
4040 host = base_url.format(tld=language_map.get(lang) or default_tld)
4141 params['url'] = host + search_url.format(page=params['pageno'] - 1,
4242 query=urlencode({'text': query}))
searx/engines/youtube_api.py+1−1
@@ -36,7 +36,7 @@ def request(query, params):
3636
3737 # add language tag if specified
3838 if params['language'] != 'all':
39− params['url'] += '&relevanceLanguage=' + params['language'].split('_')[0]
39+ params['url'] += '&relevanceLanguage=' + params['language'].split('-')[0]
4040
4141 return params
4242
searx/languages.py+388−76
@@ -1,78 +1,390 @@
1−'''
2−searx is free software: you can redistribute it and/or modify
3−it under the terms of the GNU Affero General Public License as published by
4−the Free Software Foundation, either version 3 of the License, or
5−(at your option) any later version.
6−
7−searx is distributed in the hope that it will be useful,
8−but WITHOUT ANY WARRANTY; without even the implied warranty of
9−MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10−GNU Affero General Public License for more details.
11−
12−You should have received a copy of the GNU Affero General Public License
13−along with searx. If not, see < http://www.gnu.org/licenses/ >.
14−
15−(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
16−'''
17−
1+# -*- coding: utf-8 -*-
182# list of language codes
3+# this file is generated automatically by utils/update_search_languages.py
4+
195language_codes = (
20− ("ar_XA", "Arabic", "Arabia"),
21− ("bg_BG", "Bulgarian", "Bulgaria"),
22− ("cs_CZ", "Czech", "Czech Republic"),
23− ("da_DK", "Danish", "Denmark"),
24− ("de_AT", "German", "Austria"),
25− ("de_CH", "German", "Switzerland"),
26− ("de_DE", "German", "Germany"),
27− ("el_GR", "Greek", "Greece"),
28− ("en_AU", "English", "Australia"),
29− ("en_CA", "English", "Canada"),
30− ("en_GB", "English", "United Kingdom"),
31− ("en_ID", "English", "Indonesia"),
32− ("en_IE", "English", "Ireland"),
33− ("en_IN", "English", "India"),
34− ("en_MY", "English", "Malaysia"),
35− ("en_NZ", "English", "New Zealand"),
36− ("en_PH", "English", "Philippines"),
37− ("en_SG", "English", "Singapore"),
38− ("en_US", "English", "United States"),
39− ("en_XA", "English", "Arabia"),
40− ("en_ZA", "English", "South Africa"),
41− ("es_AR", "Spanish", "Argentina"),
42− ("es_CL", "Spanish", "Chile"),
43− ("es_ES", "Spanish", "Spain"),
44− ("es_MX", "Spanish", "Mexico"),
45− ("es_US", "Spanish", "United States"),
46− ("es_XL", "Spanish", "Latin America"),
47− ("et_EE", "Estonian", "Estonia"),
48− ("fi_FI", "Finnish", "Finland"),
49− ("fr_BE", "French", "Belgium"),
50− ("fr_CA", "French", "Canada"),
51− ("fr_CH", "French", "Switzerland"),
52− ("fr_FR", "French", "France"),
53− ("he_IL", "Hebrew", "Israel"),
54− ("hr_HR", "Croatian", "Croatia"),
55− ("hu_HU", "Hungarian", "Hungary"),
56− ("it_IT", "Italian", "Italy"),
57− ("ja_JP", "Japanese", "Japan"),
58− ("ko_KR", "Korean", "Korea"),
59− ("lt_LT", "Lithuanian", "Lithuania"),
60− ("lv_LV", "Latvian", "Latvia"),
61− ("nb_NO", "Norwegian", "Norway"),
62− ("nl_BE", "Dutch", "Belgium"),
63− ("nl_NL", "Dutch", "Netherlands"),
64− ("oc_OC", "Occitan", "Occitan"),
65− ("pl_PL", "Polish", "Poland"),
66− ("pt_BR", "Portuguese", "Brazil"),
67− ("pt_PT", "Portuguese", "Portugal"),
68− ("ro_RO", "Romanian", "Romania"),
69− ("ru_RU", "Russian", "Russia"),
70− ("sk_SK", "Slovak", "Slovak Republic"),
71− ("sl_SL", "Slovenian", "Slovenia"),
72− ("sv_SE", "Swedish", "Sweden"),
73− ("th_TH", "Thai", "Thailand"),
74− ("tr_TR", "Turkish", "Turkey"),
75− ("uk_UA", "Ukrainian", "Ukraine"),
76− ("zh_CN", "Chinese", "China"),
77− ("zh_HK", "Chinese", "Hong Kong SAR"),
78− ("zh_TW", "Chinese", "Taiwan"))
6+ (u"gv", u"Gaelg", u"", u"Manx"),
7+ (u"sco", u"Scots", u"", u"Scots"),
8+ (u"scn", u"Sicilianu", u"", u"Sicilian"),
9+ (u"gu", u"ગુજરાતી", u"", u"Gujarati"),
10+ (u"gd", u"Gàidhlig", u"", u"Scottish Gaelic"),
11+ (u"ga", u"Gaeilge", u"", u"Irish"),
12+ (u"gn", u"Avañe'ẽ", u"", u"Guarani"),
13+ (u"gl", u"Galego", u"", u"Galician"),
14+ (u"als", u"Alemannisch", u"", u"Alemannic"),
15+ (u"lt-LT", u"Lietuvių", u"", u"Lithuanian"),
16+ (u"vep", u"Vepsän", u"", u"Vepsian"),
17+ (u"ty", u"Reo Mā`ohi", u"", u"Tahitian"),
18+ (u"tw", u"Twi", u"", u"Twi"),
19+ (u"tt", u"Tatarça / Татарча", u"", u"Tatar"),
20+ (u"tr", u"Türkçe", u"", u"Turkish"),
21+ (u"ts", u"Xitsonga", u"", u"Tsonga"),
22+ (u"tn", u"Setswana", u"", u"Tswana"),
23+ (u"to", u"faka Tonga", u"", u"Tongan"),
24+ (u"tl", u"Tagalog", u"", u"Tagalog"),
25+ (u"vec", u"Vèneto", u"", u"Venetian"),
26+ (u"th", u"ไทย", u"", u"Thai"),
27+ (u"ti", u"ትግርኛ", u"", u"Tigrinya"),
28+ (u"tg", u"Тоҷикӣ", u"", u"Tajik"),
29+ (u"te", u"తెలుగు", u"", u"Telugu"),
30+ (u"ta", u"தமிழ்", u"", u"Tamil"),
31+ (u"lrc", u"لۊری شومالی", u"", u"Northern Luri"),
32+ (u"en-NZ", u"English", u"", u"English"),
33+ (u"got", u"𐌲𐌿𐍄𐌹𐍃𐌺", u"", u"Gothic"),
34+ (u"vls", u"West-Vlams", u"", u"West Flemish"),
35+ (u"ro", u"Română", u"", u"Romanian"),
36+ (u"bxr", u"Буряад", u"", u"Buryat"),
37+ (u"fiu-vro", u"Võro", u"", u"Võro"),
38+ (u"diq", u"Zazaki", u"", u"Zazaki"),
39+ (u"zh", u"中文", u"", u"Chinese"),
40+ (u"pms", u"Piemontèis", u"", u"Piedmontese"),
41+ (u"za", u"Cuengh", u"", u"Zhuang"),
42+ (u"zh-HK", u"中文", u"", u"Chinese"),
43+ (u"zu", u"isiZulu", u"", u"Zulu"),
44+ (u"tet", u"Tetun", u"", u"Tetum"),
45+ (u"es-PE", u"Español", u"", u"Spanish"),
46+ (u"new", u"नेपाल भाषा", u"", u"Newar"),
47+ (u"lez", u"Лезги чІал (Lezgi č’al)", u"", u"Lezgian"),
48+ (u"glk", u"گیلکی", u"", u"Gilaki"),
49+ (u"ko-KR", u"한국어", u"", u"Korean"),
50+ (u"id-ID", u"Bahasa Indonesia", u"", u"Indonesian"),
51+ (u"cho", u"Choctaw", u"", u"Choctaw"),
52+ (u"chr", u"ᏣᎳᎩ", u"", u"Cherokee"),
53+ (u"vi", u"Tiếng Việt", u"", u"Vietnamese"),
54+ (u"chy", u"Tsetsêhestâhese", u"", u"Cheyenne"),
55+ (u"is", u"Íslenska", u"", u"Icelandic"),
56+ (u"tk", u"تركمن / Туркмен", u"", u"Turkmen"),
57+ (u"da-DK", u"Dansk", u"", u"Danish"),
58+ (u"pfl", u"Pälzisch", u"", u"Palatinate German"),
59+ (u"hu-HU", u"Magyar", u"", u"Hungarian"),
60+ (u"he-IL", u"עברית", u"", u"Hebrew"),
61+ (u"mg", u"Malagasy", u"", u"Malagasy"),
62+ (u"ml", u"മലയാളം", u"", u"Malayalam"),
63+ (u"mo", u"Молдовеняскэ", u"", u"Moldovan"),
64+ (u"mn", u"Монгол", u"", u"Mongolian"),
65+ (u"mi", u"Māori", u"", u"Maori"),
66+ (u"mh", u"Ebon", u"", u"Marshallese"),
67+ (u"mk", u"Македонски", u"", u"Macedonian"),
68+ (u"mt", u"Malti", u"", u"Maltese"),
69+ (u"ms", u"Bahasa Melayu", u"", u"Malay"),
70+ (u"mr", u"मराठी", u"", u"Marathi"),
71+ (u"mwl", u"Mirandés", u"", u"Mirandese"),
72+ (u"my", u"မြန်မာဘာသာ", u"", u"Burmese"),
73+ (u"en-PH", u"English", u"", u"English"),
74+ (u"srn", u"Sranantongo", u"", u"Sranan"),
75+ (u"pl-PL", u"Polski", u"", u"Polish"),
76+ (u"sl-SL", u"Slovenščina", u"", u"Slovenian"),
77+ (u"csb", u"Kaszëbsczi", u"", u"Kashubian"),
78+ (u"cbk-zam", u"Chavacano de Zamboanga", u"", u"Zamboanga Chavacano"),
79+ (u"nyn", u"Runyankore", u"", u""),
80+ (u"ig", u"Igbo", u"", u"Igbo"),
81+ (u"fr", u"Français", u"", u"French"),
82+ (u"lad", u"Dzhudezmo", u"", u"Ladino"),
83+ (u"fy", u"Frysk", u"", u"West Frisian"),
84+ (u"fa", u"فارسی", u"", u"Persian"),
85+ (u"ff", u"Fulfulde", u"", u"Fula"),
86+ (u"mai", u"मैथिली", u"", u"Maithili"),
87+ (u"fi", u"Suomi", u"", u"Finnish"),
88+ (u"fj", u"Na Vosa Vakaviti", u"", u"Fijian"),
89+ (u"fo", u"Føroyskt", u"", u"Faroese"),
90+ (u"ss", u"SiSwati", u"", u"Swati"),
91+ (u"roa-tara", u"Tarandíne", u"", u"Tarantino"),
92+ (u"sq", u"Shqip", u"", u"Albanian"),
93+ (u"sw", u"Kiswahili", u"", u"Swahili"),
94+ (u"sv", u"Svenska", u"", u"Swedish"),
95+ (u"su", u"Basa Sunda", u"", u"Sundanese"),
96+ (u"st", u"Sesotho", u"", u"Sesotho"),
97+ (u"sk", u"Slovenčina", u"", u"Slovak"),
98+ (u"si", u"සිංහල", u"", u"Sinhalese"),
99+ (u"sh", u"Srpskohrvatski / Српскохрватски", u"", u"Serbo-Croatian"),
100+ (u"so", u"Soomaali", u"", u"Somali"),
101+ (u"sn", u"chiShona", u"", u"Shona"),
102+ (u"sm", u"Gagana Samoa", u"", u"Samoan"),
103+ (u"sl", u"Slovenščina", u"", u"Slovenian"),
104+ (u"sc", u"Sardu", u"", u"Sardinian"),
105+ (u"pt-BR", u"português (Brasil)", u"", u""),
106+ (u"sa", u"संस्कृतम्", u"", u"Sanskrit"),
107+ (u"sg", u"Sängö", u"", u"Sango"),
108+ (u"se", u"Sámegiella", u"", u"Northern Sami"),
109+ (u"sd", u"سنڌي، سندھی ، सिन्ध", u"", u"Sindhi"),
110+ (u"fr-CH", u"Français", u"", u"French"),
111+ (u"zea", u"Zeêuws", u"", u"Zeelandic"),
112+ (u"it-CH", u"Italiano", u"", u"Italian"),
113+ (u"wuu", u"吴语", u"", u"Wu"),
114+ (u"fr-CA", u"Français", u"", u"French"),
115+ (u"ar-XA", u"العربية", u"", u"Arabic"),
116+ (u"kbd", u"Адыгэбзэ (Adighabze)", u"", u"Kabardian Circassian"),
117+ (u"no-NO", u"Norsk (Bokmål)", u"", u"Norwegian (Bokmål)"),
118+ (u"ca-ES", u"Català", u"", u"Catalan"),
119+ (u"lg", u"Luganda", u"", u"Luganda"),
120+ (u"lb", u"Lëtzebuergesch", u"", u"Luxembourgish"),
121+ (u"la", u"Latina", u"", u"Latin"),
122+ (u"ln", u"Lingala", u"", u"Lingala"),
123+ (u"lo", u"ລາວ", u"", u"Lao"),
124+ (u"de-CH", u"Deutsch", u"", u"German"),
125+ (u"li", u"Limburgs", u"", u"Limburgish"),
126+ (u"lv", u"Latviešu", u"", u"Latvian"),
127+ (u"lt", u"Lietuvių", u"", u"Lithuanian"),
128+ (u"pcm", u"Nigerian Pidgin", u"", u""),
129+ (u"pcd", u"Picard", u"", u"Picard"),
130+ (u"yi", u"ייִדיש", u"", u"Yiddish"),
131+ (u"ceb", u"Sinugboanong Binisaya", u"", u"Cebuano"),
132+ (u"yo", u"Yorùbá", u"", u"Yoruba"),
133+ (u"ro-RO", u"Română", u"", u"Romanian"),
134+ (u"bar", u"Boarisch", u"", u"Bavarian"),
135+ (u"nov", u"Novial", u"", u"Novial"),
136+ (u"sr-ME", u"srpski (Crna Gora)", u"", u""),
137+ (u"es-CL", u"Español", u"", u"Spanish"),
138+ (u"es-CO", u"Español", u"", u"Spanish"),
139+ (u"nl-NL", u"Nederlands", u"", u"Dutch"),
140+ (u"map-bms", u"Basa Banyumasan", u"", u"Banyumasan"),
141+ (u"el", u"Ελληνικά", u"", u"Greek"),
142+ (u"eo", u"Esperanto", u"", u"Esperanto"),
143+ (u"en", u"English", u"", u"English"),
144+ (u"ee", u"Eʋegbe", u"", u"Ewe"),
145+ (u"mdf", u"Мокшень (Mokshanj Kälj)", u"", u"Moksha"),
146+ (u"eu", u"Euskara", u"", u"Basque"),
147+ (u"et", u"Eesti", u"", u"Estonian"),
148+ (u"es", u"Español", u"", u"Spanish"),
149+ (u"gom", u"गोवा कोंकणी / Gova Konknni", u"", u"Goan Konkani"),
150+ (u"ru", u"Русский", u"", u"Russian"),
151+ (u"rw", u"Ikinyarwanda", u"", u"Kinyarwanda"),
152+ (u"rm", u"Rumantsch", u"", u"Romansh"),
153+ (u"rn", u"Kirundi", u"", u"Kirundi"),
154+ (u"es-419", u"español (Latinoamérica)", u"", u""),
155+ (u"dsb", u"Dolnoserbski", u"", u"Lower Sorbian"),
156+ (u"ast", u"Asturianu", u"", u"Asturian"),
157+ (u"lmo", u"Lumbaart", u"", u"Lombard"),
158+ (u"ltg", u"Latgaļu", u"", u"Latgalian"),
159+ (u"xh", u"isiXhosa", u"", u"Xhosa"),
160+ (u"en-CA", u"English", u"", u"English"),
161+ (u"koi", u"Перем Коми (Perem Komi)", u"", u"Komi-Permyak"),
162+ (u"tr-TR", u"Türkçe", u"", u"Turkish"),
163+ (u"pnt", u"Ποντιακά", u"", u"Pontic"),
164+ (u"es-XL", u"Español", u"", u"Spanish"),
165+ (u"fi-FI", u"Suomi", u"", u"Finnish"),
166+ (u"pnb", u"شاہ مکھی پنجابی (Shāhmukhī Pañjābī)", u"", u"Western Punjabi"),
167+ (u"udm", u"Удмурт кыл", u"", u"Udmurt"),
168+ (u"bem", u"Ichibemba", u"", u""),
169+ (u"roa-rup", u"Armãneashce", u"", u"Aromanian"),
170+ (u"sr-Latn", u"srpski (latinica)", u"", u""),
171+ (u"stq", u"Seeltersk", u"", u"Saterland Frisian"),
172+ (u"sr", u"Српски / Srpski", u"", u"Serbian"),
173+ (u"ang", u"Englisc", u"", u"Anglo-Saxon"),
174+ (u"ru-RU", u"Русский", u"", u"Russian"),
175+ (u"lbe", u"Лакку", u"", u"Lak"),
176+ (u"min", u"Minangkabau", u"", u"Minangkabau"),
177+ (u"es-US", u"Español", u"", u"Spanish"),
178+ (u"lij", u"Líguru", u"", u"Ligurian"),
179+ (u"kab", u"Taqbaylit", u"", u"Kabyle"),
180+ (u"kaa", u"Qaraqalpaqsha", u"", u"Karakalpak"),
181+ (u"fr-FR", u"Français", u"", u"French"),
182+ (u"tyv", u"Тыва", u"", u"Tuvan"),
183+ (u"ka", u"ქართული", u"", u"Georgian"),
184+ (u"kg", u"KiKongo", u"", u"Kongo"),
185+ (u"ckb", u"Soranî / کوردی", u"", u"Sorani"),
186+ (u"kk", u"Қазақша", u"", u"Kazakh"),
187+ (u"kj", u"Kuanyama", u"", u"Kuanyama"),
188+ (u"ki", u"Gĩkũyũ", u"", u"Kikuyu"),
189+ (u"ko", u"한국어", u"", u"Korean"),
190+ (u"kn", u"ಕನ್ನಡ", u"", u"Kannada"),
191+ (u"tpi", u"Tok Pisin", u"", u"Tok Pisin"),
192+ (u"kl", u"Kalaallisut", u"", u"Greenlandic"),
193+ (u"ks", u"कश्मीरी / كشميري", u"", u"Kashmiri"),
194+ (u"kr", u"Kanuri", u"", u"Kanuri"),
195+ (u"ext", u"Estremeñu", u"", u"Extremaduran"),
196+ (u"kw", u"Kernewek/Karnuack", u"", u"Cornish"),
197+ (u"kv", u"Коми", u"", u"Komi"),
198+ (u"mrj", u"Кырык Мары (Kyryk Mary)", u"", u"Hill Mari"),
199+ (u"ky", u"Кыргызча", u"", u"Kirghiz"),
200+ (u"szl", u"Ślůnski", u"", u"Silesian"),
201+ (u"cdo", u"Mìng-dĕ̤ng-ngṳ̄", u"", u"Min Dong"),
202+ (u"en-GB", u"English", u"", u"English"),
203+ (u"xmf", u"მარგალური (Margaluri)", u"", u"Mingrelian"),
204+ (u"jam", u"Jamaican Creole English", u"", u"Patois"),
205+ (u"ar-SA", u"العربية", u"", u"Arabic"),
206+ (u"ksh", u"Ripoarisch", u"", u"Ripuarian"),
207+ (u"ms-MY", u"Bahasa Melayu", u"", u"Malay"),
208+ (u"de", u"Deutsch", u"", u"German"),
209+ (u"da", u"Dansk", u"", u"Danish"),
210+ (u"dz", u"ཇོང་ཁ", u"", u"Dzongkha"),
211+ (u"hif", u"Fiji Hindi", u"", u"Fiji Hindi"),
212+ (u"dv", u"ދިވެހިބަސް", u"", u"Divehi"),
213+ (u"crs", u"Seychellois Creole", u"", u""),
214+ (u"qu", u"Runa Simi", u"", u"Quechua"),
215+ (u"eml", u"Emiliàn e rumagnòl", u"", u"Emilian-Romagnol"),
216+ (u"ban", u"Balinese", u"", u""),
217+ (u"crh", u"Qırımtatarca", u"", u"Crimean Tatar"),
218+ (u"arz", u"مصرى (Maṣri)", u"", u"Egyptian Arabic"),
219+ (u"rmy", u"romani - रोमानी", u"", u"Romani"),
220+ (u"arc", u"ܐܪܡܝܐ", u"", u"Aramaic"),
221+ (u"th-TH", u"ไทย", u"", u"Thai"),
222+ (u"mus", u"Muskogee", u"", u"Muscogee"),
223+ (u"lua", u"Luba-Lulua", u"", u""),
224+ (u"en-ZA", u"English", u"", u"English"),
225+ (u"wa", u"Walon", u"", u"Walloon"),
226+ (u"wo", u"Wolof", u"", u"Wolof"),
227+ (u"jv", u"Basa Jawa", u"", u"Javanese"),
228+ (u"jw", u"Javanese", u"", u""),
229+ (u"fr-BE", u"Français", u"", u"French"),
230+ (u"tum", u"chiTumbuka", u"", u"Tumbuka"),
231+ (u"ja", u"日本語", u"", u"Japanese"),
232+ (u"pt-PT", u"português (Portugal)", u"", u""),
233+ (u"ilo", u"Ilokano", u"", u"Ilokano"),
234+ (u"tlh", u"Klingon", u"", u""),
235+ (u"pdc", u"Deitsch", u"", u"Pennsylvania German"),
236+ (u"aa", u"Afar", u"", u"Afar"),
237+ (u"ch", u"Chamoru", u"", u"Chamorro"),
238+ (u"co", u"Corsu", u"", u"Corsican"),
239+ (u"simple", u"Simple English", u"", u"Simple English"),
240+ (u"ca", u"Català", u"", u"Catalan"),
241+ (u"xx-pirate", u"Pirate", u"", u""),
242+ (u"ce", u"Нохчийн", u"", u"Chechen"),
243+ (u"cy", u"Cymraeg", u"", u"Welsh"),
244+ (u"sah", u"Саха тыла (Saxa Tyla)", u"", u"Sakha"),
245+ (u"cs", u"Čeština", u"", u"Czech"),
246+ (u"cr", u"Nehiyaw", u"", u"Cree"),
247+ (u"bg-BG", u"Български", u"", u"Bulgarian"),
248+ (u"cv", u"Чăваш", u"", u"Chuvash"),
249+ (u"cu", u"Словѣньскъ", u"", u"Old Church Slavonic"),
250+ (u"ps", u"پښتو", u"", u"Pashto"),
251+ (u"pt", u"Português", u"", u"Portuguese"),
252+ (u"vi-VN", u"Tiếng Việt", u"", u"Vietnamese"),
253+ (u"frr", u"Nordfriisk", u"", u"North Frisian"),
254+ (u"frp", u"Arpitan", u"", u"Franco-Provençal"),
255+ (u"xal", u"Хальмг", u"", u"Kalmyk"),
256+ (u"pi", u"पाऴि", u"", u"Pali"),
257+ (u"it-IT", u"Italiano", u"", u"Italian"),
258+ (u"pl", u"Polski", u"", u"Polish"),
259+ (u"nrm", u"Nouormand/Normaund", u"", u"Norman"),
260+ (u"en-US", u"English", u"", u"English"),
261+ (u"gan", u"贛語", u"", u"Gan"),
262+ (u"bat-smg", u"Žemaitėška", u"", u"Samogitian"),
263+ (u"en-UK", u"English", u"", u"English"),
264+ (u"gag", u"Gagauz", u"", u"Gagauz"),
265+ (u"an", u"Aragonés", u"", u"Aragonese"),
266+ (u"gaa", u"Ga", u"", u""),
267+ (u"fur", u"Furlan", u"", u"Friulian"),
268+ (u"kr-KR", u"Kanuri", u"", u"Kanuri"),
269+ (u"zh-CN", u"中文 (简体)", u"", u""),
270+ (u"tl-PH", u"Tagalog", u"", u"Tagalog"),
271+ (u"en-IN", u"English", u"", u"English"),
272+ (u"ve", u"Tshivenda", u"", u"Venda"),
273+ (u"en-ID", u"English", u"", u"English"),
274+ (u"en-IE", u"English", u"", u"English"),
275+ (u"xx-bork", u"Bork, bork, bork!", u"", u""),
276+ (u"iu", u"ᐃᓄᒃᑎᑐᑦ", u"", u"Inuktitut"),
277+ (u"it", u"Italiano", u"", u"Italian"),
278+ (u"iw", u"עברית", u"", u""),
279+ (u"vo", u"Volapük", u"", u"Volapük"),
280+ (u"ii", u"ꆇꉙ", u"", u"Sichuan Yi"),
281+ (u"ik", u"Iñupiak", u"", u"Inupiak"),
282+ (u"io", u"Ido", u"", u"Ido"),
283+ (u"ia", u"Interlingua", u"", u"Interlingua"),
284+ (u"ja-JP", u"日本語", u"", u"Japanese"),
285+ (u"ie", u"Interlingue", u"", u"Interlingue"),
286+ (u"id", u"Bahasa Indonesia", u"", u"Indonesian"),
287+ (u"nds-nl", u"Nedersaksisch", u"", u"Dutch Low Saxon"),
288+ (u"pap", u"Papiamentu", u"", u"Papiamentu"),
289+ (u"pag", u"Pangasinan", u"", u"Pangasinan"),
290+ (u"pam", u"Kapampangan", u"", u"Kapampangan"),
291+ (u"lv-LV", u"Latviešu", u"", u"Latvian"),
292+ (u"mzn", u"مَزِروني", u"", u"Mazandarani"),
293+ (u"nl-BE", u"Nederlands", u"", u"Dutch"),
294+ (u"sk-SK", u"Slovenčina", u"", u"Slovak"),
295+ (u"zh-TW", u"中文 (繁體)", u"", u""),
296+ (u"es-MX", u"Español", u"", u"Spanish"),
297+ (u"de-DE", u"Deutsch", u"", u"German"),
298+ (u"jbo", u"Lojban", u"", u"Lojban"),
299+ (u"mfe", u"kreol morisien", u"", u""),
300+ (u"hak", u"Hak-kâ-fa / 客家話", u"", u"Hakka"),
301+ (u"ny", u"Chichewa", u"", u"Chichewa"),
302+ (u"ady", u"Адыгэбзэ", u"", u"Adyghe"),
303+ (u"haw", u"Hawai`i", u"", u"Hawaiian"),
304+ (u"el-GR", u"Ελληνικά", u"", u"Greek"),
305+ (u"bpy", u"ইমার ঠার/বিষ্ণুপ্রিয়া মণিপুরী", u"", u"Bishnupriya Manipuri"),
306+ (u"mhr", u"Олык Марий (Olyk Marij)", u"", u"Meadow Mari"),
307+ (u"ca-CT", u"Català", u"", u"Catalan"),
308+ (u"en-MY", u"English", u"", u"English"),
309+ (u"sv-SE", u"Svenska", u"", u"Swedish"),
310+ (u"de-AT", u"Deutsch", u"", u"German"),
311+ (u"xx-elmer", u"Elmer Fudd", u"", u""),
312+ (u"hsb", u"Hornjoserbsce", u"", u"Upper Sorbian"),
313+ (u"be", u"Беларуская", u"", u"Belarusian"),
314+ (u"bg", u"Български", u"", u"Bulgarian"),
315+ (u"ba", u"Башҡорт", u"", u"Bashkir"),
316+ (u"bm", u"Bamanankan", u"", u"Bambara"),
317+ (u"bn", u"বাংলা", u"", u"Bengali"),
318+ (u"bo", u"བོད་སྐད", u"", u"Tibetan"),
319+ (u"bh", u"भोजपुरी", u"", u"Bihari"),
320+ (u"bi", u"Bislama", u"", u"Bislama"),
321+ (u"rue", u"Русиньскый", u"", u"Rusyn"),
322+ (u"et-EE", u"Eesti", u"", u"Estonian"),
323+ (u"br", u"Brezhoneg", u"", u"Breton"),
324+ (u"bs", u"Bosanski", u"", u"Bosnian"),
325+ (u"om", u"Oromoo", u"", u"Oromo"),
326+ (u"ace", u"Bahsa Acèh", u"", u"Acehnese"),
327+ (u"es-AR", u"Español", u"", u"Spanish"),
328+ (u"ach", u"Acoli", u"", u""),
329+ (u"oc", u"Occitan", u"", u"Occitan"),
330+ (u"kri", u"Krio (Sierra Leone)", u"", u""),
331+ (u"be-tarask", u"Беларуская (тарашкевіца)", u"", u"Belarusian (Taraškievica)"),
332+ (u"krc", u"Къарачай-Малкъар (Qarachay-Malqar)", u"", u"Karachay-Balkar"),
333+ (u"nds", u"Plattdüütsch", u"", u"Low Saxon"),
334+ (u"os", u"Иронау", u"", u"Ossetian"),
335+ (u"or", u"ଓଡ଼ିଆ", u"", u"Oriya"),
336+ (u"nso", u"Sepedi", u"", u"Northern Sotho"),
337+ (u"bjn", u"Bahasa Banjar", u"", u"Banjar"),
338+ (u"xx-hacker", u"Hacker", u"", u""),
339+ (u"zh-min-nan", u"Bân-lâm-gú", u"", u"Min Nan"),
340+ (u"pa", u"ਪੰਜਾਬੀ", u"", u"Punjabi"),
341+ (u"loz", u"Lozi", u"", u""),
342+ (u"war", u"Winaray", u"", u"Waray-Waray"),
343+ (u"hz", u"Otsiherero", u"", u"Herero"),
344+ (u"hy", u"Հայերեն", u"", u"Armenian"),
345+ (u"hr", u"Hrvatski", u"", u"Croatian"),
346+ (u"ht", u"Krèyol ayisyen", u"", u"Haitian"),
347+ (u"hu", u"Magyar", u"", u"Hungarian"),
348+ (u"hi", u"हिन्दी", u"", u"Hindi"),
349+ (u"ho", u"Hiri Motu", u"", u"Hiri Motu"),
350+ (u"ha", u"هَوُسَ", u"", u"Hausa"),
351+ (u"bug", u"Basa Ugi", u"", u"Buginese"),
352+ (u"he", u"עברית", u"", u"Hebrew"),
353+ (u"hr-HR", u"Hrvatski", u"", u"Croatian"),
354+ (u"uz", u"O‘zbek", u"", u"Uzbek"),
355+ (u"azb", u"تۆرکجه", u"", u"South Azerbaijani"),
356+ (u"ur", u"اردو", u"", u"Urdu"),
357+ (u"uk", u"Українська", u"", u"Ukrainian"),
358+ (u"ug", u"ئۇيغۇر تىلى", u"", u"Uyghur"),
359+ (u"pih", u"Norfuk", u"", u"Norfolk"),
360+ (u"ab", u"Аҧсуа", u"", u"Abkhazian"),
361+ (u"af", u"Afrikaans", u"", u"Afrikaans"),
362+ (u"ak", u"Akana", u"", u"Akan"),
363+ (u"am", u"አማርኛ", u"", u"Amharic"),
364+ (u"myv", u"Эрзянь (Erzjanj Kelj)", u"", u"Erzya"),
365+ (u"as", u"অসমীয়া", u"", u"Assamese"),
366+ (u"ar", u"العربية", u"", u"Arabic"),
367+ (u"km", u"ភាសាខ្មែរ", u"", u"Khmer"),
368+ (u"uk-UA", u"Українська", u"", u"Ukrainian"),
369+ (u"av", u"Авар", u"", u"Avar"),
370+ (u"ay", u"Aymar", u"", u"Aymara"),
371+ (u"az", u"Azərbaycanca", u"", u"Azerbaijani"),
372+ (u"es-ES", u"Español", u"", u"Spanish"),
373+ (u"nl", u"Nederlands", u"", u"Dutch"),
374+ (u"nn", u"Nynorsk", u"", u"Norwegian (Nynorsk)"),
375+ (u"no", u"Norsk (Bokmål)", u"", u"Norwegian (Bokmål)"),
376+ (u"na", u"dorerin Naoero", u"", u"Nauruan"),
377+ (u"nah", u"Nāhuatl", u"", u"Nahuatl"),
378+ (u"ne", u"नेपाली", u"", u"Nepali"),
379+ (u"ng", u"Oshiwambo", u"", u"Ndonga"),
380+ (u"en-AU", u"English", u"", u"English"),
381+ (u"nap", u"Nnapulitano", u"", u"Neapolitan"),
382+ (u"nv", u"Diné bizaad", u"", u"Navajo"),
383+ (u"ku", u"Kurdî / كوردی", u"", u"Kurdish"),
384+ (u"cs-CZ", u"Čeština", u"", u"Czech"),
385+ (u"zh-yue", u"粵語", u"", u"Cantonese"),
386+ (u"en-SG", u"English", u"", u"English"),
387+ (u"zh-classical", u"古文 / 文言文", u"", u"Classical Chinese"),
388+ (u"bcl", u"Bikol", u"", u"Central Bicolano"),
389+ (u"en-XA", u"English", u"", u"English")
390+)
searx/query.py+7−4
@@ -71,21 +71,24 @@ class RawTextQuery(object):
7171 # check if any language-code is equal with
7272 # declared language-codes
7373 for lc in language_codes:
74− lang_id, lang_name, country = map(str.lower, lc)
74+ lang_id, lang_name, country, english_name = map(unicode.lower, lc)
7575
7676 # if correct language-code is found
7777 # set it as new search-language
7878 if lang == lang_id\
7979 or lang_id.startswith(lang)\
8080 or lang == lang_name\
81+ or lang == english_name\
8182 or lang.replace('_', ' ') == country:
8283 parse_next = True
83− self.languages.append(lang)
84− break
84+ self.languages.append(lang_id)
85+ # to ensure best match (first match is not necessarily the best one)
86+ if lang == lang_id:
87+ break
8588
8689 # this force a engine or category
8790 if query_part[0] == '!' or query_part[0] == '?':
88− prefix = query_part[1:].replace('_', ' ')
91+ prefix = query_part[1:].replace('-', ' ')
8992
9093 # check if prefix is equal with engine shortcut
9194 if prefix in engine_shortcuts:
searx/templates/courgette/preferences.html+3−3
@@ -13,9 +13,9 @@
1313 <legend>{{ _('Search language') }}</legend>
1414 <p>
1515 <select name='language'>
16− <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Automatic') }}</option>
17− {% for lang_id,lang_name,country_name in language_codes | sort(attribute=1) %}
18− <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} ({{ country_name }}) - {{ lang_id }}</option>
16+ <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
17+ {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
18+ <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>
1919 {% endfor %}
2020 </select>
2121 </p>
searx/templates/legacy/preferences.html+3−3
@@ -14,9 +14,9 @@
1414 <legend>{{ _('Search language') }}</legend>
1515 <p>
1616 <select name='language'>
17− <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Automatic') }}</option>
18− {% for lang_id,lang_name,country_name in language_codes | sort(attribute=1) %}
19− <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} ({{ country_name }}) - {{ lang_id }}</option>
17+ <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
18+ {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
19+ <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>
2020 {% endfor %}
2121 </select>
2222 </p>
searx/templates/oscar/advanced.html+1
@@ -6,4 +6,5 @@
66<div id="advanced-search-container">
77 {% include 'oscar/categories.html' %}
88 {% include 'oscar/time-range.html' %}
9+ {% include 'oscar/languages.html' %}
910</div>
searx/templates/oscar/languages.htmlnew file+12
@@ -0,0 +1,12 @@
1+{% if preferences %}
2+<select class="form-control" name='language'>
3+{% else %}
4+<select class="time_range" name='language'>
5+{% endif %}
6+ <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
7+ {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
8+ <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>
9+ {{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}
10+ </option>
11+ {% endfor %}
12+</select>
searx/templates/oscar/preferences.html+5−6
@@ -40,12 +40,7 @@
4040 {% set language_label = _('Search language') %}
4141 {% set language_info = _('What language do you prefer for search?') %}
4242 {{ preferences_item_header(language_info, language_label, rtl) }}
43− <select class="form-control" name='language'>
44− <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Automatic') }}</option>
45− {% for lang_id,lang_name,country_name in language_codes | sort(attribute=1) %}
46− <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} ({{ country_name }}) - {{ lang_id }}</option>
47− {% endfor %}
48− </select>
43+ {% include 'oscar/languages.html' %}
4944 {{ preferences_item_footer(language_info, language_label, rtl) }}
5045
5146 {% set locale_label = _('Interface language') %}
@@ -153,6 +148,7 @@
153148 <th>{{ _("Allow") }}</th>
154149 <th>{{ _("Engine name") }}</th>
155150 <th>{{ _("Shortcut") }}</th>
151+ <th>{{ _("Language support") }}</th>
156152 <th>{{ _("SafeSearch") }}</th>
157153 <th>{{ _("Time range") }}</th>
158154 <th>{{ _("Avg. time") }}</th>
@@ -161,6 +157,7 @@
161157 <th>{{ _("Max time") }}</th>
162158 <th>{{ _("Avg. time") }}</th>
163159 <th>{{ _("SafeSearch") }}</th>
160+ <th>{{ _("Language support") }}</th>
164161 <th>{{ _("Shortcut") }}</th>
165162 <th>{{ _("Engine name") }}</th>
166163 <th>{{ _("Allow") }}</th>
@@ -175,6 +172,7 @@
175172 </td>
176173 <th>{{ search_engine.name }}</th>
177174 <td>{{ shortcuts[search_engine.name] }}</td>
175+ <td><input type="checkbox" {{ "checked" if current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages else ""}} readonly="readonly" disabled="disabled"></td>
178176 <td><input type="checkbox" {{ "checked" if search_engine.safesearch==True else ""}} readonly="readonly" disabled="disabled"></td>
179177 <td><input type="checkbox" {{ "checked" if search_engine.time_range_support==True else ""}} readonly="readonly" disabled="disabled"></td>
180178 <td class="{{ 'danger' if stats[search_engine.name]['warn_time'] else '' }}">{{ 'N/A' if stats[search_engine.name].time==None else stats[search_engine.name].time }}</td>
@@ -183,6 +181,7 @@
183181 <td class="{{ 'danger' if stats[search_engine.name]['warn_timeout'] else '' }}">{{ search_engine.timeout }}</td>
184182 <td class="{{ 'danger' if stats[search_engine.name]['warn_time'] else '' }}">{{ 'N/A' if stats[search_engine.name].time==None else stats[search_engine.name].time }}</td>
185183 <td><input type="checkbox" {{ "checked" if search_engine.safesearch==True else ""}} readonly="readonly" disabled="disabled"></td>
184+ <td><input type="checkbox" {{ "checked" if current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages else ""}} readonly="readonly" disabled="disabled"></td>
186185 <td>{{ shortcuts[search_engine.name] }}</td>
187186 <th>{{ search_engine.name }}</th>
188187 <td class="onoff-checkbox">
searx/templates/pix-art/preferences.html+3−3
@@ -9,9 +9,9 @@
99 <legend>{{ _('Search language') }}</legend>
1010 <p>
1111 <select name='language'>
12− <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Automatic') }}</option>
13− {% for lang_id,lang_name,country_name in language_codes | sort(attribute=1) %}
14− <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} ({{ country_name }}) - {{ lang_id }}</option>
12+ <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
13+ {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
14+ <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>
1515 {% endfor %}
1616 </select>
1717 </p>
searx/webapp.py+8−4
@@ -330,6 +330,10 @@ def render(template_name, override_theme=None, **kwargs):
330330
331331 kwargs['safesearch'] = str(request.preferences.get_value('safesearch'))
332332
333+ kwargs['language_codes'] = language_codes
334+ if 'current_language' not in kwargs:
335+ kwargs['current_language'] = request.preferences.get_value('language')
336+
333337 # override url_for function in templates
334338 kwargs['url_for'] = url_for_theme
335339
@@ -510,6 +514,7 @@ def index():
510514 answers=result_container.answers,
511515 infoboxes=result_container.infoboxes,
512516 paging=result_container.paging,
517+ current_language=search.lang,
513518 base_url=get_base_url(),
514519 theme=get_current_theme_name(),
515520 favicons=global_favicons[themes.index(get_current_theme_name())]
@@ -552,7 +557,7 @@ def autocompleter():
552557 if not language or language == 'all':
553558 language = 'en'
554559 else:
555− language = language.split('_')[0]
560+ language = language.split('-')[0]
556561 # run autocompletion
557562 raw_results.extend(completer(raw_text_query.getSearchQuery(), language))
558563
@@ -615,9 +620,7 @@ def preferences():
615620 return render('preferences.html',
616621 locales=settings['locales'],
617622 current_locale=get_locale(),
618− current_language=lang,
619623 image_proxy=image_proxy,
620− language_codes=language_codes,
621624 engines_by_category=categories,
622625 stats=stats,
623626 answerers=[{'info': a.self_info(), 'keywords': a.keywords} for a in answerers],
@@ -627,7 +630,8 @@ def preferences():
627630 themes=themes,
628631 plugins=plugins,
629632 allowed_plugins=allowed_plugins,
630− theme=get_current_theme_name())
633+ theme=get_current_theme_name(),
634+ preferences=True)
631635
632636
633637@app.route('/image_proxy', methods=['GET'])
utils/update_languages.pynew file+99
@@ -0,0 +1,99 @@
1+# -*- coding: utf-8 -*-
2+
3+# This script generates languages.py from
4+# intersecting each engine's supported languages.
5+#
6+# The language's native names are obtained from
7+# Wikipedia's supported languages.
8+#
9+# Output file (languages.py) is written in current directory
10+# to avoid overwriting in case something goes wrong.
11+
12+from requests import get
13+from re import sub
14+from lxml.html import fromstring
15+from json import loads
16+from sys import path
17+path.append('../searx')
18+from searx.engines import engines
19+
20+# list of language names
21+wiki_languages_url = 'https://meta.wikimedia.org/wiki/List_of_Wikipedias'
22+google_languages_url = 'https://www.google.com/preferences?#languages'
23+
24+google_json_name = 'google.preferences.langMap'
25+
26+languages = {}
27+
28+# Get language names from Wikipedia.
29+def get_wikipedia_languages():
30+ response = get(wiki_languages_url)
31+ dom = fromstring(response.text)
32+ tables = dom.xpath('//table[contains(@class,"sortable")]')
33+ for table in tables:
34+ # exclude header row
35+ trs = table.xpath('.//tr')[1:]
36+ for tr in trs:
37+ td = tr.xpath('./td')
38+ code = td[3].xpath('./a')[0].text
39+ name = td[2].xpath('./a')[0].text
40+ english_name = td[1].xpath('./a')[0].text
41+
42+ if code not in languages:
43+ languages[code] = (name, '', english_name)
44+
45+# Get language names from Google.
46+def get_google_languages():
47+ response = get(google_languages_url)
48+ dom = fromstring(response.text)
49+ options = dom.xpath('//select[@name="hl"]/option')
50+ for option in options:
51+ code = option.xpath('./@value')[0]
52+ name = option.text[:-1]
53+
54+ if code not in languages:
55+ languages[code] = (name, '', '')
56+
57+# Join all language lists.
58+# iterate all languages supported by each engine
59+def join_language_lists():
60+ for engine_name in engines:
61+ for locale in engines[engine_name].supported_languages:
62+ locale = locale.replace('_', '-')
63+ if locale not in languages:
64+ # try to get language name
65+ language = languages.get(locale.split('-')[0], None)
66+ if language == None:
67+ print engine_name + ": " + locale
68+ continue
69+
70+ (name, country, english) = language
71+ languages[locale] = (name, country, english)
72+
73+# Write languages.py.
74+def write_languages_file():
75+ new_file = open('languages.py', 'w')
76+ file_content = '# -*- coding: utf-8 -*-\n'
77+ file_content += '# list of language codes\n'
78+ file_content += '# this file is generated automatically by utils/update_search_languages.py\n'
79+ file_content += '\nlanguage_codes = ('
80+ for code in languages:
81+ (name, country, english) = languages[code]
82+ file_content += '\n (u"' + code + '"'\
83+ + ', u"' + name + '"'\
84+ + ', u"' + country[1:-1] + '"'\
85+ + ', u"' + english + '"),'
86+ # remove last comma
87+ file_content = file_content[:-1]
88+ file_content += '\n)\n'
89+ new_file.write(file_content.encode('utf8'))
90+ new_file.close()
91+
92+def main():
93+ get_wikipedia_languages()
94+ get_google_languages()
95+ join_language_lists()
96+ write_languages_file()
97+
98+if __name__ == "__main__":
99+ main()