git.iko.soy/searx
on master help sign in
[enh] add supported_languages on engines and auto-generate languages.py
149802cmarc
searx/autocomplete.py+3−3
compare with master
unifiedsplit
@@ -81,17 +81,17 @@ def searx_bang(full_query):
81 engine_query = full_query.getSearchQuery()[1:]
81 engine_query = full_query.getSearchQuery()[1:]
82
82
83 for lc in language_codes:
83 for lc in language_codes:
84− lang_id, lang_name, country = map(str.lower, lc)
84+ lang_id, lang_name, country, english_name = map(str.lower, lc)
85
85
86 # check if query starts with language-id
86 # check if query starts with language-id
87 if lang_id.startswith(engine_query):
87 if lang_id.startswith(engine_query):
88 if len(engine_query) <= 2:
88 if len(engine_query) <= 2:
89− results.append(':{lang_id}'.format(lang_id=lang_id.split('_')[0]))
89+ results.append(':{lang_id}'.format(lang_id=lang_id.split('-')[0]))
90 else:
90 else:
91 results.append(':{lang_id}'.format(lang_id=lang_id))
91 results.append(':{lang_id}'.format(lang_id=lang_id))
92
92
93 # check if query starts with language name
93 # check if query starts with language name
94− if lang_name.startswith(engine_query):
94+ if lang_name.startswith(engine_query) or english_name.startswith(engine_query):
95 results.append(':{lang_name}'.format(lang_name=lang_name))
95 results.append(':{lang_name}'.format(lang_name=lang_name))
96
96
97 # check if query starts with country
97 # check if query starts with country
searx/engines/__init__.py+1
@@ -38,6 +38,7 @@ engine_shortcuts = {}
38engine_default_args = {'paging': False,
38engine_default_args = {'paging': False,
39 'categories': ['general'],
39 'categories': ['general'],
40 'language_support': True,
40 'language_support': True,
41+ 'supported_languages': [],
41 'safesearch': False,
42 'safesearch': False,
42 'timeout': settings['outgoing']['request_timeout'],
43 'timeout': settings['outgoing']['request_timeout'],
43 'shortcut': '-',
44 'shortcut': '-',
searx/engines/archlinux.py+3−2
@@ -29,8 +29,8 @@ xpath_link = './/div[@class="mw-search-result-heading"]/a'
29
29
30# cut 'en' from 'en_US', 'de' from 'de_CH', and so on
30# cut 'en' from 'en_US', 'de' from 'de_CH', and so on
31def locale_to_lang_code(locale):
31def locale_to_lang_code(locale):
32− if locale.find('_') >= 0:
32+ if locale.find('-') >= 0:
33− locale = locale.split('_')[0]
33+ locale = locale.split('-')[0]
34 return locale
34 return locale
35
35
36
36
@@ -95,6 +95,7 @@ main_langs = {
95 'uk': 'Українська',
95 'uk': 'Українська',
96 'zh': '简体中文'
96 'zh': '简体中文'
97}
97}
98+supported_languages = dict(lang_urls, **main_langs)
98
99
99
100
100# do search-request
101# do search-request
searx/engines/bing.py+1−1
@@ -32,7 +32,7 @@ def request(query, params):
32 offset = (params['pageno'] - 1) * 10 + 1
32 offset = (params['pageno'] - 1) * 10 + 1
33
33
34 if params['language'] != 'all':
34 if params['language'] != 'all':
35− query = u'language:{} {}'.format(params['language'].split('_')[0].upper(),
35+ query = u'language:{} {}'.format(params['language'].split('-')[0].upper(),
36 query.decode('utf-8')).encode('utf-8')
36 query.decode('utf-8')).encode('utf-8')
37
37
38 search_path = search_string.format(
38 search_path = search_string.format(
searx/engines/bing_images.py+2−1
@@ -19,6 +19,7 @@ from urllib import urlencode
19from lxml import html
19from lxml import html
20from json import loads
20from json import loads
21import re
21import re
22+from searx.engines.bing import supported_languages
22
23
23# engine dependent config
24# engine dependent config
24categories = ['images']
25categories = ['images']
@@ -53,7 +54,7 @@ def request(query, params):
53 if params['language'] == 'all':
54 if params['language'] == 'all':
54 language = 'en-US'
55 language = 'en-US'
55 else:
56 else:
56− language = params['language'].replace('_', '-')
57+ language = params['language']
57
58
58 search_path = search_string.format(
59 search_path = search_string.format(
59 query=urlencode({'q': query}),
60 query=urlencode({'q': query}),
searx/engines/bing_news.py+2−1
@@ -17,6 +17,7 @@ from datetime import datetime
17from dateutil import parser
17from dateutil import parser
18from lxml import etree
18from lxml import etree
19from searx.utils import list_get
19from searx.utils import list_get
20+from searx.engines.bing import supported_languages
20
21
21# engine dependent config
22# engine dependent config
22categories = ['news']
23categories = ['news']
@@ -74,7 +75,7 @@ def request(query, params):
74 if params['language'] == 'all':
75 if params['language'] == 'all':
75 language = 'en-US'
76 language = 'en-US'
76 else:
77 else:
77− language = params['language'].replace('_', '-')
78+ language = params['language']
78
79
79 params['url'] = _get_url(query, language, offset, params['time_range'])
80 params['url'] = _get_url(query, language, offset, params['time_range'])
80
81
searx/engines/duckduckgo.py+40−2
@@ -22,6 +22,13 @@ from searx.languages import language_codes
22categories = ['general']
22categories = ['general']
23paging = True
23paging = True
24language_support = True
24language_support = True
25+supported_languages = ["es-AR", "en-AU", "de-AT", "fr-BE", "nl-BE", "pt-BR", "bg-BG", "en-CA", "fr-CA", "ca-CT",
26+ "es-CL", "zh-CN", "es-CO", "hr-HR", "cs-CZ", "da-DK", "et-EE", "fi-FI", "fr-FR", "de-DE",
27+ "el-GR", "tzh-HK", "hu-HU", "en-IN", "id-ID", "en-ID", "en-IE", "he-IL", "it-IT", "jp-JP",
28+ "kr-KR", "es-XL", "lv-LV", "lt-LT", "ms-MY", "en-MY", "es-MX", "nl-NL", "en-NZ", "no-NO",
29+ "es-PE", "en-PH", "tl-PH", "pl-PL", "pt-PT", "ro-RO", "ru-RU", "ar-XA", "en-XA", "en-SG",
30+ "sk-SK", "sl-SL", "en-ZA", "es-ES", "ca-ES", "sv-SE", "de-CH", "fr-CH", "it-CH", "tzh-TW",
31+ "th-TH", "tr-TR", "uk-UA", "en-UK", "en-US", "es-US", "vi-VN"]
25time_range_support = True
32time_range_support = True
26
33
27# search-url
34# search-url
@@ -46,10 +53,23 @@ def request(query, params):
46
53
47 offset = (params['pageno'] - 1) * 30
54 offset = (params['pageno'] - 1) * 30
48
55
56+ # custom fixes for languages
49 if params['language'] == 'all':
57 if params['language'] == 'all':
50 locale = None
58 locale = None
59+ elif params['language'][:2] == 'ja':
60+ locale = 'jp-jp'
61+ elif params['language'] == 'zh-TW':
62+ locale = 'tw-tzh'
63+ elif params['language'] == 'zh-HK':
64+ locale = 'hk-tzh'
65+ elif params['language'][-2:] == 'SA':
66+ locale = 'xa' + params['language'].split('-')[0]
67+ elif params['language'][-2:] == 'GB':
68+ locale = 'uk' + params['language'].split('-')[0]
69+ elif params['language'] == 'es-419':
70+ locale = 'xl-es'
51 else:
71 else:
52− locale = params['language'].split('_')
72+ locale = params['language'].split('-')
53 if len(locale) == 2:
73 if len(locale) == 2:
54 # country code goes first
74 # country code goes first
55 locale = locale[1].lower() + '-' + locale[0].lower()
75 locale = locale[1].lower() + '-' + locale[0].lower()
@@ -58,7 +78,25 @@ def request(query, params):
58 locale = locale[0].lower()
78 locale = locale[0].lower()
59 lang_codes = [x[0] for x in language_codes]
79 lang_codes = [x[0] for x in language_codes]
60 for lc in lang_codes:
80 for lc in lang_codes:
61− lc = lc.split('_')
81+ lc = lc.split('-')
82+ if locale == lc[0] and len(lc) == 2:
83+ locale = lc[1].lower() + '-' + lc[0].lower()
84+ break
85+
86+ if locale:
87+ params['url'] = url.format(
88+ query=urlencode({'q': query, 'kl': locale}), offset=offset)
89+ else:
90+ locale = params['language'].split('-')
91+ if len(locale) == 2:
92+ # country code goes first
93+ locale = locale[1].lower() + '-' + locale[0].lower()
94+ else:
95+ # tries to get a country code from language
96+ locale = locale[0].lower()
97+ lang_codes = [x[0] for x in language_codes]
98+ for lc in lang_codes:
99+ lc = lc.split('-')
62 if locale == lc[0]:
100 if locale == lc[0]:
63 locale = lc[1].lower() + '-' + lc[0].lower()
101 locale = lc[1].lower() + '-' + lc[0].lower()
64 break
102 break
searx/engines/duckduckgo_definitions.py+2−1
@@ -4,6 +4,7 @@ from re import compile, sub
4from lxml import html
4from lxml import html
5from searx.utils import html_to_text
5from searx.utils import html_to_text
6from searx.engines.xpath import extract_text
6from searx.engines.xpath import extract_text
7+from searx.engines.duckduckgo import supported_languages
7
8
8url = 'https://api.duckduckgo.com/'\
9url = 'https://api.duckduckgo.com/'\
9 + '?{query}&format=json&pretty=0&no_redirect=1&d=1'
10 + '?{query}&format=json&pretty=0&no_redirect=1&d=1'
@@ -23,7 +24,7 @@ def result_to_text(url, text, htmlResult):
23
24
24def request(query, params):
25def request(query, params):
25 params['url'] = url.format(query=urlencode({'q': query}))
26 params['url'] = url.format(query=urlencode({'q': query}))
26− params['headers']['Accept-Language'] = params['language']
27+ params['headers']['Accept-Language'] = params['language'].split('-')[0]
27 return params
28 return params
28
29
29
30
searx/engines/gigablast.py+1−1
@@ -48,7 +48,7 @@ def request(query, params):
48 if params['language'] == 'all':
48 if params['language'] == 'all':
49 language = 'xx'
49 language = 'xx'
50 else:
50 else:
51− language = params['language'][0:2]
51+ language = params['language'].split('-')[0]
52
52
53 if params['safesearch'] >= 1:
53 if params['safesearch'] >= 1:
54 safesearch = 1
54 safesearch = 1
searx/engines/google.py+14
@@ -23,6 +23,20 @@ categories = ['general']
23paging = True
23paging = True
24language_support = True
24language_support = True
25use_locale_domain = True
25use_locale_domain = True
26+supported_languages = ['de', 'en', 'es', 'es_419', 'fr', 'hr', 'it', 'nl', 'pl', 'pt-BR',
27+ 'pt-PT', 'vi', 'tr', 'ru', 'ar', 'th', 'ko', 'zh-CN', 'zh-TW', 'ja',
28+ 'ach', 'af', 'ak', 'az', 'ms', 'ban', 'xx_bork', 'bs', 'br', 'ca',
29+ 'ceb', 'ckb', 'cs', 'sn', 'co', 'cy', 'da', 'yo', 'et', 'xx_elmer',
30+ 'eo', 'eu', 'ee', 'tl', 'fo', 'gaa', 'ga', 'gd', 'gl', 'gn', 'xx_hacker',
31+ 'ht', 'ha', 'haw', 'bem', 'ig', 'rn', 'id', 'ia', 'zu', 'is', 'jw', 'rw',
32+ 'sw', 'tlh', 'kg', 'mfe', 'kri', 'la', 'lv', 'to', 'lt', 'ln', 'loz',
33+ 'lua', 'lg', 'hu', 'mg', 'mt', 'mi', 'pcm', 'no', 'nso', 'ny', 'nn',
34+ 'uz', 'oc', 'om', 'xx_pirate', 'pt', 'ro', 'mo', 'rm', 'qu', 'nyn', 'crs',
35+ 'sq', 'sd', 'sk', 'sl', 'so', 'st', 'sr_ME', 'sr_Latn', 'su', 'fi', 'sv',
36+ 'tg', 'tt', 'tn', 'tum', 'tk', 'tw', 'fy', 'wo', 'xh', 'el', 'be', 'bg',
37+ 'ky', 'kk', 'mk', 'mn', 'sr', 'uk', 'ka', 'hy', 'yi', 'iw', 'ug', 'ur',
38+ 'ps', 'fa', 'ti', 'am', 'ne', 'mr', 'hi', 'bn', 'pa', 'gu', 'or', 'ta',
39+ 'te', 'kn', 'ml', 'si', 'lo', 'my', 'km', 'chr']
26time_range_support = True
40time_range_support = True
27
41
28# based on https://en.wikipedia.org/wiki/List_of_Google_domains and tests
42# based on https://en.wikipedia.org/wiki/List_of_Google_domains and tests
searx/engines/google_news.py+3−1
@@ -12,6 +12,8 @@
12
12
13from lxml import html
13from lxml import html
14from urllib import urlencode
14from urllib import urlencode
15+from json import loads
16+from searx.engines.google import supported_languages
15
17
16# search-url
18# search-url
17categories = ['news']
19categories = ['news']
@@ -50,7 +52,7 @@ def request(query, params):
50 search_options=urlencode(search_options))
52 search_options=urlencode(search_options))
51
53
52 if params['language'] != 'all':
54 if params['language'] != 'all':
53− language_array = params['language'].lower().split('_')
55+ language_array = params['language'].lower().split('-')
54 params['url'] += '&lr=lang_' + language_array[0]
56 params['url'] += '&lr=lang_' + language_array[0]
55
57
56 return params
58 return params
searx/engines/mediawiki.py+2−1
@@ -15,6 +15,7 @@
15from json import loads
15from json import loads
16from string import Formatter
16from string import Formatter
17from urllib import urlencode, quote
17from urllib import urlencode, quote
18+from searx.engines.wikipedia import supported_engines
18
19
19# engine dependent config
20# engine dependent config
20categories = ['general']
21categories = ['general']
@@ -46,7 +47,7 @@ def request(query, params):
46 if params['language'] == 'all':
47 if params['language'] == 'all':
47 language = 'en'
48 language = 'en'
48 else:
49 else:
49− language = params['language'].split('_')[0]
50+ language = params['language'].split('-')[0]
50
51
51 # format_string [('https://', 'language', '', None), ('.wikipedia.org/', None, None, None)]
52 # format_string [('https://', 'language', '', None), ('.wikipedia.org/', None, None, None)]
52 if any(x[1] == 'language' for x in format_strings):
53 if any(x[1] == 'language' for x in format_strings):
searx/engines/photon.py+2−2
@@ -26,7 +26,7 @@ search_string = 'api/?{query}&limit={limit}'
26result_base_url = 'https://openstreetmap.org/{osm_type}/{osm_id}'
26result_base_url = 'https://openstreetmap.org/{osm_type}/{osm_id}'
27
27
28# list of supported languages
28# list of supported languages
29−allowed_languages = ['de', 'en', 'fr', 'it']
29+supported_languages = ['de', 'en', 'fr', 'it']
30
30
31
31
32# do search-request
32# do search-request
@@ -37,7 +37,7 @@ def request(query, params):
37
37
38 if params['language'] != 'all':
38 if params['language'] != 'all':
39 language = params['language'].split('_')[0]
39 language = params['language'].split('_')[0]
40− if language in allowed_languages:
40+ if language in supported_languages:
41 params['url'] = params['url'] + "&lang=" + language
41 params['url'] = params['url'] + "&lang=" + language
42
42
43 # using searx User-Agent
43 # using searx User-Agent
searx/engines/startpage.py+1−1
@@ -47,7 +47,7 @@ def request(query, params):
47
47
48 # set language if specified
48 # set language if specified
49 if params['language'] != 'all':
49 if params['language'] != 'all':
50− params['data']['with_language'] = ('lang_' + params['language'].split('_')[0])
50+ params['data']['with_language'] = ('lang_' + params['language'].split('-')[0])
51
51
52 return params
52 return params
53
53
searx/engines/subtitleseeker.py+7−2
@@ -43,8 +43,13 @@ def response(resp):
43
43
44 search_lang = ""
44 search_lang = ""
45
45
46+ # dirty fix for languages named differenly in their site
46− if resp.search_params['language'] != 'all':
47+ if resp.search_params['language'][:2] == 'fa':
48+ search_lang = 'Farsi'
49+ elif resp.search_params['language'] == 'pt_BR':
50+ search_lang = 'Brazilian'
51+ elif resp.search_params['language'] != 'all':
47− search_lang = [lc[1]
52+ search_lang = [lc[3]
48 for lc in language_codes
53 for lc in language_codes
49 if lc[0][:2] == resp.search_params['language'].split('_')[0]][0]
54 if lc[0][:2] == resp.search_params['language'].split('_')[0]][0]
50
55
searx/engines/swisscows.py+2−2
@@ -36,8 +36,8 @@ def request(query, params):
36 ui_language = 'browser'
36 ui_language = 'browser'
37 region = 'browser'
37 region = 'browser'
38 else:
38 else:
39− region = params['language'].replace('_', '-')
39+ region = params['language']
40− ui_language = params['language'].split('_')[0]
40+ ui_language = params['language'].split('-')[0]
41
41
42 search_path = search_string.format(
42 search_path = search_string.format(
43 query=urlencode({'query': query,
43 query=urlencode({'query': query,
searx/engines/twitter.py+1−1
@@ -40,7 +40,7 @@ def request(query, params):
40
40
41 # set language if specified
41 # set language if specified
42 if params['language'] != 'all':
42 if params['language'] != 'all':
43− params['cookies']['lang'] = params['language'].split('_')[0]
43+ params['cookies']['lang'] = params['language'].split('-')[0]
44 else:
44 else:
45 params['cookies']['lang'] = 'en'
45 params['cookies']['lang'] = 'en'
46
46
searx/engines/wikidata.py+2
@@ -14,6 +14,8 @@
14from searx import logger
14from searx import logger
15from searx.poolrequests import get
15from searx.poolrequests import get
16from searx.engines.xpath import extract_text
16from searx.engines.xpath import extract_text
17+from searx.utils import format_date_by_locale
18+from searx.engines.wikipedia import supported_languages
17
19
18from json import loads
20from json import loads
19from lxml.html import fromstring
21from lxml.html import fromstring
searx/engines/wikipedia.py+33−2
@@ -13,6 +13,36 @@
13from json import loads
13from json import loads
14from urllib import urlencode, quote
14from urllib import urlencode, quote
15
15
16+supported_languages = ["en", "sv", "ceb", "de", "nl", "fr", "ru", "it", "es", "war",
17+ "pl", "vi", "ja", "pt", "zh", "uk", "ca", "fa", "no", "sh",
18+ "ar", "fi", "hu", "id", "ro", "cs", "ko", "sr", "ms", "tr",
19+ "eu", "eo", "min", "bg", "da", "kk", "sk", "hy", "he", "zh-min-nan",
20+ "lt", "hr", "sl", "et", "ce", "gl", "nn", "uz", "la", "vo",
21+ "el", "simple", "be", "az", "th", "ur", "ka", "hi", "oc", "ta",
22+ "mk", "mg", "new", "lv", "cy", "bs", "tt", "tl", "te", "pms",
23+ "be-tarask", "br", "sq", "ky", "ht", "jv", "tg", "ast", "zh-yue", "lb",
24+ "mr", "ml", "bn", "pnb", "is", "af", "sco", "ga", "ba", "fy",
25+ "cv", "lmo", "sw", "my", "an", "yo", "ne", "io", "gu", "nds",
26+ "scn", "bpy", "pa", "ku", "als", "kn", "bar", "ia", "qu", "su",
27+ "ckb", "bat-smg", "mn", "arz", "nap", "wa", "bug", "gd", "yi", "map-bms",
28+ "am", "mzn", "fo", "si", "nah", "li", "sah", "vec", "hsb", "or",
29+ "os", "mrj", "sa", "hif", "mhr", "roa-tara", "azb", "pam", "ilo",
30+ "sd", "ps", "se", "mi", "bh", "eml", "bcl", "xmf", "diq", "hak",
31+ "gan", "glk", "vls", "nds-nl", "rue", "bo", "fiu-vro", "co", "sc",
32+ "tk", "csb", "lrc", "vep", "wuu", "km", "szl", "gv", "crh", "kv",
33+ "zh-classical", "frr", "zea", "as", "so", "kw", "nso", "ay", "stq",
34+ "udm", "cdo", "nrm", "ie", "koi", "rm", "pcd", "myv", "mt", "fur",
35+ "ace", "lad", "gn", "lij", "dsb", "dv", "cbk-zam", "ext", "gom",
36+ "kab", "ksh", "ang", "mai", "mwl", "lez", "gag", "ln", "ug", "pi",
37+ "pag", "frp", "sn", "nv", "av", "pfl", "haw", "xal", "krc", "kaa",
38+ "rw", "bxr", "pdc", "to", "kl", "nov", "arc", "kbd", "lo", "bjn",
39+ "pap", "ha", "tet", "ki", "tyv", "tpi", "na", "lbe", "ig", "jbo",
40+ "roa-rup", "ty", "jam", "za", "kg", "mdf", "lg", "wo", "srn", "ab",
41+ "ltg", "zu", "sm", "chr", "om", "tn", "chy", "rmy", "cu", "tw", "tum",
42+ "xh", "bi", "rn", "pih", "got", "ss", "pnt", "bm", "ch", "mo", "ts",
43+ "ady", "iu", "st", "ee", "ny", "fj", "ks", "ak", "ik", "sg", "ve",
44+ "dz", "ff", "ti", "cr", "ng", "cho", "kj", "mh", "ho", "ii", "aa", "mus", "hz", "kr"]
45+
16# search-url
46# search-url
17base_url = 'https://{language}.wikipedia.org/'
47base_url = 'https://{language}.wikipedia.org/'
18search_postfix = 'w/api.php?'\
48search_postfix = 'w/api.php?'\
@@ -28,10 +58,11 @@ search_postfix = 'w/api.php?'\
28
58
29# set language in base_url
59# set language in base_url
30def url_lang(lang):
60def url_lang(lang):
31− if lang == 'all':
61+ lang = lang.split('-')[0]
62+ if lang == 'all' or lang not in supported_languages:
32 language = 'en'
63 language = 'en'
33 else:
64 else:
34− language = lang.split('_')[0]
65+ language = lang
35
66
36 return base_url.format(language=language)
67 return base_url.format(language=language)
37
68
searx/engines/yacy.py+1−1
@@ -53,7 +53,7 @@ def request(query, params):
53
53
54 # add language tag if specified
54 # add language tag if specified
55 if params['language'] != 'all':
55 if params['language'] != 'all':
56− params['url'] += '&lr=lang_' + params['language'].split('_')[0]
56+ params['url'] += '&lr=lang_' + params['language'].split('-')[0]
57
57
58 return params
58 return params
59
59
searx/engines/yahoo.py+11−1
@@ -20,6 +20,10 @@ from searx.engines.xpath import extract_text, extract_url
20categories = ['general']
20categories = ['general']
21paging = True
21paging = True
22language_support = True
22language_support = True
23+supported_languages = ["ar", "bg", "ca", "szh", "tzh", "hr", "cs", "da", "nl", "en",
24+ "et", "fi", "fr", "de", "el", "he", "hu", "is", "id", "it", "ja",
25+ "ko", "lv", "lt", "no", "fa", "pl", "pt", "ro", "ru", "sk", "sr",
26+ "sl", "es", "sv", "th", "tr"]
23time_range_support = True
27time_range_support = True
24
28
25# search-url
29# search-url
@@ -72,7 +76,13 @@ def _get_url(query, offset, language, time_range):
72def _get_language(params):
76def _get_language(params):
73 if params['language'] == 'all':
77 if params['language'] == 'all':
74 return 'en'
78 return 'en'
79+ elif params['language'][:2] == 'zh':
80+ if params['language'] == 'zh' or params['language'] == 'zh-CH':
81+ return 'szh'
82+ else:
83+ return 'tzh'
84+ else:
75− return params['language'].split('_')[0]
85+ return params['language'].split('-')[0]
76
86
77
87
78# do search-request
88# do search-request
searx/engines/yahoo_news.py+1−1
@@ -12,7 +12,7 @@
12from urllib import urlencode
12from urllib import urlencode
13from lxml import html
13from lxml import html
14from searx.engines.xpath import extract_text, extract_url
14from searx.engines.xpath import extract_text, extract_url
15−from searx.engines.yahoo import parse_url
15+from searx.engines.yahoo import parse_url, supported_languages
16from datetime import datetime, timedelta
16from datetime import datetime, timedelta
17import re
17import re
18from dateutil import parser
18from dateutil import parser
searx/engines/yandex.py+1−1
@@ -36,7 +36,7 @@ content_xpath = './/div[@class="text-container typo typo_text_m typo_line_m orga
36
36
37
37
38def request(query, params):
38def request(query, params):
39− lang = params['language'].split('_')[0]
39+ lang = params['language'].split('-')[0]
40 host = base_url.format(tld=language_map.get(lang) or default_tld)
40 host = base_url.format(tld=language_map.get(lang) or default_tld)
41 params['url'] = host + search_url.format(page=params['pageno'] - 1,
41 params['url'] = host + search_url.format(page=params['pageno'] - 1,
42 query=urlencode({'text': query}))
42 query=urlencode({'text': query}))
searx/engines/youtube_api.py+1−1
@@ -36,7 +36,7 @@ def request(query, params):
36
36
37 # add language tag if specified
37 # add language tag if specified
38 if params['language'] != 'all':
38 if params['language'] != 'all':
39− params['url'] += '&relevanceLanguage=' + params['language'].split('_')[0]
39+ params['url'] += '&relevanceLanguage=' + params['language'].split('-')[0]
40
40
41 return params
41 return params
42
42
searx/languages.py+388−76
@@ -1,78 +1,390 @@
1−'''
1+# -*- coding: utf-8 -*-
2−searx is free software: you can redistribute it and/or modify
3−it under the terms of the GNU Affero General Public License as published by
4−the Free Software Foundation, either version 3 of the License, or
5−(at your option) any later version.
6−
7−searx is distributed in the hope that it will be useful,
8−but WITHOUT ANY WARRANTY; without even the implied warranty of
9−MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10−GNU Affero General Public License for more details.
11−
12−You should have received a copy of the GNU Affero General Public License
13−along with searx. If not, see < http://www.gnu.org/licenses/ >.
14−
15−(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
16−'''
17−
18# list of language codes
2# list of language codes
3+# this file is generated automatically by utils/update_search_languages.py
4+
19language_codes = (
5language_codes = (
20− ("ar_XA", "Arabic", "Arabia"),
6+ (u"gv", u"Gaelg", u"", u"Manx"),
21− ("bg_BG", "Bulgarian", "Bulgaria"),
7+ (u"sco", u"Scots", u"", u"Scots"),
22− ("cs_CZ", "Czech", "Czech Republic"),
8+ (u"scn", u"Sicilianu", u"", u"Sicilian"),
23− ("da_DK", "Danish", "Denmark"),
9+ (u"gu", u"ગુજરાતી", u"", u"Gujarati"),
24− ("de_AT", "German", "Austria"),
10+ (u"gd", u"Gàidhlig", u"", u"Scottish Gaelic"),
25− ("de_CH", "German", "Switzerland"),
11+ (u"ga", u"Gaeilge", u"", u"Irish"),
26− ("de_DE", "German", "Germany"),
12+ (u"gn", u"Avañe'ẽ", u"", u"Guarani"),
27− ("el_GR", "Greek", "Greece"),
13+ (u"gl", u"Galego", u"", u"Galician"),
28− ("en_AU", "English", "Australia"),
14+ (u"als", u"Alemannisch", u"", u"Alemannic"),
29− ("en_CA", "English", "Canada"),
15+ (u"lt-LT", u"Lietuvių", u"", u"Lithuanian"),
30− ("en_GB", "English", "United Kingdom"),
16+ (u"vep", u"Vepsän", u"", u"Vepsian"),
31− ("en_ID", "English", "Indonesia"),
17+ (u"ty", u"Reo Mā`ohi", u"", u"Tahitian"),
32− ("en_IE", "English", "Ireland"),
18+ (u"tw", u"Twi", u"", u"Twi"),
33− ("en_IN", "English", "India"),
19+ (u"tt", u"Tatarça / Татарча", u"", u"Tatar"),
34− ("en_MY", "English", "Malaysia"),
20+ (u"tr", u"Türkçe", u"", u"Turkish"),
35− ("en_NZ", "English", "New Zealand"),
21+ (u"ts", u"Xitsonga", u"", u"Tsonga"),
36− ("en_PH", "English", "Philippines"),
22+ (u"tn", u"Setswana", u"", u"Tswana"),
37− ("en_SG", "English", "Singapore"),
23+ (u"to", u"faka Tonga", u"", u"Tongan"),
38− ("en_US", "English", "United States"),
24+ (u"tl", u"Tagalog", u"", u"Tagalog"),
39− ("en_XA", "English", "Arabia"),
25+ (u"vec", u"Vèneto", u"", u"Venetian"),
40− ("en_ZA", "English", "South Africa"),
26+ (u"th", u"ไทย", u"", u"Thai"),
41− ("es_AR", "Spanish", "Argentina"),
27+ (u"ti", u"ትግርኛ", u"", u"Tigrinya"),
42− ("es_CL", "Spanish", "Chile"),
28+ (u"tg", u"Тоҷикӣ", u"", u"Tajik"),
43− ("es_ES", "Spanish", "Spain"),
29+ (u"te", u"తెలుగు", u"", u"Telugu"),
44− ("es_MX", "Spanish", "Mexico"),
30+ (u"ta", u"தமிழ்", u"", u"Tamil"),
45− ("es_US", "Spanish", "United States"),
31+ (u"lrc", u"لۊری شومالی", u"", u"Northern Luri"),
46− ("es_XL", "Spanish", "Latin America"),
32+ (u"en-NZ", u"English", u"", u"English"),
47− ("et_EE", "Estonian", "Estonia"),
33+ (u"got", u"𐌲𐌿𐍄𐌹𐍃𐌺", u"", u"Gothic"),
48− ("fi_FI", "Finnish", "Finland"),
34+ (u"vls", u"West-Vlams", u"", u"West Flemish"),
49− ("fr_BE", "French", "Belgium"),
35+ (u"ro", u"Română", u"", u"Romanian"),
50− ("fr_CA", "French", "Canada"),
36+ (u"bxr", u"Буряад", u"", u"Buryat"),
51− ("fr_CH", "French", "Switzerland"),
37+ (u"fiu-vro", u"Võro", u"", u"Võro"),
52− ("fr_FR", "French", "France"),
38+ (u"diq", u"Zazaki", u"", u"Zazaki"),
53− ("he_IL", "Hebrew", "Israel"),
39+ (u"zh", u"中文", u"", u"Chinese"),
54− ("hr_HR", "Croatian", "Croatia"),
40+ (u"pms", u"Piemontèis", u"", u"Piedmontese"),
55− ("hu_HU", "Hungarian", "Hungary"),
41+ (u"za", u"Cuengh", u"", u"Zhuang"),
56− ("it_IT", "Italian", "Italy"),
42+ (u"zh-HK", u"中文", u"", u"Chinese"),
57− ("ja_JP", "Japanese", "Japan"),
43+ (u"zu", u"isiZulu", u"", u"Zulu"),
58− ("ko_KR", "Korean", "Korea"),
44+ (u"tet", u"Tetun", u"", u"Tetum"),
59− ("lt_LT", "Lithuanian", "Lithuania"),
45+ (u"es-PE", u"Español", u"", u"Spanish"),
60− ("lv_LV", "Latvian", "Latvia"),
46+ (u"new", u"नेपाल भाषा", u"", u"Newar"),
61− ("nb_NO", "Norwegian", "Norway"),
47+ (u"lez", u"Лезги чІал (Lezgi č’al)", u"", u"Lezgian"),
62− ("nl_BE", "Dutch", "Belgium"),
48+ (u"glk", u"گیلکی", u"", u"Gilaki"),
63− ("nl_NL", "Dutch", "Netherlands"),
49+ (u"ko-KR", u"한국어", u"", u"Korean"),
64− ("oc_OC", "Occitan", "Occitan"),
50+ (u"id-ID", u"Bahasa Indonesia", u"", u"Indonesian"),
65− ("pl_PL", "Polish", "Poland"),
51+ (u"cho", u"Choctaw", u"", u"Choctaw"),
66− ("pt_BR", "Portuguese", "Brazil"),
52+ (u"chr", u"ᏣᎳᎩ", u"", u"Cherokee"),
67− ("pt_PT", "Portuguese", "Portugal"),
53+ (u"vi", u"Tiếng Việt", u"", u"Vietnamese"),
68− ("ro_RO", "Romanian", "Romania"),
54+ (u"chy", u"Tsetsêhestâhese", u"", u"Cheyenne"),
69− ("ru_RU", "Russian", "Russia"),
55+ (u"is", u"Íslenska", u"", u"Icelandic"),
70− ("sk_SK", "Slovak", "Slovak Republic"),
56+ (u"tk", u"تركمن / Туркмен", u"", u"Turkmen"),
71− ("sl_SL", "Slovenian", "Slovenia"),
57+ (u"da-DK", u"Dansk", u"", u"Danish"),
72− ("sv_SE", "Swedish", "Sweden"),
58+ (u"pfl", u"Pälzisch", u"", u"Palatinate German"),
73− ("th_TH", "Thai", "Thailand"),
59+ (u"hu-HU", u"Magyar", u"", u"Hungarian"),
74− ("tr_TR", "Turkish", "Turkey"),
60+ (u"he-IL", u"עברית", u"", u"Hebrew"),
75− ("uk_UA", "Ukrainian", "Ukraine"),
61+ (u"mg", u"Malagasy", u"", u"Malagasy"),
76− ("zh_CN", "Chinese", "China"),
62+ (u"ml", u"മലയാളം", u"", u"Malayalam"),
77− ("zh_HK", "Chinese", "Hong Kong SAR"),
63+ (u"mo", u"Молдовеняскэ", u"", u"Moldovan"),
78− ("zh_TW", "Chinese", "Taiwan"))
64+ (u"mn", u"Монгол", u"", u"Mongolian"),
65+ (u"mi", u"Māori", u"", u"Maori"),
66+ (u"mh", u"Ebon", u"", u"Marshallese"),
67+ (u"mk", u"Македонски", u"", u"Macedonian"),
68+ (u"mt", u"Malti", u"", u"Maltese"),
69+ (u"ms", u"Bahasa Melayu", u"", u"Malay"),
70+ (u"mr", u"मराठी", u"", u"Marathi"),
71+ (u"mwl", u"Mirandés", u"", u"Mirandese"),
72+ (u"my", u"မြန်မာဘာသာ", u"", u"Burmese"),
73+ (u"en-PH", u"English", u"", u"English"),
74+ (u"srn", u"Sranantongo", u"", u"Sranan"),
75+ (u"pl-PL", u"Polski", u"", u"Polish"),
76+ (u"sl-SL", u"Slovenščina", u"", u"Slovenian"),
77+ (u"csb", u"Kaszëbsczi", u"", u"Kashubian"),
78+ (u"cbk-zam", u"Chavacano de Zamboanga", u"", u"Zamboanga Chavacano"),
79+ (u"nyn", u"Runyankore", u"", u""),
80+ (u"ig", u"Igbo", u"", u"Igbo"),
81+ (u"fr", u"Français", u"", u"French"),
82+ (u"lad", u"Dzhudezmo", u"", u"Ladino"),
83+ (u"fy", u"Frysk", u"", u"West Frisian"),
84+ (u"fa", u"فارسی", u"", u"Persian"),
85+ (u"ff", u"Fulfulde", u"", u"Fula"),
86+ (u"mai", u"मैथिली", u"", u"Maithili"),
87+ (u"fi", u"Suomi", u"", u"Finnish"),
88+ (u"fj", u"Na Vosa Vakaviti", u"", u"Fijian"),
89+ (u"fo", u"Føroyskt", u"", u"Faroese"),
90+ (u"ss", u"SiSwati", u"", u"Swati"),
91+ (u"roa-tara", u"Tarandíne", u"", u"Tarantino"),
92+ (u"sq", u"Shqip", u"", u"Albanian"),
93+ (u"sw", u"Kiswahili", u"", u"Swahili"),
94+ (u"sv", u"Svenska", u"", u"Swedish"),
95+ (u"su", u"Basa Sunda", u"", u"Sundanese"),
96+ (u"st", u"Sesotho", u"", u"Sesotho"),
97+ (u"sk", u"Slovenčina", u"", u"Slovak"),
98+ (u"si", u"සිංහල", u"", u"Sinhalese"),
99+ (u"sh", u"Srpskohrvatski / Српскохрватски", u"", u"Serbo-Croatian"),
100+ (u"so", u"Soomaali", u"", u"Somali"),
101+ (u"sn", u"chiShona", u"", u"Shona"),
102+ (u"sm", u"Gagana Samoa", u"", u"Samoan"),
103+ (u"sl", u"Slovenščina", u"", u"Slovenian"),
104+ (u"sc", u"Sardu", u"", u"Sardinian"),
105+ (u"pt-BR", u"português (Brasil)", u"", u""),
106+ (u"sa", u"संस्कृतम्", u"", u"Sanskrit"),
107+ (u"sg", u"Sängö", u"", u"Sango"),
108+ (u"se", u"Sámegiella", u"", u"Northern Sami"),
109+ (u"sd", u"سنڌي، سندھی ، सिन्ध", u"", u"Sindhi"),
110+ (u"fr-CH", u"Français", u"", u"French"),
111+ (u"zea", u"Zeêuws", u"", u"Zeelandic"),
112+ (u"it-CH", u"Italiano", u"", u"Italian"),
113+ (u"wuu", u"吴语", u"", u"Wu"),
114+ (u"fr-CA", u"Français", u"", u"French"),
115+ (u"ar-XA", u"العربية", u"", u"Arabic"),
116+ (u"kbd", u"Адыгэбзэ (Adighabze)", u"", u"Kabardian Circassian"),
117+ (u"no-NO", u"Norsk (Bokmål)", u"", u"Norwegian (Bokmål)"),
118+ (u"ca-ES", u"Català", u"", u"Catalan"),
119+ (u"lg", u"Luganda", u"", u"Luganda"),
120+ (u"lb", u"Lëtzebuergesch", u"", u"Luxembourgish"),
121+ (u"la", u"Latina", u"", u"Latin"),
122+ (u"ln", u"Lingala", u"", u"Lingala"),
123+ (u"lo", u"ລາວ", u"", u"Lao"),
124+ (u"de-CH", u"Deutsch", u"", u"German"),
125+ (u"li", u"Limburgs", u"", u"Limburgish"),
126+ (u"lv", u"Latviešu", u"", u"Latvian"),
127+ (u"lt", u"Lietuvių", u"", u"Lithuanian"),
128+ (u"pcm", u"Nigerian Pidgin", u"", u""),
129+ (u"pcd", u"Picard", u"", u"Picard"),
130+ (u"yi", u"ייִדיש", u"", u"Yiddish"),
131+ (u"ceb", u"Sinugboanong Binisaya", u"", u"Cebuano"),
132+ (u"yo", u"Yorùbá", u"", u"Yoruba"),
133+ (u"ro-RO", u"Română", u"", u"Romanian"),
134+ (u"bar", u"Boarisch", u"", u"Bavarian"),
135+ (u"nov", u"Novial", u"", u"Novial"),
136+ (u"sr-ME", u"srpski (Crna Gora)", u"", u""),
137+ (u"es-CL", u"Español", u"", u"Spanish"),
138+ (u"es-CO", u"Español", u"", u"Spanish"),
139+ (u"nl-NL", u"Nederlands", u"", u"Dutch"),
140+ (u"map-bms", u"Basa Banyumasan", u"", u"Banyumasan"),
141+ (u"el", u"Ελληνικά", u"", u"Greek"),
142+ (u"eo", u"Esperanto", u"", u"Esperanto"),
143+ (u"en", u"English", u"", u"English"),
144+ (u"ee", u"Eʋegbe", u"", u"Ewe"),
145+ (u"mdf", u"Мокшень (Mokshanj Kälj)", u"", u"Moksha"),
146+ (u"eu", u"Euskara", u"", u"Basque"),
147+ (u"et", u"Eesti", u"", u"Estonian"),
148+ (u"es", u"Español", u"", u"Spanish"),
149+ (u"gom", u"गोवा कोंकणी / Gova Konknni", u"", u"Goan Konkani"),
150+ (u"ru", u"Русский", u"", u"Russian"),
151+ (u"rw", u"Ikinyarwanda", u"", u"Kinyarwanda"),
152+ (u"rm", u"Rumantsch", u"", u"Romansh"),
153+ (u"rn", u"Kirundi", u"", u"Kirundi"),
154+ (u"es-419", u"español (Latinoamérica)", u"", u""),
155+ (u"dsb", u"Dolnoserbski", u"", u"Lower Sorbian"),
156+ (u"ast", u"Asturianu", u"", u"Asturian"),
157+ (u"lmo", u"Lumbaart", u"", u"Lombard"),
158+ (u"ltg", u"Latgaļu", u"", u"Latgalian"),
159+ (u"xh", u"isiXhosa", u"", u"Xhosa"),
160+ (u"en-CA", u"English", u"", u"English"),
161+ (u"koi", u"Перем Коми (Perem Komi)", u"", u"Komi-Permyak"),
162+ (u"tr-TR", u"Türkçe", u"", u"Turkish"),
163+ (u"pnt", u"Ποντιακά", u"", u"Pontic"),
164+ (u"es-XL", u"Español", u"", u"Spanish"),
165+ (u"fi-FI", u"Suomi", u"", u"Finnish"),
166+ (u"pnb", u"شاہ مکھی پنجابی (Shāhmukhī Pañjābī)", u"", u"Western Punjabi"),
167+ (u"udm", u"Удмурт кыл", u"", u"Udmurt"),
168+ (u"bem", u"Ichibemba", u"", u""),
169+ (u"roa-rup", u"Armãneashce", u"", u"Aromanian"),
170+ (u"sr-Latn", u"srpski (latinica)", u"", u""),
171+ (u"stq", u"Seeltersk", u"", u"Saterland Frisian"),
172+ (u"sr", u"Српски / Srpski", u"", u"Serbian"),
173+ (u"ang", u"Englisc", u"", u"Anglo-Saxon"),
174+ (u"ru-RU", u"Русский", u"", u"Russian"),
175+ (u"lbe", u"Лакку", u"", u"Lak"),
176+ (u"min", u"Minangkabau", u"", u"Minangkabau"),
177+ (u"es-US", u"Español", u"", u"Spanish"),
178+ (u"lij", u"Líguru", u"", u"Ligurian"),
179+ (u"kab", u"Taqbaylit", u"", u"Kabyle"),
180+ (u"kaa", u"Qaraqalpaqsha", u"", u"Karakalpak"),
181+ (u"fr-FR", u"Français", u"", u"French"),
182+ (u"tyv", u"Тыва", u"", u"Tuvan"),
183+ (u"ka", u"ქართული", u"", u"Georgian"),
184+ (u"kg", u"KiKongo", u"", u"Kongo"),
185+ (u"ckb", u"Soranî / کوردی", u"", u"Sorani"),
186+ (u"kk", u"Қазақша", u"", u"Kazakh"),
187+ (u"kj", u"Kuanyama", u"", u"Kuanyama"),
188+ (u"ki", u"Gĩkũyũ", u"", u"Kikuyu"),
189+ (u"ko", u"한국어", u"", u"Korean"),
190+ (u"kn", u"ಕನ್ನಡ", u"", u"Kannada"),
191+ (u"tpi", u"Tok Pisin", u"", u"Tok Pisin"),
192+ (u"kl", u"Kalaallisut", u"", u"Greenlandic"),
193+ (u"ks", u"कश्मीरी / كشميري", u"", u"Kashmiri"),
194+ (u"kr", u"Kanuri", u"", u"Kanuri"),
195+ (u"ext", u"Estremeñu", u"", u"Extremaduran"),
196+ (u"kw", u"Kernewek/Karnuack", u"", u"Cornish"),
197+ (u"kv", u"Коми", u"", u"Komi"),
198+ (u"mrj", u"Кырык Мары (Kyryk Mary)", u"", u"Hill Mari"),
199+ (u"ky", u"Кыргызча", u"", u"Kirghiz"),
200+ (u"szl", u"Ślůnski", u"", u"Silesian"),
201+ (u"cdo", u"Mìng-dĕ̤ng-ngṳ̄", u"", u"Min Dong"),
202+ (u"en-GB", u"English", u"", u"English"),
203+ (u"xmf", u"მარგალური (Margaluri)", u"", u"Mingrelian"),
204+ (u"jam", u"Jamaican Creole English", u"", u"Patois"),
205+ (u"ar-SA", u"العربية", u"", u"Arabic"),
206+ (u"ksh", u"Ripoarisch", u"", u"Ripuarian"),
207+ (u"ms-MY", u"Bahasa Melayu", u"", u"Malay"),
208+ (u"de", u"Deutsch", u"", u"German"),
209+ (u"da", u"Dansk", u"", u"Danish"),
210+ (u"dz", u"ཇོང་ཁ", u"", u"Dzongkha"),
211+ (u"hif", u"Fiji Hindi", u"", u"Fiji Hindi"),
212+ (u"dv", u"ދިވެހިބަސް", u"", u"Divehi"),
213+ (u"crs", u"Seychellois Creole", u"", u""),
214+ (u"qu", u"Runa Simi", u"", u"Quechua"),
215+ (u"eml", u"Emiliàn e rumagnòl", u"", u"Emilian-Romagnol"),
216+ (u"ban", u"Balinese", u"", u""),
217+ (u"crh", u"Qırımtatarca", u"", u"Crimean Tatar"),
218+ (u"arz", u"مصرى (Maṣri)", u"", u"Egyptian Arabic"),
219+ (u"rmy", u"romani - रोमानी", u"", u"Romani"),
220+ (u"arc", u"ܐܪܡܝܐ", u"", u"Aramaic"),
221+ (u"th-TH", u"ไทย", u"", u"Thai"),
222+ (u"mus", u"Muskogee", u"", u"Muscogee"),
223+ (u"lua", u"Luba-Lulua", u"", u""),
224+ (u"en-ZA", u"English", u"", u"English"),
225+ (u"wa", u"Walon", u"", u"Walloon"),
226+ (u"wo", u"Wolof", u"", u"Wolof"),
227+ (u"jv", u"Basa Jawa", u"", u"Javanese"),
228+ (u"jw", u"Javanese", u"", u""),
229+ (u"fr-BE", u"Français", u"", u"French"),
230+ (u"tum", u"chiTumbuka", u"", u"Tumbuka"),
231+ (u"ja", u"日本語", u"", u"Japanese"),
232+ (u"pt-PT", u"português (Portugal)", u"", u""),
233+ (u"ilo", u"Ilokano", u"", u"Ilokano"),
234+ (u"tlh", u"Klingon", u"", u""),
235+ (u"pdc", u"Deitsch", u"", u"Pennsylvania German"),
236+ (u"aa", u"Afar", u"", u"Afar"),
237+ (u"ch", u"Chamoru", u"", u"Chamorro"),
238+ (u"co", u"Corsu", u"", u"Corsican"),
239+ (u"simple", u"Simple English", u"", u"Simple English"),
240+ (u"ca", u"Català", u"", u"Catalan"),
241+ (u"xx-pirate", u"Pirate", u"", u""),
242+ (u"ce", u"Нохчийн", u"", u"Chechen"),
243+ (u"cy", u"Cymraeg", u"", u"Welsh"),
244+ (u"sah", u"Саха тыла (Saxa Tyla)", u"", u"Sakha"),
245+ (u"cs", u"Čeština", u"", u"Czech"),
246+ (u"cr", u"Nehiyaw", u"", u"Cree"),
247+ (u"bg-BG", u"Български", u"", u"Bulgarian"),
248+ (u"cv", u"Чăваш", u"", u"Chuvash"),
249+ (u"cu", u"Словѣньскъ", u"", u"Old Church Slavonic"),
250+ (u"ps", u"پښتو", u"", u"Pashto"),
251+ (u"pt", u"Português", u"", u"Portuguese"),
252+ (u"vi-VN", u"Tiếng Việt", u"", u"Vietnamese"),
253+ (u"frr", u"Nordfriisk", u"", u"North Frisian"),
254+ (u"frp", u"Arpitan", u"", u"Franco-Provençal"),
255+ (u"xal", u"Хальмг", u"", u"Kalmyk"),
256+ (u"pi", u"पाऴि", u"", u"Pali"),
257+ (u"it-IT", u"Italiano", u"", u"Italian"),
258+ (u"pl", u"Polski", u"", u"Polish"),
259+ (u"nrm", u"Nouormand/Normaund", u"", u"Norman"),
260+ (u"en-US", u"English", u"", u"English"),
261+ (u"gan", u"贛語", u"", u"Gan"),
262+ (u"bat-smg", u"Žemaitėška", u"", u"Samogitian"),
263+ (u"en-UK", u"English", u"", u"English"),
264+ (u"gag", u"Gagauz", u"", u"Gagauz"),
265+ (u"an", u"Aragonés", u"", u"Aragonese"),
266+ (u"gaa", u"Ga", u"", u""),
267+ (u"fur", u"Furlan", u"", u"Friulian"),
268+ (u"kr-KR", u"Kanuri", u"", u"Kanuri"),
269+ (u"zh-CN", u"中文 (简体)", u"", u""),
270+ (u"tl-PH", u"Tagalog", u"", u"Tagalog"),
271+ (u"en-IN", u"English", u"", u"English"),
272+ (u"ve", u"Tshivenda", u"", u"Venda"),
273+ (u"en-ID", u"English", u"", u"English"),
274+ (u"en-IE", u"English", u"", u"English"),
275+ (u"xx-bork", u"Bork, bork, bork!", u"", u""),
276+ (u"iu", u"ᐃᓄᒃᑎᑐᑦ", u"", u"Inuktitut"),
277+ (u"it", u"Italiano", u"", u"Italian"),
278+ (u"iw", u"עברית", u"", u""),
279+ (u"vo", u"Volapük", u"", u"Volapük"),
280+ (u"ii", u"ꆇꉙ", u"", u"Sichuan Yi"),
281+ (u"ik", u"Iñupiak", u"", u"Inupiak"),
282+ (u"io", u"Ido", u"", u"Ido"),
283+ (u"ia", u"Interlingua", u"", u"Interlingua"),
284+ (u"ja-JP", u"日本語", u"", u"Japanese"),
285+ (u"ie", u"Interlingue", u"", u"Interlingue"),
286+ (u"id", u"Bahasa Indonesia", u"", u"Indonesian"),
287+ (u"nds-nl", u"Nedersaksisch", u"", u"Dutch Low Saxon"),
288+ (u"pap", u"Papiamentu", u"", u"Papiamentu"),
289+ (u"pag", u"Pangasinan", u"", u"Pangasinan"),
290+ (u"pam", u"Kapampangan", u"", u"Kapampangan"),
291+ (u"lv-LV", u"Latviešu", u"", u"Latvian"),
292+ (u"mzn", u"مَزِروني", u"", u"Mazandarani"),
293+ (u"nl-BE", u"Nederlands", u"", u"Dutch"),
294+ (u"sk-SK", u"Slovenčina", u"", u"Slovak"),
295+ (u"zh-TW", u"中文 (繁體)", u"", u""),
296+ (u"es-MX", u"Español", u"", u"Spanish"),
297+ (u"de-DE", u"Deutsch", u"", u"German"),
298+ (u"jbo", u"Lojban", u"", u"Lojban"),
299+ (u"mfe", u"kreol morisien", u"", u""),
300+ (u"hak", u"Hak-kâ-fa / 客家話", u"", u"Hakka"),
301+ (u"ny", u"Chichewa", u"", u"Chichewa"),
302+ (u"ady", u"Адыгэбзэ", u"", u"Adyghe"),
303+ (u"haw", u"Hawai`i", u"", u"Hawaiian"),
304+ (u"el-GR", u"Ελληνικά", u"", u"Greek"),
305+ (u"bpy", u"ইমার ঠার/বিষ্ণুপ্রিয়া মণিপুরী", u"", u"Bishnupriya Manipuri"),
306+ (u"mhr", u"Олык Марий (Olyk Marij)", u"", u"Meadow Mari"),
307+ (u"ca-CT", u"Català", u"", u"Catalan"),
308+ (u"en-MY", u"English", u"", u"English"),
309+ (u"sv-SE", u"Svenska", u"", u"Swedish"),
310+ (u"de-AT", u"Deutsch", u"", u"German"),
311+ (u"xx-elmer", u"Elmer Fudd", u"", u""),
312+ (u"hsb", u"Hornjoserbsce", u"", u"Upper Sorbian"),
313+ (u"be", u"Беларуская", u"", u"Belarusian"),
314+ (u"bg", u"Български", u"", u"Bulgarian"),
315+ (u"ba", u"Башҡорт", u"", u"Bashkir"),
316+ (u"bm", u"Bamanankan", u"", u"Bambara"),
317+ (u"bn", u"বাংলা", u"", u"Bengali"),
318+ (u"bo", u"བོད་སྐད", u"", u"Tibetan"),
319+ (u"bh", u"भोजपुरी", u"", u"Bihari"),
320+ (u"bi", u"Bislama", u"", u"Bislama"),
321+ (u"rue", u"Русиньскый", u"", u"Rusyn"),
322+ (u"et-EE", u"Eesti", u"", u"Estonian"),
323+ (u"br", u"Brezhoneg", u"", u"Breton"),
324+ (u"bs", u"Bosanski", u"", u"Bosnian"),
325+ (u"om", u"Oromoo", u"", u"Oromo"),
326+ (u"ace", u"Bahsa Acèh", u"", u"Acehnese"),
327+ (u"es-AR", u"Español", u"", u"Spanish"),
328+ (u"ach", u"Acoli", u"", u""),
329+ (u"oc", u"Occitan", u"", u"Occitan"),
330+ (u"kri", u"Krio (Sierra Leone)", u"", u""),
331+ (u"be-tarask", u"Беларуская (тарашкевіца)", u"", u"Belarusian (Taraškievica)"),
332+ (u"krc", u"Къарачай-Малкъар (Qarachay-Malqar)", u"", u"Karachay-Balkar"),
333+ (u"nds", u"Plattdüütsch", u"", u"Low Saxon"),
334+ (u"os", u"Иронау", u"", u"Ossetian"),
335+ (u"or", u"ଓଡ଼ିଆ", u"", u"Oriya"),
336+ (u"nso", u"Sepedi", u"", u"Northern Sotho"),
337+ (u"bjn", u"Bahasa Banjar", u"", u"Banjar"),
338+ (u"xx-hacker", u"Hacker", u"", u""),
339+ (u"zh-min-nan", u"Bân-lâm-gú", u"", u"Min Nan"),
340+ (u"pa", u"ਪੰਜਾਬੀ", u"", u"Punjabi"),
341+ (u"loz", u"Lozi", u"", u""),
342+ (u"war", u"Winaray", u"", u"Waray-Waray"),
343+ (u"hz", u"Otsiherero", u"", u"Herero"),
344+ (u"hy", u"Հայերեն", u"", u"Armenian"),
345+ (u"hr", u"Hrvatski", u"", u"Croatian"),
346+ (u"ht", u"Krèyol ayisyen", u"", u"Haitian"),
347+ (u"hu", u"Magyar", u"", u"Hungarian"),
348+ (u"hi", u"हिन्दी", u"", u"Hindi"),
349+ (u"ho", u"Hiri Motu", u"", u"Hiri Motu"),
350+ (u"ha", u"هَوُسَ", u"", u"Hausa"),
351+ (u"bug", u"Basa Ugi", u"", u"Buginese"),
352+ (u"he", u"עברית", u"", u"Hebrew"),
353+ (u"hr-HR", u"Hrvatski", u"", u"Croatian"),
354+ (u"uz", u"O‘zbek", u"", u"Uzbek"),
355+ (u"azb", u"تۆرکجه", u"", u"South Azerbaijani"),
356+ (u"ur", u"اردو", u"", u"Urdu"),
357+ (u"uk", u"Українська", u"", u"Ukrainian"),
358+ (u"ug", u"ئۇيغۇر تىلى", u"", u"Uyghur"),
359+ (u"pih", u"Norfuk", u"", u"Norfolk"),
360+ (u"ab", u"Аҧсуа", u"", u"Abkhazian"),
361+ (u"af", u"Afrikaans", u"", u"Afrikaans"),
362+ (u"ak", u"Akana", u"", u"Akan"),
363+ (u"am", u"አማርኛ", u"", u"Amharic"),
364+ (u"myv", u"Эрзянь (Erzjanj Kelj)", u"", u"Erzya"),
365+ (u"as", u"অসমীয়া", u"", u"Assamese"),
366+ (u"ar", u"العربية", u"", u"Arabic"),
367+ (u"km", u"ភាសាខ្មែរ", u"", u"Khmer"),
368+ (u"uk-UA", u"Українська", u"", u"Ukrainian"),
369+ (u"av", u"Авар", u"", u"Avar"),
370+ (u"ay", u"Aymar", u"", u"Aymara"),
371+ (u"az", u"Azərbaycanca", u"", u"Azerbaijani"),
372+ (u"es-ES", u"Español", u"", u"Spanish"),
373+ (u"nl", u"Nederlands", u"", u"Dutch"),
374+ (u"nn", u"Nynorsk", u"", u"Norwegian (Nynorsk)"),
375+ (u"no", u"Norsk (Bokmål)", u"", u"Norwegian (Bokmål)"),
376+ (u"na", u"dorerin Naoero", u"", u"Nauruan"),
377+ (u"nah", u"Nāhuatl", u"", u"Nahuatl"),
378+ (u"ne", u"नेपाली", u"", u"Nepali"),
379+ (u"ng", u"Oshiwambo", u"", u"Ndonga"),
380+ (u"en-AU", u"English", u"", u"English"),
381+ (u"nap", u"Nnapulitano", u"", u"Neapolitan"),
382+ (u"nv", u"Diné bizaad", u"", u"Navajo"),
383+ (u"ku", u"Kurdî / كوردی", u"", u"Kurdish"),
384+ (u"cs-CZ", u"Čeština", u"", u"Czech"),
385+ (u"zh-yue", u"粵語", u"", u"Cantonese"),
386+ (u"en-SG", u"English", u"", u"English"),
387+ (u"zh-classical", u"古文 / 文言文", u"", u"Classical Chinese"),
388+ (u"bcl", u"Bikol", u"", u"Central Bicolano"),
389+ (u"en-XA", u"English", u"", u"English")
390+)
searx/query.py+7−4
@@ -71,21 +71,24 @@ class RawTextQuery(object):
71 # check if any language-code is equal with
71 # check if any language-code is equal with
72 # declared language-codes
72 # declared language-codes
73 for lc in language_codes:
73 for lc in language_codes:
74− lang_id, lang_name, country = map(str.lower, lc)
74+ lang_id, lang_name, country, english_name = map(unicode.lower, lc)
75
75
76 # if correct language-code is found
76 # if correct language-code is found
77 # set it as new search-language
77 # set it as new search-language
78 if lang == lang_id\
78 if lang == lang_id\
79 or lang_id.startswith(lang)\
79 or lang_id.startswith(lang)\
80 or lang == lang_name\
80 or lang == lang_name\
81+ or lang == english_name\
81 or lang.replace('_', ' ') == country:
82 or lang.replace('_', ' ') == country:
82 parse_next = True
83 parse_next = True
83− self.languages.append(lang)
84+ self.languages.append(lang_id)
85+ # to ensure best match (first match is not necessarily the best one)
86+ if lang == lang_id:
84− break
87+ break
85
88
86 # this force a engine or category
89 # this force a engine or category
87 if query_part[0] == '!' or query_part[0] == '?':
90 if query_part[0] == '!' or query_part[0] == '?':
88− prefix = query_part[1:].replace('_', ' ')
91+ prefix = query_part[1:].replace('-', ' ')
89
92
90 # check if prefix is equal with engine shortcut
93 # check if prefix is equal with engine shortcut
91 if prefix in engine_shortcuts:
94 if prefix in engine_shortcuts:
searx/templates/courgette/preferences.html+3−3
@@ -13,9 +13,9 @@
13 <legend>{{ _('Search language') }}</legend>
13 <legend>{{ _('Search language') }}</legend>
14 <p>
14 <p>
15 <select name='language'>
15 <select name='language'>
16− <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Automatic') }}</option>
16+ <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
17− {% for lang_id,lang_name,country_name in language_codes | sort(attribute=1) %}
17+ {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
18− <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} ({{ country_name }}) - {{ lang_id }}</option>
18+ <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>
19 {% endfor %}
19 {% endfor %}
20 </select>
20 </select>
21 </p>
21 </p>
searx/templates/legacy/preferences.html+3−3
@@ -14,9 +14,9 @@
14 <legend>{{ _('Search language') }}</legend>
14 <legend>{{ _('Search language') }}</legend>
15 <p>
15 <p>
16 <select name='language'>
16 <select name='language'>
17− <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Automatic') }}</option>
17+ <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
18− {% for lang_id,lang_name,country_name in language_codes | sort(attribute=1) %}
18+ {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
19− <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} ({{ country_name }}) - {{ lang_id }}</option>
19+ <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>
20 {% endfor %}
20 {% endfor %}
21 </select>
21 </select>
22 </p>
22 </p>
searx/templates/oscar/advanced.html+1
@@ -6,4 +6,5 @@
6<div id="advanced-search-container">
6<div id="advanced-search-container">
7 {% include 'oscar/categories.html' %}
7 {% include 'oscar/categories.html' %}
8 {% include 'oscar/time-range.html' %}
8 {% include 'oscar/time-range.html' %}
9+ {% include 'oscar/languages.html' %}
9</div>
10</div>
searx/templates/oscar/languages.htmlnew file+12
@@ -0,0 +1,12 @@
1+{% if preferences %}
2+<select class="form-control" name='language'>
3+{% else %}
4+<select class="time_range" name='language'>
5+{% endif %}
6+ <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
7+ {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
8+ <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>
9+ {{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}
10+ </option>
11+ {% endfor %}
12+</select>
searx/templates/oscar/preferences.html+5−6
@@ -40,12 +40,7 @@
40 {% set language_label = _('Search language') %}
40 {% set language_label = _('Search language') %}
41 {% set language_info = _('What language do you prefer for search?') %}
41 {% set language_info = _('What language do you prefer for search?') %}
42 {{ preferences_item_header(language_info, language_label, rtl) }}
42 {{ preferences_item_header(language_info, language_label, rtl) }}
43− <select class="form-control" name='language'>
43+ {% include 'oscar/languages.html' %}
44− <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Automatic') }}</option>
45− {% for lang_id,lang_name,country_name in language_codes | sort(attribute=1) %}
46− <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} ({{ country_name }}) - {{ lang_id }}</option>
47− {% endfor %}
48− </select>
49 {{ preferences_item_footer(language_info, language_label, rtl) }}
44 {{ preferences_item_footer(language_info, language_label, rtl) }}
50
45
51 {% set locale_label = _('Interface language') %}
46 {% set locale_label = _('Interface language') %}
@@ -153,6 +148,7 @@
153 <th>{{ _("Allow") }}</th>
148 <th>{{ _("Allow") }}</th>
154 <th>{{ _("Engine name") }}</th>
149 <th>{{ _("Engine name") }}</th>
155 <th>{{ _("Shortcut") }}</th>
150 <th>{{ _("Shortcut") }}</th>
151+ <th>{{ _("Language support") }}</th>
156 <th>{{ _("SafeSearch") }}</th>
152 <th>{{ _("SafeSearch") }}</th>
157 <th>{{ _("Time range") }}</th>
153 <th>{{ _("Time range") }}</th>
158 <th>{{ _("Avg. time") }}</th>
154 <th>{{ _("Avg. time") }}</th>
@@ -161,6 +157,7 @@
161 <th>{{ _("Max time") }}</th>
157 <th>{{ _("Max time") }}</th>
162 <th>{{ _("Avg. time") }}</th>
158 <th>{{ _("Avg. time") }}</th>
163 <th>{{ _("SafeSearch") }}</th>
159 <th>{{ _("SafeSearch") }}</th>
160+ <th>{{ _("Language support") }}</th>
164 <th>{{ _("Shortcut") }}</th>
161 <th>{{ _("Shortcut") }}</th>
165 <th>{{ _("Engine name") }}</th>
162 <th>{{ _("Engine name") }}</th>
166 <th>{{ _("Allow") }}</th>
163 <th>{{ _("Allow") }}</th>
@@ -175,6 +172,7 @@
175 </td>
172 </td>
176 <th>{{ search_engine.name }}</th>
173 <th>{{ search_engine.name }}</th>
177 <td>{{ shortcuts[search_engine.name] }}</td>
174 <td>{{ shortcuts[search_engine.name] }}</td>
175+ <td><input type="checkbox" {{ "checked" if current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages else ""}} readonly="readonly" disabled="disabled"></td>
178 <td><input type="checkbox" {{ "checked" if search_engine.safesearch==True else ""}} readonly="readonly" disabled="disabled"></td>
176 <td><input type="checkbox" {{ "checked" if search_engine.safesearch==True else ""}} readonly="readonly" disabled="disabled"></td>
179 <td><input type="checkbox" {{ "checked" if search_engine.time_range_support==True else ""}} readonly="readonly" disabled="disabled"></td>
177 <td><input type="checkbox" {{ "checked" if search_engine.time_range_support==True else ""}} readonly="readonly" disabled="disabled"></td>
180 <td class="{{ 'danger' if stats[search_engine.name]['warn_time'] else '' }}">{{ 'N/A' if stats[search_engine.name].time==None else stats[search_engine.name].time }}</td>
178 <td class="{{ 'danger' if stats[search_engine.name]['warn_time'] else '' }}">{{ 'N/A' if stats[search_engine.name].time==None else stats[search_engine.name].time }}</td>
@@ -183,6 +181,7 @@
183 <td class="{{ 'danger' if stats[search_engine.name]['warn_timeout'] else '' }}">{{ search_engine.timeout }}</td>
181 <td class="{{ 'danger' if stats[search_engine.name]['warn_timeout'] else '' }}">{{ search_engine.timeout }}</td>
184 <td class="{{ 'danger' if stats[search_engine.name]['warn_time'] else '' }}">{{ 'N/A' if stats[search_engine.name].time==None else stats[search_engine.name].time }}</td>
182 <td class="{{ 'danger' if stats[search_engine.name]['warn_time'] else '' }}">{{ 'N/A' if stats[search_engine.name].time==None else stats[search_engine.name].time }}</td>
185 <td><input type="checkbox" {{ "checked" if search_engine.safesearch==True else ""}} readonly="readonly" disabled="disabled"></td>
183 <td><input type="checkbox" {{ "checked" if search_engine.safesearch==True else ""}} readonly="readonly" disabled="disabled"></td>
184+ <td><input type="checkbox" {{ "checked" if current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages else ""}} readonly="readonly" disabled="disabled"></td>
186 <td>{{ shortcuts[search_engine.name] }}</td>
185 <td>{{ shortcuts[search_engine.name] }}</td>
187 <th>{{ search_engine.name }}</th>
186 <th>{{ search_engine.name }}</th>
188 <td class="onoff-checkbox">
187 <td class="onoff-checkbox">
searx/templates/pix-art/preferences.html+3−3
@@ -9,9 +9,9 @@
9 <legend>{{ _('Search language') }}</legend>
9 <legend>{{ _('Search language') }}</legend>
10 <p>
10 <p>
11 <select name='language'>
11 <select name='language'>
12− <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Automatic') }}</option>
12+ <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
13− {% for lang_id,lang_name,country_name in language_codes | sort(attribute=1) %}
13+ {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
14− <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} ({{ country_name }}) - {{ lang_id }}</option>
14+ <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>
15 {% endfor %}
15 {% endfor %}
16 </select>
16 </select>
17 </p>
17 </p>
searx/webapp.py+8−4
@@ -330,6 +330,10 @@ def render(template_name, override_theme=None, **kwargs):
330
330
331 kwargs['safesearch'] = str(request.preferences.get_value('safesearch'))
331 kwargs['safesearch'] = str(request.preferences.get_value('safesearch'))
332
332
333+ kwargs['language_codes'] = language_codes
334+ if 'current_language' not in kwargs:
335+ kwargs['current_language'] = request.preferences.get_value('language')
336+
333 # override url_for function in templates
337 # override url_for function in templates
334 kwargs['url_for'] = url_for_theme
338 kwargs['url_for'] = url_for_theme
335
339
@@ -510,6 +514,7 @@ def index():
510 answers=result_container.answers,
514 answers=result_container.answers,
511 infoboxes=result_container.infoboxes,
515 infoboxes=result_container.infoboxes,
512 paging=result_container.paging,
516 paging=result_container.paging,
517+ current_language=search.lang,
513 base_url=get_base_url(),
518 base_url=get_base_url(),
514 theme=get_current_theme_name(),
519 theme=get_current_theme_name(),
515 favicons=global_favicons[themes.index(get_current_theme_name())]
520 favicons=global_favicons[themes.index(get_current_theme_name())]
@@ -552,7 +557,7 @@ def autocompleter():
552 if not language or language == 'all':
557 if not language or language == 'all':
553 language = 'en'
558 language = 'en'
554 else:
559 else:
555− language = language.split('_')[0]
560+ language = language.split('-')[0]
556 # run autocompletion
561 # run autocompletion
557 raw_results.extend(completer(raw_text_query.getSearchQuery(), language))
562 raw_results.extend(completer(raw_text_query.getSearchQuery(), language))
558
563
@@ -615,9 +620,7 @@ def preferences():
615 return render('preferences.html',
620 return render('preferences.html',
616 locales=settings['locales'],
621 locales=settings['locales'],
617 current_locale=get_locale(),
622 current_locale=get_locale(),
618− current_language=lang,
619 image_proxy=image_proxy,
623 image_proxy=image_proxy,
620− language_codes=language_codes,
621 engines_by_category=categories,
624 engines_by_category=categories,
622 stats=stats,
625 stats=stats,
623 answerers=[{'info': a.self_info(), 'keywords': a.keywords} for a in answerers],
626 answerers=[{'info': a.self_info(), 'keywords': a.keywords} for a in answerers],
@@ -627,7 +630,8 @@ def preferences():
627 themes=themes,
630 themes=themes,
628 plugins=plugins,
631 plugins=plugins,
629 allowed_plugins=allowed_plugins,
632 allowed_plugins=allowed_plugins,
630− theme=get_current_theme_name())
633+ theme=get_current_theme_name(),
634+ preferences=True)
631
635
632
636
633@app.route('/image_proxy', methods=['GET'])
637@app.route('/image_proxy', methods=['GET'])
utils/update_languages.pynew file+99
@@ -0,0 +1,99 @@
1+# -*- coding: utf-8 -*-
2+
3+# This script generates languages.py from
4+# intersecting each engine's supported languages.
5+#
6+# The language's native names are obtained from
7+# Wikipedia's supported languages.
8+#
9+# Output file (languages.py) is written in current directory
10+# to avoid overwriting in case something goes wrong.
11+
12+from requests import get
13+from re import sub
14+from lxml.html import fromstring
15+from json import loads
16+from sys import path
17+path.append('../searx')
18+from searx.engines import engines
19+
20+# list of language names
21+wiki_languages_url = 'https://meta.wikimedia.org/wiki/List_of_Wikipedias'
22+google_languages_url = 'https://www.google.com/preferences?#languages'
23+
24+google_json_name = 'google.preferences.langMap'
25+
26+languages = {}
27+
28+# Get language names from Wikipedia.
29+def get_wikipedia_languages():
30+ response = get(wiki_languages_url)
31+ dom = fromstring(response.text)
32+ tables = dom.xpath('//table[contains(@class,"sortable")]')
33+ for table in tables:
34+ # exclude header row
35+ trs = table.xpath('.//tr')[1:]
36+ for tr in trs:
37+ td = tr.xpath('./td')
38+ code = td[3].xpath('./a')[0].text
39+ name = td[2].xpath('./a')[0].text
40+ english_name = td[1].xpath('./a')[0].text
41+
42+ if code not in languages:
43+ languages[code] = (name, '', english_name)
44+
45+# Get language names from Google.
46+def get_google_languages():
47+ response = get(google_languages_url)
48+ dom = fromstring(response.text)
49+ options = dom.xpath('//select[@name="hl"]/option')
50+ for option in options:
51+ code = option.xpath('./@value')[0]
52+ name = option.text[:-1]
53+
54+ if code not in languages:
55+ languages[code] = (name, '', '')
56+
57+# Join all language lists.
58+# iterate all languages supported by each engine
59+def join_language_lists():
60+ for engine_name in engines:
61+ for locale in engines[engine_name].supported_languages:
62+ locale = locale.replace('_', '-')
63+ if locale not in languages:
64+ # try to get language name
65+ language = languages.get(locale.split('-')[0], None)
66+ if language == None:
67+ print engine_name + ": " + locale
68+ continue
69+
70+ (name, country, english) = language
71+ languages[locale] = (name, country, english)
72+
73+# Write languages.py.
74+def write_languages_file():
75+ new_file = open('languages.py', 'w')
76+ file_content = '# -*- coding: utf-8 -*-\n'
77+ file_content += '# list of language codes\n'
78+ file_content += '# this file is generated automatically by utils/update_search_languages.py\n'
79+ file_content += '\nlanguage_codes = ('
80+ for code in languages:
81+ (name, country, english) = languages[code]
82+ file_content += '\n (u"' + code + '"'\
83+ + ', u"' + name + '"'\
84+ + ', u"' + country[1:-1] + '"'\
85+ + ', u"' + english + '"),'
86+ # remove last comma
87+ file_content = file_content[:-1]
88+ file_content += '\n)\n'
89+ new_file.write(file_content.encode('utf8'))
90+ new_file.close()
91+
92+def main():
93+ get_wikipedia_languages()
94+ get_google_languages()
95+ join_language_lists()
96+ write_languages_file()
97+
98+if __name__ == "__main__":
99+ main()