git.iko.soy/searx
on master help sign in
Add language support for more engines.
commita11948c71bfe7b2aac6e50e7634874d5073c7d84 authormarc <a01200356@itesm.mx> dateSat 29 Oct 2016, 21:04 (10 years ago) parenta112308
15 files changed+66−39
compare with master
unifiedsplit
searx/engines/dailymotion.py+18 searx/engines/duckduckgo.py+1−21 searx/engines/gigablast.py+1−1 searx/engines/qwant.py+14−1 searx/engines/startpage.py+5 searx/engines/swisscows.py+8 searx/engines/yandex.py+3−1 searx/languages.py+3−1 tests/unit/engines/test_duckduckgo.py+1−1 tests/unit/engines/test_duckduckgo_definitions.py+4 tests/unit/engines/test_google.py+1−1 tests/unit/engines/test_qwant.py+1−1 tests/unit/engines/test_swisscows.py+1−1 tests/unit/engines/test_wikipedia.py+1−1 utils/update_languages.py+4−9
searx/engines/dailymotion.py+18
@@ -20,6 +20,24 @@ from datetime import datetime
2020categories = ['videos']
2121paging = True
2222language_support = True
23+supported_languages = ["af", "ak", "am", "ar", "an", "as", "av", "ae", "ay", "az",
24+ "ba", "bm", "be", "bn", "bi", "bo", "bs", "br", "bg", "ca",
25+ "cs", "ch", "ce", "cu", "cv", "kw", "co", "cr", "cy", "da",
26+ "de", "dv", "dz", "el", "en", "eo", "et", "eu", "ee", "fo",
27+ "fa", "fj", "fi", "fr", "fy", "ff", "gd", "ga", "gl", "gv",
28+ "gn", "gu", "ht", "ha", "sh", "he", "hz", "hi", "ho", "hr",
29+ "hu", "hy", "ig", "io", "ii", "iu", "ie", "ia", "id", "ik",
30+ "is", "it", "jv", "ja", "kl", "kn", "ks", "ka", "kr", "kk",
31+ "km", "ki", "rw", "ky", "kv", "kg", "ko", "kj", "ku", "lo",
32+ "la", "lv", "li", "ln", "lt", "lb", "lu", "lg", "mh", "ml",
33+ "mr", "mk", "mg", "mt", "mn", "mi", "ms", "my", "na", "nv",
34+ "nr", "nd", "ng", "ne", "nl", "nn", "nb", "no", "ny", "oc",
35+ "oj", "or", "om", "os", "pa", "pi", "pl", "pt", "ps", "qu",
36+ "rm", "ro", "rn", "ru", "sg", "sa", "si", "sk", "sl", "se",
37+ "sm", "sn", "sd", "so", "st", "es", "sq", "sc", "sr", "ss",
38+ "su", "sw", "sv", "ty", "ta", "tt", "te", "tg", "tl", "th",
39+ "ti", "to", "tn", "ts", "tk", "tr", "tw", "ug", "uk", "ur",
40+ "uz", "ve", "vi", "vo", "wa", "wo", "xh", "yi", "yo", "za", "zh", "zu"]
2341
2442# search-url
2543# see http://www.dailymotion.com/doc/api/obj-video.html
searx/engines/duckduckgo.py+1−21
@@ -16,7 +16,6 @@
1616from urllib import urlencode
1717from lxml.html import fromstring
1818from searx.engines.xpath import extract_text
19−from searx.languages import language_codes
2019
2120# engine dependent config
2221categories = ['general']
@@ -76,26 +75,7 @@ def request(query, params):
7675 else:
7776 # tries to get a country code from language
7877 locale = locale[0].lower()
79− lang_codes = [x[0] for x in language_codes]
80− for lc in lang_codes:
81− lc = lc.split('-')
82− if locale == lc[0] and len(lc) == 2:
83− locale = lc[1].lower() + '-' + lc[0].lower()
84− break
85−
86− if locale:
87− params['url'] = url.format(
88− query=urlencode({'q': query, 'kl': locale}), offset=offset)
89− else:
90− locale = params['language'].split('-')
91− if len(locale) == 2:
92− # country code goes first
93− locale = locale[1].lower() + '-' + locale[0].lower()
94− else:
95− # tries to get a country code from language
96− locale = locale[0].lower()
97− lang_codes = [x[0] for x in language_codes]
98− for lc in lang_codes:
78+ for lc in supported_languages:
9979 lc = lc.split('-')
10080 if locale == lc[0]:
10181 locale = lc[1].lower() + '-' + lc[0].lower()
searx/engines/gigablast.py+1−1
@@ -44,7 +44,7 @@ supported_languages = ["en", "fr", "es", "ru", "tr", "ja", "zh-CN", "zh-TW", "ko
4444 "nl", "it", "fi", "sv", "no", "pt", "vi", "ar", "he", "id", "el",
4545 "th", "hi", "bn", "pl", "tl", "la", "eo", "ca", "bg", "tx", "sr",
4646 "hu", "da", "lt", "cs", "gl", "ka", "gd", "go", "ro", "ga", "lv",
47− "hy", "is", "ag", "gv", "io", "fa", "te", "vv", "mg", "ku", "lb", "et"]
47+ "hy", "is", "ag", "gv", "io", "fa", "te", "vv", "mg", "ku", "lb", "et"]
4848
4949
5050# do search-request
searx/engines/qwant.py+14−1
@@ -20,6 +20,11 @@ from searx.utils import html_to_text
2020categories = None
2121paging = True
2222language_support = True
23+supported_languages = ["fr-FR", "de-DE", "en-GB", "it-IT", "es-ES", "pt-PT", "de-CH", "fr-CH", "it-CH", "de-AT",
24+ "fr-BE", "nl-BE", "nl-NL", "da-DK", "fi-FI", "sv-SE", "en-IE", "no-NO", "pl-PL", "ru-RU",
25+ "el-GR", "bg-BG", "cs-CZ", "et-EE", "hu-HU", "ro-RO", "en-US", "en-CA", "fr-CA", "pt-BR",
26+ "es-AR", "es-CL", "es-MX", "ja-JP", "en-SG", "en-IN", "en-MY", "ms-MY", "ko-KR", "tl-PH",
27+ "th-TH", "he-IL", "tr-TR", "en-AU", "en-NZ"]
2328
2429category_to_keyword = {'general': 'web',
2530 'images': 'images',
@@ -46,7 +51,15 @@ def request(query, params):
4651
4752 # add language tag if specified
4853 if params['language'] != 'all':
49− params['url'] += '&locale=' + params['language'].lower()
54+ locale = params['language'].split('-')
55+ if len(locale) == 2 and params['language'] in supported_languages:
56+ params['url'] += '&locale=' + params['language'].replace('-', '_').lower()
57+ else:
58+ # try to get a country code for language
59+ for lang in supported_languages:
60+ if locale[0] == lang.split('-')[0]:
61+ params['url'] += '&locale=' + lang.replace('-', '_').lower()
62+ break
5063
5164 return params
5265
searx/engines/startpage.py+5
@@ -24,6 +24,11 @@ categories = ['general']
2424
2525# paging = False
2626language_support = True
27+supported_languages = ["af", "de", "ar", "hy", "be", "bg", "ca", "cs", "zh-CN", "zh-TW",
28+ "ko", "hr", "da", "sk", "sl", "es", "eo", "et", "fi", "fr",
29+ "el", "iw", "hi", "nl", "hu", "id", "en", "is", "it", "ja",
30+ "lv", "lt", "no", "fa", "pl", "pt", "ro", "ru", "sr", "sw",
31+ "sv", "tl", "th", "tr", "uk", "vi"]
2732
2833# search-url
2934base_url = 'https://startpage.com/'
searx/engines/swisscows.py+8
@@ -18,6 +18,12 @@ import re
1818categories = ['general', 'images']
1919paging = True
2020language_support = True
21+supported_languages = ["ar-SA", "es-AR", "en-AU", "de-AT", "fr-BE", "nl-BE", "pt-BR", "bg-BG", "en-CA", "fr-CA",
22+ "es-CL", "zh-CN", "hr-HR", "cs-CZ", "da-DK", "et-EE", "fi-FI", "fr-FR", "de-DE", "el-GR",
23+ "zh-HK", "hu-HU", "en-IN", "en-IE", "he-IL", "it-IT", "ja-JP", "ko-KR", "lv-LV", "lt-LT",
24+ "en-MY", "es-MX", "nl-NL", "en-NZ", "nb-NO", "en-PH", "pl-PL", "pt-PT", "ro-RO", "ru-RU",
25+ "en-SG", "sk-SK", "sl-SI", "en-ZA", "es-ES", "sv-SE", "de-CH", "fr-CH", "zh-TW", "th-TH",
26+ "tr-TR", "uk-UA", "en-GB", "en-US", "es-US"]
2127
2228# search-url
2329base_url = 'https://swisscows.ch/'
@@ -35,6 +41,8 @@ def request(query, params):
3541 if params['language'] == 'all':
3642 ui_language = 'browser'
3743 region = 'browser'
44+ elif params['language'].split('-')[0] == 'no':
45+ region = 'nb-NO'
3846 else:
3947 region = params['language']
4048 ui_language = params['language'].split('-')[0]
searx/engines/yandex.py+3−1
@@ -22,7 +22,9 @@ language_support = True # TODO
2222
2323default_tld = 'com'
2424language_map = {'ru': 'ru',
25− 'ua': 'uk',
25+ 'ua': 'ua',
26+ 'be': 'by',
27+ 'kk': 'kz',
2628 'tr': 'com.tr'}
2729
2830# search-url
searx/languages.py+3−1
@@ -100,7 +100,7 @@ language_codes = (
100100 (u"sa", u"संस्कृतम्", u"", u"Sanskrit"),
101101 (u"he-IL", u"עברית", u"", u"Hebrew"),
102102 (u"se", u"Sámegiella", u"", u"Northern Sami"),
103− (u"sd", u"سنڌي، سندھی ، सिन्ध", u"", u"Sindhi"),
103+ (u"sd", u"سنڌي ،सिन्ध", u"", u"Sindhi"),
104104 (u"fr-CH", u"Français", u"", u"French"),
105105 (u"zea", u"Zeêuws", u"", u"Zeelandic"),
106106 (u"it-CH", u"Italiano", u"", u"Italian"),
@@ -191,6 +191,7 @@ language_codes = (
191191 (u"jam", u"Jamaican Creole English", u"", u"Patois"),
192192 (u"udm", u"Удмурт кыл", u"", u"Udmurt"),
193193 (u"ksh", u"Ripoarisch", u"", u"Ripuarian"),
194+ (u"sl-SI", u"Slovenščina", u"", u"Slovenian"),
194195 (u"ms-MY", u"Bahasa Melayu", u"", u"Malay"),
195196 (u"de", u"Deutsch", u"", u"German"),
196197 (u"da", u"Dansk", u"", u"Danish"),
@@ -284,6 +285,7 @@ language_codes = (
284285 (u"mhr", u"Олык Марий (Olyk Marij)", u"", u"Meadow Mari"),
285286 (u"ca-CT", u"Català", u"", u"Catalan"),
286287 (u"en-MY", u"English", u"", u"English"),
288+ (u"olo", u"Livvi-Karelian", u"", u"Livvinkarjala"),
287289 (u"sv-SE", u"Svenska", u"", u"Swedish"),
288290 (u"de-AT", u"Deutsch", u"", u"German"),
289291 (u"hsb", u"Hornjoserbsce", u"", u"Upper Sorbian"),
tests/unit/engines/test_duckduckgo.py+1−1
@@ -11,7 +11,7 @@ class TestDuckduckgoEngine(SearxTestCase):
1111 query = 'test_query'
1212 dicto = defaultdict(dict)
1313 dicto['pageno'] = 1
14− dicto['language'] = 'de_CH'
14+ dicto['language'] = 'de-CH'
1515 dicto['time_range'] = ''
1616 params = duckduckgo.request(query, dicto)
1717 self.assertIn('url', params)
tests/unit/engines/test_duckduckgo_definitions.py+4
@@ -21,10 +21,14 @@ class TestDDGDefinitionsEngine(SearxTestCase):
2121 query = 'test_query'
2222 dicto = defaultdict(dict)
2323 dicto['pageno'] = 1
24+ dicto['language'] = 'es'
2425 params = duckduckgo_definitions.request(query, dicto)
2526 self.assertIn('url', params)
2627 self.assertIn(query, params['url'])
2728 self.assertIn('duckduckgo.com', params['url'])
29+ self.assertIn('headers', params)
30+ self.assertIn('Accept-Language', params['headers'])
31+ self.assertIn('es', params['headers']['Accept-Language'])
2832
2933 def test_response(self):
3034 self.assertRaises(AttributeError, duckduckgo_definitions.response, None)
tests/unit/engines/test_google.py+1−1
@@ -18,7 +18,7 @@ class TestGoogleEngine(SearxTestCase):
1818 query = 'test_query'
1919 dicto = defaultdict(dict)
2020 dicto['pageno'] = 1
21− dicto['language'] = 'fr_FR'
21+ dicto['language'] = 'fr-FR'
2222 dicto['time_range'] = ''
2323 params = google.request(query, dicto)
2424 self.assertIn('url', params)
tests/unit/engines/test_qwant.py+1−1
@@ -10,7 +10,7 @@ class TestQwantEngine(SearxTestCase):
1010 query = 'test_query'
1111 dicto = defaultdict(dict)
1212 dicto['pageno'] = 0
13− dicto['language'] = 'fr_FR'
13+ dicto['language'] = 'fr-FR'
1414 qwant.categories = ['']
1515 params = qwant.request(query, dicto)
1616 self.assertIn('url', params)
tests/unit/engines/test_swisscows.py+1−1
@@ -10,7 +10,7 @@ class TestSwisscowsEngine(SearxTestCase):
1010 query = 'test_query'
1111 dicto = defaultdict(dict)
1212 dicto['pageno'] = 1
13− dicto['language'] = 'de_DE'
13+ dicto['language'] = 'de-DE'
1414 params = swisscows.request(query, dicto)
1515 self.assertTrue('url' in params)
1616 self.assertTrue(query in params['url'])
tests/unit/engines/test_wikipedia.py+1−1
@@ -10,7 +10,7 @@ class TestWikipediaEngine(SearxTestCase):
1010 def test_request(self):
1111 query = 'test_query'
1212 dicto = defaultdict(dict)
13− dicto['language'] = 'fr_FR'
13+ dicto['language'] = 'fr-FR'
1414 params = wikipedia.request(query, dicto)
1515 self.assertIn('url', params)
1616 self.assertIn(query, params['url'])
utils/update_languages.py+4−9
@@ -41,7 +41,6 @@ def valid_code(lang_code):
4141 if len(lang_code) > 2 or len(lang_code[0]) > 3:
4242 return False
4343 if len(lang_code) == 2 and len(lang_code[1]) > 2:
44− print lang_code
4544 return False
4645
4746 return True
@@ -62,8 +61,8 @@ def get_wikipedia_languages():
6261 english_name = td[1].xpath('./a')[0].text
6362 articles = int(td[4].xpath('./a/b')[0].text.replace(',',''))
6463
65− # exclude languages with few articles and language variants
66− if code not in languages and articles >= 100 and valid_code(code):
64+ # exclude language variants and languages with few articles
65+ if code not in languages and articles >= 1000 and valid_code(code):
6766 languages[code] = (name, '', english_name)
6867
6968
@@ -90,7 +89,7 @@ def join_language_lists():
9089 # try to get language name
9190 language = languages.get(locale.split('-')[0], None)
9291 if language == None:
93− # print engine_name + ": " + locale
92+ print engine_name + ": " + locale
9493 continue
9594
9695 (name, country, english) = language
@@ -117,12 +116,8 @@ def write_languages_file():
117116 new_file.close()
118117
119118
120−def main():
119+if __name__ == "__main__":
121120 get_wikipedia_languages()
122121 get_google_languages()
123122 join_language_lists()
124123 write_languages_file()
125−
126−
127−if __name__ == "__main__":
128− main()