@@ -14,9 +14,12 @@ from lxml import html
1414from dateutil import parser
1515from datetime import datetime, timedelta
1616import re
17+from unicodedata import normalize, combining
18+from babel import Locale
19+from babel.localedata import locale_identifiers
1720from searx.engines.xpath import extract_text
1821from searx.languages import language_codes
19−from searx.utils import eval_xpath
22+from searx.utils import eval_xpath, match_language
2023
2124# engine dependent config
2225categories = ['general']
@@ -26,6 +29,7 @@ categories = ['general']
2629
2730paging = True
2831language_support = True
32+supported_languages_url = 'https://www.startpage.com/do/settings'
2933
3034# search-url
3135base_url = 'https://startpage.com/'
@@ -54,12 +58,11 @@ def request(query, params):
5458
5559 # set language if specified
5660 if params['language'] != 'all':
57− language = 'english'
58− for lc, _, _, lang in language_codes:
59− if lc == params['language']:
60− language = lang
61− params['data']['language'] = language
62− params['data']['lui'] = language
61+ lang_code = match_language(params['language'], supported_languages, fallback=None)
62+ if lang_code:
63+ language_name = supported_languages[lang_code]['alias']
64+ params['data']['language'] = language_name
65+ params['data']['lui'] = language_name
6366
6467 return params
6568
@@ -132,3 +135,55 @@ def response(resp):
132135
133136 # return results
134137 return results
138+
139+
140+# get supported languages from their site
141+def _fetch_supported_languages(resp):
142+ # startpage's language selector is a mess
143+ # each option has a displayed name and a value, either of which may represent the language name
144+ # in the native script, the language name in English, an English transliteration of the native name,
145+ # the English name of the writing script used by the language, or occasionally something else entirely.
146+
147+ # this cases are so special they need to be hardcoded, a couple of them are mispellings
148+ language_names = {
149+ 'english_uk': 'en-GB',
150+ 'fantizhengwen': ['zh-TW', 'zh-HK'],
151+ 'hangul': 'ko',
152+ 'malayam': 'ml',
153+ 'norsk': 'nb',
154+ 'sinhalese': 'si',
155+ 'sudanese': 'su'
156+ }
157+
158+ # get the English name of every language known by babel
159+ language_names.update({name.lower(): lang_code for lang_code, name in Locale('en')._data['languages'].items()})
160+
161+ # get the native name of every language known by babel
162+ for lang_code in filter(lambda lang_code: lang_code.find('_') == -1, locale_identifiers()):
163+ native_name = Locale(lang_code).get_language_name().lower()
164+ # add native name exactly as it is
165+ language_names[native_name] = lang_code
166+
167+ # add "normalized" language name (i.e. français becomes francais and español becomes espanol)
168+ unaccented_name = ''.join(filter(lambda c: not combining(c), normalize('NFKD', native_name)))
169+ if len(unaccented_name) == len(unaccented_name.encode()):
170+ # add only if result is ascii (otherwise "normalization" didn't work)
171+ language_names[unaccented_name] = lang_code
172+
173+ dom = html.fromstring(resp.text)
174+ sp_lang_names = []
175+ for option in dom.xpath('//form[@id="settings-form"]//select[@name="language"]/option'):
176+ sp_lang_names.append((option.get('value'), extract_text(option).lower()))
177+
178+ supported_languages = {}
179+ for sp_option_value, sp_option_text in sp_lang_names:
180+ lang_code = language_names.get(sp_option_value) or language_names.get(sp_option_text)
181+ if isinstance(lang_code, str):
182+ supported_languages[lang_code] = {'alias': sp_option_value}
183+ elif isinstance(lang_code, list):
184+ for lc in lang_code:
185+ supported_languages[lc] = {'alias': sp_option_value}
186+ else:
187+ print('Unknown language option in Startpage: {} ({})'.format(sp_option_value, sp_option_text))
188+
189+ return supported_languages