49b8450 · searx · git.iko.soy

git.iko.soy/searx

Merge pull request #973 from MarcAbonce/languages

Remove 'all' option from search languages

commit49b845051f027cf9d29a20821968103a0f55c9d9 authorAdam Tauber <asciimoo@gmail.com> dateWed 6 Dec 2017, 14:34 (9 years ago) parentdf0b8ee 4d17703

56 files changed+166−249

compare with master

unifiedsplit

searx/engines/archlinux.py+3−3

@@ -26,7 +26,7 @@ xpath_results = '//ul[@class="mw-search-results"]/li'

2626xpath_link = './/div[@class="mw-search-result-heading"]/a'

2727

2828

29−# cut 'en' from 'en_US', 'de' from 'de_CH', and so on

29+# cut 'en' from 'en-US', 'de' from 'de-CH', and so on

3030def locale_to_lang_code(locale):

3131 if locale.find('-') >= 0:

3232 locale = locale.split('-')[0]

@@ -36,7 +36,7 @@ def locale_to_lang_code(locale):

3636# wikis for some languages were moved off from the main site, we need to make

3737# requests to correct URLs to be able to get results in those languages

3838lang_urls = {

39− 'all': {

39+ 'en': {

4040 'base': 'https://wiki.archlinux.org',

4141 'search': '/index.php?title=Special:Search&offset={offset}&{query}'

4242 },

@@ -67,7 +67,7 @@ lang_urls = {

6767def get_lang_urls(language):

6868 if language in lang_urls:

6969 return lang_urls[language]

70− return lang_urls['all']

70+ return lang_urls['en']

7171

7272

7373# Language names to build search requests for

searx/engines/bing.py+1−4

@@ -32,10 +32,7 @@ search_string = 'search?{query}&first={offset}'

3232def request(query, params):

3333 offset = (params['pageno'] - 1) * 10 + 1

3434

35− if params['language'] != 'all':

36− lang = params['language'].split('-')[0].upper()

37− else:

38− lang = 'EN'

35+ lang = params['language'].split('-')[0].upper()

3936

4037 query = u'language:{} {}'.format(lang, query.decode('utf-8')).encode('utf-8')

4138

searx/engines/bing_news.py+1−4

@@ -71,10 +71,7 @@ def request(query, params):

7171

7272 offset = (params['pageno'] - 1) * 10 + 1

7373

74− if params['language'] == 'all':

75− language = 'en-US'

76− else:

77− language = params['language']

74+ language = params['language']

7875

7976 params['url'] = _get_url(query, language, offset, params['time_range'])

8077

searx/engines/dailymotion.py+1−4

@@ -32,10 +32,7 @@ supported_languages_url = 'https://api.dailymotion.com/languages'

3232

3333# do search-request

3434def request(query, params):

35− if params['language'] == 'all':

36− locale = 'en-US'

37− else:

38− locale = params['language']

35+ locale = params['language']

3936

4037 params['url'] = search_url.format(

4138 query=urlencode({'search': query, 'localization': locale}),

searx/engines/duckduckgo.py+3−9

@@ -44,9 +44,7 @@ content_xpath = './/a[@class="result__snippet"]'

4444# match query's language to a region code that duckduckgo will accept

4545def get_region_code(lang, lang_list=None):

4646 # custom fixes for languages

47− if lang == 'all':

48− region_code = None

49− elif lang[:2] == 'ja':

47+ if lang[:2] == 'ja':

5048 region_code = 'jp-jp'

5149 elif lang[:2] == 'sl':

5250 region_code = 'sl-sl'

@@ -82,12 +80,8 @@ def request(query, params):

8280 offset = (params['pageno'] - 1) * 30

8381

8482 region_code = get_region_code(params['language'])

85− if region_code:

86− params['url'] = url.format(

87− query=urlencode({'q': query, 'kl': region_code}), offset=offset, dc_param=offset)

88− else:

89− params['url'] = url.format(

90− query=urlencode({'q': query}), offset=offset, dc_param=offset)

83+ params['url'] = url.format(

84+ query=urlencode({'q': query, 'kl': region_code}), offset=offset, dc_param=offset)

9185

9286 if params['time_range'] in time_range_dict:

9387 params['url'] += time_range_url.format(range=time_range_dict[params['time_range']])

searx/engines/duckduckgo_images.py+2−6

@@ -53,12 +53,8 @@ def request(query, params):

5353 safesearch = params['safesearch'] - 1

5454

5555 region_code = get_region_code(params['language'], lang_list=supported_languages)

56− if region_code:

57− params['url'] = images_url.format(

58− query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd)

59− else:

60− params['url'] = images_url.format(

61− query=urlencode({'q': query}), offset=offset, safesearch=safesearch, vqd=vqd)

56+ params['url'] = images_url.format(

57+ query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd)

6258

6359 return params

6460

searx/engines/faroo.py+1−4

@@ -40,10 +40,7 @@ def request(query, params):

4040 offset = (params['pageno'] - 1) * number_of_results + 1

4141 categorie = search_category.get(params['category'], 'web')

4242

43− if params['language'] == 'all':

44− language = 'en'

45− else:

46− language = params['language'].split('_')[0]

43+ language = params['language'].split('-')[0]

4744

4845 # if language is not supported, put it in english

4946 if language != 'en' and\

searx/engines/gigablast.py+3−6

@@ -49,12 +49,9 @@ supported_languages_url = 'https://gigablast.com/search?&rxikd=1'

4949def request(query, params):

5050 offset = (params['pageno'] - 1) * number_of_results

5151

52− if params['language'] == 'all':

53− language = 'xx'

54− else:

55− language = params['language'].replace('-', '_').lower()

56− if language.split('-')[0] != 'zh':

57− language = language.split('-')[0]

52+ language = params['language'].replace('-', '_').lower()

53+ if language.split('-')[0] != 'zh':

54+ language = language.split('-')[0]

5855

5956 if params['safesearch'] >= 1:

6057 safesearch = 1

searx/engines/google.py+2−1

@@ -165,7 +165,8 @@ def extract_text_from_dom(result, xpath):

165165def request(query, params):

166166 offset = (params['pageno'] - 1) * 10

167167

168− if params['language'] == 'all':

168+ # temporary fix until a way of supporting en-US is found

169+ if params['language'] == 'en-US':

169170 params['language'] = 'en-GB'

170171

171172 if params['language'][:2] == 'jv':

searx/engines/google_news.py+2−3

@@ -50,9 +50,8 @@ def request(query, params):

5050 params['url'] = search_url.format(query=urlencode({'q': query}),

5151 search_options=urlencode(search_options))

5252

53− if params['language'] != 'all':

54− language_array = params['language'].lower().split('-')

55− params['url'] += '&lr=lang_' + language_array[0]

53+ language_array = params['language'].lower().split('-')

54+ params['url'] += '&lr=lang_' + language_array[0]

5655

5756 return params

5857

searx/engines/mediawiki.py+1−4

@@ -45,10 +45,7 @@ def request(query, params):

4545

4646 format_strings = list(Formatter().parse(base_url))

4747

48− if params['language'] == 'all':

49− language = 'en'

50− else:

51− language = params['language'].split('-')[0]

48+ language = params['language'].split('-')[0]

5249

5350 # format_string [('https://', 'language', '', None), ('.wikipedia.org/', None, None, None)]

5451 if any(x[1] == 'language' for x in format_strings):

searx/engines/photon.py+3−4

@@ -35,10 +35,9 @@ def request(query, params):

3535 search_string.format(query=urlencode({'q': query}),

3636 limit=number_of_results)

3737

38− if params['language'] != 'all':

39− language = params['language'].split('_')[0]

40− if language in supported_languages:

41− params['url'] = params['url'] + "&lang=" + language

38+ language = params['language'].split('-')[0]

39+ if language in supported_languages:

40+ params['url'] = params['url'] + "&lang=" + language

4241

4342 # using searx User-Agent

4443 params['headers']['User-Agent'] = searx_useragent()

searx/engines/qwant.py+11−12

@@ -44,18 +44,17 @@ def request(query, params):

4444 query=urlencode({'q': query}),

4545 offset=offset)

4646

47− # add language tag if specified

48− if params['language'] != 'all':

49− if params['language'] == 'no' or params['language'].startswith('no-'):

50− params['language'] = params['language'].replace('no', 'nb', 1)

51− if params['language'].find('-') < 0:

52− # tries to get a country code from language

53− for lang in supported_languages:

54− lc = lang.split('-')

55− if params['language'] == lc[0]:

56− params['language'] = lang

57− break

58− params['url'] += '&locale=' + params['language'].replace('-', '_').lower()

47+ # add language tag

48+ if params['language'] == 'no' or params['language'].startswith('no-'):

49+ params['language'] = params['language'].replace('no', 'nb', 1)

50+ if params['language'].find('-') < 0:

51+ # tries to get a country code from language

52+ for lang in supported_languages:

53+ lc = lang.split('-')

54+ if params['language'] == lc[0]:

55+ params['language'] = lang

56+ break

57+ params['url'] += '&locale=' + params['language'].replace('-', '_').lower()

5958

6059 return params

6160

searx/engines/startpage.py+2−3

@@ -45,9 +45,8 @@ def request(query, params):

4545 params['data'] = {'query': query,

4646 'startat': offset}

4747

48− # set language if specified

49− if params['language'] != 'all':

50− params['data']['with_language'] = ('lang_' + params['language'].split('-')[0])

48+ # set language

49+ params['data']['with_language'] = ('lang_' + params['language'].split('-')[0])

5150

5251 return params

5352

searx/engines/subtitleseeker.py+1−1

@@ -48,7 +48,7 @@ def response(resp):

4848 search_lang = 'Farsi'

4949 elif resp.search_params['language'] == 'pt-BR':

5050 search_lang = 'Brazilian'

51− elif resp.search_params['language'] != 'all':

51+ else:

5252 search_lang = [lc[3]

5353 for lc in language_codes

5454 if lc[0].split('-')[0] == resp.search_params['language'].split('-')[0]]

searx/engines/swisscows.py+1−4

@@ -35,10 +35,7 @@ regex_img_url_remove_start = re.compile(b'^https?://i\.swisscows\.ch/\?link=')

3535

3636# do search-request

3737def request(query, params):

38− if params['language'] == 'all':

39− ui_language = 'browser'

40− region = 'browser'

41− elif params['language'].split('-')[0] == 'no':

38+ if params['language'].split('-')[0] == 'no':

4239 region = 'nb-NO'

4340 else:

4441 region = params['language']

searx/engines/twitter.py+1−6

@@ -37,12 +37,7 @@ timestamp_xpath = './/span[contains(@class,"_timestamp")]'

3737# do search-request

3838def request(query, params):

3939 params['url'] = search_url + urlencode({'q': query})

40−

41− # set language if specified

42− if params['language'] != 'all':

43− params['cookies']['lang'] = params['language'].split('-')[0]

44− else:

45− params['cookies']['lang'] = 'en'

40+ params['cookies']['lang'] = params['language'].split('-')[0]

4641

4742 return params

4843

searx/engines/wikidata.py−4

@@ -57,8 +57,6 @@ calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'

5757

5858def request(query, params):

5959 language = params['language'].split('-')[0]

60− if language == 'all':

61− language = 'en'

6260

6361 params['url'] = url_search.format(

6462 query=urlencode({'label': query, 'language': language}))

@@ -71,8 +69,6 @@ def response(resp):

7169 wikidata_ids = html.xpath(wikidata_ids_xpath)

7270

7371 language = resp.search_params['language'].split('-')[0]

74− if language == 'all':

75− language = 'en'

7672

7773 # TODO: make requests asynchronous to avoid timeout when result_count > 1

7874 for wikidata_id in wikidata_ids[:result_count]:

searx/engines/wikipedia.py+1−1

@@ -31,7 +31,7 @@ supported_languages_url = 'https://meta.wikimedia.org/wiki/List_of_Wikipedias'

3131# set language in base_url

3232def url_lang(lang):

3333 lang = lang.split('-')[0]

34− if lang == 'all' or lang not in supported_languages:

34+ if lang not in supported_languages:

3535 language = 'en'

3636 else:

3737 language = lang

searx/engines/yacy.py+1−3

@@ -51,9 +51,7 @@ def request(query, params):

5151 limit=number_of_results,

5252 search_type=search_type)

5353

54− # add language tag if specified

55− if params['language'] != 'all':

56− params['url'] += '&lr=lang_' + params['language'].split('-')[0]

54+ params['url'] += '&lr=lang_' + params['language'].split('-')[0]

5755

5856 return params

5957

searx/engines/yahoo.py+1−3

@@ -71,9 +71,7 @@ def _get_url(query, offset, language, time_range):

7171

7272

7373def _get_language(params):

74− if params['language'] == 'all':

75− return 'en'

76− elif params['language'][:2] == 'zh':

74+ if params['language'][:2] == 'zh':

7775 if params['language'] == 'zh' or params['language'] == 'zh-CH':

7876 return 'szh'

7977 else:

searx/engines/yahoo_news.py+1−4

@@ -38,10 +38,7 @@ suggestion_xpath = '//div[contains(@class,"VerALSOTRY")]//a'

3838def request(query, params):

3939 offset = (params['pageno'] - 1) * 10 + 1

4040

41− if params['language'] == 'all':

42− language = 'en'

43− else:

44− language = params['language'].split('_')[0]

41+ language = params['language'].split('-')[0]

4542

4643 params['url'] = search_url.format(offset=offset,

4744 query=urlencode({'p': query}),

searx/engines/youtube_api.py+1−3

@@ -34,9 +34,7 @@ def request(query, params):

3434 params['url'] = search_url.format(query=urlencode({'q': query}),

3535 api_key=api_key)

3636

37− # add language tag if specified

38− if params['language'] != 'all':

39− params['url'] += '&relevanceLanguage=' + params['language'].split('-')[0]

37+ params['url'] += '&relevanceLanguage=' + params['language'].split('-')[0]

4038

4139 return params

4240

searx/preferences.py−1

@@ -12,7 +12,6 @@ if version[0] == '3':

1212

1313COOKIE_MAX_AGE = 60 * 60 * 24 * 365 * 5 # 5 years

1414LANGUAGE_CODES = [l[0] for l in languages]

15−LANGUAGE_CODES.append('all')

1615DISABLED = 0

1716ENABLED = 1

1817DOI_RESOLVERS = list(settings['doi_resolvers'])

searx/query.py+19−15

@@ -73,11 +73,6 @@ class RawTextQuery(object):

7373 if query_part[0] == ':':

7474 lang = query_part[1:].lower().replace('_', '-')

7575

76− # user may set a valid, yet not selectable language

77− if VALID_LANGUAGE_CODE.match(lang):

78− self.languages.append(lang)

79− parse_next = True

80−

8176 # check if any language-code is equal with

8277 # declared language-codes

8378 for lc in language_codes:

@@ -85,16 +80,25 @@ class RawTextQuery(object):

8580

8681 # if correct language-code is found

8782 # set it as new search-language

88− if lang == lang_id\

89− or lang_id.startswith(lang)\

90− or lang == lang_name\

91− or lang == english_name\

92− or lang.replace('-', ' ') == country:

93− parse_next = True

94− self.languages.append(lang_id)

95− # to ensure best match (first match is not necessarily the best one)

96− if lang == lang_id:

97− break

83+ if (lang == lang_id

84+ or lang == lang_name

85+ or lang == english_name

86+ or lang.replace('-', ' ') == country)\

87+ and lang not in self.languages:

88+ parse_next = True

89+ lang_parts = lang_id.split('-')

90+ if len(lang_parts) == 2:

91+ self.languages.append(lang_parts[0] + '-' + lang_parts[1].upper())

92+ else:

93+ self.languages.append(lang_id)

94+ # to ensure best match (first match is not necessarily the best one)

95+ if lang == lang_id:

96+ break

97+

98+ # user may set a valid, yet not selectable language

99+ if not self.languages and VALID_LANGUAGE_CODE.match(lang):

100+ self.languages.append(lang)

101+ parse_next = True

98102

99103 # this force a engine or category

100104 if query_part[0] == '!' or query_part[0] == '?':

searx/search.py+5−1

@@ -24,7 +24,7 @@ from flask_babel import gettext

2424import requests.exceptions

2525import searx.poolrequests as requests_lib

2626from searx.engines import (

27− categories, engines

27+ categories, engines, settings

2828)

2929from searx.answerers import ask

3030from searx.utils import gen_useragent

@@ -220,6 +220,10 @@ def get_search_query_from_webapp(preferences, form):

220220 else:

221221 query_lang = preferences.get_value('language')

222222

223+ # provides backwards compatibility for requests using old language default

224+ if query_lang == 'all':

225+ query_lang = settings['search']['language']

226+

223227 # check language

224228 if not VALID_LANGUAGE_CODE.match(query_lang):

225229 raise SearxParameterException('language', query_lang)

searx/settings.yml+1−1

@@ -5,7 +5,7 @@ general:

55search:

66 safe_search : 0 # Filter results. 0: None, 1: Moderate, 2: Strict

77 autocomplete : "" # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "startpage", "wikipedia" - leave blank to turn it off by default

8− language : "all"

8+ language : "en-US"

1010server:

1111 port : 8888

searx/settings_robot.yml+1−1

@@ -5,7 +5,7 @@ general:

55search:

66 safe_search : 0

77 autocomplete : ""

8− language: "all"

8+ language: "en-US"

1010server:

1111 port : 11111

searx/templates/courgette/preferences.html−1

@@ -13,7 +13,6 @@

1313 <legend>{{ _('Search language') }}</legend>

1414 <p>

1515 <select name='language'>

16− <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>

1716 {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}

1817 <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>

1918 {% endfor %}

searx/templates/legacy/preferences.html−1

@@ -14,7 +14,6 @@

1414 <legend>{{ _('Search language') }}</legend>

1515 <p>

1616 <select name='language'>

17− <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>

1817 {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}

1918 <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>

2019 {% endfor %}

searx/templates/oscar/languages.html+5−6

@@ -3,10 +3,9 @@

33{% else %}

44<select class="time_range custom-select form-control" id='language' name='language'>

55{% endif %}

6− <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>

7− {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}

8− <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>

9− {{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}

10− </option>

11− {% endfor %}

6+ {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}

7+ <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>

8+ {{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}

9+ </option>

10+ {% endfor %}

1211</select>

searx/templates/oscar/preferences.html+2−2

@@ -187,7 +187,7 @@

187187 </td>

188188 <th>{{ search_engine.name }}</th>

189189 <td class="name">{{ shortcuts[search_engine.name] }}</td>

190− <td>{{ support_toggle(current_language == 'all' or current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages) }}</td>

190+ <td>{{ support_toggle(current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages) }}</td>

191191 <td>{{ support_toggle(search_engine.safesearch==True) }}</td>

192192 <td>{{ support_toggle(search_engine.time_range_support==True) }}</td>

193193 <td class="{{ 'danger' if stats[search_engine.name]['warn_time'] else '' }}">{{ 'N/A' if stats[search_engine.name].time==None else stats[search_engine.name].time }}</td>

@@ -197,7 +197,7 @@

197197 <td class="{{ 'danger' if stats[search_engine.name]['warn_time'] else '' }}">{{ 'N/A' if stats[search_engine.name].time==None else stats[search_engine.name].time }}</td>

198198 <td>{{ support_toggle(search_engine.time_range_support==True) }}</td>

199199 <td>{{ support_toggle(search_engine.safesearch==True) }}</td>

200− <td>{{ support_toggle(current_language == 'all' or current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages) }}</td>

200+ <td>{{ support_toggle(current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages) }}</td>

201201 <td>{{ shortcuts[search_engine.name] }}</td>

202202 <th>{{ search_engine.name }}</th>

203203 <td class="onoff-checkbox">

searx/templates/pix-art/preferences.html−1

@@ -9,7 +9,6 @@

99 <legend>{{ _('Search language') }}</legend>

1010 <p>

1111 <select name='language'>

12− <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>

1312 {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}

1413 <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>

1514 {% endfor %}

searx/webapp.py+2−2

@@ -630,8 +630,8 @@ def autocompleter():

630630 if len(raw_results) <= 3 and completer:

631631 # get language from cookie

632632 language = request.preferences.get_value('language')

633− if not language or language == 'all':

634− language = 'en'

633+ if not language:

634+ language = settings['search']['language']

635635 else:

636636 language = language.split('-')[0]

637637 # run autocompletion

tests/unit/engines/test_archlinux.py+1−1

@@ -4,7 +4,7 @@ from searx.engines import archlinux

44from searx.testing import SearxTestCase

66domains = {

7− 'all': 'https://wiki.archlinux.org',

7+ 'en': 'https://wiki.archlinux.org',

88 'de': 'https://wiki.archlinux.de',

99 'fr': 'https://wiki.archlinux.fr',

1010 'ja': 'https://wiki.archlinuxjp.org',

tests/unit/engines/test_bing.py+1−5

@@ -10,17 +10,13 @@ class TestBingEngine(SearxTestCase):

1010 query = u'test_query'

1111 dicto = defaultdict(dict)

1212 dicto['pageno'] = 0

13− dicto['language'] = 'fr_FR'

13+ dicto['language'] = 'fr-FR'

1414 params = bing.request(query.encode('utf-8'), dicto)

1515 self.assertTrue('url' in params)

1616 self.assertTrue(query in params['url'])

1717 self.assertTrue('language%3AFR' in params['url'])

1818 self.assertTrue('bing.com' in params['url'])

1919

20− dicto['language'] = 'all'

21− params = bing.request(query.encode('utf-8'), dicto)

22− self.assertTrue('language' in params['url'])

23−

2420 def test_response(self):

2521 self.assertRaises(AttributeError, bing.response, None)

2622 self.assertRaises(AttributeError, bing.response, [])

tests/unit/engines/test_bing_news.py−4

@@ -19,10 +19,6 @@ class TestBingNewsEngine(SearxTestCase):

1919 self.assertIn('bing.com', params['url'])

2020 self.assertIn('fr', params['url'])

2121

22− dicto['language'] = 'all'

23− params = bing_news.request(query, dicto)

24− self.assertIn('en', params['url'])

25−

2622 def test_no_url_in_request_year_time_range(self):

2723 dicto = defaultdict(dict)

2824 query = 'test_query'

tests/unit/engines/test_dailymotion.py−4

@@ -18,10 +18,6 @@ class TestDailymotionEngine(SearxTestCase):

1818 self.assertTrue('dailymotion.com' in params['url'])

1919 self.assertTrue('fr' in params['url'])

2020

21− dicto['language'] = 'all'

22− params = dailymotion.request(query, dicto)

23− self.assertTrue('en' in params['url'])

24−

2521 def test_response(self):

2622 self.assertRaises(AttributeError, dailymotion.response, None)

2723 self.assertRaises(AttributeError, dailymotion.response, [])

tests/unit/engines/test_duckduckgo_images.py+1−1

@@ -15,7 +15,7 @@ class TestDuckduckgoImagesEngine(SearxTestCase):

1515 dicto['is_test'] = True

1616 dicto['pageno'] = 1

1717 dicto['safesearch'] = 0

18− dicto['language'] = 'all'

18+ dicto['language'] = 'en-US'

1919 params = duckduckgo_images.request(query, dicto)

2020 self.assertIn('url', params)

2121 self.assertIn(query, params['url'])

tests/unit/engines/test_faroo.py+2−6

@@ -11,7 +11,7 @@ class TestFarooEngine(SearxTestCase):

1111 query = 'test_query'

1212 dicto = defaultdict(dict)

1313 dicto['pageno'] = 1

14− dicto['language'] = 'fr_FR'

14+ dicto['language'] = 'fr-FR'

1515 dicto['category'] = 'general'

1616 params = faroo.request(query, dicto)

1717 self.assertIn('url', params)

@@ -20,11 +20,7 @@ class TestFarooEngine(SearxTestCase):

2020 self.assertIn('en', params['url'])

2121 self.assertIn('web', params['url'])

2222

23− dicto['language'] = 'all'

24− params = faroo.request(query, dicto)

25− self.assertIn('en', params['url'])

26−

27− dicto['language'] = 'de_DE'

23+ dicto['language'] = 'de-DE'

2824 params = faroo.request(query, dicto)

2925 self.assertIn('de', params['url'])

3026

tests/unit/engines/test_gigablast.py+1−6

@@ -11,16 +11,11 @@ class TestGigablastEngine(SearxTestCase):

1111 dicto = defaultdict(dict)

1212 dicto['pageno'] = 0

1313 dicto['safesearch'] = 0

14− dicto['language'] = 'all'

14+ dicto['language'] = 'en-US'

1515 params = gigablast.request(query, dicto)

1616 self.assertTrue('url' in params)

1717 self.assertTrue(query in params['url'])

1818 self.assertTrue('gigablast.com' in params['url'])

19− self.assertTrue('xx' in params['url'])

20−

21− dicto['language'] = 'en-US'

22− params = gigablast.request(query, dicto)

23− self.assertTrue('en' in params['url'])

2419 self.assertFalse('en-US' in params['url'])

2520

2621 def test_response(self):

tests/unit/engines/test_google.py+1−1

@@ -26,7 +26,7 @@ class TestGoogleEngine(SearxTestCase):

2626 self.assertIn('google.fr', params['url'])

2727 self.assertIn('fr', params['headers']['Accept-Language'])

2828

29− dicto['language'] = 'all'

29+ dicto['language'] = 'en-US'

3030 params = google.request(query, dicto)

3131 self.assertIn('google.co', params['url'])

3232 self.assertIn('en', params['headers']['Accept-Language'])

tests/unit/engines/test_google_news.py+1−6

@@ -12,18 +12,13 @@ class TestGoogleNewsEngine(SearxTestCase):

1212 query = 'test_query'

1313 dicto = defaultdict(dict)

1414 dicto['pageno'] = 1

15− dicto['language'] = 'fr_FR'

15+ dicto['language'] = 'fr-FR'

1616 dicto['time_range'] = 'w'

1717 params = google_news.request(query, dicto)

1818 self.assertIn('url', params)

1919 self.assertIn(query, params['url'])

2020 self.assertIn('fr', params['url'])

2121

22− dicto['language'] = 'all'

23− params = google_news.request(query, dicto)

24− self.assertIn('url', params)

25− self.assertNotIn('fr', params['url'])

26−

2722 def test_response(self):

2823 self.assertRaises(AttributeError, google_news.response, None)

2924 self.assertRaises(AttributeError, google_news.response, [])

tests/unit/engines/test_mediawiki.py−4

@@ -18,10 +18,6 @@ class TestMediawikiEngine(SearxTestCase):

1818 self.assertIn('wikipedia.org', params['url'])

1919 self.assertIn('fr', params['url'])

2020

21− dicto['language'] = 'all'

22− params = mediawiki.request(query, dicto)

23− self.assertIn('en', params['url'])

24−

2521 mediawiki.base_url = "http://test.url/"

2622 mediawiki.search_url = mediawiki.base_url +\

2723 'w/api.php?action=query'\

tests/unit/engines/test_qwant.py+2−1

@@ -19,10 +19,11 @@ class TestQwantEngine(SearxTestCase):

1919 self.assertIn('qwant.com', params['url'])

2020 self.assertIn('fr_fr', params['url'])

2121

22− dicto['language'] = 'all'

22+ dicto['language'] = 'en-US'

2323 qwant.categories = ['news']

2424 params = qwant.request(query, dicto)

2525 self.assertFalse('fr' in params['url'])

26+ self.assertIn('en_us', params['url'])

2627 self.assertIn('news', params['url'])

2728

2829 qwant.supported_languages = ['en', 'fr-FR', 'fr-CA']

tests/unit/engines/test_startpage.py−4

@@ -21,10 +21,6 @@ class TestStartpageEngine(SearxTestCase):

2121 self.assertIn('with_language', params['data'])

2222 self.assertIn('lang_fr', params['data']['with_language'])

2323

24− dicto['language'] = 'all'

25− params = startpage.request(query, dicto)

26− self.assertNotIn('with_language', params['data'])

27−

2824 def test_response(self):

2925 self.assertRaises(AttributeError, startpage.response, None)

3026 self.assertRaises(AttributeError, startpage.response, [])

tests/unit/engines/test_subtitleseeker.py−36

@@ -73,42 +73,6 @@ class TestSubtitleseekerEngine(SearxTestCase):

7373 results = subtitleseeker.response(response)

7474 self.assertEqual(results[0]['url'], 'http://this.is.the.url/Brazilian/')

7575

76− html = """

77− <div class="boxRows">

78− <div class="boxRowsInner" style="width:600px;">

79− <img src="http://static.subtitleseeker.com/images/movie.gif"

80− style="width:16px; height:16px;" class="icon">

81− <a href="http://this.is.the.url/"

82− class="blue" title="Title subtitle" >

83− This is the Title

84− </a>

85− </div>

86− <div class="boxRowsInner f12b red" style="width:70px;">

87− 1998

88− </div>

89− <div class="boxRowsInner grey-web f12" style="width:120px;">

90− <img src="http://static.subtitleseeker.com/images/basket_put.png"

91− style="width:16px; height:16px;" class="icon">

92− 1039 Subs

93− </div>

94− <div class="boxRowsInner grey-web f10" style="width:130px;">

95− <img src="http://static.subtitleseeker.com/images/arrow_refresh_small.png"

96− style="width:16px; height:16px;" class="icon">

97− 1 hours ago

98− </div>

99− <div class="clear"></div>

100− </div>

101− """

102− dicto['language'] = 'all'

103− response = mock.Mock(text=html, search_params=dicto)

104− results = subtitleseeker.response(response)

105− self.assertEqual(type(results), list)

106− self.assertEqual(len(results), 1)

107− self.assertEqual(results[0]['title'], 'This is the Title')

108− self.assertEqual(results[0]['url'], 'http://this.is.the.url/')

109− self.assertIn('1998', results[0]['content'])

110− self.assertIn('1039 Subs', results[0]['content'])

111−

11276 html = """

11377 <div class="boxRows">

11478 <div class="boxRowsInner" style="width:600px;">

tests/unit/engines/test_swisscows.py−5

@@ -18,11 +18,6 @@ class TestSwisscowsEngine(SearxTestCase):

1818 self.assertTrue('uiLanguage=de' in params['url'])

1919 self.assertTrue('region=de-DE' in params['url'])

2020

21− dicto['language'] = 'all'

22− params = swisscows.request(query, dicto)

23− self.assertTrue('uiLanguage=browser' in params['url'])

24− self.assertTrue('region=browser' in params['url'])

25−

2621 dicto['category'] = 'images'

2722 params = swisscows.request(query, dicto)

2823 self.assertIn('image', params['url'])

tests/unit/engines/test_twitter.py−6

@@ -20,12 +20,6 @@ class TestTwitterEngine(SearxTestCase):

2020 self.assertIn('lang', params['cookies'])

2121 self.assertIn('fr', params['cookies']['lang'])

2222

23− dicto['language'] = 'all'

24− params = twitter.request(query, dicto)

25− self.assertIn('cookies', params)

26− self.assertIn('lang', params['cookies'])

27− self.assertIn('en', params['cookies']['lang'])

28−

2923 def test_response(self):

3024 self.assertRaises(AttributeError, twitter.response, None)

3125 self.assertRaises(AttributeError, twitter.response, [])

tests/unit/engines/test_wikidata.py+3−3

@@ -11,14 +11,14 @@ class TestWikidataEngine(SearxTestCase):

1111 def test_request(self):

1212 query = 'test_query'

1313 dicto = defaultdict(dict)

14− dicto['language'] = 'all'

14+ dicto['language'] = 'en-US'

1515 params = wikidata.request(query, dicto)

1616 self.assertIn('url', params)

1717 self.assertIn(query, params['url'])

1818 self.assertIn('wikidata.org', params['url'])

1919 self.assertIn('en', params['url'])

2020

21− dicto['language'] = 'es_ES'

21+ dicto['language'] = 'es-ES'

2222 params = wikidata.request(query, dicto)

2323 self.assertIn(query, params['url'])

2424 self.assertIn('es', params['url'])

@@ -30,7 +30,7 @@ class TestWikidataEngine(SearxTestCase):

3030 self.assertRaises(AttributeError, wikidata.response, '')

3131 self.assertRaises(AttributeError, wikidata.response, '[]')

3232

33− response = mock.Mock(text='<html></html>', search_params={"language": "all"})

33+ response = mock.Mock(text='<html></html>', search_params={"language": "en"})

3434 self.assertEqual(wikidata.response(response), [])

3535

3636 def test_getDetail(self):

tests/unit/engines/test_wikipedia.py−4

@@ -25,10 +25,6 @@ class TestWikipediaEngine(SearxTestCase):

2525 self.assertIn('Test_Query', params['url'])

2626 self.assertNotIn('test_query', params['url'])

2727

28− dicto['language'] = 'all'

29− params = wikipedia.request(query, dicto)

30− self.assertIn('en', params['url'])

31−

3228 dicto['language'] = 'xx'

3329 params = wikipedia.request(query, dicto)

3430 self.assertIn('en', params['url'])

tests/unit/engines/test_yacy.py−5

@@ -17,11 +17,6 @@ class TestYacyEngine(SearxTestCase):

1717 self.assertIn('localhost', params['url'])

1818 self.assertIn('fr', params['url'])

1919

20− dicto['language'] = 'all'

21− params = yacy.request(query, dicto)

22− self.assertIn('url', params)

23− self.assertNotIn('lr=lang_', params['url'])

24−

2520 def test_response(self):

2621 self.assertRaises(AttributeError, yacy.response, None)

2722 self.assertRaises(AttributeError, yacy.response, [])

tests/unit/engines/test_yahoo.py−7

@@ -39,13 +39,6 @@ class TestYahooEngine(SearxTestCase):

3939 self.assertIn('sB', params['cookies'])

4040 self.assertIn('fr', params['cookies']['sB'])

4141

42− dicto['language'] = 'all'

43− params = yahoo.request(query, dicto)

44− self.assertIn('cookies', params)

45− self.assertIn('sB', params['cookies'])

46− self.assertIn('en', params['cookies']['sB'])

47− self.assertIn('en', params['url'])

48−

4942 def test_no_url_in_request_year_time_range(self):

5043 dicto = defaultdict(dict)

5144 query = 'test_query'

tests/unit/engines/test_yahoo_news.py−7

@@ -22,13 +22,6 @@ class TestYahooNewsEngine(SearxTestCase):

2222 self.assertIn('sB', params['cookies'])

2323 self.assertIn('fr', params['cookies']['sB'])

2424

25− dicto['language'] = 'all'

26− params = yahoo_news.request(query, dicto)

27− self.assertIn('cookies', params)

28− self.assertIn('sB', params['cookies'])

29− self.assertIn('en', params['cookies']['sB'])

30− self.assertIn('en', params['url'])

31−

3225 def test_sanitize_url(self):

3326 url = "test.url"

3427 self.assertEqual(url, yahoo_news.sanitize_url(url))

tests/unit/test_preferences.py+10−5

@@ -90,25 +90,30 @@ class TestSettings(SearxTestCase):

9090

9191 # search language settings

9292 def test_lang_setting_valid_choice(self):

93− setting = SearchLanguageSetting('all', choices=['all', 'de', 'en'])

93+ setting = SearchLanguageSetting('en', choices=['de', 'en'])

9494 setting.parse('de')

9595 self.assertEquals(setting.get_value(), 'de')

9696

9797 def test_lang_setting_invalid_choice(self):

98− setting = SearchLanguageSetting('all', choices=['all', 'de', 'en'])

98+ setting = SearchLanguageSetting('en', choices=['de', 'en'])

9999 setting.parse('xx')

100− self.assertEquals(setting.get_value(), 'all')

100+ self.assertEquals(setting.get_value(), 'en')

101101

102102 def test_lang_setting_old_cookie_choice(self):

103− setting = SearchLanguageSetting('all', choices=['all', 'es', 'es-ES'])

103+ setting = SearchLanguageSetting('en', choices=['en', 'es', 'es-ES'])

104104 setting.parse('es_XA')

105105 self.assertEquals(setting.get_value(), 'es')

106106

107107 def test_lang_setting_old_cookie_format(self):

108− setting = SearchLanguageSetting('all', choices=['all', 'es', 'es-ES'])

108+ setting = SearchLanguageSetting('en', choices=['en', 'es', 'es-ES'])

109109 setting.parse('es_ES')

110110 self.assertEquals(setting.get_value(), 'es-ES')

111111

112+ def test_lang_setting_old_default(self):

113+ setting = SearchLanguageSetting('en', choices=['en', 'es', 'de'])

114+ setting.parse('all')

115+ self.assertEquals(setting.get_value(), 'en')

116+

112117 # plugins settings

113118 def test_plugins_setting_all_default_enabled(self):

114119 plugin1 = PluginStub('plugin1', True)

tests/unit/test_query.pynew file+64

@@ -0,0 +1,64 @@

1+from searx.query import RawTextQuery

2+from searx.testing import SearxTestCase

5+class TestQuery(SearxTestCase):

7+ def test_simple_query(self):

8+ query_text = 'the query'

9+ query = RawTextQuery(query_text, [])

10+ query.parse_query()

11+

12+ self.assertEquals(query.getFullQuery(), query_text)

13+ self.assertEquals(len(query.query_parts), 1)

14+ self.assertEquals(len(query.languages), 0)

15+ self.assertFalse(query.specific)

16+

17+ def test_language_code(self):

18+ language = 'es-ES'

19+ query_text = 'the query'

20+ full_query = ':' + language + ' ' + query_text

21+ query = RawTextQuery(full_query, [])

22+ query.parse_query()

23+

24+ self.assertEquals(query.getFullQuery(), full_query)

25+ self.assertEquals(len(query.query_parts), 3)

26+ self.assertEquals(len(query.languages), 1)

27+ self.assertIn(language, query.languages)

28+ self.assertFalse(query.specific)

29+

30+ def test_language_name(self):

31+ language = 'english'

32+ query_text = 'the query'

33+ full_query = ':' + language + ' ' + query_text

34+ query = RawTextQuery(full_query, [])

35+ query.parse_query()

36+

37+ self.assertEquals(query.getFullQuery(), full_query)

38+ self.assertEquals(len(query.query_parts), 3)

39+ self.assertIn('en', query.languages)

40+ self.assertFalse(query.specific)

41+

42+ def test_unlisted_language_code(self):

43+ language = 'all'

44+ query_text = 'the query'

45+ full_query = ':' + language + ' ' + query_text

46+ query = RawTextQuery(full_query, [])

47+ query.parse_query()

48+

49+ self.assertEquals(query.getFullQuery(), full_query)

50+ self.assertEquals(len(query.query_parts), 3)

51+ self.assertIn('all', query.languages)

52+ self.assertFalse(query.specific)

53+

54+ def test_invalid_language_code(self):

55+ language = 'not_a_language'

56+ query_text = 'the query'

57+ full_query = ':' + language + ' ' + query_text

58+ query = RawTextQuery(full_query, [])

59+ query.parse_query()

60+

61+ self.assertEquals(query.getFullQuery(), full_query)

62+ self.assertEquals(len(query.query_parts), 1)

63+ self.assertEquals(len(query.languages), 0)

64+ self.assertFalse(query.specific)