b63d645 · searx · git.iko.soy

searx/engines/archlinux.py+2−2

compare with master

unifiedsplit

@@ -36,7 +36,7 @@ def locale_to_lang_code(locale):

36# wikis for some languages were moved off from the main site, we need to make

37# requests to correct URLs to be able to get results in those languages

38lang_urls = {

39− 'en': {

39+ 'all': {

40 'base': 'https://wiki.archlinux.org',

41 'search': '/index.php?title=Special:Search&offset={offset}&{query}'

42 },

@@ -67,7 +67,7 @@ lang_urls = {

67def get_lang_urls(language):

68 if language in lang_urls:

69 return lang_urls[language]

70− return lang_urls['en']

70+ return lang_urls['all']

71

72

73# Language names to build search requests for

searx/engines/bing.py+4−1

@@ -34,7 +34,10 @@ search_string = 'search?{query}&first={offset}'

34def request(query, params):

35 offset = (params['pageno'] - 1) * 10 + 1

36

37+ if params['language'] == 'all':

38+ lang = 'EN'

39+ else:

37− lang = match_language(params['language'], supported_languages, language_aliases)

40+ lang = match_language(params['language'], supported_languages, language_aliases)

38

41

39 query = u'language:{} {}'.format(lang.split('-')[0].upper(), query.decode('utf-8')).encode('utf-8')

42 query = u'language:{} {}'.format(lang.split('-')[0].upper(), query.decode('utf-8')).encode('utf-8')

40

43

searx/engines/bing_news.py+4−1

@@ -71,7 +71,10 @@ def request(query, params):

71

72 offset = (params['pageno'] - 1) * 10 + 1

73

74+ if params['language'] == 'all':

75+ language = 'en-US'

76+ else:

74− language = match_language(params['language'], supported_languages, language_aliases)

77+ language = match_language(params['language'], supported_languages, language_aliases)

75

78

76 params['url'] = _get_url(query, language, offset, params['time_range'])

79 params['url'] = _get_url(query, language, offset, params['time_range'])

77

80

searx/engines/dailymotion.py+4−1

@@ -33,7 +33,10 @@ supported_languages_url = 'https://api.dailymotion.com/languages'

33

34# do search-request

35def request(query, params):

36+ if params['language'] == 'all':

37+ locale = 'en-US'

38+ else:

36− locale = match_language(params['language'], supported_languages)

39+ locale = match_language(params['language'], supported_languages)

37

40

38 params['url'] = search_url.format(

41 params['url'] = search_url.format(

39 query=urlencode({'search': query, 'localization': locale}),

42 query=urlencode({'search': query, 'localization': locale}),

searx/engines/duckduckgo.py+9−3

@@ -54,6 +54,9 @@ content_xpath = './/a[@class="result__snippet"]'

54

55# match query's language to a region code that duckduckgo will accept

56def get_region_code(lang, lang_list=[]):

57+ if lang == 'all':

58+ return None

59+

57 lang_code = match_language(lang, lang_list, language_aliases, 'wt-WT')

60 lang_code = match_language(lang, lang_list, language_aliases, 'wt-WT')

58 lang_parts = lang_code.split('-')

61 lang_parts = lang_code.split('-')

59

62

@@ -61,7 +64,6 @@ def get_region_code(lang, lang_list=[]):

61 return lang_parts[1].lower() + '-' + lang_parts[0].lower()

64 return lang_parts[1].lower() + '-' + lang_parts[0].lower()

62

65

63

66

64−# do search-request

65def request(query, params):

67def request(query, params):

66 if params['time_range'] and params['time_range'] not in time_range_dict:

68 if params['time_range'] and params['time_range'] not in time_range_dict:

67 return params

69 return params

@@ -69,8 +71,12 @@ def request(query, params):

69 offset = (params['pageno'] - 1) * 30

71 offset = (params['pageno'] - 1) * 30

70

72

71 region_code = get_region_code(params['language'], supported_languages)

73 region_code = get_region_code(params['language'], supported_languages)

74+ if region_code:

72− params['url'] = url.format(

75+ params['url'] = url.format(

73− query=urlencode({'q': query, 'kl': region_code}), offset=offset, dc_param=offset)

76+ query=urlencode({'q': query, 'kl': region_code}), offset=offset, dc_param=offset)

77+ else:

78+ params['url'] = url.format(

79+ query=urlencode({'q': query}), offset=offset, dc_param=offset)

74

80

75 if params['time_range'] in time_range_dict:

81 if params['time_range'] in time_range_dict:

76 params['url'] += time_range_url.format(range=time_range_dict[params['time_range']])

82 params['url'] += time_range_url.format(range=time_range_dict[params['time_range']])

searx/engines/duckduckgo_images.py+6−2

@@ -56,8 +56,12 @@ def request(query, params):

56 safesearch = params['safesearch'] - 1

57

58 region_code = get_region_code(params['language'], lang_list=supported_languages)

59+ if region_code:

59− params['url'] = images_url.format(

60+ params['url'] = images_url.format(

60− query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd)

61+ query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd)

62+ else:

63+ params['url'] = images_url.format(

64+ query=urlencode({'q': query}), offset=offset, safesearch=safesearch, vqd=vqd)

61

65

62 return params

66 return params

63

67

searx/engines/faroo.py+4−1

@@ -40,7 +40,10 @@ def request(query, params):

40 offset = (params['pageno'] - 1) * number_of_results + 1

41 categorie = search_category.get(params['category'], 'web')

42

43+ if params['language'] == 'all':

44+ language = 'en'

45+ else:

43− language = params['language'].split('-')[0]

46+ language = params['language'].split('-')[0]

44

47

45 # if language is not supported, put it in english

48 # if language is not supported, put it in english

46 if language != 'en' and\

49 if language != 'en' and\

searx/engines/gigablast.py+6−3

@@ -50,9 +50,12 @@ supported_languages_url = 'https://gigablast.com/search?&rxikd=1'

50def request(query, params):

51 offset = (params['pageno'] - 1) * number_of_results

52

53+ if params['language'] == 'all':

54+ language = 'xx'

55+ else:

53− language = params['language'].replace('-', '_').lower()

56+ language = params['language'].replace('-', '_').lower()

54− if language.split('-')[0] != 'zh':

57+ if language.split('-')[0] != 'zh':

55− language = language.split('-')[0]

58+ language = language.split('-')[0]

56

59

57 if params['safesearch'] >= 1:

60 if params['safesearch'] >= 1:

58 safesearch = 1

61 safesearch = 1

searx/engines/google.py+5−1

@@ -166,7 +166,11 @@ def extract_text_from_dom(result, xpath):

166def request(query, params):

167 offset = (params['pageno'] - 1) * 10

168

169+ if params['language'] == 'all' or params['language'] == 'en-US':

170+ language = 'en-GB'

171+ else:

169− language = match_language(params['language'], supported_languages, language_aliases)

172+ language = match_language(params['language'], supported_languages, language_aliases)

173+

170 language_array = language.split('-')

174 language_array = language.split('-')

171 if params['language'].find('-') > 0:

175 if params['language'].find('-') > 0:

172 country = params['language'].split('-')[1]

176 country = params['language'].split('-')[1]

searx/engines/google_news.py+4−3

@@ -51,9 +51,10 @@ def request(query, params):

51 params['url'] = search_url.format(query=urlencode({'q': query}),

52 search_options=urlencode(search_options))

53

54+ if params['language'] != 'all':

54− language = match_language(params['language'], supported_languages, language_aliases).split('-')[0]

55+ language = match_language(params['language'], supported_languages, language_aliases).split('-')[0]

55− if language:

56+ if language:

56− params['url'] += '&lr=lang_' + language

57+ params['url'] += '&lr=lang_' + language

57

58

58 return params

59 return params

59

60

searx/engines/mediawiki.py+4−1

@@ -45,7 +45,10 @@ def request(query, params):

45

46 format_strings = list(Formatter().parse(base_url))

47

48+ if params['language'] == 'all':

49+ language = 'en'

50+ else:

48− language = params['language'].split('-')[0]

51+ language = params['language'].split('-')[0]

49

52

50 # format_string [('https://', 'language', '', None), ('.wikipedia.org/', None, None, None)]

53 # format_string [('https://', 'language', '', None), ('.wikipedia.org/', None, None, None)]

51 if any(x[1] == 'language' for x in format_strings):

54 if any(x[1] == 'language' for x in format_strings):

searx/engines/photon.py+4−3

@@ -35,9 +35,10 @@ def request(query, params):

35 search_string.format(query=urlencode({'q': query}),

36 limit=number_of_results)

37

38+ if params['language'] != 'all':

38− language = params['language'].split('-')[0]

39+ language = params['language'].split('_')[0]

39− if language in supported_languages:

40+ if language in supported_languages:

40− params['url'] = params['url'] + "&lang=" + language

41+ params['url'] = params['url'] + "&lang=" + language

41

42

42 # using searx User-Agent

43 # using searx User-Agent

43 params['headers']['User-Agent'] = searx_useragent()

44 params['headers']['User-Agent'] = searx_useragent()

searx/engines/qwant.py+3−2

@@ -46,8 +46,9 @@ def request(query, params):

46 offset=offset)

47

48 # add language tag

49+ if params['language'] != 'all':

49− language = match_language(params['language'], supported_languages, language_aliases)

50+ language = match_language(params['language'], supported_languages, language_aliases)

50− params['url'] += '&locale=' + language.replace('-', '_').lower()

51+ params['url'] += '&locale=' + language.replace('-', '_').lower()

51

52

52 return params

53 return params

53

54

searx/engines/startpage.py+3−2

@@ -46,8 +46,9 @@ def request(query, params):

46 params['data'] = {'query': query,

47 'startat': offset}

48

49− # set language

49+ # set language if specified

50+ if params['language'] != 'all':

50− params['data']['with_language'] = ('lang_' + params['language'].split('-')[0])

51+ params['data']['with_language'] = ('lang_' + params['language'].split('-')[0])

51

52

52 return params

53 return params

53

54

searx/engines/subtitleseeker.py+1−1

@@ -48,7 +48,7 @@ def response(resp):

48 search_lang = 'Farsi'

49 elif resp.search_params['language'] == 'pt-BR':

50 search_lang = 'Brazilian'

51− else:

51+ elif resp.search_params['language'] != 'all':

52 search_lang = [lc[3]

53 for lc in language_codes

54 if lc[0].split('-')[0] == resp.search_params['language'].split('-')[0]]

searx/engines/swisscows.py+6−2

@@ -36,8 +36,12 @@ regex_img_url_remove_start = re.compile(b'^https?://i\.swisscows\.ch/\?link=')

36

37# do search-request

38def request(query, params):

39+ if params['language'] == 'all':

40+ ui_language = 'browser'

41+ region = 'browser'

42+ else:

39− region = match_language(params['language'], supported_languages, language_aliases)

43+ region = match_language(params['language'], supported_languages, language_aliases)

40− ui_language = region.split('-')[0]

44+ ui_language = region.split('-')[0]

41

45

42 search_path = search_string.format(

46 search_path = search_string.format(

43 query=urlencode({'query': query, 'uiLanguage': ui_language, 'region': region}),

47 query=urlencode({'query': query, 'uiLanguage': ui_language, 'region': region}),

searx/engines/twitter.py+6−1

@@ -37,7 +37,12 @@ timestamp_xpath = './/span[contains(@class,"_timestamp")]'

37# do search-request

38def request(query, params):

39 params['url'] = search_url + urlencode({'q': query})

40+

41+ # set language if specified

42+ if params['language'] != 'all':

40− params['cookies']['lang'] = params['language'].split('-')[0]

43+ params['cookies']['lang'] = params['language'].split('-')[0]

44+ else:

45+ params['cookies']['lang'] = 'en'

41

46

42 return params

47 return params

43

48

searx/engines/wikidata.py+4−1

@@ -68,7 +68,10 @@ def response(resp):

68 html = fromstring(resp.text)

69 search_results = html.xpath(wikidata_ids_xpath)

70

71+ if resp.search_params['language'].split('-')[0] == 'all':

72+ language = 'en'

73+ else:

71− language = match_language(resp.search_params['language'], supported_languages, language_aliases).split('-')[0]

74+ language = match_language(resp.search_params['language'], supported_languages, language_aliases).split('-')[0]

72

75

73 # TODO: make requests asynchronous to avoid timeout when result_count > 1

76 # TODO: make requests asynchronous to avoid timeout when result_count > 1

74 for search_result in search_results[:result_count]:

77 for search_result in search_results[:result_count]:

searx/engines/wikipedia.py+3

@@ -31,6 +31,9 @@ supported_languages_url = 'https://meta.wikimedia.org/wiki/List_of_Wikipedias'

31

32# set language in base_url

33def url_lang(lang):

34+ lang_pre = lang.split('-')[0]

35+ if lang_pre == 'all' or lang_pre not in supported_languages:

36+ return 'en'

34 return match_language(lang, supported_languages, language_aliases).split('-')[0]

37 return match_language(lang, supported_languages, language_aliases).split('-')[0]

35

38

36

39

searx/engines/yacy.py+3−1

@@ -51,7 +51,9 @@ def request(query, params):

51 limit=number_of_results,

52 search_type=search_type)

53

54+ # add language tag if specified

55+ if params['language'] != 'all':

54− params['url'] += '&lr=lang_' + params['language'].split('-')[0]

56+ params['url'] += '&lr=lang_' + params['language'].split('-')[0]

55

57

56 return params

58 return params

57

59

searx/engines/yahoo.py+13−4

@@ -73,16 +73,25 @@ def _get_url(query, offset, language, time_range):

73 lang=language)

74

75

76+def _get_language(params):

77+ if params['language'] == 'all':

78+ return 'en'

79+

80+ language = match_language(params['language'], supported_languages, language_aliases)

81+ if language not in language_aliases.values():

82+ language = language.split('-')[0]

83+ language = language.replace('-', '_').lower()

84+

85+ return language

86+

87+

76# do search-request

88# do search-request

77def request(query, params):

89def request(query, params):

78 if params['time_range'] and params['time_range'] not in time_range_dict:

90 if params['time_range'] and params['time_range'] not in time_range_dict:

79 return params

91 return params

80

92

81 offset = (params['pageno'] - 1) * 10 + 1

93 offset = (params['pageno'] - 1) * 10 + 1

82− language = match_language(params['language'], supported_languages, language_aliases)

94+ language = _get_language(params)

83− if language not in language_aliases.values():

84− language = language.split('-')[0]

85− language = language.replace('-', '_').lower()

86

95

87 params['url'] = _get_url(query, offset, language, params['time_range'])

96 params['url'] = _get_url(query, offset, language, params['time_range'])

88

97

searx/engines/yahoo_news.py+4−1

@@ -41,7 +41,10 @@ suggestion_xpath = '//div[contains(@class,"VerALSOTRY")]//a'

41def request(query, params):

42 offset = (params['pageno'] - 1) * 10 + 1

43

44+ if params['language'] == 'all':

45+ language = 'en'

46+ else:

44− language = match_language(params['language'], supported_languages, language_aliases).split('-')[0]

47+ language = match_language(params['language'], supported_languages, language_aliases).split('-')[0]

45

48

46 params['url'] = search_url.format(offset=offset,

49 params['url'] = search_url.format(offset=offset,

47 query=urlencode({'p': query}),

50 query=urlencode({'p': query}),

searx/engines/youtube_api.py+3−1

@@ -34,7 +34,9 @@ def request(query, params):

34 params['url'] = search_url.format(query=urlencode({'q': query}),

35 api_key=api_key)

36

37+ # add language tag if specified

38+ if params['language'] != 'all':

37− params['url'] += '&relevanceLanguage=' + params['language'].split('-')[0]

39+ params['url'] += '&relevanceLanguage=' + params['language'].split('-')[0]

38

40

39 return params

41 return params

40

42

searx/preferences.py+1

@@ -12,6 +12,7 @@ if version[0] == '3':

12

13COOKIE_MAX_AGE = 60 * 60 * 24 * 365 * 5 # 5 years

14LANGUAGE_CODES = [l[0] for l in languages]

15+LANGUAGE_CODES.append('all')

15DISABLED = 0

16DISABLED = 0

16ENABLED = 1

17ENABLED = 1

17DOI_RESOLVERS = list(settings['doi_resolvers'])

18DOI_RESOLVERS = list(settings['doi_resolvers'])

searx/search.py+1−5

@@ -24,7 +24,7 @@ from flask_babel import gettext

24import requests.exceptions

25import searx.poolrequests as requests_lib

26from searx.engines import (

27− categories, engines, settings

27+ categories, engines

28)

29from searx.answerers import ask

30from searx.utils import gen_useragent

@@ -221,10 +221,6 @@ def get_search_query_from_webapp(preferences, form):

221 else:

222 query_lang = preferences.get_value('language')

223

224− # provides backwards compatibility for requests using old language default

225− if query_lang == 'all':

226− query_lang = settings['search']['language']

227−

228 # check language

224 # check language

229 if not VALID_LANGUAGE_CODE.match(query_lang):

225 if not VALID_LANGUAGE_CODE.match(query_lang):

230 raise SearxParameterException('language', query_lang)

226 raise SearxParameterException('language', query_lang)

searx/settings_robot.yml+1−1

@@ -5,7 +5,7 @@ general:

5search:

6 safe_search : 0

7 autocomplete : ""

8− language: "en-US"

8+ language: "all"

9

10server:

11 port : 11111

searx/templates/courgette/preferences.html+1

@@ -13,6 +13,7 @@

13 <legend>{{ _('Search language') }}</legend>

14 <p>

15 <select name='language'>

16+ <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>

16 {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}

17 {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}

17 <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>

18 <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>

18 {% endfor %}

19 {% endfor %}

searx/templates/legacy/preferences.html+1

@@ -14,6 +14,7 @@

14 <legend>{{ _('Search language') }}</legend>

15 <p>

16 <select name='language'>

17+ <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>

17 {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}

18 {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}

18 <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>

19 <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>

19 {% endfor %}

20 {% endfor %}

searx/templates/oscar/languages.html+6−5

@@ -3,9 +3,10 @@

3{% else %}

4<select class="time_range custom-select form-control" id='language' name='language'>

5{% endif %}

6+ <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>

6− {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}

7+ {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}

7− <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>

8+ <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>

8− {{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}

9+ {{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}

9− </option>

10+ </option>

10− {% endfor %}

11+ {% endfor %}

11</select>

12</select>

searx/templates/pix-art/preferences.html+1

@@ -9,6 +9,7 @@

9 <legend>{{ _('Search language') }}</legend>

10 <p>

11 <select name='language'>

12+ <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>

12 {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}

13 {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}

13 <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>

14 <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>

14 {% endfor %}

15 {% endfor %}

searx/webapp.py+11−6

@@ -637,8 +637,8 @@ def autocompleter():

637 if len(raw_results) <= 3 and completer:

638 # get language from cookie

639 language = request.preferences.get_value('language')

640− if not language:

640+ if not language or language == 'all':

641− language = settings['search']['language']

641+ language = 'en'

642 else:

643 language = language.split('-')[0]

644 # run autocompletion

@@ -691,10 +691,7 @@ def preferences():

691 'warn_time': False}

692 if e.timeout > settings['outgoing']['request_timeout']:

693 stats[e.name]['warn_timeout'] = True

694− if match_language(request.preferences.get_value('language'),

695− getattr(e, 'supported_languages', []),

696− getattr(e, 'language_aliases', {}), None):

697− stats[e.name]['supports_selected_language'] = True

694+ stats[e.name]['supports_selected_language'] = _is_selected_language_supported(e, request.preferences)

698

695

699 # get first element [0], the engine time,

696 # get first element [0], the engine time,

700 # and then the second element [1] : the time (the first one is the label)

697 # and then the second element [1] : the time (the first one is the label)

@@ -725,6 +722,14 @@ def preferences():

725 preferences=True)

722 preferences=True)

726

723

727

724

725+def _is_selected_language_supported(engine, preferences):

726+ language = preferences.get_value('language')

727+ return (language == 'all'

728+ or match_language(language,

729+ getattr(engine, 'supported_languages', []),

730+ getattr(engine, 'language_aliases', {}), None))

731+

732+

728@app.route('/image_proxy', methods=['GET'])

733@app.route('/image_proxy', methods=['GET'])

729def image_proxy():

734def image_proxy():

730 url = request.args.get('url').encode('utf-8')

735 url = request.args.get('url').encode('utf-8')

tests/unit/engines/test_archlinux.py+1−1

@@ -4,7 +4,7 @@ from searx.engines import archlinux

4from searx.testing import SearxTestCase

5

6domains = {

7− 'en': 'https://wiki.archlinux.org',

7+ 'all': 'https://wiki.archlinux.org',

8 'de': 'https://wiki.archlinux.de',

9 'fr': 'https://wiki.archlinux.fr',

10 'ja': 'https://wiki.archlinuxjp.org',

tests/unit/engines/test_bing.py+4

@@ -18,6 +18,10 @@ class TestBingEngine(SearxTestCase):

18 self.assertTrue('language%3AFR' in params['url'])

19 self.assertTrue('bing.com' in params['url'])

20

21+ dicto['language'] = 'all'

22+ params = bing.request(query.encode('utf-8'), dicto)

23+ self.assertTrue('language' in params['url'])

24+

21 def test_response(self):

25 def test_response(self):

22 self.assertRaises(AttributeError, bing.response, None)

26 self.assertRaises(AttributeError, bing.response, None)

23 self.assertRaises(AttributeError, bing.response, [])

27 self.assertRaises(AttributeError, bing.response, [])

tests/unit/engines/test_bing_news.py+4

@@ -20,6 +20,10 @@ class TestBingNewsEngine(SearxTestCase):

20 self.assertIn('bing.com', params['url'])

21 self.assertIn('fr', params['url'])

22

23+ dicto['language'] = 'all'

24+ params = bing_news.request(query, dicto)

25+ self.assertIn('en', params['url'])

26+

23 def test_no_url_in_request_year_time_range(self):

27 def test_no_url_in_request_year_time_range(self):

24 dicto = defaultdict(dict)

28 dicto = defaultdict(dict)

25 query = 'test_query'

29 query = 'test_query'

tests/unit/engines/test_dailymotion.py+4

@@ -19,6 +19,10 @@ class TestDailymotionEngine(SearxTestCase):

19 self.assertTrue('dailymotion.com' in params['url'])

20 self.assertTrue('fr' in params['url'])

21

22+ dicto['language'] = 'all'

23+ params = dailymotion.request(query, dicto)

24+ self.assertTrue('en' in params['url'])

25+

22 def test_response(self):

26 def test_response(self):

23 self.assertRaises(AttributeError, dailymotion.response, None)

27 self.assertRaises(AttributeError, dailymotion.response, None)

24 self.assertRaises(AttributeError, dailymotion.response, [])

28 self.assertRaises(AttributeError, dailymotion.response, [])

tests/unit/engines/test_duckduckgo_images.py+1−1

@@ -14,7 +14,7 @@ class TestDuckduckgoImagesEngine(SearxTestCase):

14 dicto['is_test'] = True

15 dicto['pageno'] = 1

16 dicto['safesearch'] = 0

17− dicto['language'] = 'en-US'

17+ dicto['language'] = 'all'

18 params = duckduckgo_images.request(query, dicto)

19 self.assertIn('url', params)

20 self.assertIn(query, params['url'])

tests/unit/engines/test_faroo.py+4

@@ -20,6 +20,10 @@ class TestFarooEngine(SearxTestCase):

20 self.assertIn('en', params['url'])

21 self.assertIn('web', params['url'])

22

23+ dicto['language'] = 'all'

24+ params = faroo.request(query, dicto)

25+ self.assertIn('en', params['url'])

26+

23 dicto['language'] = 'de-DE'

27 dicto['language'] = 'de-DE'

24 params = faroo.request(query, dicto)

28 params = faroo.request(query, dicto)

25 self.assertIn('de', params['url'])

29 self.assertIn('de', params['url'])

tests/unit/engines/test_gigablast.py+6−1

@@ -11,11 +11,16 @@ class TestGigablastEngine(SearxTestCase):

11 dicto = defaultdict(dict)

12 dicto['pageno'] = 0

13 dicto['safesearch'] = 0

14− dicto['language'] = 'en-US'

14+ dicto['language'] = 'all'

15 params = gigablast.request(query, dicto)

16 self.assertTrue('url' in params)

17 self.assertTrue(query in params['url'])

18 self.assertTrue('gigablast.com' in params['url'])

19+ self.assertTrue('xx' in params['url'])

20+

21+ dicto['language'] = 'en-US'

22+ params = gigablast.request(query, dicto)

23+ self.assertTrue('en' in params['url'])

19 self.assertFalse('en-US' in params['url'])

24 self.assertFalse('en-US' in params['url'])

20

25

21 def test_response(self):

26 def test_response(self):

tests/unit/engines/test_google_news.py+5

@@ -21,6 +21,11 @@ class TestGoogleNewsEngine(SearxTestCase):

21 self.assertIn(query, params['url'])

22 self.assertIn('fr', params['url'])

23

24+ dicto['language'] = 'all'

25+ params = google_news.request(query, dicto)

26+ self.assertIn('url', params)

27+ self.assertNotIn('fr', params['url'])

28+

24 def test_response(self):

29 def test_response(self):

25 self.assertRaises(AttributeError, google_news.response, None)

30 self.assertRaises(AttributeError, google_news.response, None)

26 self.assertRaises(AttributeError, google_news.response, [])

31 self.assertRaises(AttributeError, google_news.response, [])

tests/unit/engines/test_mediawiki.py+4

@@ -18,6 +18,10 @@ class TestMediawikiEngine(SearxTestCase):

18 self.assertIn('wikipedia.org', params['url'])

19 self.assertIn('fr', params['url'])

20

21+ dicto['language'] = 'all'

22+ params = mediawiki.request(query, dicto)

23+ self.assertIn('en', params['url'])

24+

21 mediawiki.base_url = "http://test.url/"

25 mediawiki.base_url = "http://test.url/"

22 mediawiki.search_url = mediawiki.base_url +\

26 mediawiki.search_url = mediawiki.base_url +\

23 'w/api.php?action=query'\

27 'w/api.php?action=query'\

tests/unit/engines/test_qwant.py+1−2

@@ -21,11 +21,10 @@ class TestQwantEngine(SearxTestCase):

21 self.assertIn('qwant.com', params['url'])

22 self.assertIn('fr_fr', params['url'])

23

24− dicto['language'] = 'en-US'

24+ dicto['language'] = 'all'

25 qwant.categories = ['news']

26 params = qwant.request(query, dicto)

27 self.assertFalse('fr' in params['url'])

28− self.assertIn('en_us', params['url'])

29 self.assertIn('news', params['url'])

28 self.assertIn('news', params['url'])

30

29

31 dicto['language'] = 'fr'

30 dicto['language'] = 'fr'

tests/unit/engines/test_startpage.py+4

@@ -21,6 +21,10 @@ class TestStartpageEngine(SearxTestCase):

21 self.assertIn('with_language', params['data'])

22 self.assertIn('lang_fr', params['data']['with_language'])

23

24+ dicto['language'] = 'all'

25+ params = startpage.request(query, dicto)

26+ self.assertNotIn('with_language', params['data'])

27+

24 def test_response(self):

28 def test_response(self):

25 self.assertRaises(AttributeError, startpage.response, None)

29 self.assertRaises(AttributeError, startpage.response, None)

26 self.assertRaises(AttributeError, startpage.response, [])

30 self.assertRaises(AttributeError, startpage.response, [])

tests/unit/engines/test_subtitleseeker.py+36

@@ -73,6 +73,42 @@ class TestSubtitleseekerEngine(SearxTestCase):

73 results = subtitleseeker.response(response)

74 self.assertEqual(results[0]['url'], 'http://this.is.the.url/Brazilian/')

75

76+ html = """

77+ <div class="boxRows">

78+ <div class="boxRowsInner" style="width:600px;">

79+ <img src="http://static.subtitleseeker.com/images/movie.gif"

80+ style="width:16px; height:16px;" class="icon">

81+ <a href="http://this.is.the.url/"

82+ class="blue" title="Title subtitle" >

83+ This is the Title

84+ </a>

85+ </div>

86+ <div class="boxRowsInner f12b red" style="width:70px;">

87+ 1998

88+ </div>

89+ <div class="boxRowsInner grey-web f12" style="width:120px;">

90+ <img src="http://static.subtitleseeker.com/images/basket_put.png"

91+ style="width:16px; height:16px;" class="icon">

92+ 1039 Subs

93+ </div>

94+ <div class="boxRowsInner grey-web f10" style="width:130px;">

95+ <img src="http://static.subtitleseeker.com/images/arrow_refresh_small.png"

96+ style="width:16px; height:16px;" class="icon">

97+ 1 hours ago

98+ </div>

99+ <div class="clear"></div>

100+ </div>

101+ """

102+ dicto['language'] = 'all'

103+ response = mock.Mock(text=html, search_params=dicto)

104+ results = subtitleseeker.response(response)

105+ self.assertEqual(type(results), list)

106+ self.assertEqual(len(results), 1)

107+ self.assertEqual(results[0]['title'], 'This is the Title')

108+ self.assertEqual(results[0]['url'], 'http://this.is.the.url/')

109+ self.assertIn('1998', results[0]['content'])

110+ self.assertIn('1039 Subs', results[0]['content'])

111+

76 html = """

112 html = """

77 <div class="boxRows">

113 <div class="boxRows">

78 <div class="boxRowsInner" style="width:600px;">

114 <div class="boxRowsInner" style="width:600px;">

tests/unit/engines/test_swisscows.py+5

@@ -20,6 +20,11 @@ class TestSwisscowsEngine(SearxTestCase):

20 self.assertTrue('uiLanguage=de' in params['url'])

21 self.assertTrue('region=de-DE' in params['url'])

22

23+ dicto['language'] = 'all'

24+ params = swisscows.request(query, dicto)

25+ self.assertTrue('uiLanguage=browser' in params['url'])

26+ self.assertTrue('region=browser' in params['url'])

27+

23 dicto['category'] = 'images'

28 dicto['category'] = 'images'

24 params = swisscows.request(query, dicto)

29 params = swisscows.request(query, dicto)

25 self.assertIn('image', params['url'])

30 self.assertIn('image', params['url'])

tests/unit/engines/test_twitter.py+6

@@ -20,6 +20,12 @@ class TestTwitterEngine(SearxTestCase):

20 self.assertIn('lang', params['cookies'])

21 self.assertIn('fr', params['cookies']['lang'])

22

23+ dicto['language'] = 'all'

24+ params = twitter.request(query, dicto)

25+ self.assertIn('cookies', params)

26+ self.assertIn('lang', params['cookies'])

27+ self.assertIn('en', params['cookies']['lang'])

28+

23 def test_response(self):

29 def test_response(self):

24 self.assertRaises(AttributeError, twitter.response, None)

30 self.assertRaises(AttributeError, twitter.response, None)

25 self.assertRaises(AttributeError, twitter.response, [])

31 self.assertRaises(AttributeError, twitter.response, [])

tests/unit/engines/test_wikidata.py+2

@@ -11,11 +11,13 @@ class TestWikidataEngine(SearxTestCase):

11 def test_request(self):

12 query = 'test_query'

13 dicto = defaultdict(dict)

14+ dicto['language'] = 'all'

14 params = wikidata.request(query, dicto)

15 params = wikidata.request(query, dicto)

15 self.assertIn('url', params)

16 self.assertIn('url', params)

16 self.assertIn(query, params['url'])

17 self.assertIn(query, params['url'])

17 self.assertIn('wikidata.org', params['url'])

18 self.assertIn('wikidata.org', params['url'])

18

19

20+ dicto['language'] = 'es_ES'

19 params = wikidata.request(query, dicto)

21 params = wikidata.request(query, dicto)

20 self.assertIn(query, params['url'])

22 self.assertIn(query, params['url'])

21

23

tests/unit/engines/test_wikipedia.py+6

@@ -26,9 +26,15 @@ class TestWikipediaEngine(SearxTestCase):

26 self.assertIn('Test_Query', params['url'])

27 self.assertNotIn('test_query', params['url'])

28

29+<<<<<<< HEAD

29 dicto['language'] = 'nb'

30 dicto['language'] = 'nb'

30 params = wikipedia.request(query, dicto)

31 params = wikipedia.request(query, dicto)

31 self.assertIn('no.wikipedia.org', params['url'])

32 self.assertIn('no.wikipedia.org', params['url'])

33+=======

34+ dicto['language'] = 'all'

35+ params = wikipedia.request(query, dicto)

36+ self.assertIn('en', params['url'])

37+>>>>>>> Revert "remove 'all' option from search languages"

32

38

33 dicto['language'] = 'xx'

39 dicto['language'] = 'xx'

34 params = wikipedia.request(query, dicto)

40 params = wikipedia.request(query, dicto)

tests/unit/engines/test_yacy.py+5

@@ -17,6 +17,11 @@ class TestYacyEngine(SearxTestCase):

17 self.assertIn('localhost', params['url'])

18 self.assertIn('fr', params['url'])

19

20+ dicto['language'] = 'all'

21+ params = yacy.request(query, dicto)

22+ self.assertIn('url', params)

23+ self.assertNotIn('lr=lang_', params['url'])

24+

20 def test_response(self):

25 def test_response(self):

21 self.assertRaises(AttributeError, yacy.response, None)

26 self.assertRaises(AttributeError, yacy.response, None)

22 self.assertRaises(AttributeError, yacy.response, [])

27 self.assertRaises(AttributeError, yacy.response, [])

tests/unit/engines/test_yahoo.py+7

@@ -50,6 +50,13 @@ class TestYahooEngine(SearxTestCase):

50 self.assertIn('zh_cht', params['url'])

51 self.assertIn('zh_cht', params['cookies']['sB'])

52

53+ dicto['language'] = 'all'

54+ params = yahoo.request(query, dicto)

55+ self.assertIn('cookies', params)

56+ self.assertIn('sB', params['cookies'])

57+ self.assertIn('en', params['cookies']['sB'])

58+ self.assertIn('en', params['url'])

59+

53 def test_no_url_in_request_year_time_range(self):

60 def test_no_url_in_request_year_time_range(self):

54 dicto = defaultdict(dict)

61 dicto = defaultdict(dict)

55 query = 'test_query'

62 query = 'test_query'

tests/unit/engines/test_yahoo_news.py+7

@@ -23,6 +23,13 @@ class TestYahooNewsEngine(SearxTestCase):

23 self.assertIn('sB', params['cookies'])

24 self.assertIn('fr', params['cookies']['sB'])

25

26+ dicto['language'] = 'all'

27+ params = yahoo_news.request(query, dicto)

28+ self.assertIn('cookies', params)

29+ self.assertIn('sB', params['cookies'])

30+ self.assertIn('en', params['cookies']['sB'])

31+ self.assertIn('en', params['url'])

32+

26 def test_sanitize_url(self):

33 def test_sanitize_url(self):

27 url = "test.url"

34 url = "test.url"

28 self.assertEqual(url, yahoo_news.sanitize_url(url))

35 self.assertEqual(url, yahoo_news.sanitize_url(url))

tests/unit/test_preferences.py+5−10

@@ -90,30 +90,25 @@ class TestSettings(SearxTestCase):

90

91 # search language settings

92 def test_lang_setting_valid_choice(self):

93− setting = SearchLanguageSetting('en', choices=['de', 'en'])

93+ setting = SearchLanguageSetting('all', choices=['all', 'de', 'en'])

94 setting.parse('de')

95 self.assertEquals(setting.get_value(), 'de')

96

97 def test_lang_setting_invalid_choice(self):

98− setting = SearchLanguageSetting('en', choices=['de', 'en'])

98+ setting = SearchLanguageSetting('all', choices=['all', 'de', 'en'])

99 setting.parse('xx')

100− self.assertEquals(setting.get_value(), 'en')

100+ self.assertEquals(setting.get_value(), 'all')

101

102 def test_lang_setting_old_cookie_choice(self):

103− setting = SearchLanguageSetting('en', choices=['en', 'es', 'es-ES'])

103+ setting = SearchLanguageSetting('all', choices=['all', 'es', 'es-ES'])

104 setting.parse('es_XA')

105 self.assertEquals(setting.get_value(), 'es')

106

107 def test_lang_setting_old_cookie_format(self):

108− setting = SearchLanguageSetting('en', choices=['en', 'es', 'es-ES'])

108+ setting = SearchLanguageSetting('all', choices=['all', 'es', 'es-ES'])

109 setting.parse('es_ES')

110 self.assertEquals(setting.get_value(), 'es-ES')

111

112− def test_lang_setting_old_default(self):

113− setting = SearchLanguageSetting('en', choices=['en', 'es', 'de'])

114− setting.parse('all')

115− self.assertEquals(setting.get_value(), 'en')

116−

117 # plugins settings

112 # plugins settings

118 def test_plugins_setting_all_default_enabled(self):

113 def test_plugins_setting_all_default_enabled(self):

119 plugin1 = PluginStub('plugin1', True)

114 plugin1 = PluginStub('plugin1', True)