49b8450 · searx · git.iko.soy

searx/engines/archlinux.py+3−3

compare with master

unifiedsplit

@@ -26,7 +26,7 @@ xpath_results = '//ul[@class="mw-search-results"]/li'

26xpath_link = './/div[@class="mw-search-result-heading"]/a'

27

28

29−# cut 'en' from 'en_US', 'de' from 'de_CH', and so on

29+# cut 'en' from 'en-US', 'de' from 'de-CH', and so on

30def locale_to_lang_code(locale):

31 if locale.find('-') >= 0:

32 locale = locale.split('-')[0]

@@ -36,7 +36,7 @@ def locale_to_lang_code(locale):

36# wikis for some languages were moved off from the main site, we need to make

37# requests to correct URLs to be able to get results in those languages

38lang_urls = {

39− 'all': {

39+ 'en': {

40 'base': 'https://wiki.archlinux.org',

41 'search': '/index.php?title=Special:Search&offset={offset}&{query}'

42 },

@@ -67,7 +67,7 @@ lang_urls = {

67def get_lang_urls(language):

68 if language in lang_urls:

69 return lang_urls[language]

70− return lang_urls['all']

70+ return lang_urls['en']

71

72

73# Language names to build search requests for

searx/engines/bing.py+1−4

@@ -32,10 +32,7 @@ search_string = 'search?{query}&first={offset}'

32def request(query, params):

33 offset = (params['pageno'] - 1) * 10 + 1

34

35− if params['language'] != 'all':

36− lang = params['language'].split('-')[0].upper()

35+ lang = params['language'].split('-')[0].upper()

37− else:

38− lang = 'EN'

39

36

40 query = u'language:{} {}'.format(lang, query.decode('utf-8')).encode('utf-8')

37 query = u'language:{} {}'.format(lang, query.decode('utf-8')).encode('utf-8')

41

38

searx/engines/bing_news.py+1−4

@@ -71,10 +71,7 @@ def request(query, params):

71

72 offset = (params['pageno'] - 1) * 10 + 1

73

74− if params['language'] == 'all':

75− language = 'en-US'

76− else:

77− language = params['language']

74+ language = params['language']

78

75

79 params['url'] = _get_url(query, language, offset, params['time_range'])

76 params['url'] = _get_url(query, language, offset, params['time_range'])

80

77

searx/engines/dailymotion.py+1−4

@@ -32,10 +32,7 @@ supported_languages_url = 'https://api.dailymotion.com/languages'

32

33# do search-request

34def request(query, params):

35− if params['language'] == 'all':

36− locale = 'en-US'

37− else:

38− locale = params['language']

35+ locale = params['language']

39

36

40 params['url'] = search_url.format(

37 params['url'] = search_url.format(

41 query=urlencode({'search': query, 'localization': locale}),

38 query=urlencode({'search': query, 'localization': locale}),

searx/engines/duckduckgo.py+3−9

@@ -44,9 +44,7 @@ content_xpath = './/a[@class="result__snippet"]'

44# match query's language to a region code that duckduckgo will accept

45def get_region_code(lang, lang_list=None):

46 # custom fixes for languages

47− if lang == 'all':

47+ if lang[:2] == 'ja':

48− region_code = None

49− elif lang[:2] == 'ja':

50 region_code = 'jp-jp'

48 region_code = 'jp-jp'

51 elif lang[:2] == 'sl':

49 elif lang[:2] == 'sl':

52 region_code = 'sl-sl'

50 region_code = 'sl-sl'

@@ -82,12 +80,8 @@ def request(query, params):

82 offset = (params['pageno'] - 1) * 30

80 offset = (params['pageno'] - 1) * 30

83

81

84 region_code = get_region_code(params['language'])

82 region_code = get_region_code(params['language'])

85− if region_code:

86− params['url'] = url.format(

83+ params['url'] = url.format(

87− query=urlencode({'q': query, 'kl': region_code}), offset=offset, dc_param=offset)

84+ query=urlencode({'q': query, 'kl': region_code}), offset=offset, dc_param=offset)

88− else:

89− params['url'] = url.format(

90− query=urlencode({'q': query}), offset=offset, dc_param=offset)

91

85

92 if params['time_range'] in time_range_dict:

86 if params['time_range'] in time_range_dict:

93 params['url'] += time_range_url.format(range=time_range_dict[params['time_range']])

87 params['url'] += time_range_url.format(range=time_range_dict[params['time_range']])

searx/engines/duckduckgo_images.py+2−6

@@ -53,12 +53,8 @@ def request(query, params):

53 safesearch = params['safesearch'] - 1

54

55 region_code = get_region_code(params['language'], lang_list=supported_languages)

56− if region_code:

57− params['url'] = images_url.format(

56+ params['url'] = images_url.format(

58− query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd)

57+ query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd)

59− else:

60− params['url'] = images_url.format(

61− query=urlencode({'q': query}), offset=offset, safesearch=safesearch, vqd=vqd)

62

58

63 return params

59 return params

64

60

searx/engines/faroo.py+1−4

@@ -40,10 +40,7 @@ def request(query, params):

40 offset = (params['pageno'] - 1) * number_of_results + 1

41 categorie = search_category.get(params['category'], 'web')

42

43− if params['language'] == 'all':

44− language = 'en'

45− else:

46− language = params['language'].split('_')[0]

43+ language = params['language'].split('-')[0]

47

44

48 # if language is not supported, put it in english

45 # if language is not supported, put it in english

49 if language != 'en' and\

46 if language != 'en' and\

searx/engines/gigablast.py+3−6

@@ -49,12 +49,9 @@ supported_languages_url = 'https://gigablast.com/search?&rxikd=1'

49def request(query, params):

50 offset = (params['pageno'] - 1) * number_of_results

51

52− if params['language'] == 'all':

53− language = 'xx'

54− else:

55− language = params['language'].replace('-', '_').lower()

52+ language = params['language'].replace('-', '_').lower()

56− if language.split('-')[0] != 'zh':

53+ if language.split('-')[0] != 'zh':

57− language = language.split('-')[0]

54+ language = language.split('-')[0]

58

55

59 if params['safesearch'] >= 1:

56 if params['safesearch'] >= 1:

60 safesearch = 1

57 safesearch = 1

searx/engines/google.py+2−1

@@ -165,7 +165,8 @@ def extract_text_from_dom(result, xpath):

165def request(query, params):

166 offset = (params['pageno'] - 1) * 10

167

168+ # temporary fix until a way of supporting en-US is found

168− if params['language'] == 'all':

169+ if params['language'] == 'en-US':

169 params['language'] = 'en-GB'

170 params['language'] = 'en-GB'

170

171

171 if params['language'][:2] == 'jv':

172 if params['language'][:2] == 'jv':

searx/engines/google_news.py+2−3

@@ -50,9 +50,8 @@ def request(query, params):

50 params['url'] = search_url.format(query=urlencode({'q': query}),

51 search_options=urlencode(search_options))

52

53− if params['language'] != 'all':

54− language_array = params['language'].lower().split('-')

53+ language_array = params['language'].lower().split('-')

55− params['url'] += '&lr=lang_' + language_array[0]

54+ params['url'] += '&lr=lang_' + language_array[0]

56

55

57 return params

56 return params

58

57

searx/engines/mediawiki.py+1−4

@@ -45,10 +45,7 @@ def request(query, params):

45

46 format_strings = list(Formatter().parse(base_url))

47

48− if params['language'] == 'all':

49− language = 'en'

50− else:

51− language = params['language'].split('-')[0]

48+ language = params['language'].split('-')[0]

52

49

53 # format_string [('https://', 'language', '', None), ('.wikipedia.org/', None, None, None)]

50 # format_string [('https://', 'language', '', None), ('.wikipedia.org/', None, None, None)]

54 if any(x[1] == 'language' for x in format_strings):

51 if any(x[1] == 'language' for x in format_strings):

searx/engines/photon.py+3−4

@@ -35,10 +35,9 @@ def request(query, params):

35 search_string.format(query=urlencode({'q': query}),

36 limit=number_of_results)

37

38− if params['language'] != 'all':

39− language = params['language'].split('_')[0]

38+ language = params['language'].split('-')[0]

40− if language in supported_languages:

39+ if language in supported_languages:

41− params['url'] = params['url'] + "&lang=" + language

40+ params['url'] = params['url'] + "&lang=" + language

42

41

43 # using searx User-Agent

42 # using searx User-Agent

44 params['headers']['User-Agent'] = searx_useragent()

43 params['headers']['User-Agent'] = searx_useragent()

searx/engines/qwant.py+11−12

@@ -44,18 +44,17 @@ def request(query, params):

44 query=urlencode({'q': query}),

45 offset=offset)

46

47− # add language tag if specified

47+ # add language tag

48− if params['language'] != 'all':

49− if params['language'] == 'no' or params['language'].startswith('no-'):

48+ if params['language'] == 'no' or params['language'].startswith('no-'):

50− params['language'] = params['language'].replace('no', 'nb', 1)

49+ params['language'] = params['language'].replace('no', 'nb', 1)

51− if params['language'].find('-') < 0:

50+ if params['language'].find('-') < 0:

52− # tries to get a country code from language

51+ # tries to get a country code from language

53− for lang in supported_languages:

52+ for lang in supported_languages:

54− lc = lang.split('-')

53+ lc = lang.split('-')

55− if params['language'] == lc[0]:

54+ if params['language'] == lc[0]:

56− params['language'] = lang

55+ params['language'] = lang

57− break

56+ break

58− params['url'] += '&locale=' + params['language'].replace('-', '_').lower()

57+ params['url'] += '&locale=' + params['language'].replace('-', '_').lower()

59

58

60 return params

59 return params

61

60

searx/engines/startpage.py+2−3

@@ -45,9 +45,8 @@ def request(query, params):

45 params['data'] = {'query': query,

46 'startat': offset}

47

48− # set language if specified

48+ # set language

49− if params['language'] != 'all':

50− params['data']['with_language'] = ('lang_' + params['language'].split('-')[0])

49+ params['data']['with_language'] = ('lang_' + params['language'].split('-')[0])

51

50

52 return params

51 return params

53

52

searx/engines/subtitleseeker.py+1−1

@@ -48,7 +48,7 @@ def response(resp):

48 search_lang = 'Farsi'

49 elif resp.search_params['language'] == 'pt-BR':

50 search_lang = 'Brazilian'

51− elif resp.search_params['language'] != 'all':

51+ else:

52 search_lang = [lc[3]

53 for lc in language_codes

54 if lc[0].split('-')[0] == resp.search_params['language'].split('-')[0]]

searx/engines/swisscows.py+1−4

@@ -35,10 +35,7 @@ regex_img_url_remove_start = re.compile(b'^https?://i\.swisscows\.ch/\?link=')

35

36# do search-request

37def request(query, params):

38− if params['language'] == 'all':

38+ if params['language'].split('-')[0] == 'no':

39− ui_language = 'browser'

40− region = 'browser'

41− elif params['language'].split('-')[0] == 'no':

42 region = 'nb-NO'

39 region = 'nb-NO'

43 else:

40 else:

44 region = params['language']

41 region = params['language']

searx/engines/twitter.py+1−6

@@ -37,12 +37,7 @@ timestamp_xpath = './/span[contains(@class,"_timestamp")]'

37# do search-request

38def request(query, params):

39 params['url'] = search_url + urlencode({'q': query})

40−

41− # set language if specified

42− if params['language'] != 'all':

43− params['cookies']['lang'] = params['language'].split('-')[0]

40+ params['cookies']['lang'] = params['language'].split('-')[0]

44− else:

45− params['cookies']['lang'] = 'en'

46

41

47 return params

42 return params

48

43

searx/engines/wikidata.py−4

@@ -57,8 +57,6 @@ calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'

57

58def request(query, params):

59 language = params['language'].split('-')[0]

60− if language == 'all':

61− language = 'en'

62

60

63 params['url'] = url_search.format(

61 params['url'] = url_search.format(

64 query=urlencode({'label': query, 'language': language}))

62 query=urlencode({'label': query, 'language': language}))

@@ -71,8 +69,6 @@ def response(resp):

71 wikidata_ids = html.xpath(wikidata_ids_xpath)

69 wikidata_ids = html.xpath(wikidata_ids_xpath)

72

70

73 language = resp.search_params['language'].split('-')[0]

71 language = resp.search_params['language'].split('-')[0]

74− if language == 'all':

75− language = 'en'

76

72

77 # TODO: make requests asynchronous to avoid timeout when result_count > 1

73 # TODO: make requests asynchronous to avoid timeout when result_count > 1

78 for wikidata_id in wikidata_ids[:result_count]:

74 for wikidata_id in wikidata_ids[:result_count]:

searx/engines/wikipedia.py+1−1

@@ -31,7 +31,7 @@ supported_languages_url = 'https://meta.wikimedia.org/wiki/List_of_Wikipedias'

31# set language in base_url

32def url_lang(lang):

33 lang = lang.split('-')[0]

34− if lang == 'all' or lang not in supported_languages:

34+ if lang not in supported_languages:

35 language = 'en'

36 else:

37 language = lang

searx/engines/yacy.py+1−3

@@ -51,9 +51,7 @@ def request(query, params):

51 limit=number_of_results,

52 search_type=search_type)

53

54− # add language tag if specified

55− if params['language'] != 'all':

56− params['url'] += '&lr=lang_' + params['language'].split('-')[0]

54+ params['url'] += '&lr=lang_' + params['language'].split('-')[0]

57

55

58 return params

56 return params

59

57

searx/engines/yahoo.py+1−3

@@ -71,9 +71,7 @@ def _get_url(query, offset, language, time_range):

71

72

73def _get_language(params):

74− if params['language'] == 'all':

74+ if params['language'][:2] == 'zh':

75− return 'en'

76− elif params['language'][:2] == 'zh':

77 if params['language'] == 'zh' or params['language'] == 'zh-CH':

75 if params['language'] == 'zh' or params['language'] == 'zh-CH':

78 return 'szh'

76 return 'szh'

79 else:

77 else:

searx/engines/yahoo_news.py+1−4

@@ -38,10 +38,7 @@ suggestion_xpath = '//div[contains(@class,"VerALSOTRY")]//a'

38def request(query, params):

39 offset = (params['pageno'] - 1) * 10 + 1

40

41− if params['language'] == 'all':

42− language = 'en'

43− else:

44− language = params['language'].split('_')[0]

41+ language = params['language'].split('-')[0]

45

42

46 params['url'] = search_url.format(offset=offset,

43 params['url'] = search_url.format(offset=offset,

47 query=urlencode({'p': query}),

44 query=urlencode({'p': query}),

searx/engines/youtube_api.py+1−3

@@ -34,9 +34,7 @@ def request(query, params):

34 params['url'] = search_url.format(query=urlencode({'q': query}),

35 api_key=api_key)

36

37− # add language tag if specified

38− if params['language'] != 'all':

39− params['url'] += '&relevanceLanguage=' + params['language'].split('-')[0]

37+ params['url'] += '&relevanceLanguage=' + params['language'].split('-')[0]

40

38

41 return params

39 return params

42

40

searx/preferences.py−1

@@ -12,7 +12,6 @@ if version[0] == '3':

12

13COOKIE_MAX_AGE = 60 * 60 * 24 * 365 * 5 # 5 years

14LANGUAGE_CODES = [l[0] for l in languages]

15−LANGUAGE_CODES.append('all')

16DISABLED = 0

15DISABLED = 0

17ENABLED = 1

16ENABLED = 1

18DOI_RESOLVERS = list(settings['doi_resolvers'])

17DOI_RESOLVERS = list(settings['doi_resolvers'])

searx/query.py+19−15

@@ -73,11 +73,6 @@ class RawTextQuery(object):

73 if query_part[0] == ':':

74 lang = query_part[1:].lower().replace('_', '-')

75

76− # user may set a valid, yet not selectable language

77− if VALID_LANGUAGE_CODE.match(lang):

78− self.languages.append(lang)

79− parse_next = True

80−

81 # check if any language-code is equal with

76 # check if any language-code is equal with

82 # declared language-codes

77 # declared language-codes

83 for lc in language_codes:

78 for lc in language_codes:

@@ -85,16 +80,25 @@ class RawTextQuery(object):

85

80

86 # if correct language-code is found

81 # if correct language-code is found

87 # set it as new search-language

82 # set it as new search-language

88− if lang == lang_id\

83+ if (lang == lang_id

89− or lang_id.startswith(lang)\

90− or lang == lang_name\

84+ or lang == lang_name

91− or lang == english_name\

85+ or lang == english_name

92− or lang.replace('-', ' ') == country:

86+ or lang.replace('-', ' ') == country)\

87+ and lang not in self.languages:

93− parse_next = True

88+ parse_next = True

89+ lang_parts = lang_id.split('-')

90+ if len(lang_parts) == 2:

94− self.languages.append(lang_id)

91+ self.languages.append(lang_parts[0] + '-' + lang_parts[1].upper())

92+ else:

93+ self.languages.append(lang_id)

95− # to ensure best match (first match is not necessarily the best one)

94+ # to ensure best match (first match is not necessarily the best one)

96− if lang == lang_id:

95+ if lang == lang_id:

97− break

96+ break

97+

98+ # user may set a valid, yet not selectable language

99+ if not self.languages and VALID_LANGUAGE_CODE.match(lang):

100+ self.languages.append(lang)

101+ parse_next = True

98

102

99 # this force a engine or category

103 # this force a engine or category

100 if query_part[0] == '!' or query_part[0] == '?':

104 if query_part[0] == '!' or query_part[0] == '?':

searx/search.py+5−1

@@ -24,7 +24,7 @@ from flask_babel import gettext

24import requests.exceptions

25import searx.poolrequests as requests_lib

26from searx.engines import (

27− categories, engines

27+ categories, engines, settings

28)

29from searx.answerers import ask

30from searx.utils import gen_useragent

@@ -220,6 +220,10 @@ def get_search_query_from_webapp(preferences, form):

220 else:

221 query_lang = preferences.get_value('language')

222

223+ # provides backwards compatibility for requests using old language default

224+ if query_lang == 'all':

225+ query_lang = settings['search']['language']

226+

223 # check language

227 # check language

224 if not VALID_LANGUAGE_CODE.match(query_lang):

228 if not VALID_LANGUAGE_CODE.match(query_lang):

225 raise SearxParameterException('language', query_lang)

229 raise SearxParameterException('language', query_lang)

searx/settings.yml+1−1

@@ -5,7 +5,7 @@ general:

5search:

6 safe_search : 0 # Filter results. 0: None, 1: Moderate, 2: Strict

7 autocomplete : "" # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "startpage", "wikipedia" - leave blank to turn it off by default

8− language : "all"

8+ language : "en-US"

9

10server:

11 port : 8888

searx/settings_robot.yml+1−1

@@ -5,7 +5,7 @@ general:

5search:

6 safe_search : 0

7 autocomplete : ""

8− language: "all"

8+ language: "en-US"

9

10server:

11 port : 11111

searx/templates/courgette/preferences.html−1

@@ -13,7 +13,6 @@

13 <legend>{{ _('Search language') }}</legend>

14 <p>

15 <select name='language'>

16− <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>

17 {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}

16 {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}

18 <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>

17 <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>

19 {% endfor %}

18 {% endfor %}

searx/templates/legacy/preferences.html−1

@@ -14,7 +14,6 @@

14 <legend>{{ _('Search language') }}</legend>

15 <p>

16 <select name='language'>

17− <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>

18 {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}

17 {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}

19 <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>

18 <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>

20 {% endfor %}

19 {% endfor %}

searx/templates/oscar/languages.html+5−6

@@ -3,10 +3,9 @@

3{% else %}

4<select class="time_range custom-select form-control" id='language' name='language'>

5{% endif %}

6− <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>

7− {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}

6+ {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}

8− <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>

7+ <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>

9− {{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}

8+ {{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}

10− </option>

9+ </option>

11− {% endfor %}

10+ {% endfor %}

12</select>

11</select>

searx/templates/oscar/preferences.html+2−2

@@ -187,7 +187,7 @@

187 </td>

188 <th>{{ search_engine.name }}</th>

189 <td class="name">{{ shortcuts[search_engine.name] }}</td>

190− <td>{{ support_toggle(current_language == 'all' or current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages) }}</td>

190+ <td>{{ support_toggle(current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages) }}</td>

191 <td>{{ support_toggle(search_engine.safesearch==True) }}</td>

192 <td>{{ support_toggle(search_engine.time_range_support==True) }}</td>

193 <td class="{{ 'danger' if stats[search_engine.name]['warn_time'] else '' }}">{{ 'N/A' if stats[search_engine.name].time==None else stats[search_engine.name].time }}</td>

@@ -197,7 +197,7 @@

197 <td class="{{ 'danger' if stats[search_engine.name]['warn_time'] else '' }}">{{ 'N/A' if stats[search_engine.name].time==None else stats[search_engine.name].time }}</td>

198 <td>{{ support_toggle(search_engine.time_range_support==True) }}</td>

199 <td>{{ support_toggle(search_engine.safesearch==True) }}</td>

200− <td>{{ support_toggle(current_language == 'all' or current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages) }}</td>

200+ <td>{{ support_toggle(current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages) }}</td>

201 <td>{{ shortcuts[search_engine.name] }}</td>

202 <th>{{ search_engine.name }}</th>

203 <td class="onoff-checkbox">

searx/templates/pix-art/preferences.html−1

@@ -9,7 +9,6 @@

9 <legend>{{ _('Search language') }}</legend>

10 <p>

11 <select name='language'>

12− <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>

13 {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}

12 {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}

14 <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>

13 <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>

15 {% endfor %}

14 {% endfor %}

searx/webapp.py+2−2

@@ -630,8 +630,8 @@ def autocompleter():

630 if len(raw_results) <= 3 and completer:

631 # get language from cookie

632 language = request.preferences.get_value('language')

633− if not language or language == 'all':

633+ if not language:

634− language = 'en'

634+ language = settings['search']['language']

635 else:

636 language = language.split('-')[0]

637 # run autocompletion

tests/unit/engines/test_archlinux.py+1−1

@@ -4,7 +4,7 @@ from searx.engines import archlinux

4from searx.testing import SearxTestCase

5

6domains = {

7− 'all': 'https://wiki.archlinux.org',

7+ 'en': 'https://wiki.archlinux.org',

8 'de': 'https://wiki.archlinux.de',

9 'fr': 'https://wiki.archlinux.fr',

10 'ja': 'https://wiki.archlinuxjp.org',

tests/unit/engines/test_bing.py+1−5

@@ -10,17 +10,13 @@ class TestBingEngine(SearxTestCase):

10 query = u'test_query'

11 dicto = defaultdict(dict)

12 dicto['pageno'] = 0

13− dicto['language'] = 'fr_FR'

13+ dicto['language'] = 'fr-FR'

14 params = bing.request(query.encode('utf-8'), dicto)

15 self.assertTrue('url' in params)

16 self.assertTrue(query in params['url'])

17 self.assertTrue('language%3AFR' in params['url'])

18 self.assertTrue('bing.com' in params['url'])

19

20− dicto['language'] = 'all'

21− params = bing.request(query.encode('utf-8'), dicto)

22− self.assertTrue('language' in params['url'])

23−

24 def test_response(self):

20 def test_response(self):

25 self.assertRaises(AttributeError, bing.response, None)

21 self.assertRaises(AttributeError, bing.response, None)

26 self.assertRaises(AttributeError, bing.response, [])

22 self.assertRaises(AttributeError, bing.response, [])

tests/unit/engines/test_bing_news.py−4

@@ -19,10 +19,6 @@ class TestBingNewsEngine(SearxTestCase):

19 self.assertIn('bing.com', params['url'])

20 self.assertIn('fr', params['url'])

21

22− dicto['language'] = 'all'

23− params = bing_news.request(query, dicto)

24− self.assertIn('en', params['url'])

25−

26 def test_no_url_in_request_year_time_range(self):

22 def test_no_url_in_request_year_time_range(self):

27 dicto = defaultdict(dict)

23 dicto = defaultdict(dict)

28 query = 'test_query'

24 query = 'test_query'

tests/unit/engines/test_dailymotion.py−4

@@ -18,10 +18,6 @@ class TestDailymotionEngine(SearxTestCase):

18 self.assertTrue('dailymotion.com' in params['url'])

19 self.assertTrue('fr' in params['url'])

20

21− dicto['language'] = 'all'

22− params = dailymotion.request(query, dicto)

23− self.assertTrue('en' in params['url'])

24−

25 def test_response(self):

21 def test_response(self):

26 self.assertRaises(AttributeError, dailymotion.response, None)

22 self.assertRaises(AttributeError, dailymotion.response, None)

27 self.assertRaises(AttributeError, dailymotion.response, [])

23 self.assertRaises(AttributeError, dailymotion.response, [])

tests/unit/engines/test_duckduckgo_images.py+1−1

@@ -15,7 +15,7 @@ class TestDuckduckgoImagesEngine(SearxTestCase):

15 dicto['is_test'] = True

16 dicto['pageno'] = 1

17 dicto['safesearch'] = 0

18− dicto['language'] = 'all'

18+ dicto['language'] = 'en-US'

19 params = duckduckgo_images.request(query, dicto)

20 self.assertIn('url', params)

21 self.assertIn(query, params['url'])

tests/unit/engines/test_faroo.py+2−6

@@ -11,7 +11,7 @@ class TestFarooEngine(SearxTestCase):

11 query = 'test_query'

12 dicto = defaultdict(dict)

13 dicto['pageno'] = 1

14− dicto['language'] = 'fr_FR'

14+ dicto['language'] = 'fr-FR'

15 dicto['category'] = 'general'

16 params = faroo.request(query, dicto)

17 self.assertIn('url', params)

@@ -20,11 +20,7 @@ class TestFarooEngine(SearxTestCase):

20 self.assertIn('en', params['url'])

21 self.assertIn('web', params['url'])

22

23− dicto['language'] = 'all'

23+ dicto['language'] = 'de-DE'

24− params = faroo.request(query, dicto)

25− self.assertIn('en', params['url'])

26−

27− dicto['language'] = 'de_DE'

28 params = faroo.request(query, dicto)

24 params = faroo.request(query, dicto)

29 self.assertIn('de', params['url'])

25 self.assertIn('de', params['url'])

30

26

tests/unit/engines/test_gigablast.py+1−6

@@ -11,16 +11,11 @@ class TestGigablastEngine(SearxTestCase):

11 dicto = defaultdict(dict)

12 dicto['pageno'] = 0

13 dicto['safesearch'] = 0

14− dicto['language'] = 'all'

14+ dicto['language'] = 'en-US'

15 params = gigablast.request(query, dicto)

16 self.assertTrue('url' in params)

17 self.assertTrue(query in params['url'])

18 self.assertTrue('gigablast.com' in params['url'])

19− self.assertTrue('xx' in params['url'])

20−

21− dicto['language'] = 'en-US'

22− params = gigablast.request(query, dicto)

23− self.assertTrue('en' in params['url'])

24 self.assertFalse('en-US' in params['url'])

19 self.assertFalse('en-US' in params['url'])

25

20

26 def test_response(self):

21 def test_response(self):

tests/unit/engines/test_google.py+1−1

@@ -26,7 +26,7 @@ class TestGoogleEngine(SearxTestCase):

26 self.assertIn('google.fr', params['url'])

27 self.assertIn('fr', params['headers']['Accept-Language'])

28

29− dicto['language'] = 'all'

29+ dicto['language'] = 'en-US'

30 params = google.request(query, dicto)

31 self.assertIn('google.co', params['url'])

32 self.assertIn('en', params['headers']['Accept-Language'])

tests/unit/engines/test_google_news.py+1−6

@@ -12,18 +12,13 @@ class TestGoogleNewsEngine(SearxTestCase):

12 query = 'test_query'

13 dicto = defaultdict(dict)

14 dicto['pageno'] = 1

15− dicto['language'] = 'fr_FR'

15+ dicto['language'] = 'fr-FR'

16 dicto['time_range'] = 'w'

17 params = google_news.request(query, dicto)

18 self.assertIn('url', params)

19 self.assertIn(query, params['url'])

20 self.assertIn('fr', params['url'])

21

22− dicto['language'] = 'all'

23− params = google_news.request(query, dicto)

24− self.assertIn('url', params)

25− self.assertNotIn('fr', params['url'])

26−

27 def test_response(self):

22 def test_response(self):

28 self.assertRaises(AttributeError, google_news.response, None)

23 self.assertRaises(AttributeError, google_news.response, None)

29 self.assertRaises(AttributeError, google_news.response, [])

24 self.assertRaises(AttributeError, google_news.response, [])

tests/unit/engines/test_mediawiki.py−4

@@ -18,10 +18,6 @@ class TestMediawikiEngine(SearxTestCase):

18 self.assertIn('wikipedia.org', params['url'])

19 self.assertIn('fr', params['url'])

20

21− dicto['language'] = 'all'

22− params = mediawiki.request(query, dicto)

23− self.assertIn('en', params['url'])

24−

25 mediawiki.base_url = "http://test.url/"

21 mediawiki.base_url = "http://test.url/"

26 mediawiki.search_url = mediawiki.base_url +\

22 mediawiki.search_url = mediawiki.base_url +\

27 'w/api.php?action=query'\

23 'w/api.php?action=query'\

tests/unit/engines/test_qwant.py+2−1

@@ -19,10 +19,11 @@ class TestQwantEngine(SearxTestCase):

19 self.assertIn('qwant.com', params['url'])

20 self.assertIn('fr_fr', params['url'])

21

22− dicto['language'] = 'all'

22+ dicto['language'] = 'en-US'

23 qwant.categories = ['news']

24 params = qwant.request(query, dicto)

25 self.assertFalse('fr' in params['url'])

26+ self.assertIn('en_us', params['url'])

26 self.assertIn('news', params['url'])

27 self.assertIn('news', params['url'])

27

28

28 qwant.supported_languages = ['en', 'fr-FR', 'fr-CA']

29 qwant.supported_languages = ['en', 'fr-FR', 'fr-CA']

tests/unit/engines/test_startpage.py−4

@@ -21,10 +21,6 @@ class TestStartpageEngine(SearxTestCase):

21 self.assertIn('with_language', params['data'])

22 self.assertIn('lang_fr', params['data']['with_language'])

23

24− dicto['language'] = 'all'

25− params = startpage.request(query, dicto)

26− self.assertNotIn('with_language', params['data'])

27−

28 def test_response(self):

24 def test_response(self):

29 self.assertRaises(AttributeError, startpage.response, None)

25 self.assertRaises(AttributeError, startpage.response, None)

30 self.assertRaises(AttributeError, startpage.response, [])

26 self.assertRaises(AttributeError, startpage.response, [])

tests/unit/engines/test_subtitleseeker.py−36

@@ -73,42 +73,6 @@ class TestSubtitleseekerEngine(SearxTestCase):

73 results = subtitleseeker.response(response)

74 self.assertEqual(results[0]['url'], 'http://this.is.the.url/Brazilian/')

75

76− html = """

77− <div class="boxRows">

78− <div class="boxRowsInner" style="width:600px;">

79− <img src="http://static.subtitleseeker.com/images/movie.gif"

80− style="width:16px; height:16px;" class="icon">

81− <a href="http://this.is.the.url/"

82− class="blue" title="Title subtitle" >

83− This is the Title

84− </a>

85− </div>

86− <div class="boxRowsInner f12b red" style="width:70px;">

87− 1998

88− </div>

89− <div class="boxRowsInner grey-web f12" style="width:120px;">

90− <img src="http://static.subtitleseeker.com/images/basket_put.png"

91− style="width:16px; height:16px;" class="icon">

92− 1039 Subs

93− </div>

94− <div class="boxRowsInner grey-web f10" style="width:130px;">

95− <img src="http://static.subtitleseeker.com/images/arrow_refresh_small.png"

96− style="width:16px; height:16px;" class="icon">

97− 1 hours ago

98− </div>

99− <div class="clear"></div>

100− </div>

101− """

102− dicto['language'] = 'all'

103− response = mock.Mock(text=html, search_params=dicto)

104− results = subtitleseeker.response(response)

105− self.assertEqual(type(results), list)

106− self.assertEqual(len(results), 1)

107− self.assertEqual(results[0]['title'], 'This is the Title')

108− self.assertEqual(results[0]['url'], 'http://this.is.the.url/')

109− self.assertIn('1998', results[0]['content'])

110− self.assertIn('1039 Subs', results[0]['content'])

111−

112 html = """

76 html = """

113 <div class="boxRows">

77 <div class="boxRows">

114 <div class="boxRowsInner" style="width:600px;">

78 <div class="boxRowsInner" style="width:600px;">

tests/unit/engines/test_swisscows.py−5

@@ -18,11 +18,6 @@ class TestSwisscowsEngine(SearxTestCase):

18 self.assertTrue('uiLanguage=de' in params['url'])

19 self.assertTrue('region=de-DE' in params['url'])

20

21− dicto['language'] = 'all'

22− params = swisscows.request(query, dicto)

23− self.assertTrue('uiLanguage=browser' in params['url'])

24− self.assertTrue('region=browser' in params['url'])

25−

26 dicto['category'] = 'images'

21 dicto['category'] = 'images'

27 params = swisscows.request(query, dicto)

22 params = swisscows.request(query, dicto)

28 self.assertIn('image', params['url'])

23 self.assertIn('image', params['url'])

tests/unit/engines/test_twitter.py−6

@@ -20,12 +20,6 @@ class TestTwitterEngine(SearxTestCase):

20 self.assertIn('lang', params['cookies'])

21 self.assertIn('fr', params['cookies']['lang'])

22

23− dicto['language'] = 'all'

24− params = twitter.request(query, dicto)

25− self.assertIn('cookies', params)

26− self.assertIn('lang', params['cookies'])

27− self.assertIn('en', params['cookies']['lang'])

28−

29 def test_response(self):

23 def test_response(self):

30 self.assertRaises(AttributeError, twitter.response, None)

24 self.assertRaises(AttributeError, twitter.response, None)

31 self.assertRaises(AttributeError, twitter.response, [])

25 self.assertRaises(AttributeError, twitter.response, [])

tests/unit/engines/test_wikidata.py+3−3

@@ -11,14 +11,14 @@ class TestWikidataEngine(SearxTestCase):

11 def test_request(self):

12 query = 'test_query'

13 dicto = defaultdict(dict)

14− dicto['language'] = 'all'

14+ dicto['language'] = 'en-US'

15 params = wikidata.request(query, dicto)

16 self.assertIn('url', params)

17 self.assertIn(query, params['url'])

18 self.assertIn('wikidata.org', params['url'])

19 self.assertIn('en', params['url'])

20

21− dicto['language'] = 'es_ES'

21+ dicto['language'] = 'es-ES'

22 params = wikidata.request(query, dicto)

23 self.assertIn(query, params['url'])

24 self.assertIn('es', params['url'])

@@ -30,7 +30,7 @@ class TestWikidataEngine(SearxTestCase):

30 self.assertRaises(AttributeError, wikidata.response, '')

31 self.assertRaises(AttributeError, wikidata.response, '[]')

32

33− response = mock.Mock(text='<html></html>', search_params={"language": "all"})

33+ response = mock.Mock(text='<html></html>', search_params={"language": "en"})

34 self.assertEqual(wikidata.response(response), [])

35

36 def test_getDetail(self):

tests/unit/engines/test_wikipedia.py−4

@@ -25,10 +25,6 @@ class TestWikipediaEngine(SearxTestCase):

25 self.assertIn('Test_Query', params['url'])

26 self.assertNotIn('test_query', params['url'])

27

28− dicto['language'] = 'all'

29− params = wikipedia.request(query, dicto)

30− self.assertIn('en', params['url'])

31−

32 dicto['language'] = 'xx'

28 dicto['language'] = 'xx'

33 params = wikipedia.request(query, dicto)

29 params = wikipedia.request(query, dicto)

34 self.assertIn('en', params['url'])

30 self.assertIn('en', params['url'])

tests/unit/engines/test_yacy.py−5

@@ -17,11 +17,6 @@ class TestYacyEngine(SearxTestCase):

17 self.assertIn('localhost', params['url'])

18 self.assertIn('fr', params['url'])

19

20− dicto['language'] = 'all'

21− params = yacy.request(query, dicto)

22− self.assertIn('url', params)

23− self.assertNotIn('lr=lang_', params['url'])

24−

25 def test_response(self):

20 def test_response(self):

26 self.assertRaises(AttributeError, yacy.response, None)

21 self.assertRaises(AttributeError, yacy.response, None)

27 self.assertRaises(AttributeError, yacy.response, [])

22 self.assertRaises(AttributeError, yacy.response, [])

tests/unit/engines/test_yahoo.py−7

@@ -39,13 +39,6 @@ class TestYahooEngine(SearxTestCase):

39 self.assertIn('sB', params['cookies'])

40 self.assertIn('fr', params['cookies']['sB'])

41

42− dicto['language'] = 'all'

43− params = yahoo.request(query, dicto)

44− self.assertIn('cookies', params)

45− self.assertIn('sB', params['cookies'])

46− self.assertIn('en', params['cookies']['sB'])

47− self.assertIn('en', params['url'])

48−

49 def test_no_url_in_request_year_time_range(self):

42 def test_no_url_in_request_year_time_range(self):

50 dicto = defaultdict(dict)

43 dicto = defaultdict(dict)

51 query = 'test_query'

44 query = 'test_query'

tests/unit/engines/test_yahoo_news.py−7

@@ -22,13 +22,6 @@ class TestYahooNewsEngine(SearxTestCase):

22 self.assertIn('sB', params['cookies'])

23 self.assertIn('fr', params['cookies']['sB'])

24

25− dicto['language'] = 'all'

26− params = yahoo_news.request(query, dicto)

27− self.assertIn('cookies', params)

28− self.assertIn('sB', params['cookies'])

29− self.assertIn('en', params['cookies']['sB'])

30− self.assertIn('en', params['url'])

31−

32 def test_sanitize_url(self):

25 def test_sanitize_url(self):

33 url = "test.url"

26 url = "test.url"

34 self.assertEqual(url, yahoo_news.sanitize_url(url))

27 self.assertEqual(url, yahoo_news.sanitize_url(url))

tests/unit/test_preferences.py+10−5

@@ -90,25 +90,30 @@ class TestSettings(SearxTestCase):

90

91 # search language settings

92 def test_lang_setting_valid_choice(self):

93− setting = SearchLanguageSetting('all', choices=['all', 'de', 'en'])

93+ setting = SearchLanguageSetting('en', choices=['de', 'en'])

94 setting.parse('de')

95 self.assertEquals(setting.get_value(), 'de')

96

97 def test_lang_setting_invalid_choice(self):

98− setting = SearchLanguageSetting('all', choices=['all', 'de', 'en'])

98+ setting = SearchLanguageSetting('en', choices=['de', 'en'])

99 setting.parse('xx')

100− self.assertEquals(setting.get_value(), 'all')

100+ self.assertEquals(setting.get_value(), 'en')

101

102 def test_lang_setting_old_cookie_choice(self):

103− setting = SearchLanguageSetting('all', choices=['all', 'es', 'es-ES'])

103+ setting = SearchLanguageSetting('en', choices=['en', 'es', 'es-ES'])

104 setting.parse('es_XA')

105 self.assertEquals(setting.get_value(), 'es')

106

107 def test_lang_setting_old_cookie_format(self):

108− setting = SearchLanguageSetting('all', choices=['all', 'es', 'es-ES'])

108+ setting = SearchLanguageSetting('en', choices=['en', 'es', 'es-ES'])

109 setting.parse('es_ES')

110 self.assertEquals(setting.get_value(), 'es-ES')

111

112+ def test_lang_setting_old_default(self):

113+ setting = SearchLanguageSetting('en', choices=['en', 'es', 'de'])

114+ setting.parse('all')

115+ self.assertEquals(setting.get_value(), 'en')

116+

112 # plugins settings

117 # plugins settings

113 def test_plugins_setting_all_default_enabled(self):

118 def test_plugins_setting_all_default_enabled(self):

114 plugin1 = PluginStub('plugin1', True)

119 plugin1 = PluginStub('plugin1', True)

tests/unit/test_query.pynew file+64

@@ -0,0 +1,64 @@

1+from searx.query import RawTextQuery

2+from searx.testing import SearxTestCase

3+

4+

5+class TestQuery(SearxTestCase):

6+

7+ def test_simple_query(self):

8+ query_text = 'the query'

9+ query = RawTextQuery(query_text, [])

10+ query.parse_query()

11+

12+ self.assertEquals(query.getFullQuery(), query_text)

13+ self.assertEquals(len(query.query_parts), 1)

14+ self.assertEquals(len(query.languages), 0)

15+ self.assertFalse(query.specific)

16+

17+ def test_language_code(self):

18+ language = 'es-ES'

19+ query_text = 'the query'

20+ full_query = ':' + language + ' ' + query_text

21+ query = RawTextQuery(full_query, [])

22+ query.parse_query()

23+

24+ self.assertEquals(query.getFullQuery(), full_query)

25+ self.assertEquals(len(query.query_parts), 3)

26+ self.assertEquals(len(query.languages), 1)

27+ self.assertIn(language, query.languages)

28+ self.assertFalse(query.specific)

29+

30+ def test_language_name(self):

31+ language = 'english'

32+ query_text = 'the query'

33+ full_query = ':' + language + ' ' + query_text

34+ query = RawTextQuery(full_query, [])

35+ query.parse_query()

36+

37+ self.assertEquals(query.getFullQuery(), full_query)

38+ self.assertEquals(len(query.query_parts), 3)

39+ self.assertIn('en', query.languages)

40+ self.assertFalse(query.specific)

41+

42+ def test_unlisted_language_code(self):

43+ language = 'all'

44+ query_text = 'the query'

45+ full_query = ':' + language + ' ' + query_text

46+ query = RawTextQuery(full_query, [])

47+ query.parse_query()

48+

49+ self.assertEquals(query.getFullQuery(), full_query)

50+ self.assertEquals(len(query.query_parts), 3)

51+ self.assertIn('all', query.languages)

52+ self.assertFalse(query.specific)

53+

54+ def test_invalid_language_code(self):

55+ language = 'not_a_language'

56+ query_text = 'the query'

57+ full_query = ':' + language + ' ' + query_text

58+ query = RawTextQuery(full_query, [])

59+ query.parse_query()

60+

61+ self.assertEquals(query.getFullQuery(), full_query)

62+ self.assertEquals(len(query.query_parts), 1)

63+ self.assertEquals(len(query.languages), 0)

64+ self.assertFalse(query.specific)