5568f24 · searx · git.iko.soy

git.iko.soy/searx

[fix] check language aliases when setting search language

commit5568f24d6ca9ae9a7eca27c107982b2689e40105 authorMarc Abonce Seguin <marc-abonce@mailbox.org> dateSun 25 Nov 2018, 23:32 (8 years ago) parent3c95d64

16 files changed+29−11

compare with master

unifiedsplit

searx/engines/bing_images.py+1−1 searx/engines/bing_videos.py+1−1 searx/engines/google.py+1−1 searx/engines/google_news.py+1−1 searx/engines/qwant.py+1−1 searx/engines/swisscows.py+1−1 searx/engines/wikidata.py+1−1 searx/engines/wikipedia.py+1−1 tests/unit/engines/test_bing_images.py+1 tests/unit/engines/test_bing_videos.py+1 tests/unit/engines/test_google.py+8−1 tests/unit/engines/test_google_news.py+1 tests/unit/engines/test_qwant.py+1 tests/unit/engines/test_swisscows.py+1 tests/unit/engines/test_wikidata.py+1 tests/unit/engines/test_wikipedia.py+7−2

searx/engines/bing_images.py+1−1

@@ -55,7 +55,7 @@ def request(query, params):

5555 query=urlencode({'q': query}),

5656 offset=offset)

5757

58− language = match_language(params['language'], supported_languages).lower()

58+ language = match_language(params['language'], supported_languages, language_aliases).lower()

5959

6060 params['cookies']['SRCHHPGUSR'] = \

6161 'ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE')

searx/engines/bing_videos.py+1−1

@@ -48,7 +48,7 @@ def request(query, params):

4848 'ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE')

4949

5050 # language cookie

51− language = match_language(params['language'], supported_languages).lower()

51+ language = match_language(params['language'], supported_languages, language_aliases).lower()

5252 params['cookies']['_EDGE_S'] = 'mkt=' + language + '&F=1'

5353

5454 # query and paging

searx/engines/google.py+1−1

@@ -166,7 +166,7 @@ def extract_text_from_dom(result, xpath):

166166def request(query, params):

167167 offset = (params['pageno'] - 1) * 10

168168

169− language = match_language(params['language'], supported_languages)

169+ language = match_language(params['language'], supported_languages, language_aliases)

170170 language_array = language.split('-')

171171 if params['language'].find('-') > 0:

172172 country = params['language'].split('-')[1]

searx/engines/google_news.py+1−1

@@ -51,7 +51,7 @@ def request(query, params):

5151 params['url'] = search_url.format(query=urlencode({'q': query}),

5252 search_options=urlencode(search_options))

5353

54− language = match_language(params['language'], supported_languages).split('-')[0]

54+ language = match_language(params['language'], supported_languages, language_aliases).split('-')[0]

5555 if language:

5656 params['url'] += '&lr=lang_' + language

5757

searx/engines/qwant.py+1−1

@@ -46,7 +46,7 @@ def request(query, params):

4646 offset=offset)

4747

4848 # add language tag

49− language = match_language(params['language'], supported_languages)

49+ language = match_language(params['language'], supported_languages, language_aliases)

5050 params['url'] += '&locale=' + language.replace('-', '_').lower()

5151

5252 return params

searx/engines/swisscows.py+1−1

@@ -36,7 +36,7 @@ regex_img_url_remove_start = re.compile(b'^https?://i\.swisscows\.ch/\?link=')

3636

3737# do search-request

3838def request(query, params):

39− region = match_language(params['language'], supported_languages)

39+ region = match_language(params['language'], supported_languages, language_aliases)

4040 ui_language = region.split('-')[0]

4141

4242 search_path = search_string.format(

searx/engines/wikidata.py+1−1

@@ -68,7 +68,7 @@ def response(resp):

6868 html = fromstring(resp.text)

6969 search_results = html.xpath(wikidata_ids_xpath)

7070

71− language = match_language(resp.search_params['language'], supported_languages).split('-')[0]

71+ language = match_language(resp.search_params['language'], supported_languages, language_aliases).split('-')[0]

7272

7373 # TODO: make requests asynchronous to avoid timeout when result_count > 1

7474 for search_result in search_results[:result_count]:

searx/engines/wikipedia.py+1−1

@@ -31,7 +31,7 @@ supported_languages_url = 'https://meta.wikimedia.org/wiki/List_of_Wikipedias'

3131

3232# set language in base_url

3333def url_lang(lang):

34− return match_language(lang, supported_languages).split('-')[0]

34+ return match_language(lang, supported_languages, language_aliases).split('-')[0]

3535

3636

3737# do search-request

tests/unit/engines/test_bing_images.py+1

@@ -9,6 +9,7 @@ class TestBingImagesEngine(SearxTestCase):

1010 def test_request(self):

1111 bing_images.supported_languages = ['fr-FR', 'en-US']

12+ bing_images.language_aliases = {}

1213 query = 'test_query'

1314 dicto = defaultdict(dict)

1415 dicto['pageno'] = 1

tests/unit/engines/test_bing_videos.py+1

@@ -9,6 +9,7 @@ class TestBingVideosEngine(SearxTestCase):

1010 def test_request(self):

1111 bing_videos.supported_languages = ['fr-FR', 'en-US']

12+ bing_videos.language_aliases = {}

1213 query = 'test_query'

1314 dicto = defaultdict(dict)

1415 dicto['pageno'] = 1

tests/unit/engines/test_google.py+8−1

@@ -15,7 +15,8 @@ class TestGoogleEngine(SearxTestCase):

1515 return response

1616

1717 def test_request(self):

18− google.supported_languages = ['en', 'fr', 'zh-CN']

18+ google.supported_languages = ['en', 'fr', 'zh-CN', 'iw']

19+ google.language_aliases = {'he': 'iw'}

1920

2021 query = 'test_query'

2122 dicto = defaultdict(dict)

@@ -41,6 +42,12 @@ class TestGoogleEngine(SearxTestCase):

4142 self.assertIn('zh-CN', params['url'])

4243 self.assertIn('zh-CN', params['headers']['Accept-Language'])

4344

45+ dicto['language'] = 'he'

46+ params = google.request(query, dicto)

47+ self.assertIn('google.com', params['url'])

48+ self.assertIn('iw', params['url'])

49+ self.assertIn('iw', params['headers']['Accept-Language'])

50+

4451 def test_response(self):

4552 self.assertRaises(AttributeError, google.response, None)

4653 self.assertRaises(AttributeError, google.response, [])

tests/unit/engines/test_google_news.py+1

@@ -10,6 +10,7 @@ class TestGoogleNewsEngine(SearxTestCase):

1010

1111 def test_request(self):

1212 google_news.supported_languages = ['en-US', 'fr-FR']

13+ google_news.language_aliases = {}

1314 query = 'test_query'

1415 dicto = defaultdict(dict)

1516 dicto['pageno'] = 1

tests/unit/engines/test_qwant.py+1

@@ -8,6 +8,7 @@ class TestQwantEngine(SearxTestCase):

99 def test_request(self):

1010 qwant.supported_languages = ['en-US', 'fr-CA', 'fr-FR']

11+ qwant.language_aliases = {}

1112 query = 'test_query'

1213 dicto = defaultdict(dict)

1314 dicto['pageno'] = 0

tests/unit/engines/test_swisscows.py+1

@@ -8,6 +8,7 @@ class TestSwisscowsEngine(SearxTestCase):

99 def test_request(self):

1010 swisscows.supported_languages = ['de-AT', 'de-DE']

11+ swisscows.language_aliases = {}

1112 query = 'test_query'

1213 dicto = defaultdict(dict)

1314 dicto['pageno'] = 1

tests/unit/engines/test_wikidata.py+1

@@ -27,6 +27,7 @@ class TestWikidataEngine(SearxTestCase):

2727 self.assertRaises(AttributeError, wikidata.response, '[]')

2828

2929 wikidata.supported_languages = ['en', 'es']

30+ wikidata.language_aliases = {}

3031 response = mock.Mock(text='<html></html>', search_params={"language": "en"})

3132 self.assertEqual(wikidata.response(response), [])

3233

tests/unit/engines/test_wikipedia.py+7−2

@@ -8,7 +8,8 @@ from searx.testing import SearxTestCase

88class TestWikipediaEngine(SearxTestCase):

1010 def test_request(self):

11− wikipedia.supported_languages = ['fr', 'en']

11+ wikipedia.supported_languages = ['fr', 'en', 'no']

12+ wikipedia.language_aliases = {'nb': 'no'}

1213

1314 query = 'test_query'

1415 dicto = defaultdict(dict)

@@ -25,9 +26,13 @@ class TestWikipediaEngine(SearxTestCase):

2526 self.assertIn('Test_Query', params['url'])

2627 self.assertNotIn('test_query', params['url'])

2728

29+ dicto['language'] = 'nb'

30+ params = wikipedia.request(query, dicto)

31+ self.assertIn('no.wikipedia.org', params['url'])

32+

2833 dicto['language'] = 'xx'

2934 params = wikipedia.request(query, dicto)

30− self.assertIn('en', params['url'])

35+ self.assertIn('en.wikipedia.org', params['url'])

3136

3237 def test_response(self):

3338 dicto = defaultdict(dict)