on master help sign in
[fix] language support for bing images and videos
commita524dbb823e88482a762d56ac1ed352641f3f0c3 authormarc <a01200356@itesm.mx> dateTue 10 Oct 2017, 16:49 (9 years ago) parent1adc8d6
4 files changed+61−14
searx/engines/bing_images.py+47−9 searx/engines/bing_videos.py+4−1 tests/unit/engines/test_bing_images.py+8−4 tests/unit/engines/test_bing_videos.py+2
searx/engines/bing_images.py+47−9
@@ -18,7 +18,6 @@
1818from lxml import html
1919from json import loads
2020import re
21−from searx.engines.bing import _fetch_supported_languages, supported_languages_url
2221from searx.url_utils import urlencode
2322
2423# engine dependent config
@@ -26,6 +25,8 @@ categories = ['images']
2625paging = True
2726safesearch = True
2827time_range_support = True
28+language_support = True
29+supported_languages_url = 'https://www.bing.com/account/general'
2930
3031# search-url
3132base_url = 'https://www.bing.com/'
@@ -45,23 +46,41 @@ safesearch_types = {2: 'STRICT',
4546_quote_keys_regex = re.compile('({|,)([a-z][a-z0-9]*):(")', re.I | re.U)
4647
4748
49+# get supported region code
50+def get_region_code(lang, lang_list=None):
51+ region = None
52+ if lang in (lang_list or supported_languages):
53+ region = lang
54+ elif lang.startswith('no'):
55+ region = 'nb-NO'
56+ else:
57+ # try to get a supported country code with language
58+ lang = lang.split('-')[0]
59+ for lc in (lang_list or supported_languages):
60+ if lang == lc.split('-')[0]:
61+ region = lc
62+ break
63+ if region:
64+ return region.lower()
65+ else:
66+ return 'en-us'
67+
68+
4869# do search-request
4970def request(query, params):
5071 offset = (params['pageno'] - 1) * 10 + 1
5172
52− # required for cookie
53− if params['language'] == 'all':
54− language = 'en-US'
55− else:
56− language = params['language']
57−
5873 search_path = search_string.format(
5974 query=urlencode({'q': query}),
6075 offset=offset)
6176
77+ language = get_region_code(params['language'])
78+
6279 params['cookies']['SRCHHPGUSR'] = \
63− 'NEWWND=0&NRSLT=-1&SRCHLANG=' + language.split('-')[0] +\
64− '&ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE')
80+ 'ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE')
81+
82+ params['cookies']['_EDGE_S'] = 'mkt=' + language +\
83+ '&ui=' + language + '&F=1'
6584
6685 params['url'] = base_url + search_path
6786 if params['time_range'] in time_range_dict:
@@ -106,3 +125,22 @@ def response(resp):
106125
107126 # return results
108127 return results
128+
129+
130+# get supported languages from their site
131+def _fetch_supported_languages(resp):
132+ supported_languages = []
133+ dom = html.fromstring(resp.text)
134+
135+ regions_xpath = '//div[@id="region-section-content"]' \
136+ + '//ul[@class="b_vList"]/li/a/@href'
137+
138+ regions = dom.xpath(regions_xpath)
139+ for region in regions:
140+ code = re.search('setmkt=[^\&]+', region).group()[7:]
141+ if code == 'nb-NO':
142+ code = 'no-NO'
143+
144+ supported_languages.append(code)
145+
146+ return supported_languages
searx/engines/bing_videos.py+4−1
@@ -12,6 +12,7 @@
1212
1313from json import loads
1414from lxml import html
15+from searx.engines.bing_images import _fetch_supported_languages, supported_languages_url, get_region_code
1516from searx.engines.xpath import extract_text
1617from searx.url_utils import urlencode
1718
@@ -21,6 +22,7 @@ paging = True
2122safesearch = True
2223time_range_support = True
2324number_of_results = 10
25+language_support = True
2426
2527search_url = 'https://www.bing.com/videos/asyncv2?{query}&async=content&'\
2628 'first={offset}&count={number_of_results}&CW=1366&CH=25&FORM=R5VR5'
@@ -45,7 +47,8 @@ def request(query, params):
4547 'ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE')
4648
4749 # language cookie
48− params['cookies']['_EDGE_S'] = 'mkt=' + params['language'].lower() + '&F=1'
50+ region = get_region_code(params['language'], lang_list=supported_languages)
51+ params['cookies']['_EDGE_S'] = 'mkt=' + region + '&F=1'
4952
5053 # query and paging
5154 params['url'] = search_url.format(query=urlencode({'q': query}),
tests/unit/engines/test_bing_images.py+8−4
@@ -8,10 +8,12 @@ from searx.testing import SearxTestCase
88class TestBingImagesEngine(SearxTestCase):
99
1010 def test_request(self):
11+ bing_images.supported_languages = ['fr-FR', 'en-US']
12+
1113 query = 'test_query'
1214 dicto = defaultdict(dict)
1315 dicto['pageno'] = 1
14− dicto['language'] = 'fr_FR'
16+ dicto['language'] = 'fr-FR'
1517 dicto['safesearch'] = 1
1618 dicto['time_range'] = ''
1719 params = bing_images.request(query, dicto)
@@ -19,12 +21,14 @@ class TestBingImagesEngine(SearxTestCase):
1921 self.assertTrue(query in params['url'])
2022 self.assertTrue('bing.com' in params['url'])
2123 self.assertTrue('SRCHHPGUSR' in params['cookies'])
22− self.assertTrue('fr' in params['cookies']['SRCHHPGUSR'])
24+ self.assertTrue('DEMOTE' in params['cookies']['SRCHHPGUSR'])
25+ self.assertTrue('_EDGE_S' in params['cookies'])
26+ self.assertTrue('fr-fr' in params['cookies']['_EDGE_S'])
2327
2428 dicto['language'] = 'all'
2529 params = bing_images.request(query, dicto)
26− self.assertIn('SRCHHPGUSR', params['cookies'])
27− self.assertIn('en', params['cookies']['SRCHHPGUSR'])
30+ self.assertTrue('_EDGE_S' in params['cookies'])
31+ self.assertTrue('en' in params['cookies']['_EDGE_S'])
2832
2933 def test_response(self):
3034 self.assertRaises(AttributeError, bing_images.response, None)
tests/unit/engines/test_bing_videos.py+2
@@ -8,6 +8,8 @@ from searx.testing import SearxTestCase
88class TestBingVideosEngine(SearxTestCase):
99
1010 def test_request(self):
11+ bing_videos.supported_languages = ['fr-FR', 'en-US']
12+
1113 query = 'test_query'
1214 dicto = defaultdict(dict)
1315 dicto['pageno'] = 1