on master help sign in
[fix] bing unicode encode error - fixes #408
604f32fAdam Tauber
searx/engines/bing.py+1−1
@@ -52,7 +52,7 @@ def request(query, params):
52def response(resp):
52def response(resp):
53 results = []
53 results = []
54
54
55− dom = html.fromstring(resp.content)
55+ dom = html.fromstring(resp.text)
56
56
57 # parse results
57 # parse results
58 for result in dom.xpath('//div[@class="sa_cc"]'):
58 for result in dom.xpath('//div[@class="sa_cc"]'):
searx/engines/bing_images.py+1−1
@@ -63,7 +63,7 @@ def request(query, params):
63def response(resp):
63def response(resp):
64 results = []
64 results = []
65
65
66− dom = html.fromstring(resp.content)
66+ dom = html.fromstring(resp.text)
67
67
68 # init regex for yaml-parsing
68 # init regex for yaml-parsing
69 p = re.compile('({|,)([a-z]+):(")')
69 p = re.compile('({|,)([a-z]+):(")')
searx/engines/bing_news.py+1−1
@@ -68,7 +68,7 @@ def request(query, params):
68def response(resp):
68def response(resp):
69 results = []
69 results = []
70
70
71− rss = etree.fromstring(resp.content)
71+ rss = etree.fromstring(resp.text)
72
72
73 ns = rss.nsmap
73 ns = rss.nsmap
74
74
searx/tests/engines/test_bing.py+4−4
@@ -29,10 +29,10 @@ class TestBingEngine(SearxTestCase):
29 self.assertRaises(AttributeError, bing.response, '')
29 self.assertRaises(AttributeError, bing.response, '')
30 self.assertRaises(AttributeError, bing.response, '[]')
30 self.assertRaises(AttributeError, bing.response, '[]')
31
31
32− response = mock.Mock(content='<html></html>')
32+ response = mock.Mock(text='<html></html>')
33 self.assertEqual(bing.response(response), [])
33 self.assertEqual(bing.response(response), [])
34
34
35− response = mock.Mock(content='<html></html>')
35+ response = mock.Mock(text='<html></html>')
36 self.assertEqual(bing.response(response), [])
36 self.assertEqual(bing.response(response), [])
37
37
38 html = """
38 html = """
@@ -54,7 +54,7 @@ class TestBingEngine(SearxTestCase):
54 </div>
54 </div>
55 </div>
55 </div>
56 """
56 """
57− response = mock.Mock(content=html)
57+ response = mock.Mock(text=html)
58 results = bing.response(response)
58 results = bing.response(response)
59 self.assertEqual(type(results), list)
59 self.assertEqual(type(results), list)
60 self.assertEqual(len(results), 1)
60 self.assertEqual(len(results), 1)
@@ -81,7 +81,7 @@ class TestBingEngine(SearxTestCase):
81 </div>
81 </div>
82 </li>
82 </li>
83 """
83 """
84− response = mock.Mock(content=html)
84+ response = mock.Mock(text=html)
85 results = bing.response(response)
85 results = bing.response(response)
86 self.assertEqual(type(results), list)
86 self.assertEqual(type(results), list)
87 self.assertEqual(len(results), 1)
87 self.assertEqual(len(results), 1)
searx/tests/engines/test_bing_images.py+5−5
@@ -31,10 +31,10 @@ class TestBingImagesEngine(SearxTestCase):
31 self.assertRaises(AttributeError, bing_images.response, '')
31 self.assertRaises(AttributeError, bing_images.response, '')
32 self.assertRaises(AttributeError, bing_images.response, '[]')
32 self.assertRaises(AttributeError, bing_images.response, '[]')
33
33
34− response = mock.Mock(content='<html></html>')
34+ response = mock.Mock(text='<html></html>')
35 self.assertEqual(bing_images.response(response), [])
35 self.assertEqual(bing_images.response(response), [])
36
36
37− response = mock.Mock(content='<html></html>')
37+ response = mock.Mock(text='<html></html>')
38 self.assertEqual(bing_images.response(response), [])
38 self.assertEqual(bing_images.response(response), [])
39
39
40 html = """
40 html = """
@@ -52,7 +52,7 @@ oh:&quot;238&quot;,tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%2
52 </div>
52 </div>
53 """
53 """
54 html = html.replace('\r\n', '').replace('\n', '').replace('\r', '')
54 html = html.replace('\r\n', '').replace('\n', '').replace('\r', '')
55− response = mock.Mock(content=html)
55+ response = mock.Mock(text=html)
56 results = bing_images.response(response)
56 results = bing_images.response(response)
57 self.assertEqual(type(results), list)
57 self.assertEqual(type(results), list)
58 self.assertEqual(len(results), 1)
58 self.assertEqual(len(results), 1)
@@ -75,7 +75,7 @@ oh:&quot;238&quot;,tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%2
75 style="height:144px;" width="178" height="144"/>
75 style="height:144px;" width="178" height="144"/>
76 </a>
76 </a>
77 """
77 """
78− response = mock.Mock(content=html)
78+ response = mock.Mock(text=html)
79 results = bing_images.response(response)
79 results = bing_images.response(response)
80 self.assertEqual(type(results), list)
80 self.assertEqual(type(results), list)
81 self.assertEqual(len(results), 0)
81 self.assertEqual(len(results), 0)
@@ -263,7 +263,7 @@ oh:&quot;238&quot;,tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%2
263 </div>
263 </div>
264 """
264 """
265 html = html.replace('\r\n', '').replace('\n', '').replace('\r', '')
265 html = html.replace('\r\n', '').replace('\n', '').replace('\r', '')
266− response = mock.Mock(content=html)
266+ response = mock.Mock(text=html)
267 results = bing_images.response(response)
267 results = bing_images.response(response)
268 self.assertEqual(type(results), list)
268 self.assertEqual(type(results), list)
269 self.assertEqual(len(results), 10)
269 self.assertEqual(len(results), 10)
searx/tests/engines/test_bing_news.py+6−6
@@ -28,10 +28,10 @@ class TestBingNewsEngine(SearxTestCase):
28 self.assertRaises(AttributeError, bing_news.response, '')
28 self.assertRaises(AttributeError, bing_news.response, '')
29 self.assertRaises(AttributeError, bing_news.response, '[]')
29 self.assertRaises(AttributeError, bing_news.response, '[]')
30
30
31− response = mock.Mock(content='<html></html>')
31+ response = mock.Mock(text='<html></html>')
32 self.assertEqual(bing_news.response(response), [])
32 self.assertEqual(bing_news.response(response), [])
33
33
34− response = mock.Mock(content='<html></html>')
34+ response = mock.Mock(text='<html></html>')
35 self.assertEqual(bing_news.response(response), [])
35 self.assertEqual(bing_news.response(response), [])
36
36
37 html = """<?xml version="1.0" encoding="utf-8" ?>
37 html = """<?xml version="1.0" encoding="utf-8" ?>
@@ -66,7 +66,7 @@ class TestBingNewsEngine(SearxTestCase):
66 </item>
66 </item>
67 </channel>
67 </channel>
68</rss>""" # noqa
68</rss>""" # noqa
69− response = mock.Mock(content=html)
69+ response = mock.Mock(text=html)
70 results = bing_news.response(response)
70 results = bing_news.response(response)
71 self.assertEqual(type(results), list)
71 self.assertEqual(type(results), list)
72 self.assertEqual(len(results), 2)
72 self.assertEqual(len(results), 2)
@@ -105,7 +105,7 @@ class TestBingNewsEngine(SearxTestCase):
105 </item>
105 </item>
106 </channel>
106 </channel>
107</rss>""" # noqa
107</rss>""" # noqa
108− response = mock.Mock(content=html)
108+ response = mock.Mock(text=html)
109 results = bing_news.response(response)
109 results = bing_news.response(response)
110 self.assertEqual(type(results), list)
110 self.assertEqual(type(results), list)
111 self.assertEqual(len(results), 1)
111 self.assertEqual(len(results), 1)
@@ -128,11 +128,11 @@ class TestBingNewsEngine(SearxTestCase):
128 </channel>
128 </channel>
129</rss>""" # noqa
129</rss>""" # noqa
130
130
131− response = mock.Mock(content=html)
131+ response = mock.Mock(text=html)
132 results = bing_news.response(response)
132 results = bing_news.response(response)
133 self.assertEqual(type(results), list)
133 self.assertEqual(type(results), list)
134 self.assertEqual(len(results), 0)
134 self.assertEqual(len(results), 0)
135
135
136 html = """<?xml version="1.0" encoding="utf-8" ?>gabarge"""
136 html = """<?xml version="1.0" encoding="utf-8" ?>gabarge"""
137− response = mock.Mock(content=html)
137+ response = mock.Mock(text=html)
138 self.assertRaises(lxml.etree.XMLSyntaxError, bing_news.response, response)
138 self.assertRaises(lxml.etree.XMLSyntaxError, bing_news.response, response)