git.iko.soy/searx
on update_data_update_firefox_version.py help sign in
searx/tests/unit/test_utils.py
228 lines 9.6 kB history raw
1# -*- coding: utf-8 -*-
2import lxml.etree
3from lxml import html
5from searx.testing import SearxTestCase
6from searx.exceptions import SearxXPathSyntaxException, SearxEngineXPathException
7from searx import utils
10class TestUtils(SearxTestCase):
12 def test_gen_useragent(self):
13 self.assertIsInstance(utils.gen_useragent(), str)
14 self.assertIsNotNone(utils.gen_useragent())
15 self.assertTrue(utils.gen_useragent().startswith('Mozilla'))
17 def test_searx_useragent(self):
18 self.assertIsInstance(utils.searx_useragent(), str)
19 self.assertIsNotNone(utils.searx_useragent())
20 self.assertTrue(utils.searx_useragent().startswith('searx'))
22 def test_html_to_text(self):
23 html_str = """
24 <a href="/testlink" class="link_access_account">
25 <style>
26 .toto {
27 color: red;
28 }
29 </style>
30 <span class="toto">
31 <span>
32 <img src="test.jpg" />
33 </span>
34 </span>
35 <span class="titi">
36 Test text
37 </span>
38 <script>value='dummy';</script>
39 </a>
40 """
41 self.assertIsInstance(utils.html_to_text(html_str), str)
42 self.assertIsNotNone(utils.html_to_text(html_str))
43 self.assertEqual(utils.html_to_text(html_str), "Test text")
45 def test_extract_text(self):
46 html_str = """
47 <a href="/testlink" class="link_access_account">
48 <span class="toto">
49 <span>
50 <img src="test.jpg" />
51 </span>
52 </span>
53 <span class="titi">
54 Test text
55 </span>
56 </a>
57 """
58 dom = html.fromstring(html_str)
59 self.assertEqual(utils.extract_text(dom), 'Test text')
60 self.assertEqual(utils.extract_text(dom.xpath('//span')), 'Test text')
61 self.assertEqual(utils.extract_text(dom.xpath('//span/text()')), 'Test text')
62 self.assertEqual(utils.extract_text(dom.xpath('count(//span)')), '3.0')
63 self.assertEqual(utils.extract_text(dom.xpath('boolean(//span)')), 'True')
64 self.assertEqual(utils.extract_text(dom.xpath('//img/@src')), 'test.jpg')
65 self.assertEqual(utils.extract_text(dom.xpath('//unexistingtag')), '')
66 self.assertEqual(utils.extract_text(None, allow_none=True), None)
67 with self.assertRaises(ValueError):
68 utils.extract_text(None)
69 with self.assertRaises(ValueError):
70 utils.extract_text({})
72 def test_extract_url(self):
73 def f(html_str, search_url):
74 return utils.extract_url(html.fromstring(html_str), search_url)
75 self.assertEqual(f('<span id="42">https://example.com</span>', 'http://example.com/'), 'https://example.com/')
76 self.assertEqual(f('https://example.com', 'http://example.com/'), 'https://example.com/')
77 self.assertEqual(f('//example.com', 'http://example.com/'), 'http://example.com/')
78 self.assertEqual(f('//example.com', 'https://example.com/'), 'https://example.com/')
79 self.assertEqual(f('/path?a=1', 'https://example.com'), 'https://example.com/path?a=1')
80 with self.assertRaises(lxml.etree.ParserError):
81 f('', 'https://example.com')
82 with self.assertRaises(Exception):
83 utils.extract_url([], 'https://example.com')
85 def test_html_to_text_invalid(self):
86 html = '<p><b>Lorem ipsum</i>dolor sit amet</p>'
87 self.assertEqual(utils.html_to_text(html), "Lorem ipsum")
89 def test_match_language(self):
90 self.assertEqual(utils.match_language('es', ['es']), 'es')
91 self.assertEqual(utils.match_language('es', [], fallback='fallback'), 'fallback')
92 self.assertEqual(utils.match_language('ja', ['jp'], {'ja': 'jp'}), 'jp')
94 aliases = {'en-GB': 'en-UK', 'he': 'iw'}
96 # guess country
97 self.assertEqual(utils.match_language('de-DE', ['de']), 'de')
98 self.assertEqual(utils.match_language('de', ['de-DE']), 'de-DE')
99 self.assertEqual(utils.match_language('es-CO', ['es-AR', 'es-ES', 'es-MX']), 'es-ES')
100 self.assertEqual(utils.match_language('es-CO', ['es-MX']), 'es-MX')
101 self.assertEqual(utils.match_language('en-UK', ['en-AU', 'en-GB', 'en-US']), 'en-GB')
102 self.assertEqual(utils.match_language('en-GB', ['en-AU', 'en-UK', 'en-US'], aliases), 'en-UK')
104 # language aliases
105 self.assertEqual(utils.match_language('iw', ['he']), 'he')
106 self.assertEqual(utils.match_language('he', ['iw'], aliases), 'iw')
107 self.assertEqual(utils.match_language('iw-IL', ['he']), 'he')
108 self.assertEqual(utils.match_language('he-IL', ['iw'], aliases), 'iw')
109 self.assertEqual(utils.match_language('iw', ['he-IL']), 'he-IL')
110 self.assertEqual(utils.match_language('he', ['iw-IL'], aliases), 'iw-IL')
111 self.assertEqual(utils.match_language('iw-IL', ['he-IL']), 'he-IL')
112 self.assertEqual(utils.match_language('he-IL', ['iw-IL'], aliases), 'iw-IL')
114 def test_ecma_unscape(self):
115 self.assertEqual(utils.ecma_unescape('text%20with%20space'), 'text with space')
116 self.assertEqual(utils.ecma_unescape('text using %xx: %F3'),
117 'text using %xx: ó')
118 self.assertEqual(utils.ecma_unescape('text using %u: %u5409, %u4E16%u754c'),
119 'text using %u: 吉, 世界')
122class TestHTMLTextExtractor(SearxTestCase):
124 def setUp(self):
125 self.html_text_extractor = utils.HTMLTextExtractor()
127 def test__init__(self):
128 self.assertEqual(self.html_text_extractor.result, [])
130 def test_handle_charref(self):
131 self.html_text_extractor.handle_charref('xF')
132 self.assertIn('\x0f', self.html_text_extractor.result)
133 self.html_text_extractor.handle_charref('XF')
134 self.assertIn('\x0f', self.html_text_extractor.result)
136 self.html_text_extractor.handle_charref('97')
137 self.assertIn('a', self.html_text_extractor.result)
139 def test_handle_entityref(self):
140 entity = 'test'
141 self.html_text_extractor.handle_entityref(entity)
142 self.assertIn(entity, self.html_text_extractor.result)
144 def test_invalid_html(self):
145 text = '<p><b>Lorem ipsum</i>dolor sit amet</p>'
146 with self.assertRaises(utils.HTMLTextExtractorException):
147 self.html_text_extractor.feed(text)
150class TestXPathUtils(SearxTestCase):
152 TEST_DOC = """<ul>
153 <li>Text in <b>bold</b> and <i>italic</i> </li>
154 <li>Another <b>text</b> <img src="data:image/gif;base64,R0lGODlhAQABAIAAAAUEBAAAACwAAAAAAQABAAACAkQBADs="></li>
155 </ul>"""
157 def test_get_xpath_cache(self):
158 xp1 = utils.get_xpath('//a')
159 xp2 = utils.get_xpath('//div')
160 xp3 = utils.get_xpath('//a')
162 self.assertEqual(id(xp1), id(xp3))
163 self.assertNotEqual(id(xp1), id(xp2))
165 def test_get_xpath_type(self):
166 utils.get_xpath(lxml.etree.XPath('//a'))
168 with self.assertRaises(TypeError):
169 utils.get_xpath([])
171 def test_get_xpath_invalid(self):
172 invalid_xpath = '//a[0].text'
173 with self.assertRaises(SearxXPathSyntaxException) as context:
174 utils.get_xpath(invalid_xpath)
176 self.assertEqual(context.exception.message, 'Invalid expression')
177 self.assertEqual(context.exception.xpath_str, invalid_xpath)
179 def test_eval_xpath_unregistered_function(self):
180 doc = html.fromstring(TestXPathUtils.TEST_DOC)
182 invalid_function_xpath = 'int(//a)'
183 with self.assertRaises(SearxEngineXPathException) as context:
184 utils.eval_xpath(doc, invalid_function_xpath)
186 self.assertEqual(context.exception.message, 'Unregistered function')
187 self.assertEqual(context.exception.xpath_str, invalid_function_xpath)
189 def test_eval_xpath(self):
190 doc = html.fromstring(TestXPathUtils.TEST_DOC)
192 self.assertEqual(utils.eval_xpath(doc, '//p'), [])
193 self.assertEqual(utils.eval_xpath(doc, '//i/text()'), ['italic'])
194 self.assertEqual(utils.eval_xpath(doc, 'count(//i)'), 1.0)
196 def test_eval_xpath_list(self):
197 doc = html.fromstring(TestXPathUtils.TEST_DOC)
199 # check a not empty list
200 self.assertEqual(utils.eval_xpath_list(doc, '//i/text()'), ['italic'])
202 # check min_len parameter
203 with self.assertRaises(SearxEngineXPathException) as context:
204 utils.eval_xpath_list(doc, '//p', min_len=1)
205 self.assertEqual(context.exception.message, 'len(xpath_str) < 1')
206 self.assertEqual(context.exception.xpath_str, '//p')
208 def test_eval_xpath_getindex(self):
209 doc = html.fromstring(TestXPathUtils.TEST_DOC)
211 # check index 0
212 self.assertEqual(utils.eval_xpath_getindex(doc, '//i/text()', 0), 'italic')
214 # default is 'something'
215 self.assertEqual(utils.eval_xpath_getindex(doc, '//i/text()', 1, default='something'), 'something')
217 # default is None
218 self.assertEqual(utils.eval_xpath_getindex(doc, '//i/text()', 1, default=None), None)
220 # index not found
221 with self.assertRaises(SearxEngineXPathException) as context:
222 utils.eval_xpath_getindex(doc, '//i/text()', 1)
223 self.assertEqual(context.exception.message, 'index 1 not found')
225 # not a list
226 with self.assertRaises(SearxEngineXPathException) as context:
227 utils.eval_xpath_getindex(doc, 'count(//i)', 1)
228 self.assertEqual(context.exception.message, 'the result is not a list')