git.iko.soy/searx
on eecfff2 help sign in
searx/tests/unit/test_utils.py
97 lines 4.1 kB history raw
1# -*- coding: utf-8 -*-
2from searx.testing import SearxTestCase
3from searx import utils
6class TestUtils(SearxTestCase):
8 def test_gen_useragent(self):
9 self.assertIsInstance(utils.gen_useragent(), str)
10 self.assertIsNotNone(utils.gen_useragent())
11 self.assertTrue(utils.gen_useragent().startswith('Mozilla'))
13 def test_searx_useragent(self):
14 self.assertIsInstance(utils.searx_useragent(), str)
15 self.assertIsNotNone(utils.searx_useragent())
16 self.assertTrue(utils.searx_useragent().startswith('searx'))
18 def test_html_to_text(self):
19 html = """
20 <a href="/testlink" class="link_access_account">
21 <span class="toto">
22 <span>
23 <img src="test.jpg" />
24 </span>
25 </span>
26 <span class="titi">
27 Test text
28 </span>
29 </a>
30 """
31 self.assertIsInstance(utils.html_to_text(html), str)
32 self.assertIsNotNone(utils.html_to_text(html))
33 self.assertEqual(utils.html_to_text(html), "Test text")
35 def test_html_to_text_invalid(self):
36 html = '<p><b>Lorem ipsum</i>dolor sit amet</p>'
37 self.assertEqual(utils.html_to_text(html), "Lorem ipsum")
39 def test_match_language(self):
40 self.assertEqual(utils.match_language('es', ['es']), 'es')
41 self.assertEqual(utils.match_language('es', [], fallback='fallback'), 'fallback')
42 self.assertEqual(utils.match_language('ja', ['jp'], {'ja': 'jp'}), 'jp')
44 aliases = {'en-GB': 'en-UK', 'he': 'iw'}
46 # guess country
47 self.assertEqual(utils.match_language('de-DE', ['de']), 'de')
48 self.assertEqual(utils.match_language('de', ['de-DE']), 'de-DE')
49 self.assertEqual(utils.match_language('es-CO', ['es-AR', 'es-ES', 'es-MX']), 'es-ES')
50 self.assertEqual(utils.match_language('es-CO', ['es-MX']), 'es-MX')
51 self.assertEqual(utils.match_language('en-UK', ['en-AU', 'en-GB', 'en-US']), 'en-GB')
52 self.assertEqual(utils.match_language('en-GB', ['en-AU', 'en-UK', 'en-US'], aliases), 'en-UK')
54 # language aliases
55 self.assertEqual(utils.match_language('iw', ['he']), 'he')
56 self.assertEqual(utils.match_language('he', ['iw'], aliases), 'iw')
57 self.assertEqual(utils.match_language('iw-IL', ['he']), 'he')
58 self.assertEqual(utils.match_language('he-IL', ['iw'], aliases), 'iw')
59 self.assertEqual(utils.match_language('iw', ['he-IL']), 'he-IL')
60 self.assertEqual(utils.match_language('he', ['iw-IL'], aliases), 'iw-IL')
61 self.assertEqual(utils.match_language('iw-IL', ['he-IL']), 'he-IL')
62 self.assertEqual(utils.match_language('he-IL', ['iw-IL'], aliases), 'iw-IL')
64 def test_ecma_unscape(self):
65 self.assertEqual(utils.ecma_unescape('text%20with%20space'), 'text with space')
66 self.assertEqual(utils.ecma_unescape('text using %xx: %F3'),
67 u'text using %xx: ó')
68 self.assertEqual(utils.ecma_unescape('text using %u: %u5409, %u4E16%u754c'),
69 u'text using %u: 吉, 世界')
72class TestHTMLTextExtractor(SearxTestCase):
74 def setUp(self):
75 self.html_text_extractor = utils.HTMLTextExtractor()
77 def test__init__(self):
78 self.assertEqual(self.html_text_extractor.result, [])
80 def test_handle_charref(self):
81 self.html_text_extractor.handle_charref('xF')
82 self.assertIn('\x0f', self.html_text_extractor.result)
83 self.html_text_extractor.handle_charref('XF')
84 self.assertIn('\x0f', self.html_text_extractor.result)
86 self.html_text_extractor.handle_charref('97')
87 self.assertIn('a', self.html_text_extractor.result)
89 def test_handle_entityref(self):
90 entity = 'test'
91 self.html_text_extractor.handle_entityref(entity)
92 self.assertIn(entity, self.html_text_extractor.result)
94 def test_invalid_html(self):
95 text = '<p><b>Lorem ipsum</i>dolor sit amet</p>'
96 with self.assertRaises(utils.HTMLTextExtractorException):
97 self.html_text_extractor.feed(text)