git.iko.soy/searx
on e30dc2f help sign in
searx/tests/unit/test_utils.py
138 lines 5.9 kB history raw
1# -*- coding: utf-8 -*-
2import lxml.etree
3from lxml import html
5from searx.testing import SearxTestCase
6from searx import utils
9class TestUtils(SearxTestCase):
11 def test_gen_useragent(self):
12 self.assertIsInstance(utils.gen_useragent(), str)
13 self.assertIsNotNone(utils.gen_useragent())
14 self.assertTrue(utils.gen_useragent().startswith('Mozilla'))
16 def test_searx_useragent(self):
17 self.assertIsInstance(utils.searx_useragent(), str)
18 self.assertIsNotNone(utils.searx_useragent())
19 self.assertTrue(utils.searx_useragent().startswith('searx'))
21 def test_html_to_text(self):
22 html_str = """
23 <a href="/testlink" class="link_access_account">
24 <style>
25 .toto {
26 color: red;
27 }
28 </style>
29 <span class="toto">
30 <span>
31 <img src="test.jpg" />
32 </span>
33 </span>
34 <span class="titi">
35 Test text
36 </span>
37 <script>value='dummy';</script>
38 </a>
39 """
40 self.assertIsInstance(utils.html_to_text(html_str), str)
41 self.assertIsNotNone(utils.html_to_text(html_str))
42 self.assertEqual(utils.html_to_text(html_str), "Test text")
44 def test_extract_text(self):
45 html_str = """
46 <a href="/testlink" class="link_access_account">
47 <span class="toto">
48 <span>
49 <img src="test.jpg" />
50 </span>
51 </span>
52 <span class="titi">
53 Test text
54 </span>
55 </a>
56 """
57 dom = html.fromstring(html_str)
58 self.assertEqual(utils.extract_text(dom), 'Test text')
59 self.assertEqual(utils.extract_text(dom.xpath('//span')), 'Test text')
60 self.assertEqual(utils.extract_text(dom.xpath('//img/@src')), 'test.jpg')
61 self.assertEqual(utils.extract_text(dom.xpath('//unexistingtag')), '')
63 def test_extract_url(self):
64 def f(html_str, search_url):
65 return utils.extract_url(html.fromstring(html_str), search_url)
66 self.assertEqual(f('<span id="42">https://example.com</span>', 'http://example.com/'), 'https://example.com/')
67 self.assertEqual(f('https://example.com', 'http://example.com/'), 'https://example.com/')
68 self.assertEqual(f('//example.com', 'http://example.com/'), 'http://example.com/')
69 self.assertEqual(f('//example.com', 'https://example.com/'), 'https://example.com/')
70 self.assertEqual(f('/path?a=1', 'https://example.com'), 'https://example.com/path?a=1')
71 with self.assertRaises(lxml.etree.ParserError):
72 f('', 'https://example.com')
73 with self.assertRaises(Exception):
74 utils.extract_url([], 'https://example.com')
76 def test_html_to_text_invalid(self):
77 html = '<p><b>Lorem ipsum</i>dolor sit amet</p>'
78 self.assertEqual(utils.html_to_text(html), "Lorem ipsum")
80 def test_match_language(self):
81 self.assertEqual(utils.match_language('es', ['es']), 'es')
82 self.assertEqual(utils.match_language('es', [], fallback='fallback'), 'fallback')
83 self.assertEqual(utils.match_language('ja', ['jp'], {'ja': 'jp'}), 'jp')
85 aliases = {'en-GB': 'en-UK', 'he': 'iw'}
87 # guess country
88 self.assertEqual(utils.match_language('de-DE', ['de']), 'de')
89 self.assertEqual(utils.match_language('de', ['de-DE']), 'de-DE')
90 self.assertEqual(utils.match_language('es-CO', ['es-AR', 'es-ES', 'es-MX']), 'es-ES')
91 self.assertEqual(utils.match_language('es-CO', ['es-MX']), 'es-MX')
92 self.assertEqual(utils.match_language('en-UK', ['en-AU', 'en-GB', 'en-US']), 'en-GB')
93 self.assertEqual(utils.match_language('en-GB', ['en-AU', 'en-UK', 'en-US'], aliases), 'en-UK')
95 # language aliases
96 self.assertEqual(utils.match_language('iw', ['he']), 'he')
97 self.assertEqual(utils.match_language('he', ['iw'], aliases), 'iw')
98 self.assertEqual(utils.match_language('iw-IL', ['he']), 'he')
99 self.assertEqual(utils.match_language('he-IL', ['iw'], aliases), 'iw')
100 self.assertEqual(utils.match_language('iw', ['he-IL']), 'he-IL')
101 self.assertEqual(utils.match_language('he', ['iw-IL'], aliases), 'iw-IL')
102 self.assertEqual(utils.match_language('iw-IL', ['he-IL']), 'he-IL')
103 self.assertEqual(utils.match_language('he-IL', ['iw-IL'], aliases), 'iw-IL')
105 def test_ecma_unscape(self):
106 self.assertEqual(utils.ecma_unescape('text%20with%20space'), 'text with space')
107 self.assertEqual(utils.ecma_unescape('text using %xx: %F3'),
108 u'text using %xx: ó')
109 self.assertEqual(utils.ecma_unescape('text using %u: %u5409, %u4E16%u754c'),
110 u'text using %u: 吉, 世界')
113class TestHTMLTextExtractor(SearxTestCase):
115 def setUp(self):
116 self.html_text_extractor = utils.HTMLTextExtractor()
118 def test__init__(self):
119 self.assertEqual(self.html_text_extractor.result, [])
121 def test_handle_charref(self):
122 self.html_text_extractor.handle_charref('xF')
123 self.assertIn('\x0f', self.html_text_extractor.result)
124 self.html_text_extractor.handle_charref('XF')
125 self.assertIn('\x0f', self.html_text_extractor.result)
127 self.html_text_extractor.handle_charref('97')
128 self.assertIn('a', self.html_text_extractor.result)
130 def test_handle_entityref(self):
131 entity = 'test'
132 self.html_text_extractor.handle_entityref(entity)
133 self.assertIn(entity, self.html_text_extractor.result)
135 def test_invalid_html(self):
136 text = '<p><b>Lorem ipsum</i>dolor sit amet</p>'
137 with self.assertRaises(utils.HTMLTextExtractorException):
138 self.html_text_extractor.feed(text)