git.iko.soy/searx
on master help sign in
[fix] searx.utils.HTMLTextExtractor: invalid HTML don't raise an Exception
6deb850Alexandre Flament
searx/utils.py+9−2
compare with master
unifiedsplit
@@ -77,6 +77,10 @@ def highlight_content(content, query):
77 return content
77 return content
78
78
79
79
80+class HTMLTextExtractorException(Exception):
81+ pass
82+
83+
80class HTMLTextExtractor(HTMLParser):
84class HTMLTextExtractor(HTMLParser):
81
85
82 def __init__(self):
86 def __init__(self):
@@ -92,7 +96,7 @@ class HTMLTextExtractor(HTMLParser):
92 return
96 return
93
97
94 if tag != self.tags[-1]:
98 if tag != self.tags[-1]:
95− raise Exception("invalid html")
99+ raise HTMLTextExtractorException()
96
100
97 self.tags.pop()
101 self.tags.pop()
98
102
@@ -128,7 +132,10 @@ def html_to_text(html):
128 html = html.replace('\n', ' ')
132 html = html.replace('\n', ' ')
129 html = ' '.join(html.split())
133 html = ' '.join(html.split())
130 s = HTMLTextExtractor()
134 s = HTMLTextExtractor()
135+ try:
131− s.feed(html)
136+ s.feed(html)
137+ except HTMLTextExtractorException:
138+ logger.debug("HTMLTextExtractor: invalid HTML\n%s", html)
132 return s.get_text()
139 return s.get_text()
133
140
134
141
tests/unit/test_utils.py+9
@@ -52,6 +52,10 @@ class TestUtils(SearxTestCase):
52 self.assertIsNotNone(utils.html_to_text(html))
52 self.assertIsNotNone(utils.html_to_text(html))
53 self.assertEqual(utils.html_to_text(html), "Test text")
53 self.assertEqual(utils.html_to_text(html), "Test text")
54
54
55+ def test_html_to_text_invalid(self):
56+ html = '<p><b>Lorem ipsum</i>dolor sit amet</p>'
57+ self.assertEqual(utils.html_to_text(html), "Lorem ipsum")
58+
55 def test_prettify_url(self):
59 def test_prettify_url(self):
56 data = (('https://searx.me/', 'https://searx.me/'),
60 data = (('https://searx.me/', 'https://searx.me/'),
57 ('https://searx.me/ű', 'https://searx.me/ű'),
61 ('https://searx.me/ű', 'https://searx.me/ű'),
@@ -116,6 +120,11 @@ class TestHTMLTextExtractor(SearxTestCase):
116 self.html_text_extractor.handle_entityref(entity)
120 self.html_text_extractor.handle_entityref(entity)
117 self.assertIn(entity, self.html_text_extractor.result)
121 self.assertIn(entity, self.html_text_extractor.result)
118
122
123+ def test_invalid_html(self):
124+ text = '<p><b>Lorem ipsum</i>dolor sit amet</p>'
125+ with self.assertRaises(utils.HTMLTextExtractorException):
126+ self.html_text_extractor.feed(text)
127+
119
128
120class TestUnicodeWriter(SearxTestCase):
129class TestUnicodeWriter(SearxTestCase):
121
130