git.iko.soy/searx
on master help sign in
Merge pull request #2190 from dalf/fix-htmltextextractor

[fix] searx.utils.HTMLTextExtractor: invalid HTML don't raise an Exception

commit530fc4bda7ef03b0b80e1587699c8ff29fd79aea authorAlexandre Flament <alex@al-f.net> dateSat 19 Sep 2020, 15:59 (6 years ago) parentae07f4a6deb850
2 files changed+18−2
compare with master
unifiedsplit
searx/utils.py+9−2 tests/unit/test_utils.py+9
searx/utils.py+9−2
@@ -77,6 +77,10 @@ def highlight_content(content, query):
7777 return content
7878
7979
80+class HTMLTextExtractorException(Exception):
81+ pass
82+
83+
8084class HTMLTextExtractor(HTMLParser):
8185
8286 def __init__(self):
@@ -92,7 +96,7 @@ class HTMLTextExtractor(HTMLParser):
9296 return
9397
9498 if tag != self.tags[-1]:
95− raise Exception("invalid html")
99+ raise HTMLTextExtractorException()
96100
97101 self.tags.pop()
98102
@@ -128,7 +132,10 @@ def html_to_text(html):
128132 html = html.replace('\n', ' ')
129133 html = ' '.join(html.split())
130134 s = HTMLTextExtractor()
131− s.feed(html)
135+ try:
136+ s.feed(html)
137+ except HTMLTextExtractorException:
138+ logger.debug("HTMLTextExtractor: invalid HTML\n%s", html)
132139 return s.get_text()
133140
134141
tests/unit/test_utils.py+9
@@ -52,6 +52,10 @@ class TestUtils(SearxTestCase):
5252 self.assertIsNotNone(utils.html_to_text(html))
5353 self.assertEqual(utils.html_to_text(html), "Test text")
5454
55+ def test_html_to_text_invalid(self):
56+ html = '<p><b>Lorem ipsum</i>dolor sit amet</p>'
57+ self.assertEqual(utils.html_to_text(html), "Lorem ipsum")
58+
5559 def test_prettify_url(self):
5660 data = (('https://searx.me/', 'https://searx.me/'),
5761 ('https://searx.me/ű', 'https://searx.me/ű'),
@@ -116,6 +120,11 @@ class TestHTMLTextExtractor(SearxTestCase):
116120 self.html_text_extractor.handle_entityref(entity)
117121 self.assertIn(entity, self.html_text_extractor.result)
118122
123+ def test_invalid_html(self):
124+ text = '<p><b>Lorem ipsum</i>dolor sit amet</p>'
125+ with self.assertRaises(utils.HTMLTextExtractorException):
126+ self.html_text_extractor.feed(text)
127+
119128
120129class TestUnicodeWriter(SearxTestCase):
121130