@@ -23,6 +23,9 @@ ua_os = ('Windows NT 6.3; WOW64',
2323
2424ua = "Mozilla/5.0 ({os}) Gecko/20100101 Firefox/{version}"
2525
26+blocked_tags = ('script',
27+ 'style')
28+
2629
2730def gen_useragent():
2831 # TODO
@@ -67,11 +70,29 @@ class HTMLTextExtractor(HTMLParser):
6770 def __init__(self):
6871 HTMLParser.__init__(self)
6972 self.result = []
73+ self.tags = []
74+
75+ def handle_starttag(self, tag, attrs):
76+ print tag
77+ self.tags.append(tag)
78+
79+ def handle_endtag(self, tag):
80+ print tag,tag
81+ if tag != self.tags[-1]:
82+ raise Exception("invalid html")
83+ self.tags.pop()
84+
85+ def is_valid_tag(self):
86+ return not self.tags or self.tags[-1] not in blocked_tags
7087
7188 def handle_data(self, d):
89+ if not self.is_valid_tag():
90+ return
7291 self.result.append(d)
7392
7493 def handle_charref(self, number):
94+ if not self.is_valid_tag():
95+ return
7596 if number[0] in (u'x', u'X'):
7697 codepoint = int(number[1:], 16)
7798 else:
@@ -79,6 +100,8 @@ class HTMLTextExtractor(HTMLParser):
79100 self.result.append(unichr(codepoint))
80101
81102 def handle_entityref(self, name):
103+ if not self.is_valid_tag():
104+ return
82105 # codepoint = htmlentitydefs.name2codepoint[name]
83106 # self.result.append(unichr(codepoint))
84107 self.result.append(name)