@@ -1,5 +1,8 @@
11from HTMLParser import HTMLParser
22import htmlentitydefs
3+import csv
4+import codecs
5+import cStringIO
36
47class HTMLTextExtractor(HTMLParser):
58 def __init__(self):
@@ -24,3 +27,33 @@ def html_to_text(html):
2427 s = HTMLTextExtractor()
2528 s.feed(html)
2629 return s.get_text()
30+
31+
32+class UnicodeWriter:
33+ """
34+ A CSV writer which will write rows to CSV file "f",
35+ which is encoded in the given encoding.
36+ """
37+
38+ def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
39+ # Redirect output to a queue
40+ self.queue = cStringIO.StringIO()
41+ self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
42+ self.stream = f
43+ self.encoder = codecs.getincrementalencoder(encoding)()
44+
45+ def writerow(self, row):
46+ self.writer.writerow([(s.encode("utf-8").strip() if type(s) == str or type(s) == unicode else str(s)) for s in row])
47+ # Fetch UTF-8 output from the queue ...
48+ data = self.queue.getvalue()
49+ data = data.decode("utf-8")
50+ # ... and reencode it into the target encoding
51+ data = self.encoder.encode(data)
52+ # write to the target stream
53+ self.stream.write(data)
54+ # empty queue
55+ self.queue.truncate(0)
56+
57+ def writerows(self, rows):
58+ for row in rows:
59+ self.writerow(row)