git.iko.soy/searx
on master help sign in
[enh] csv output support
a192438asciimoo
searx/utils.py+33
compare with master
unifiedsplit
@@ -1,5 +1,8 @@
1from HTMLParser import HTMLParser
1from HTMLParser import HTMLParser
2import htmlentitydefs
2import htmlentitydefs
3+import csv
4+import codecs
5+import cStringIO
3
6
4class HTMLTextExtractor(HTMLParser):
7class HTMLTextExtractor(HTMLParser):
5 def __init__(self):
8 def __init__(self):
@@ -24,3 +27,33 @@ def html_to_text(html):
24 s = HTMLTextExtractor()
27 s = HTMLTextExtractor()
25 s.feed(html)
28 s.feed(html)
26 return s.get_text()
29 return s.get_text()
30+
31+
32+class UnicodeWriter:
33+ """
34+ A CSV writer which will write rows to CSV file "f",
35+ which is encoded in the given encoding.
36+ """
37+
38+ def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
39+ # Redirect output to a queue
40+ self.queue = cStringIO.StringIO()
41+ self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
42+ self.stream = f
43+ self.encoder = codecs.getincrementalencoder(encoding)()
44+
45+ def writerow(self, row):
46+ self.writer.writerow([(s.encode("utf-8").strip() if type(s) == str or type(s) == unicode else str(s)) for s in row])
47+ # Fetch UTF-8 output from the queue ...
48+ data = self.queue.getvalue()
49+ data = data.decode("utf-8")
50+ # ... and reencode it into the target encoding
51+ data = self.encoder.encode(data)
52+ # write to the target stream
53+ self.stream.write(data)
54+ # empty queue
55+ self.queue.truncate(0)
56+
57+ def writerows(self, rows):
58+ for row in rows:
59+ self.writerow(row)
searx/webapp.py+13
@@ -26,6 +26,8 @@ from flask import Flask, request, render_template, url_for, Response, make_respo
26from searx.engines import search, categories, engines, get_engines_stats
26from searx.engines import search, categories, engines, get_engines_stats
27from searx import settings
27from searx import settings
28import json
28import json
29+import cStringIO
30+from searx.utils import UnicodeWriter
29
31
30
32
31app = Flask(__name__)
33app = Flask(__name__)
@@ -104,6 +106,17 @@ def index():
104 result['pretty_url'] = result['url']
106 result['pretty_url'] = result['url']
105 if request_data.get('format') == 'json':
107 if request_data.get('format') == 'json':
106 return Response(json.dumps({'query': query, 'results': results}), mimetype='application/json')
108 return Response(json.dumps({'query': query, 'results': results}), mimetype='application/json')
109+ elif request_data.get('format') == 'csv':
110+ csv = UnicodeWriter(cStringIO.StringIO())
111+ if len(results):
112+ keys = results[0].keys()
113+ csv.writerow(keys)
114+ for row in results:
115+ csv.writerow([row[key] for key in keys])
116+ csv.stream.seek(0)
117+ response = Response(csv.stream.read(), mimetype='application/csv', )
118+ response.headers.add('Content-Disposition', 'attachment;Filename=searx_-_{0}.csv'.format(query))
119+ return response
107 template = render('results.html'
120 template = render('results.html'
108 ,results=results
121 ,results=results
109 ,q=request_data['q']
122 ,q=request_data['q']