on master help sign in
[enh] robots.txt added
822bf2fasciimoo
searx/webapp.py+8
@@ -148,6 +148,14 @@ def stats():
148 stats = get_engines_stats()
148 stats = get_engines_stats()
149 return render('stats.html', stats=stats)
149 return render('stats.html', stats=stats)
150
150
151+@app.route('/robots.txt', methods=['GET'])
152+def robots():
153+ return Response("""User-agent: *
154+Allow: /
155+Allow: /about
156+Disallow: /stats
157+""", mimetype='text/plain')
158+
151@app.route('/opensearch.xml', methods=['GET'])
159@app.route('/opensearch.xml', methods=['GET'])
152def opensearch():
160def opensearch():
153 global opensearch_xml
161 global opensearch_xml