git.iko.soy/searx
on master help sign in
Merge pull request #1767 from return42/docs
fb6ff5aAdam Tauber
.gitignore+3
compare with master
unifiedsplit
@@ -19,5 +19,8 @@ node_modules/
19
19
20.tx/
20.tx/
21
21
22+build/
23+dist/
22local/
24local/
25+gh-pages/
23searx.egg-info/
26searx.egg-info/
Makefile+23
@@ -1,19 +1,28 @@
1# -*- coding: utf-8; mode: makefile-gmake -*-
1# -*- coding: utf-8; mode: makefile-gmake -*-
2
2
3+export GIT_URL=https://github.com/asciimoo/searx
4+export SEARX_URL=https://searx.me
5+export DOCS_URL=https://asciimoo.github.io/searx
6+
3PYOBJECTS = searx
7PYOBJECTS = searx
8+DOC = docs
4PY_SETUP_EXTRAS ?= \[test\]
9PY_SETUP_EXTRAS ?= \[test\]
5
10
6include utils/makefile.include
11include utils/makefile.include
7include utils/makefile.python
12include utils/makefile.python
13+include utils/makefile.sphinx
8
14
9all: clean install
15all: clean install
10
16
11PHONY += help
17PHONY += help
12help:
18help:
13 @echo ' test - run developer tests'
19 @echo ' test - run developer tests'
20+ @echo ' docs - build documentation'
21+ @echo ' docs-live - autobuild HTML documentation while editing'
14 @echo ' run - run developer instance'
22 @echo ' run - run developer instance'
15 @echo ' install - developer install (./local)'
23 @echo ' install - developer install (./local)'
16 @echo ' uninstall - uninstall (./local)'
24 @echo ' uninstall - uninstall (./local)'
25+ @echo ' gh-pages - build docs & deploy on gh-pages branch'
17 @echo ''
26 @echo ''
18 @$(MAKE) -s -f utils/makefile.include make-help
27 @$(MAKE) -s -f utils/makefile.include make-help
19 @echo ''
28 @echo ''
@@ -40,6 +49,20 @@ run: pyenvinstall
40 ) &
49 ) &
41 $(PY_ENV)/bin/python ./searx/webapp.py
50 $(PY_ENV)/bin/python ./searx/webapp.py
42
51
52+# docs
53+# ----
54+
55+PHONY += docs
56+docs: pyenvinstall sphinx-doc
57+ $(call cmd,sphinx,html,docs,docs)
58+
59+PHONY += docs-live
60+docs-live: pyenvinstall sphinx-live
61+ $(call cmd,sphinx_autobuild,html,docs,docs)
62+
63+$(GH_PAGES)::
64+ @echo "doc available at --> $(DOCS_URL)"
65+
43# test
66# test
44# ----
67# ----
45
68
docs/_themes/searx/static/searx.cssnew file+30
@@ -0,0 +1,30 @@
1+@import url("pocoo.css");
2+
3+a, a.reference, a.footnote-reference {
4+ color: #004b6b;
5+ border-color: #004b6b;
6+}
7+
8+a:hover {
9+ color: #6d4100;
10+ border-color: #6d4100;
11+}
12+
13+p.version-warning {
14+ background-color: #004b6b;
15+}
16+
17+div.sidebar {
18+ background-color: whitesmoke;
19+ border-color: lightsteelblue;
20+ border-radius: 3pt;
21+}
22+
23+p.sidebar-title, .sidebar p {
24+ margin: 6pt;
25+}
26+
27+.sidebar li {
28+ list-style-type: disclosure-closed;
29+}
30+
docs/_themes/searx/theme.confnew file+6
@@ -0,0 +1,6 @@
1+[theme]
2+inherit = pocoo
3+stylesheet = searx.css
4+
5+[options]
6+touch_icon =
docs/admin/api.rstnew file+96
@@ -0,0 +1,96 @@
1+.. _adminapi:
2+
3+==================
4+Administration API
5+==================
6+
7+Get configuration data
8+======================
9+
10+.. code:: http
11+
12+ GET /config HTTP/1.1
13+
14+Sample response
15+---------------
16+
17+.. code:: json
18+
19+ {
20+ "autocomplete": "",
21+ "categories": [
22+ "map",
23+ "it",
24+ "images",
25+ ],
26+ "default_locale": "",
27+ "default_theme": "oscar",
28+ "engines": [
29+ {
30+ "categories": [
31+ "map"
32+ ],
33+ "enabled": true,
34+ "name": "openstreetmap",
35+ "shortcut": "osm"
36+ },
37+ {
38+ "categories": [
39+ "it"
40+ ],
41+ "enabled": true,
42+ "name": "arch linux wiki",
43+ "shortcut": "al"
44+ },
45+ {
46+ "categories": [
47+ "images"
48+ ],
49+ "enabled": true,
50+ "name": "google images",
51+ "shortcut": "goi"
52+ },
53+ {
54+ "categories": [
55+ "it"
56+ ],
57+ "enabled": false,
58+ "name": "bitbucket",
59+ "shortcut": "bb"
60+ },
61+ ],
62+ "instance_name": "searx",
63+ "locales": {
64+ "de": "Deutsch (German)",
65+ "en": "English",
66+ "eo": "Esperanto (Esperanto)",
67+ },
68+ "plugins": [
69+ {
70+ "enabled": true,
71+ "name": "HTTPS rewrite"
72+ },
73+ {
74+ "enabled": false,
75+ "name": "Vim-like hotkeys"
76+ }
77+ ],
78+ "safe_search": 0
79+ }
80+
81+
82+Embed search bar
83+================
84+
85+The search bar can be embedded into websites. Just paste the example into the
86+HTML of the site. URL of the searx instance and values are customizable.
87+
88+.. code:: html
89+
90+ <form method="post" action="https://searx.me/">
91+ <!-- search --> <input type="text" name="q" />
92+ <!-- categories --> <input type="hidden" name="categories" value="general,social media" />
93+ <!-- language --> <input type="hidden" name="lang" value="all" />
94+ <!-- locale --> <input type="hidden" name="locale" value="en" />
95+ <!-- date filter --> <input type="hidden" name="time_range" value="month" />
96+ </form>
docs/admin/filtron.rstnew file+148
@@ -0,0 +1,148 @@
1+==========================
2+How to protect an instance
3+==========================
4+
5+Searx depens on external search services. To avoid the abuse of these services
6+it is advised to limit the number of requests processed by searx.
7+
8+An application firewall, ``filtron`` solves exactly this problem. Information
9+on how to install it can be found at the `project page of filtron
10+<https://github.com/asciimoo/filtron>`__.
11+
12+
13+Sample configuration of filtron
14+===============================
15+
16+An example configuration can be find below. This configuration limits the access
17+of:
18+
19+- scripts or applications (roboagent limit)
20+- webcrawlers (botlimit)
21+- IPs which send too many requests (IP limit)
22+- too many json, csv, etc. requests (rss/json limit)
23+- the same UserAgent of if too many requests (useragent limit)
24+
25+.. code:: json
26+
27+ [{
28+ "name":"search request",
29+ "filters":[
30+ "Param:q",
31+ "Path=^(/|/search)$"
32+ ],
33+ "interval":"<time-interval-in-sec (int)>",
34+ "limit":"<max-request-number-in-interval (int)>",
35+ "subrules":[
36+ {
37+ "name":"roboagent limit",
38+ "interval":"<time-interval-in-sec (int)>",
39+ "limit":"<max-request-number-in-interval (int)>",
40+ "filters":[
41+ "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"
42+ ],
43+ "actions":[
44+ {
45+ "name":"block",
46+ "params":{
47+ "message":"Rate limit exceeded"
48+ }
49+ }
50+ ]
51+ },
52+ {
53+ "name":"botlimit",
54+ "limit":0,
55+ "stop":true,
56+ "filters":[
57+ "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
58+ ],
59+ "actions":[
60+ {
61+ "name":"block",
62+ "params":{
63+ "message":"Rate limit exceeded"
64+ }
65+ }
66+ ]
67+ },
68+ {
69+ "name":"IP limit",
70+ "interval":"<time-interval-in-sec (int)>",
71+ "limit":"<max-request-number-in-interval (int)>",
72+ "stop":true,
73+ "aggregations":[
74+ "Header:X-Forwarded-For"
75+ ],
76+ "actions":[
77+ {
78+ "name":"block",
79+ "params":{
80+ "message":"Rate limit exceeded"
81+ }
82+ }
83+ ]
84+ },
85+ {
86+ "name":"rss/json limit",
87+ "interval":"<time-interval-in-sec (int)>",
88+ "limit":"<max-request-number-in-interval (int)>",
89+ "stop":true,
90+ "filters":[
91+ "Param:format=(csv|json|rss)"
92+ ],
93+ "actions":[
94+ {
95+ "name":"block",
96+ "params":{
97+ "message":"Rate limit exceeded"
98+ }
99+ }
100+ ]
101+ },
102+ {
103+ "name":"useragent limit",
104+ "interval":"<time-interval-in-sec (int)>",
105+ "limit":"<max-request-number-in-interval (int)>",
106+ "aggregations":[
107+ "Header:User-Agent"
108+ ],
109+ "actions":[
110+ {
111+ "name":"block",
112+ "params":{
113+ "message":"Rate limit exceeded"
114+ }
115+ }
116+ ]
117+ }
118+ ]
119+ }]
120+
121+
122+
123+Route request through filtron
124+=============================
125+
126+Filtron can be started using the following command:
127+
128+.. code:: sh
129+
130+ $ filtron -rules rules.json
131+
132+It listens on ``127.0.0.1:4004`` and forwards filtered requests to
133+``127.0.0.1:8888`` by default.
134+
135+Use it along with ``nginx`` with the following example configuration.
136+
137+.. code:: nginx
138+
139+ location / {
140+ proxy_set_header Host $http_host;
141+ proxy_set_header X-Real-IP $remote_addr;
142+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
143+ proxy_set_header X-Scheme $scheme;
144+ proxy_pass http://127.0.0.1:4004/;
145+ }
146+
147+Requests are coming from port 4004 going through filtron and then forwarded to
148+port 8888 where a searx is being run.
docs/admin/index.rstnew file+11
@@ -0,0 +1,11 @@
1+===========================
2+Administrator documentation
3+===========================
4+
5+.. toctree::
6+ :maxdepth: 1
7+
8+ installation
9+ api
10+ filtron
11+ morty
docs/admin/installation.rstnew file+341
@@ -0,0 +1,341 @@
1+.. _installation:
2+
3+============
4+Installation
5+============
6+
7+.. contents::
8+ :depth: 3
9+
10+Basic installation
11+==================
12+
13+Step by step installation for Debian/Ubuntu with virtualenv. For Ubuntu, be sure
14+to have enable universe repository.
15+
16+Install packages:
17+
18+.. code:: sh
19+
20+ $ sudo -H apt-get install \
21+ git build-essential libxslt-dev \
22+ python-dev python-virtualenv python-babel \
23+ zlib1g-dev libffi-dev libssl-dev
24+
25+Install searx:
26+
27+.. code:: sh
28+
29+ cd /usr/local
30+ sudo -H git clone https://github.com/asciimoo/searx.git
31+ sudo -H useradd searx -d /usr/local/searx
32+ sudo -H chown searx:searx -R /usr/local/searx
33+
34+Install dependencies in a virtualenv:
35+
36+.. code:: sh
37+
38+ cd /usr/local/searx
39+ sudo -H -u searx -i
40+
41+.. code:: sh
42+
43+ (searx)$ virtualenv searx-ve
44+ (searx)$ . ./searx-ve/bin/activate
45+ (searx)$ ./manage.sh update_packages
46+
47+Configuration
48+==============
49+
50+.. code:: sh
51+
52+ sed -i -e "s/ultrasecretkey/`openssl rand -hex 16`/g" searx/settings.yml
53+
54+Edit searx/settings.yml if necessary.
55+
56+Check
57+=====
58+
59+Start searx:
60+
61+.. code:: sh
62+
63+ python searx/webapp.py
64+
65+Go to http://localhost:8888
66+
67+If everything works fine, disable the debug option in settings.yml:
68+
69+.. code:: sh
70+
71+ sed -i -e "s/debug : True/debug : False/g" searx/settings.yml
72+
73+At this point searx is not demonized ; uwsgi allows this.
74+
75+You can exit the virtualenv and the searx user bash (enter exit command
76+twice).
77+
78+uwsgi
79+=====
80+
81+Install packages:
82+
83+.. code:: sh
84+
85+ sudo -H apt-get install \
86+ uwsgi uwsgi-plugin-python
87+
88+Create the configuration file ``/etc/uwsgi/apps-available/searx.ini`` with this
89+content:
90+
91+.. code:: ini
92+
93+ [uwsgi]
94+ # Who will run the code
95+ uid = searx
96+ gid = searx
97+
98+ # disable logging for privacy
99+ disable-logging = true
100+
101+ # Number of workers (usually CPU count)
102+ workers = 4
103+
104+ # The right granted on the created socket
105+ chmod-socket = 666
106+
107+ # Plugin to use and interpretor config
108+ single-interpreter = true
109+ master = true
110+ plugin = python
111+ lazy-apps = true
112+ enable-threads = true
113+
114+ # Module to import
115+ module = searx.webapp
116+
117+ # Virtualenv and python path
118+ virtualenv = /usr/local/searx/searx-ve/
119+ pythonpath = /usr/local/searx/
120+ chdir = /usr/local/searx/searx/
121+
122+Activate the uwsgi application and restart:
123+
124+.. code:: sh
125+
126+ cd /etc/uwsgi/apps-enabled
127+ ln -s ../apps-available/searx.ini
128+ /etc/init.d/uwsgi restart
129+
130+Web server
131+==========
132+
133+with nginx
134+----------
135+
136+If nginx is not installed (uwsgi will not work with the package
137+nginx-light):
138+
139+.. code:: sh
140+
141+ sudo -H apt-get install nginx
142+
143+Hosted at /
144+~~~~~~~~~~~
145+
146+Create the configuration file ``/etc/nginx/sites-available/searx`` with this
147+content:
148+
149+.. code:: nginx
150+
151+ server {
152+ listen 80;
153+ server_name searx.example.com;
154+ root /usr/local/searx;
155+
156+ location / {
157+ include uwsgi_params;
158+ uwsgi_pass unix:/run/uwsgi/app/searx/socket;
159+ }
160+ }
161+
162+Create a symlink to sites-enabled:
163+
164+.. code:: sh
165+
166+ sudo -H ln -s /etc/nginx/sites-available/searx /etc/nginx/sites-enabled/searx
167+
168+Restart service:
169+
170+.. code:: sh
171+
172+ sudo -H service nginx restart
173+ sudo -H service uwsgi restart
174+
175+from subdirectory URL (/searx)
176+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
177+
178+Add this configuration in the server config file
179+``/etc/nginx/sites-enabled/default``:
180+
181+.. code:: nginx
182+
183+ location = /searx { rewrite ^ /searx/; }
184+ location /searx {
185+ try_files $uri @searx;
186+ }
187+ location @searx {
188+ uwsgi_param SCRIPT_NAME /searx;
189+ include uwsgi_params;
190+ uwsgi_modifier1 30;
191+ uwsgi_pass unix:/run/uwsgi/app/searx/socket;
192+ }
193+
194+
195+**OR** using reverse proxy (Please, note that reverse proxy advised to be used
196+in case of single-user or low-traffic instances.)
197+
198+.. code:: nginx
199+
200+ location /searx {
201+ proxy_pass http://127.0.0.1:8888;
202+ proxy_set_header Host $host;
203+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
204+ proxy_set_header X-Scheme $scheme;
205+ proxy_set_header X-Script-Name /searx;
206+ proxy_buffering off;
207+ }
208+
209+
210+Enable ``base_url`` in ``searx/settings.yml``
211+
212+.. code:: yaml
213+
214+ base_url : http://your.domain.tld/searx/
215+
216+Restart service:
217+
218+.. code:: sh
219+
220+ sudo -H service nginx restart
221+ sudo -H service uwsgi restart
222+
223+disable logs
224+^^^^^^^^^^^^
225+
226+for better privacy you can disable nginx logs about searx.
227+
228+how to proceed: below ``uwsgi_pass`` in ``/etc/nginx/sites-available/default``
229+add:
230+
231+.. code:: nginx
232+
233+ access_log /dev/null;
234+ error_log /dev/null;
235+
236+Restart service:
237+
238+.. code:: sh
239+
240+ sudo -H service nginx restart
241+
242+with apache
243+-----------
244+
245+Add wsgi mod:
246+
247+.. code:: sh
248+
249+ sudo -H apt-get install libapache2-mod-uwsgi
250+ sudo -H a2enmod uwsgi
251+
252+Add this configuration in the file ``/etc/apache2/apache2.conf``:
253+
254+.. code:: apache
255+
256+ <Location />
257+ Options FollowSymLinks Indexes
258+ SetHandler uwsgi-handler
259+ uWSGISocket /run/uwsgi/app/searx/socket
260+ </Location>
261+
262+Note that if your instance of searx is not at the root, you should change
263+``<Location />`` by the location of your instance, like ``<Location /searx>``.
264+
265+Restart Apache:
266+
267+.. code:: sh
268+
269+ sudo -H /etc/init.d/apache2 restart
270+
271+disable logs
272+~~~~~~~~~~~~
273+
274+For better privacy you can disable Apache logs.
275+
276+.. warning::
277+
278+ You can only disable logs for the whole (virtual) server not for a specific
279+ path.
280+
281+Go back to ``/etc/apache2/apache2.conf`` and above ``<Location />`` add:
282+
283+.. code:: apache
284+
285+ CustomLog /dev/null combined
286+
287+Restart Apache:
288+
289+.. code:: sh
290+
291+ sudo -H /etc/init.d/apache2 restart
292+
293+How to update
294+=============
295+
296+.. code:: sh
297+
298+ cd /usr/local/searx
299+ sudo -H -u searx -i
300+
301+.. code:: sh
302+
303+ (searx)$ . ./searx-ve/bin/activate
304+ (searx)$ git stash
305+ (searx)$ git pull origin master
306+ (searx)$ git stash apply
307+ (searx)$ ./manage.sh update_packages
308+
309+.. code:: sh
310+
311+ sudo -H service uwsgi restart
312+
313+Docker
314+======
315+
316+Make sure you have installed Docker. For instance, you can deploy searx like this:
317+
318+.. code:: sh
319+
320+ docker pull wonderfall/searx
321+ docker run -d --name searx -p $PORT:8888 wonderfall/searx
322+
323+Go to ``http://localhost:$PORT``.
324+
325+See https://hub.docker.com/r/wonderfall/searx/ for more informations. It's also
326+possible to build searx from the embedded Dockerfile.
327+
328+.. code:: sh
329+
330+ git clone https://github.com/asciimoo/searx.git
331+ cd searx
332+ docker build -t whatever/searx .
333+
334+References
335+==========
336+
337+* https://about.okhin.fr/posts/Searx/ with some additions
338+
339+* How to: `Setup searx in a couple of hours with a free SSL certificate
340+ <https://www.reddit.com/r/privacytoolsIO/comments/366kvn/how_to_setup_your_own_privacy_respecting_search/>`__
341+
docs/admin/morty.rstnew file+26
@@ -0,0 +1,26 @@
1+=========================
2+How to setup result proxy
3+=========================
4+
5+.. _morty: https://github.com/asciimoo/morty
6+.. _morty's README: https://github.com/asciimoo/morty
7+
8+By default searx can only act as an image proxy for result images, but it is
9+possible to proxify all the result URLs with an external service, morty_.
10+
11+To use this feature, morty has to be installed and activated in searx's
12+``settings.yml``.
13+
14+Add the following snippet to your ``settings.yml`` and restart searx:
15+
16+.. code:: yaml
17+
18+ result_proxy:
19+ url : http://127.0.0.1:3000/
20+ key : your_morty_proxy_key
21+
22+``url``
23+ Is the address of the running morty service.
24+
25+``key``
26+ Is an optional argument, see `morty's README`_ for more information.
docs/blog/admin.rstnew file+43
@@ -0,0 +1,43 @@
1+=============================================================
2+Searx admin interface
3+=============================================================
4+
5+.. _searx-admin: https://github.com/kvch/searx-admin#searx-admin
6+.. _NLnet Foundation: https://nlnet.nl/
7+
8+ manage your instance from your browser
9+
10+.. sidebar:: Installation
11+
12+ Installation guide can be found in the repository of searx-admin_.
13+
14+One of the biggest advantages of searx is being extremely customizable. But at
15+first it can be daunting to newcomers. A barrier of taking advantage of this
16+feature is our ugly settings file which is sometimes hard to understand and
17+edit.
18+
19+To make self-hosting searx more accessible a new tool is introduced, called
20+``searx-admin``. It is a web application which is capable of managing your
21+instance and manipulating its settings via a web UI. It aims to replace editing
22+of ``settings.yml`` for less experienced administrators or people who prefer
23+graphical admin interfaces.
24+
25+.. figure:: searx-admin-engines.png
26+ :alt: Screenshot of engine list
27+
28+ Configuration page of engines
29+
30+Since ``searx-admin`` acts as a supervisor for searx, we have decided to
31+implement it as a standalone tool instead of part of searx. Another reason for
32+making it a standalone tool is that the codebase and dependencies of searx
33+should not grow because of a fully optional feature, which does not affect
34+existing instances.
35+
36+
37+Acknowledgements
38+================
39+
40+This development was sponsored by `NLnet Foundation`_.
41+
42+| Happy hacking.
43+| kvch // 2017.08.22 21:25
docs/blog/index.rstnew file+10
@@ -0,0 +1,10 @@
1+====
2+Blog
3+====
4+
5+.. toctree::
6+ :maxdepth: 1
7+
8+ python3
9+ admin
10+ intro-offline
docs/blog/intro-offline.rstnew file+77
@@ -0,0 +1,77 @@
1+===============================
2+Preparation for offline engines
3+===============================
4+
5+Offline engines
6+===============
7+
8+To extend the functionality of searx, offline engines are going to be
9+introduced. An offline engine is an engine which does not need Internet
10+connection to perform a search and does not use HTTP to communicate.
11+
12+Offline engines can be configured as online engines, by adding those to the
13+`engines` list of :origin:`settings.yml <searx/settings.yml>`. Thus, searx
14+finds the engine file and imports it.
15+
16+Example skeleton for the new engines:
17+
18+.. code:: python
19+
20+ from subprocess import PIPE, Popen
21+
22+ categories = ['general']
23+ offline = True
24+
25+ def init(settings):
26+ pass
27+
28+ def search(query, params):
29+ process = Popen(['ls', query], stdout=PIPE)
30+ return_code = process.wait()
31+ if return_code != 0:
32+ raise RuntimeError('non-zero return code', return_code)
33+
34+ results = []
35+ line = process.stdout.readline()
36+ while line:
37+ result = parse_line(line)
38+ results.append(results)
39+
40+ line = process.stdout.readline()
41+
42+ return results
43+
44+
45+Development progress
46+====================
47+
48+First, a proposal has been created as a Github issue. Then it was moved to the
49+wiki as a design document. You can read it here: :wiki:`Offline-engines`.
50+
51+In this development step, searx core was prepared to accept and perform offline
52+searches. Offline search requests are scheduled together with regular offline
53+requests.
54+
55+As offline searches can return arbitrary results depending on the engine, the
56+current result templates were insufficient to present such results. Thus, a new
57+template is introduced which is caplable of presenting arbitrary key value pairs
58+as a table. You can check out the pull request for more details see
59+:pull:`1700`.
60+
61+Next steps
62+==========
63+
64+Today, it is possible to create/run an offline engine. However, it is going to be publicly available for everyone who knows the searx instance. So the next step is to introduce token based access for engines. This way administrators are able to limit the access to private engines.
65+
66+Acknowledgement
67+===============
68+
69+This development was sponsored by `Search and Discovery Fund`_ of `NLnet Foundation`_ .
70+
71+.. _Search and Discovery Fund: https://nlnet.nl/discovery
72+.. _NLnet Foundation: https://nlnet.nl/
73+
74+
75+| Happy hacking.
76+| kvch // 2019.10.21 17:03
77+
docs/blog/python3.rstnew file+68
@@ -0,0 +1,68 @@
1+============================
2+Introducing Python 3 support
3+============================
4+
5+.. _Python 2.7 clock: https://pythonclock.org/
6+
7+.. sidebar:: Python 2.7 to 3 upgrade
8+
9+ This chapter exists of historical reasons. Python 2.7 release schedule ends
10+ (`Python 2.7 clock`_) after 11 years Python 3 exists
11+
12+As most operation systems are coming with Python3 installed by default. So it is
13+time for searx to support Python3. But don't worry support of Python2.7 won't be
14+dropped.
15+
16+.. image:: searxpy3.png
17+ :scale: 50 %
18+ :alt: hurray
19+ :align: center
20+
21+
22+How to run searx using Python 3
23+===============================
24+
25+Please make sure that you run at least Python 3.5.
26+
27+To run searx, first a Python3 virtualenv should be created. After entering the
28+virtualenv, dependencies must be installed. Then run searx with python3 instead
29+of the usual python command.
30+
31+.. code:: sh
32+
33+ virtualenv -p python3 venv3
34+ source venv3/bin/activate
35+ pip3 install -r requirements.txt
36+ python3 searx/webapp.py
37+
38+
39+If you want to run searx using Python2.7, you don't have to do anything
40+differently as before.
41+
42+Fun facts
43+=========
44+
45+- 115 files were changed when implementing the support for both Python versions.
46+
47+- All of the dependencies was compatible except for the robotframework used for
48+ browser tests. Thus, these tests were migrated to splinter. So from now on
49+ both versions are being tested on Travis and can be tested locally.
50+
51+If you found bugs
52+=================
53+
54+Please open an issue on `GitHub`_. Make sure that you mention your Python
55+version in your issue, so we can investigate it properly.
56+
57+.. _GitHub: https://github.com/asciimoo/searx/issues
58+
59+Acknowledgment
60+==============
61+
62+This development was sponsored by `NLnet Foundation`_.
63+
64+.. _NLnet Foundation: https://nlnet.nl/
65+
66+
67+| Happy hacking.
68+| kvch // 2017.05.13 22:57
docs/blog/searx-admin-engines.pngnew file

Binary file not shown.

docs/blog/searxpy3.pngnew file

Binary file not shown.

docs/conf.pynew file+83
@@ -0,0 +1,83 @@
1+# -*- coding: utf-8 -*-
2+
3+import sys, os
4+from searx.version import VERSION_STRING
5+from pallets_sphinx_themes import ProjectLink
6+
7+GIT_URL = os.environ.get("GIT_URL", "https://github.com/asciimoo/searx")
8+SEARX_URL = os.environ.get("SEARX_URL", "https://searx.me")
9+DOCS_URL = os.environ.get("DOCS_URL", "https://asciimoo.github.io/searx/")
10+
11+# Project --------------------------------------------------------------
12+
13+project = u'searx'
14+copyright = u'2015-2019, Adam Tauber, Noémi Ványi'
15+author = u'Adam Tauber'
16+release, version = VERSION_STRING, VERSION_STRING
17+
18+# General --------------------------------------------------------------
19+
20+master_doc = "index"
21+source_suffix = '.rst'
22+
23+# usage:: lorem :patch:`f373169` ipsum
24+extlinks = {}
25+
26+# upstream links
27+extlinks['wiki'] = ('https://github.com/asciimoo/searx/wiki/%s', ' ')
28+extlinks['pull'] = ('https://github.com/asciimoo/searx/pull/%s', 'PR ')
29+
30+# links to custom brand
31+extlinks['origin'] = (GIT_URL + '/blob/master/%s', 'git://')
32+extlinks['patch'] = (GIT_URL + '/commit/%s', '#')
33+extlinks['search'] = (SEARX_URL + '/%s', '#')
34+extlinks['docs'] = (DOCS_URL + '/%s', 'docs: ')
35+
36+extensions = [
37+ 'sphinx.ext.extlinks',
38+ 'sphinx.ext.viewcode',
39+ "sphinx.ext.autodoc",
40+ "sphinx.ext.intersphinx",
41+ "pallets_sphinx_themes",
42+ "sphinx_issues", # https://github.com/sloria/sphinx-issues/blob/master/README.rst
43+]
44+
45+intersphinx_mapping = {
46+ "python": ("https://docs.python.org/3/", None),
47+ # "flask": ("https://flask.palletsprojects.com/", None),
48+ # "werkzeug": ("https://werkzeug.palletsprojects.com/", None),
49+ # "jinja": ("https://jinja.palletsprojects.com/", None),
50+}
51+
52+issues_github_path = "asciimoo/searx"
53+
54+# HTML -----------------------------------------------------------------
55+
56+sys.path.append(os.path.abspath('_themes'))
57+
58+html_theme_path = ['_themes']
59+html_theme = "searx"
60+
61+html_theme_options = {"index_sidebar_logo": True}
62+html_context = {
63+ "project_links": [
64+ ProjectLink("Source", GIT_URL),
65+ ProjectLink("Wiki", "https://github.com/asciimoo/searx/wiki"),
66+ ProjectLink("Public instances", "https://github.com/asciimoo/searx/wiki/Searx-instances"),
67+ ProjectLink("Twitter", "https://twitter.com/Searx_engine"),
68+ ]
69+}
70+html_sidebars = {
71+ "**": ["project.html", "relations.html", "searchbox.html"],
72+}
73+singlehtml_sidebars = {"index": ["project.html", "localtoc.html"]}
74+html_static_path = ["static"]
75+html_logo = "static/img/searx_logo_small.png"
76+html_title = "Searx Documentation ({})".format("Searx-{}.tex".format(VERSION_STRING))
77+html_show_sourcelink = False
78+
79+# LaTeX ----------------------------------------------------------------
80+
81+latex_documents = [
82+ (master_doc, "searx-{}.tex".format(VERSION_STRING), html_title, author, "manual")
83+]
docs/dev/contribution_guide.rstnew file+147
@@ -0,0 +1,147 @@
1+=================
2+How to contribute
3+=================
4+
5+Prime directives: Privacy, Hackability
6+======================================
7+
8+Searx has two prime directives, **privacy-by-design and hackability** . The
9+hackability comes in three levels:
10+
11+- support of search engines
12+- plugins to alter search behaviour
13+- hacking searx itself
14+
15+Note the lack of "world domination" among the directives. Searx has no
16+intention of wide mass-adoption, rounded corners, etc. The prime directive
17+"privacy" deserves a separate chapter, as it's quite uncommon unfortunately.
18+
19+Privacy-by-design
20+-----------------
21+
22+Searx was born out of the need for a **privacy-respecting** search tool which
23+can be extended easily to maximize both, its search and its privacy protecting
24+capabilities.
25+
26+A few widely used features work differently or turned off by default or not
27+implemented at all **as a consequence of privacy-by-design**.
28+
29+If a feature reduces the privacy preserving aspects of searx, it should be
30+switched off by default or should not implemented at all. There are plenty of
31+search engines already providing such features. If a feature reduces the
32+protection of searx, users must be informed about the effect of choosing to
33+enable it. Features that protect privacy but differ from the expectations of
34+the user should also be explained.
35+
36+Also, if you think that something works weird with searx, it's might be because
37+of the tool you use is designed in a way to interfere with the privacy respect.
38+Submitting a bugreport to the vendor of the tool that misbehaves might be a good
39+feedback to reconsider the disrespect to its customers (e.g. ``GET`` vs ``POST``
40+requests in various browsers).
41+
42+Remember the other prime directive of searx is to be hackable, so if the above
43+privacy concerns do not fancy you, simply fork it.
44+
45+ *Happy hacking.*
46+
47+Code
48+====
49+
50+.. _PEP8: https://www.python.org/dev/peps/pep-0008/
51+
52+
53+In order to submit a patch, please follow the steps below:
54+
55+- Follow coding conventions.
56+
57+ - PEP8_ standards apply, except the convention of line length
58+ - Maximum line length is 120 characters
59+
60+- Check if your code breaks existing tests. If so, update the tests or fix your
61+ code.
62+
63+- If your code can be unit-tested, add unit tests.
64+
65+- Add yourself to the :origin:`AUTHORS.rst` file.
66+
67+- Create a pull request.
68+
69+For more help on getting started with searx development, see :ref:`devquickstart`.
70+
71+
72+Translation
73+===========
74+
75+Translation currently takes place on :ref:`transifex <translation>`.
76+
77+.. caution::
78+
79+ Please, do not update translation files in the repo.
80+
81+
82+Documentation
83+=============
84+
85+.. _Sphinx: http://www.sphinx-doc.org
86+.. _reST: http://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html
87+
88+.. sidebar:: The reST sources
89+
90+ has been moved from ``gh-branch`` into ``master`` (:origin:`docs`).
91+
92+The documentation is built using Sphinx_. So in order to be able to generate
93+the required files, you have to install it on your system. Much easier, use
94+Makefile our targets.
95+
96+Here is an example which makes a complete rebuild:
97+
98+.. code:: sh
99+
100+ $ make docs-clean docs
101+ ...
102+ The HTML pages are in dist/docs.
103+
104+
105+live build
106+----------
107+
108+.. sidebar:: docs-clean
109+
110+ It is recommended to assert a complete rebuild before deploying (use
111+ ``docs-clean``).
112+
113+Live build is like WYSIWYG, If you want to edit the documentation, its
114+recommended to use. The Makefile target ``docs-live`` builds the docs, opens URL
115+in your favorite browser and rebuilds every time a reST file has been changed.
116+
117+.. code:: sh
118+
119+ $ make docs-live
120+ ...
121+ The HTML pages are in dist/docs.
122+ ... Serving on http://0.0.0.0:8080
123+ ... Start watching changes
124+
125+
126+
127+deploy on github.io
128+-------------------
129+
130+To deploy documentation at :docs:`github.io <.>` use Makefile target
131+``gh-pages``, which will builds the documentation, clones searx into a sub
132+folder ``gh-pages``, cleans it, copies the doc build into and runs all the
133+needed git add, commit and push:
134+
135+.. code:: sh
136+
137+ $ make docs-clean gh-pages
138+ ...
139+ SPHINX docs --> file://<...>/dist/docs
140+ The HTML pages are in dist/docs.
141+ ...
142+ Cloning into 'gh-pages' ...
143+ ...
144+ cd gh-pages; git checkout gh-pages >/dev/null
145+ Switched to a new branch 'gh-pages'
146+ ...
147+ doc available at --> https://asciimoo.github.io/searx
docs/dev/engine_overview.rstnew file+265
@@ -0,0 +1,265 @@
1+
2+.. _engines-dev:
3+
4+===============
5+Engine overview
6+===============
7+
8+.. _metasearch-engine: https://en.wikipedia.org/wiki/Metasearch_engine
9+
10+searx is a metasearch-engine_, so it uses different search engines to provide
11+better results.
12+
13+Because there is no general search API which could be used for every search
14+engine, an adapter has to be built between searx and the external search
15+engines. Adapters are stored under the folder :origin:`searx/engines`.
16+
17+.. contents::
18+ :depth: 3
19+ :backlinks: entry
20+
21+general engine configuration
22+============================
23+
24+It is required to tell searx the type of results the engine provides. The
25+arguments can be set in the engine file or in the settings file
26+(normally ``settings.yml``). The arguments in the settings file override
27+the ones in the engine file.
28+
29+It does not matter if an option is stored in the engine file or in the
30+settings. However, the standard way is the following:
31+
32+
33+engine file
34+-----------
35+
36+======================= =========== ===========================================
37+argument type information
38+======================= =========== ===========================================
39+categories list pages, in which the engine is working
40+paging boolean support multible pages
41+language_support boolean support language choosing
42+time_range_support boolean support search time range
43+offline boolean engine runs offline
44+======================= =========== ===========================================
45+
46+
47+settings.yml
48+------------
49+
50+======================= =========== ===========================================
51+argument type information
52+======================= =========== ===========================================
53+name string name of search-engine
54+engine string name of searx-engine
55+ (filename without ``.py``)
56+shortcut string shortcut of search-engine
57+timeout string specific timeout for search-engine
58+======================= =========== ===========================================
59+
60+
61+overrides
62+---------
63+
64+A few of the options have default values in the engine, but are often
65+overwritten by the settings. If ``None`` is assigned to an option in the engine
66+file, it has to be redefined in the settings, otherwise searx will not start
67+with that engine.
68+
69+The naming of overrides is arbitrary. But the recommended overrides are the
70+following:
71+
72+======================= =========== ===========================================
73+argument type information
74+======================= =========== ===========================================
75+base_url string base-url, can be overwritten to use same
76+ engine on other URL
77+number_of_results int maximum number of results per request
78+language string ISO code of language and country like en_US
79+api_key string api-key if required by engine
80+======================= =========== ===========================================
81+
82+example code
83+------------
84+
85+.. code:: python
86+
87+ # engine dependent config
88+ categories = ['general']
89+ paging = True
90+ language_support = True
91+
92+
93+making a request
94+================
95+
96+To perform a search an URL have to be specified. In addition to specifying an
97+URL, arguments can be passed to the query.
98+
99+passed arguments
100+----------------
101+
102+These arguments can be used to construct the search query. Furthermore,
103+parameters with default value can be redefined for special purposes.
104+
105+====================== ============ ========================================================================
106+argument type default-value, information
107+====================== ============ ========================================================================
108+url string ``''``
109+method string ``'GET'``
110+headers set ``{}``
111+data set ``{}``
112+cookies set ``{}``
113+verify boolean ``True``
114+headers.User-Agent string a random User-Agent
115+category string current category, like ``'general'``
116+started datetime current date-time
117+pageno int current pagenumber
118+language string specific language code like ``'en_US'``, or ``'all'`` if unspecified
119+====================== ============ ========================================================================
120+
121+parsed arguments
122+----------------
123+
124+The function ``def request(query, params):`` always returns the ``params``
125+variable. Inside searx, the following paramters can be used to specify a search
126+request:
127+
128+============ =========== =========================================================
129+argument type information
130+============ =========== =========================================================
131+url string requested url
132+method string HTTP request method
133+headers set HTTP header information
134+data set HTTP data information (parsed if ``method != 'GET'``)
135+cookies set HTTP cookies
136+verify boolean Performing SSL-Validity check
137+============ =========== =========================================================
138+
139+
140+example code
141+------------
142+
143+.. code:: python
144+
145+ # search-url
146+ base_url = 'https://example.com/'
147+ search_string = 'search?{query}&page={page}'
148+
149+ # do search-request
150+ def request(query, params):
151+ search_path = search_string.format(
152+ query=urlencode({'q': query}),
153+ page=params['pageno'])
154+
155+ params['url'] = base_url + search_path
156+
157+ return params
158+
159+
160+returned results
161+================
162+
163+Searx is able to return results of different media-types. Currently the
164+following media-types are supported:
165+
166+- default_
167+- images_
168+- videos_
169+- torrent_
170+- map_
171+
172+To set another media-type as default, the parameter ``template`` must be set to
173+the desired type.
174+
175+default
176+-------
177+
178+========================= =====================================================
179+result-parameter information
180+========================= =====================================================
181+url string, url of the result
182+title string, title of the result
183+content string, general result-text
184+publishedDate :py:class:`datetime.datetime`, time of publish
185+========================= =====================================================
186+
187+images
188+------
189+
190+To use this template, the parameter:
191+
192+========================= =====================================================
193+result-parameter information
194+========================= =====================================================
195+template is set to ``images.html``
196+url string, url to the result site
197+title string, title of the result *(partly implemented)*
198+content *(partly implemented)*
199+publishedDate :py:class:`datetime.datetime`,
200+ time of publish *(partly implemented)*
201+img\_src string, url to the result image
202+thumbnail\_src string, url to a small-preview image
203+========================= =====================================================
204+
205+videos
206+------
207+
208+========================= =====================================================
209+result-parameter information
210+========================= =====================================================
211+template is set to ``videos.html``
212+url string, url of the result
213+title string, title of the result
214+content *(not implemented yet)*
215+publishedDate :py:class:`datetime.datetime`, time of publish
216+thumbnail string, url to a small-preview image
217+========================= =====================================================
218+
219+torrent
220+-------
221+
222+.. _magnetlink: https://en.wikipedia.org/wiki/Magnet_URI_scheme
223+
224+========================= =====================================================
225+result-parameter information
226+========================= =====================================================
227+template is set to ``torrent.html``
228+url string, url of the result
229+title string, title of the result
230+content string, general result-text
231+publishedDate :py:class:`datetime.datetime`,
232+ time of publish *(not implemented yet)*
233+seed int, number of seeder
234+leech int, number of leecher
235+filesize int, size of file in bytes
236+files int, number of files
237+magnetlink string, magnetlink_ of the result
238+torrentfile string, torrentfile of the result
239+========================= =====================================================
240+
241+
242+map
243+---
244+
245+========================= =====================================================
246+result-parameter information
247+========================= =====================================================
248+url string, url of the result
249+title string, title of the result
250+content string, general result-text
251+publishedDate :py:class:`datetime.datetime`, time of publish
252+latitude latitude of result (in decimal format)
253+longitude longitude of result (in decimal format)
254+boundingbox boundingbox of result (array of 4. values
255+ ``[lat-min, lat-max, lon-min, lon-max]``)
256+geojson geojson of result (http://geojson.org)
257+osm.type type of osm-object (if OSM-Result)
258+osm.id id of osm-object (if OSM-Result)
259+address.name name of object
260+address.road street name of object
261+address.house_number house number of object
262+address.locality city, place of object
263+address.postcode postcode of object
264+address.country country of object
265+========================= =====================================================
docs/dev/index.rstnew file+13
@@ -0,0 +1,13 @@
1+=======================
2+Developer documentation
3+=======================
4+
5+.. toctree::
6+ :maxdepth: 1
7+
8+ quickstart
9+ contribution_guide
10+ engine_overview
11+ search_api
12+ plugins
13+ translation
docs/dev/plugins.rstnew file+48
@@ -0,0 +1,48 @@
1+=======
2+Plugins
3+=======
4+
5+Plugins can extend or replace functionality of various components of searx.
6+
7+Example plugin
8+==============
9+
10+.. code:: python
11+
12+ name = 'Example plugin'
13+ description = 'This plugin extends the suggestions with the word "example"'
14+ default_on = False # disabled by default
15+
16+ js_dependencies = tuple() # optional, list of static js files
17+ css_dependencies = tuple() # optional, list of static css files
18+
19+
20+ # attach callback to the post search hook
21+ # request: flask request object
22+ # ctx: the whole local context of the post search hook
23+ def post_search(request, ctx):
24+ ctx['search'].suggestions.add('example')
25+ return True
26+
27+Plugin entry points
28+===================
29+
30+Entry points (hooks) define when a plugin runs. Right now only three hooks are
31+implemented. So feel free to implement a hook if it fits the behaviour of your
32+plugin.
33+
34+Pre search hook
35+---------------
36+
37+Runs BEFORE the search request. Function to implement: ``pre_search``
38+
39+Post search hook
40+----------------
41+
42+Runs AFTER the search request. Function to implement: ``post_search``
43+
44+Result hook
45+-----------
46+
47+Runs when a new result is added to the result list. Function to implement:
48+``on_result``
docs/dev/quickstart.rstnew file+110
@@ -0,0 +1,110 @@
1+.. _devquickstart:
2+
3+======================
4+Development Quickstart
5+======================
6+
7+This quickstart guide gets your environment set up with searx. Furthermore, it
8+gives a short introduction to the ``manage.sh`` script.
9+
10+How to setup your development environment
11+=========================================
12+
13+First, clone the source code of searx to the desired folder. In this case the
14+source is cloned to ``~/myprojects/searx``. Then create and activate the
15+searx-ve virtualenv and install the required packages using manage.sh.
16+
17+.. code:: sh
18+
19+ cd ~/myprojects
20+ git clone https://github.com/asciimoo/searx.git
21+ cd searx
22+ virtualenv searx-ve
23+ . ./searx-ve/bin/activate
24+ ./manage.sh update_dev_packages
25+
26+
27+How to run tests
28+================
29+
30+Tests can be run using the ``manage.sh`` script. Following tests and checks are
31+available:
32+
33+- Unit tests
34+- Selenium tests
35+- PEP8 validation
36+- Unit test coverage check
37+
38+For example unit tests are run with the command below:
39+
40+.. code:: sh
41+
42+ ./manage.sh unit_tests
43+
44+For further test options, please consult the help of the ``manage.sh`` script.
45+
46+
47+How to compile styles and javascript
48+====================================
49+
50+.. _less: http://lesscss.org/
51+.. _NodeJS: https://nodejs.org
52+
53+How to build styles
54+-------------------
55+
56+Less_ is required to build the styles of searx. Less_ can be installed using
57+either NodeJS_ or Apt.
58+
59+.. code:: sh
60+
61+ sudo -H apt-get install nodejs
62+ sudo -H npm install -g less
63+
64+OR
65+
66+.. code:: sh
67+
68+ sudo -H apt-get install node-less
69+
70+After satisfying the requirements styles can be build using ``manage.sh``
71+
72+.. code:: sh
73+
74+ ./manage.sh styles
75+
76+
77+How to build the source of the oscar theme
78+==========================================
79+
80+.. _grunt: https://gruntjs.com/
81+
82+Grunt_ must be installed in order to build the javascript sources. It depends on
83+NodeJS, so first Node has to be installed.
84+
85+.. code:: sh
86+
87+ sudo -H apt-get install nodejs
88+ sudo -H npm install -g grunt-cli
89+
90+After installing grunt, the files can be built using the following command:
91+
92+.. code:: sh
93+
94+ ./manage.sh grunt_build
95+
96+
97+Tips for debugging/development
98+==============================
99+
100+Turn on debug logging
101+ Whether you are working on a new engine or trying to eliminate a bug, it is
102+ always a good idea to turn on debug logging. When debug logging is enabled a
103+ stack trace appears, instead of the cryptic ``Internal Server Error``
104+ message. It can be turned on by setting ``debug: False`` to ``debug: True`` in
105+ :origin:`settings.yml <searx/settings.yml>`.
106+
107+Run ``./manage.sh tests`` before creating a PR.
108+ Failing build on Travis is common because of PEP8 checks. So a new commit
109+ must be created containing these format fixes. This phase can be skipped if
110+ ``./manage.sh tests`` is run locally before creating a PR.
docs/dev/search_api.rstnew file+112
@@ -0,0 +1,112 @@
1+==========
2+Search API
3+==========
4+
5+The search supports both ``GET`` and ``POST``.
6+
7+Furthermore, two enpoints ``/`` and ``/search`` are available for querying.
8+
9+
10+``GET /``
11+
12+``GET /search``
13+
14+Parameters
15+==========
16+
17+``q`` : required
18+ The search query. This string is passed to external search services. Thus,
19+ searx supports syntax of each search service. For example, ``site:github.com
20+ searx`` is a valid query for Google. However, if simply the query above is
21+ passed to any search engine which does not filter its results based on this
22+ syntax, you might not get the results you wanted.
23+
24+ See more at :ref:`search-syntax`
25+
26+``categories`` : optional
27+ Comma separated list, specifies the active search categories
28+
29+``engines``: optional
30+ Comma separated list, specifies the active search engines.
31+
32+``lang``: default ``all``
33+ Code of the language.
34+
35+``pageno``: default ``1``
36+ Search page number.
37+
38+``time_range``: optional
39+ [ ``day``, ``month``, ``year`` ]
40+
41+ Time range of search for engines which support it. See if an engine supports
42+ time range search in the preferences page of an instance.
43+
44+``format``: optional
45+ [ ``json``, ``csv``, ``rss`` ]
46+
47+ Output format of results.
48+
49+``results_on_new_tab``: default ``0``
50+ [ ``0``, ``1`` ]
51+
52+ Open search results on new tab.
53+
54+``image_proxy``: default ``False``
55+ [ ``True``, ``False`` ]
56+
57+ Proxy image results through searx.
58+
59+``autocomplete``: default *empty*
60+ [ ``google``, ``dbpedia``, ``duckduckgo``, ``startpage``, ``wikipedia`` ]
61+
62+ Service which completes words as you type.
63+
64+``safesearch``: default ``None``
65+ [ ``0``, ``1``, ``None`` ]
66+
67+ Filter search results of engines which support safe search. See if an engine
68+ supports safe search in the preferences page of an instance.
69+
70+``theme``: default ``oscar``
71+ [ ``oscar``, ``simple``, ``legacy``, ``pix-art``, ``courgette`` ]
72+
73+ Theme of instance.
74+
75+ Please note, available themes depend on an instance. It is possible that an
76+ instance administrator deleted, created or renamed themes on his/her instance.
77+ See the available options in the preferences page of the instance.
78+
79+``oscar-style``: default ``logicodev``
80+ [ ``pointhi``, ``logicodev`` ]
81+
82+ Style of Oscar theme. It is only parsed if the theme of an instance is
83+ ``oscar``.
84+
85+ Please note, available styles depend on an instance. It is possible that an
86+ instance administrator deleted, created or renamed styles on his/her
87+ instance. See the available options in the preferences page of the instance.
88+
89+``enabled_plugins``: optional
90+ List of enabled plugins.
91+
92+ :default: ``HTTPS_rewrite``, ``Self_Informations``,
93+ ``Search_on_category_select``, ``Tracker_URL_remover``
94+
95+ :values: [ ``DOAI_rewrite``, ``HTTPS_rewrite``, ``Infinite_scroll``,
96+ ``Vim-like_hotkeys``, ``Self_Informations``, ``Tracker_URL_remover``,
97+ ``Search_on_category_select`` ]
98+
99+``disabled_plugins``: optional
100+ List of disabled plugins.
101+
102+ :default: ``DOAI_rewrite``, ``Infinite_scroll``, ``Vim-like_hotkeys``
103+ :values: ``DOAI_rewrite``, ``HTTPS_rewrite``, ``Infinite_scroll``,
104+ ``Vim-like_hotkeys``, ``Self_Informations``, ``Tracker_URL_remover``,
105+ ``Search_on_category_select``
106+
107+``enabled_engines``: optional : *all* :origin:`engines <searx/engines>`
108+ List of enabled engines.
109+
110+``disabled_engines``: optional : *all* :origin:`engines <searx/engines>`
111+ List of disabled engines.
112+
docs/dev/translation.rstnew file+71
@@ -0,0 +1,71 @@
1+.. _translation:
2+
3+===========
4+Translation
5+===========
6+
7+.. _searx@transifex: https://www.transifex.com/asciimoo/searx/
8+
9+Translation currently takes place on `searx@transifex`_
10+
11+Requirements
12+============
13+
14+* Transifex account
15+* Installed CLI tool of Transifex
16+
17+Init Transifex project
18+======================
19+
20+After installing ``transifex`` using pip, run the following command to
21+initialize the project.
22+
23+.. code:: sh
24+
25+ tx init # Transifex instance: https://www.transifex.com/asciimoo/searx/
26+
27+
28+After ``$HOME/.transifexrc`` is created, get a Transifex API key and insert it
29+into the configuration file.
30+
31+Create a configuration file for ``tx`` named ``$HOME/.tx/config``.
32+
33+.. code:: ini
34+
35+ [main]
36+ host = https://www.transifex.com
37+ [searx.messagespo]
38+ file_filter = searx/translations/<lang>/LC_MESSAGES/messages.po
39+ source_file = messages.pot
40+ source_lang = en
41+ type = PO
42+
43+
44+Then run ``tx set``:
45+
46+.. code:: shell
47+
48+ tx set --auto-local -r searx.messagespo 'searx/translations/<lang>/LC_MESSAGES/messages.po' \
49+ --source-lang en --type PO --source-file messages.pot --execute
50+
51+
52+Update translations
53+===================
54+
55+To retrieve the latest translations, pull it from Transifex.
56+
57+.. code:: sh
58+
59+ tx pull -a
60+
61+Then check the new languages. If strings translated are not enough, delete those
62+folders, because those should not be compiled. Call the command below to compile
63+the ``.po`` files.
64+
65+.. code:: shell
66+
67+ pybabel compile -d searx/translations
68+
69+
70+After the compilation is finished commit the ``.po`` and ``.mo`` files and
71+create a PR.
docs/index.rstnew file+32
@@ -0,0 +1,32 @@
1+================
2+Welcome to searx
3+================
4+
5+Search without being tracked.
6+
7+.. sidebar:: Features
8+
9+ - Self hosted
10+ - No user tracking
11+ - No user profiling
12+ - About 70 supported search engines
13+ - Easy integration with any search engine
14+ - Cookies are not used by default
15+ - Secure, encrypted connections (HTTPS/SSL)
16+ - Hosted by organizations, such as *La Quadrature du Net*, which promote
17+ digital rights
18+
19+Searx is a free internet metasearch engine which aggregates results from more
20+than 70 search services. Users are neither tracked nor profiled. Additionally,
21+searx can be used over Tor for online anonymity.
22+
23+Get started with searx by using one of the :wiki:`Searx-instances`. If you
24+don't trust anyone, you can set up your own, see :ref:`installation`.
25+
26+.. toctree::
27+ :maxdepth: 2
28+
29+ user/index
30+ admin/index
31+ dev/index
32+ blog/index
docs/static/img/searx_logo_small.pngnew file

Binary file not shown.

docs/user/index.rstnew file+9
@@ -0,0 +1,9 @@
1+==================
2+User documentation
3+==================
4+
5+.. toctree::
6+ :maxdepth: 1
7+
8+ search_syntax
9+ own-instance
docs/user/own-instance.rstnew file+77
@@ -0,0 +1,77 @@
1+===========================
2+Why use a private instance?
3+===========================
4+
5+"Is it worth to run my own instance?" is a common question among searx users.
6+Before answering this question, see what options a searx user has.
7+
8+Public instances are open to everyone who has access to its URL. Usually, these
9+are operated by unknown parties (from the users' point of view). Private
10+instances can be used by a select group of people. It is for example a searx of
11+group of friends or a company which can be accessed through VPN. Also it can be
12+single user one which runs on the user's laptop.
13+
14+To gain more insight on how these instances work let's dive into how searx
15+protects its users.
16+
17+How does searx protect privacy?
18+===============================
19+
20+Searx protects the privacy of its users in multiple ways regardless of the type
21+of the instance (private, public). Removal of private data from search requests
22+comes in three forms:
23+
24+ 1. removal of private data from requests going to search services
25+ 2. not forwarding anything from a third party services through search services
26+ (e.g. advertisement)
27+ 3. removal of private data from requests going to the result pages
28+
29+Removing private data means not sending cookies to external search engines and
30+generating a random browser profile for every request. Thus, it does not matter
31+if a public or private instance handles the request, because it is anonymized in
32+both cases. IP addresses will be the IP of the instance. But searx can be
33+configured to use proxy or Tor. `Result proxy
34+<https://github.com/asciimoo/morty>`__ is supported, too.
35+
36+Searx does not serve ads or tracking content unlike most search services. So
37+private data is not forwarded to third parties who might monetize it. Besides
38+protecting users from search services, both referring page and search query are
39+hidden from visited result pages.
40+
41+
42+What are the consequences of using public instances?
43+----------------------------------------------------
44+
45+If someone uses a public instance, he/she has to trust the administrator of that
46+instance. This means that the user of the public instance does not know whether
47+his/her requests are logged, aggregated and sent or sold to a third party.
48+
49+Also, public instances without proper protection are more vulnerable to abusing
50+the search service, In this case the external service in exchange returns
51+CAPTCHAs or bans the IP of the instance. Thus, search requests return less
52+results.
53+
54+I see. What about private instances?
55+------------------------------------
56+
57+If users run their own instances, everything is in their control: the source
58+code, logging settings and private data. Unknown instance administrators do not
59+have to be trusted.
60+
61+Furthermore, as the default settings of their instance is editable, there is no
62+need to use cookies to tailor searx to their needs. So preferences will not be
63+reset to defaults when clearing browser cookies. As settings are stored on
64+their computer, it will not be accessible to others as long as their computer is
65+not compromised.
66+
67+Conclusion
68+==========
69+
70+Always use an instance which is operated by people you trust. The privacy
71+features of searx are available to users no matter what kind of instance they
72+use.
73+
74+If someone is on the go or just wants to try searx for the first time public
75+instances are the best choices. Additionally, public instance are making a
76+world a better place, because those who cannot or do not want to run an
77+instance, have access to a privacy respecting search service.
docs/user/search_syntax.rstnew file+42
@@ -0,0 +1,42 @@
1+
2+.. _search-syntax:
3+
4+=============
5+Search syntax
6+=============
7+
8+Searx allows you to modify the default categories, engines and search language
9+via the search query.
10+
11+Prefix ``!``
12+ to set Category/engine
13+
14+Prefix: ``:``
15+ to set language
16+
17+Prefix: ``?``
18+ to add engines and categories to the currently selected categories
19+
20+Abbrevations of the engines and languages are also accepted. Engine/category
21+modifiers are chainable and inclusive (e.g. with :search:`!it !ddg !wp qwer
22+<?q=%21it%20%21ddg%20%21wp%20qwer>` search in IT category **and** duckduckgo
23+**and** wikipedia for ``qwer``).
24+
25+See the :search:`/preferences page <preferences>` for the list of engines,
26+categories and languages.
27+
28+Examples
29+========
30+
31+Search in wikipedia for ``qwer``:
32+
33+- :search:`!wp qwer <?q=%21wp%20qwer>` or
34+- :search:`!wikipedia qwer :search:<?q=%21wikipedia%20qwer>`
35+
36+Image search:
37+
38+- :search:`!images Cthulhu <?q=%21images%20Cthulhu>`
39+
40+Custom language in wikipedia:
41+
42+- :search:`:hu !wp hackerspace <?q=%3Ahu%20%21wp%20hackerspace>`
requirements-dev.txt+3
@@ -1,3 +1,6 @@
1+pallets-sphinx-themes
2+Sphinx
3+sphinx-issues
1mock==2.0.0
4mock==2.0.0
2nose2[coverage_plugin]
5nose2[coverage_plugin]
3cov-core==1.15.0
6cov-core==1.15.0
utils/makefile.python+2−2
@@ -155,7 +155,7 @@ quiet_cmd_virtualenv = PYENV usage: $ source ./$@/bin/activate
155 if [ ! -d "./$(PY_ENV)" ];then \
155 if [ ! -d "./$(PY_ENV)" ];then \
156 $(VIRTUALENV) $(VIRTUALENV_VERBOSE) $(VTENV_OPTS) $2; \
156 $(VIRTUALENV) $(VIRTUALENV_VERBOSE) $(VTENV_OPTS) $2; \
157 else \
157 else \
158− echo " PYENV using virtualenv from $2"; \
158+ echo "PYENV using virtualenv from $2"; \
159 fi
159 fi
160
160
161# $2 path to lint
161# $2 path to lint
@@ -263,7 +263,7 @@ pydebug: $(PY_ENV)
263# install / uninstall python objects into virtualenv (PYENV)
263# install / uninstall python objects into virtualenv (PYENV)
264pyenv-install: $(PY_ENV)
264pyenv-install: $(PY_ENV)
265 @$(PY_ENV_BIN)/pip $(PIP_VERBOSE) install -e .
265 @$(PY_ENV_BIN)/pip $(PIP_VERBOSE) install -e .
266− @echo " ACTIVATE $(call normpath,$(PY_ENV_ACT)) "
266+ @echo "ACTIVATE $(call normpath,$(PY_ENV_ACT)) "
267
267
268pyenv-uninstall: $(PY_ENV)
268pyenv-uninstall: $(PY_ENV)
269 @$(PY_ENV_BIN)/pip $(PIP_VERBOSE) uninstall --yes .
269 @$(PY_ENV_BIN)/pip $(PIP_VERBOSE) uninstall --yes .
utils/makefile.sphinxnew file+215
@@ -0,0 +1,215 @@
1+# -*- coding: utf-8; mode: makefile-gmake -*-
2+
3+# You can set these variables from the command line.
4+SPHINXOPTS ?=
5+SPHINXBUILD ?= $(PY_ENV_BIN)/sphinx-build
6+SPHINX_CONF ?= conf.py
7+
8+DOCS_FOLDER ?= docs
9+DOCS_BUILD ?= build/docs
10+DOCS_DIST ?= dist/docs
11+GH_PAGES ?= gh-pages
12+
13+BOOKS_FOLDER ?= docs
14+BOOKS_DIST ?= dist/books
15+
16+ifeq ($(KBUILD_VERBOSE),1)
17+ SPHINX_VERBOSE = "-v"
18+else
19+ SPHINX_VERBOSE =
20+endif
21+
22+## SPHINXVERS variable
23+## ===================
24+##
25+## .. _requirement-specifiers: https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers
26+##
27+## Sphinx version to use, when building documentation. Set this when calling
28+## build target. The default value is empty (install latest), to select a
29+## specific version use a requirement-specifiers_. E.g. to build your target
30+## 'doc' with a select sphinx-doc_ version 1.7.9::
31+##
32+## make SPHINXVERS='==1.7.9' docs
33+##
34+## To build with latest 1.7::
35+##
36+## make SPHINXVERS='>=1.7,<1.8' docs
37+##
38+SPHINXVERS ?=
39+
40+docs-help:
41+ @echo 'makefile.sphinx:'
42+ @echo ' docs-clean - clean intermediate doc objects'
43+ @echo ' $(GH_PAGES) - create & upload github pages'
44+ @echo ' sphinx-pdf - run sphinx latex & pdf targets'
45+ echo ''
46+ @echo ' books/{name}.html : build only the HTML of document {name}'
47+ @echo ' valid values for books/{name}.html are:'
48+ @echo ' $(BOOKS_HTML)' | $(FMT)
49+ @echo ' books/{name}.pdf : build only the PDF of document {name}'
50+ @echo ' valid values for books/{name}.pdf are:'
51+ @echo ' $(BOOKS_PDF) ' | $(FMT)
52+
53+# ------------------------------------------------------------------------------
54+# requirements
55+# ------------------------------------------------------------------------------
56+
57+sphinx-doc: $(PY_ENV)
58+ @echo "PYENV installing Sphinx$(SPHINXVERS)"
59+ $(Q)$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) 'Sphinx$(SPHINXVERS)'
60+
61+sphinx-live: $(PY_ENV)
62+ @echo "PYENV installing Sphinx$(SPHINXVERS)"
63+ $(Q)$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) 'Sphinx$(SPHINXVERS)' sphinx-autobuild
64+
65+
66+PHONY += msg-texlive texlive
67+
68+ifeq ($(shell which xelatex >/dev/null 2>&1; echo $$?), 1)
69+texlive: msg-TeXLive
70+ $(error The 'xelatex' command was not found)
71+else
72+texlive:
73+ @:
74+endif
75+
76+msg-texlive:
77+ $(Q)echo "\n\
78+The TeX/PDF output and the *math* extension require TexLive and latexmk:\n\n\
79+ Make sure you have a updated TeXLive with XeTeX engine installed, grab it\n\
80+ it from https://www.tug.org/texlive or install it from your package manager.\n\n\
81+ Install latexmk from your package manager or visit https://ctan.org/pkg/latexmk\n\n\
82+ Sphinx-doc produce (Xe)LaTeX files which might use additional TeX-packages\n\
83+ and fonts. To process these LaTeX files, a TexLive installation with the\n\
84+ additional packages is required. On debian based OS these requirements\n\
85+ are installed by::\n\n\
86+ sudo -H apt-get install\n\
87+ latexmk\n\
88+ texlive-base texlive-xetex texlive-latex-recommended\n\
89+ texlive-extra-utils dvipng ttf-dejavu\n"
90+
91+# ------------------------------------------------------------------------------
92+# commands
93+# ------------------------------------------------------------------------------
94+
95+# $2 sphinx builder e.g. "html"
96+# $3 path where configuration file (conf.py) is located
97+# $4 sourcedir
98+# $5 dest subfolder e.g. "man" for man pages at $(DOCS_DIST)/man
99+
100+quiet_cmd_sphinx = SPHINX $@ --> file://$(abspath $(DOCS_DIST)/$5)
101+ cmd_sphinx = SPHINX_CONF=$(abspath $4/$(SPHINX_CONF))\
102+ $(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\
103+ -b $2 -c $3 -d $(DOCS_BUILD)/.doctrees $4 $(DOCS_DIST)/$5
104+
105+quiet_cmd_sphinx_autobuild = SPHINX $@ --> file://$(abspath $(DOCS_DIST)/$5)
106+ cmd_sphinx_autobuild = PATH="$(PY_ENV_BIN):$(PATH)" $(PY_ENV_BIN)/sphinx-autobuild $(SPHINX_VERBOSE) --poll -B --host 0.0.0.0 --port 8080 $(SPHINXOPTS)\
107+ -b $2 -c $3 -d $(DOCS_BUILD)/.doctrees $4 $(DOCS_DIST)/$5
108+
109+quiet_cmd_sphinx_clean = CLEAN $@
110+ cmd_sphinx_clean = rm -rf $(DOCS_BUILD) $(DOCS_DIST) $(GH_PAGES)/* $(GH_PAGES)/.buildinfo
111+
112+# ------------------------------------------------------------------------------
113+# targets
114+# ------------------------------------------------------------------------------
115+
116+# build PDF of whole documentation in: $(DOCS_DIST)/pdf
117+
118+PHONY += sphinx-pdf
119+sphinx-pdf: sphinx-latex
120+ $(Q)cd $(DOCS_BUILD)/latex/; make all-pdf
121+ $(Q)mkdir -p $(DOCS_DIST)/pdf
122+ $(Q)cp $(DOCS_BUILD)/latex/*.pdf $(DOCS_DIST)/pdf
123+ @echo "SPHINX *.pdf --> file://$(abspath $(DOCS_DIST)/pdf)"
124+
125+PHONY += sphinx-latex
126+sphinx-latex: texlive sphinx-doc
127+ $(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\
128+ -b latex \
129+ -c $(DOCS_FOLDER) \
130+ -d $(DOCS_BUILD)/.doctrees \
131+ $(DOCS_FOLDER) \
132+ $(DOCS_BUILD)/latex
133+
134+# Sphinx projects, we call them *books* (what is more common). Books are
135+# folders under $(BOOKS_FOLDER) containing a conf.py file. The HTML output goes
136+# to folder $(BOOKS_DIST)/<name> while PDF is placed (BOOKS_DIST)/<name>/pdf
137+
138+BOOKS=$(patsubst $(BOOKS_FOLDER)/%/conf.py,books/%,$(wildcard $(BOOKS_FOLDER)/*/conf.py))
139+
140+# fine grained targets
141+BOOKS_HTML = $(patsubst %,%.html,$(BOOKS))
142+BOOKS_CLEAN = $(patsubst %,%.clean,$(BOOKS))
143+BOOKS_LATEX = $(patsubst %,%.latex,$(BOOKS))
144+BOOKS_PDF = $(patsubst %,%.pdf,$(BOOKS))
145+BOOKS_LIVE = $(patsubst %,%.live,$(BOOKS))
146+
147+$(BOOKS_DIST):
148+ mkdir -p $(BOOKS_DIST)
149+
150+PHONY += $(BOOKS_HTML)
151+$(BOOKS_HTML): sphinx-doc | $(BOOKS_DIST)
152+ SPHINX_CONF=$(patsubst books/%.html,%,$@)/conf.py \
153+ $(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\
154+ -b html \
155+ -c $(DOCS_FOLDER) \
156+ -d $(DOCS_BUILD)/books/$(patsubst books/%.html,%,$@)/.doctrees \
157+ $(patsubst books/%.html,%,$@) \
158+ $(BOOKS_DIST)/$(patsubst books/%.html,%,$@)
159+ @echo "SPHINX $@ --> file://$(abspath $(BOOKS_DIST)/$(patsubst books/%.html,%,$@))"
160+
161+PHONY += $(BOOKS_HTML)
162+$(BOOKS_LIVE): sphinx-live | $(BOOKS_DIST)
163+ PATH="$(PY_ENV_BIN):$(PATH)" \
164+ SPHINX_CONF=$(patsubst books/%.live,%,$@)/conf.py \
165+ $(PY_ENV_BIN)/sphinx-autobuild --poll -B --host 0.0.0.0 --port 8080 $(SPHINX_VERBOSE) $(SPHINXOPTS)\
166+ -b html \
167+ -c $(DOCS_FOLDER) \
168+ -d $(DOCS_BUILD)/books/$(patsubst books/%.live,%,$@)/.doctrees \
169+ $(patsubst books/%.live,%,$@) \
170+ $(BOOKS_DIST)/$(patsubst books/%.live,%,$@)
171+
172+$(BOOKS_PDF): %.pdf : %.latex
173+ $(Q)cd $(DOCS_BUILD)/latex/$(patsubst books/%.pdf,%,$@); make all-pdf
174+ $(Q)mkdir -p $(BOOKS_DIST)/$(patsubst books/%.pdf,%,$@)/pdf
175+ $(Q)cp -v $(DOCS_BUILD)/latex/$(patsubst books/%.pdf,%,$@)/*.pdf $(BOOKS_DIST)/$(patsubst books/%.pdf,%,$@)/pdf
176+ @echo "SPHINX $@ --> file://$(abspath $(BOOKS_DIST)/$(patsubst books/%.pdf,%,$@))/pdf"
177+
178+PHONY += $(BOOKS_LATEX)
179+$(BOOKS_LATEX): sphinx-doc | $(BOOKS_DIST)
180+ SPHINX_CONF=$(patsubst books/%.latex,%,$@)/conf.py \
181+ $(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\
182+ -b latex \
183+ -c $(DOCS_FOLDER) \
184+ -d $(DOCS_BUILD)/books/$(patsubst books/%.latex,%,$@)/.doctrees \
185+ $(patsubst books/%.latex,%,$@) \
186+ $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@)
187+ @echo "SPHINX $@ --> file://$(abspath $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@))"
188+
189+$(BOOKS_CLEAN):
190+ $(Q)rm -rf $(BOOKS_DIST)/$(patsubst books/%.clean,%,$@) \
191+ $(DOCS_BUILD)/books/$(patsubst books/%.clean,%,$@) \
192+ $(DOCS_BUILD)/latex/$(patsubst books/%.clean,%,$@)
193+
194+# github pages
195+
196+PHONY += $(GH_PAGES)
197+$(GH_PAGES)::
198+ $(MAKE) docs
199+ [ -d "gh-pages/.git" ] || git clone $(GIT_URL) gh-pages
200+ -cd $(GH_PAGES); git checkout gh-pages >/dev/null
201+ -cd $(GH_PAGES); ls -A | grep -v '.git$$' | xargs rm -rf
202+ cp -r $(DOCS_DIST)/* $(GH_PAGES)/
203+ touch $(GH_PAGES)/.nojekyll
204+ echo "<html><head><META http-equiv='refresh' content='0;URL=index.html'></head></html>" > $(GH_PAGES)/404.html
205+ cd $(GH_PAGES);\
206+ git add --all . ;\
207+ git commit -m "gh-pages: updated" ;\
208+ git push origin gh-pages
209+
210+
211+PHONY += docs-clean
212+docs-clean: $(BOOKS_CLEAN)
213+ $(call cmd,sphinx_clean)
214+
215+.PHONY: $(PHONY)