on master help sign in
Merge branch 'master' into csp-oscar-theme
ca1c3bdMarkus Heiser
.config.shnew file+55
@@ -0,0 +1,55 @@
1+# -*- coding: utf-8; mode: sh -*-
2+# SPDX-License-Identifier: AGPL-3.0-or-later
3+# shellcheck shell=bash disable=SC2034
4+#
5+# This environment is used by ./utils scripts like filtron.sh or searx.sh. The
6+# default values are *most flexible* and *best maintained*, you normally not
7+# need to change the defaults (except PUBLIC_URL).
8+#
9+# Before you change any value here you have to uninstall any previous
10+# installation. Further is it recommended to backup your changes simply by
11+# adding them to you local brand (git branch)::
12+#
13+# git add .config
14+
15+# The public URL of the searx instance: PUBLIC_URL="https://mydomain.xy/searx"
16+# The default is taken from ./utils/brand.env.
17+
18+PUBLIC_URL="${SEARX_URL}"
19+
20+if [[ ${PUBLIC_URL} == "https://searx.me" ]]; then
21+ # hint: Linux containers do not have DNS entries, lets use IPs
22+ PUBLIC_URL="http://$(primary_ip)/searx"
23+fi
24+
25+# searx.sh
26+# ---------
27+
28+# SEARX_INTERNAL_URL="127.0.0.1:8888"
29+
30+# Only change, if you maintain a searx brand in your searx fork.
31+# GIT_BRANCH="${GIT_BRANCH:-master}"
32+
33+# filtron.sh
34+# ----------
35+
36+# FILTRON_API="127.0.0.1:4005"
37+# FILTRON_LISTEN="127.0.0.1:4004"
38+# FILTRON_TARGET="127.0.0.1:8888"
39+
40+# morty.sh
41+# --------
42+
43+# morty listen address
44+# MORTY_LISTEN="127.0.0.1:3000"
45+# PUBLIC_URL_PATH_MORTY="/morty/"
46+
47+# system services
48+# ---------------
49+
50+# Common $HOME folder of the service accounts
51+# SERVICE_HOME_BASE="/usr/local"
52+
53+# **experimental**: Set SERVICE_USER to run all services by one account, but be
54+# aware that removing discrete components might conflict!
55+# SERVICE_USER=searx
.gitignore+1
@@ -1,6 +1,7 @@
1# to sync with .dockerignore
1# to sync with .dockerignore
2.coverage
2.coverage
3coverage/
3coverage/
4+cache/
4.installed.cfg
5.installed.cfg
5engines.cfg
6engines.cfg
6env
7env
Makefile+52−13
@@ -1,24 +1,31 @@
1# -*- coding: utf-8; mode: makefile-gmake -*-
1# -*- coding: utf-8; mode: makefile-gmake -*-
2+.DEFAULT_GOAL=help
2
3
4+# START Makefile setup
3export GIT_URL=https://github.com/asciimoo/searx
5export GIT_URL=https://github.com/asciimoo/searx
6+export GIT_BRANCH=master
4export SEARX_URL=https://searx.me
7export SEARX_URL=https://searx.me
5export DOCS_URL=https://asciimoo.github.io/searx
8export DOCS_URL=https://asciimoo.github.io/searx
9+# END Makefile setup
10+
11+include utils/makefile.include
6
12
7PYOBJECTS = searx
13PYOBJECTS = searx
8DOC = docs
14DOC = docs
9PY_SETUP_EXTRAS ?= \[test\]
15PY_SETUP_EXTRAS ?= \[test\]
10
16
11−PYDIST=./dist/py
12−PYBUILD=./build/py
13−
14−include utils/makefile.include
15include utils/makefile.python
17include utils/makefile.python
16include utils/makefile.sphinx
18include utils/makefile.sphinx
17
19
18all: clean install
20all: clean install
19
21
20−PHONY += help
22+PHONY += help-min help-all help
23+
21−help:
24+help: help-min
25+ @echo ''
26+ @echo 'to get more help: make help-all'
27+
28+help-min:
22 @echo ' test - run developer tests'
29 @echo ' test - run developer tests'
23 @echo ' docs - build documentation'
30 @echo ' docs - build documentation'
24 @echo ' docs-live - autobuild HTML documentation while editing'
31 @echo ' docs-live - autobuild HTML documentation while editing'
@@ -33,9 +40,18 @@ help:
33 @echo ' docker - build Docker image'
40 @echo ' docker - build Docker image'
34 @echo ' node.env - download & install npm dependencies locally'
41 @echo ' node.env - download & install npm dependencies locally'
35 @echo ''
42 @echo ''
43+ @echo 'environment'
44+ @echo ' SEARX_URL = $(SEARX_URL)'
45+ @echo ' GIT_URL = $(GIT_URL)'
46+ @echo ' DOCS_URL = $(DOCS_URL)'
47+ @echo ''
36− @$(MAKE) -s -f utils/makefile.include make-help
48+ @$(MAKE) -e -s make-help
49+
50+help-all: help-min
51+ @echo ''
52+ @$(MAKE) -e -s python-help
37 @echo ''
53 @echo ''
38− @$(MAKE) -s -f utils/makefile.python python-help
54+ @$(MAKE) -e -s docs-help
39
55
40PHONY += install
56PHONY += install
41install: buildenv pyenvinstall
57install: buildenv pyenvinstall
@@ -44,7 +60,7 @@ PHONY += uninstall
44uninstall: pyenvuninstall
60uninstall: pyenvuninstall
45
61
46PHONY += clean
62PHONY += clean
47−clean: pyclean node.clean test.clean
63+clean: pyclean docs-clean node.clean test.clean
48 $(call cmd,common_clean)
64 $(call cmd,common_clean)
49
65
50PHONY += run
66PHONY += run
@@ -61,14 +77,24 @@ run: buildenv pyenvinstall
61# docs
77# docs
62# ----
78# ----
63
79
80+sphinx-doc-prebuilds:: buildenv pyenvinstall prebuild-includes
81+
64PHONY += docs
82PHONY += docs
65−docs: buildenv pyenvinstall sphinx-doc
83+docs: sphinx-doc-prebuilds sphinx-doc
66 $(call cmd,sphinx,html,docs,docs)
84 $(call cmd,sphinx,html,docs,docs)
67
85
68PHONY += docs-live
86PHONY += docs-live
69−docs-live: buildenv pyenvinstall sphinx-live
87+docs-live: sphinx-doc-prebuilds sphinx-live
70 $(call cmd,sphinx_autobuild,html,docs,docs)
88 $(call cmd,sphinx_autobuild,html,docs,docs)
71
89
90+PHONY += prebuild-includes
91+prebuild-includes:
92+ $(Q)mkdir -p $(DOCS_BUILD)/includes
93+ $(Q)./utils/searx.sh doc | cat > $(DOCS_BUILD)/includes/searx.rst
94+ $(Q)./utils/filtron.sh doc | cat > $(DOCS_BUILD)/includes/filtron.rst
95+ $(Q)./utils/morty.sh doc | cat > $(DOCS_BUILD)/includes/morty.rst
96+
97+
72$(GH_PAGES)::
98$(GH_PAGES)::
73 @echo "doc available at --> $(DOCS_URL)"
99 @echo "doc available at --> $(DOCS_URL)"
74
100
@@ -94,12 +120,14 @@ useragents.update: pyenvinstall
94buildenv:
120buildenv:
95 $(Q)echo "build searx/brand.py"
121 $(Q)echo "build searx/brand.py"
96 $(Q)echo "GIT_URL = '$(GIT_URL)'" > searx/brand.py
122 $(Q)echo "GIT_URL = '$(GIT_URL)'" > searx/brand.py
123+ $(Q)echo "GIT_BRANCH = '$(GIT_BRANCH)'" >> searx/brand.py
97 $(Q)echo "ISSUE_URL = 'https://github.com/asciimoo/searx/issues'" >> searx/brand.py
124 $(Q)echo "ISSUE_URL = 'https://github.com/asciimoo/searx/issues'" >> searx/brand.py
98 $(Q)echo "SEARX_URL = '$(SEARX_URL)'" >> searx/brand.py
125 $(Q)echo "SEARX_URL = '$(SEARX_URL)'" >> searx/brand.py
99 $(Q)echo "DOCS_URL = '$(DOCS_URL)'" >> searx/brand.py
126 $(Q)echo "DOCS_URL = '$(DOCS_URL)'" >> searx/brand.py
100 $(Q)echo "PUBLIC_INSTANCES = 'https://searx.space'" >> searx/brand.py
127 $(Q)echo "PUBLIC_INSTANCES = 'https://searx.space'" >> searx/brand.py
101 $(Q)echo "build utils/brand.env"
128 $(Q)echo "build utils/brand.env"
102 $(Q)echo "export GIT_URL='$(GIT_URL)'" > utils/brand.env
129 $(Q)echo "export GIT_URL='$(GIT_URL)'" > utils/brand.env
130+ $(Q)echo "export GIT_BRANCH='$(GIT_BRANCH)'" >> utils/brand.env
103 $(Q)echo "export ISSUE_URL='https://github.com/asciimoo/searx/issues'" >> utils/brand.env
131 $(Q)echo "export ISSUE_URL='https://github.com/asciimoo/searx/issues'" >> utils/brand.env
104 $(Q)echo "export SEARX_URL='$(SEARX_URL)'" >> utils/brand.env
132 $(Q)echo "export SEARX_URL='$(SEARX_URL)'" >> utils/brand.env
105 $(Q)echo "export DOCS_URL='$(DOCS_URL)'" >> utils/brand.env
133 $(Q)echo "export DOCS_URL='$(DOCS_URL)'" >> utils/brand.env
@@ -182,8 +210,7 @@ gecko.driver:
182# test
210# test
183# ----
211# ----
184
212
185−PHONY += test test.pylint test.pep8 test.unit test.coverage test.robot
213+PHONY += test test.sh test.pylint test.pep8 test.unit test.coverage test.robot
186−
187test: buildenv test.pylint test.pep8 test.unit gecko.driver test.robot
214test: buildenv test.pylint test.pep8 test.unit gecko.driver test.robot
188
215
189ifeq ($(PY),2)
216ifeq ($(PY),2)
@@ -191,6 +218,7 @@ test.pylint:
191 @echo "LINT skip liniting py2"
218 @echo "LINT skip liniting py2"
192else
219else
193# TODO: balance linting with pylint
220# TODO: balance linting with pylint
221+
194test.pylint: pyenvinstall
222test.pylint: pyenvinstall
195 $(call cmd,pylint,\
223 $(call cmd,pylint,\
196 searx/preferences.py \
224 searx/preferences.py \
@@ -202,6 +230,17 @@ endif
202# E402 module level import not at top of file
230# E402 module level import not at top of file
203# W503 line break before binary operator
231# W503 line break before binary operator
204
232
233+# ubu1604: uses shellcheck v0.3.7 (from 04/2015), no longer supported!
234+test.sh:
235+ shellcheck -x -s bash utils/brand.env
236+ shellcheck -x utils/lib.sh
237+ shellcheck -x utils/filtron.sh
238+ shellcheck -x utils/searx.sh
239+ shellcheck -x utils/morty.sh
240+ shellcheck -x utils/lxc.sh
241+ shellcheck -x utils/lxc-searx.env
242+ shellcheck -x .config.sh
243+
205test.pep8: pyenvinstall
244test.pep8: pyenvinstall
206 @echo "TEST pep8"
245 @echo "TEST pep8"
207 $(Q)$(PY_ENV_ACT); pep8 --exclude=searx/static --max-line-length=120 --ignore "E402,W503" searx tests
246 $(Q)$(PY_ENV_ACT); pep8 --exclude=searx/static --max-line-length=120 --ignore "E402,W503" searx tests
docs/_themes/searx/static/searx.css+40−1
@@ -33,7 +33,7 @@ p.sidebar-title, .sidebar p {
33/* admonitions
33/* admonitions
34*/
34*/
35
35
36−div.admonition, div.topic {
36+div.admonition, div.topic, div.toctree-wrapper {
37 background-color: #fafafa;
37 background-color: #fafafa;
38 margin: 8px 0px;
38 margin: 8px 0px;
39 padding: 1em;
39 padding: 1em;
@@ -42,6 +42,16 @@ div.admonition, div.topic {
42 border-right: none;
42 border-right: none;
43 border-bottom: none;
43 border-bottom: none;
44 border-left: 5pt solid #ccc;
44 border-left: 5pt solid #ccc;
45+ list-style-type: disclosure-closed;
46+}
47+
48+div.toctree-wrapper p.caption {
49+ font-weight: normal;
50+ font-size: 24px;
51+ margin: 0 0 10px 0;
52+ padding: 0;
53+ line-height: 1;
54+ display: inline;
45}
55}
46
56
47p.admonition-title:after {
57p.admonition-title:after {
@@ -128,3 +138,32 @@ caption {
128 caption-side: top;
138 caption-side: top;
129 text-align: left;
139 text-align: left;
130}
140}
141+
142+/* bugs since sphinx 3.1
143+
144+See sphinx-doc project, PR 7838 & 7484 with elementary patch to the basic CSS:
145+
146+- https://github.com/sphinx-doc/sphinx/issues/7838#issuecomment-646009605
147+- https://github.com/sphinx-doc/sphinx/pull/7484#issuecomment-646058972
148+
149+*/
150+
151+li > p:first-child {
152+ margin-top: 0;
153+}
154+
155+li > p:last-child {
156+ margin-bottom: 0;
157+}
158+
159+div.admonition dl {
160+ margin-bottom: 0;
161+}
162+
163+div.sidebar {
164+ clear: none;
165+}
166+
167+div.admonition, div.topic, pre {
168+ clear: none;
169+}
docs/admin/arch_public.dot+4−4
@@ -4,11 +4,11 @@ digraph G {
4 edge [fontname="Sans"];
4 edge [fontname="Sans"];
5
5
6 browser [label="Browser", shape=Mdiamond];
6 browser [label="Browser", shape=Mdiamond];
7− rp [label="Reverse Proxy", href="url to configure reverse proxy"];
7+ rp [label="Reverse Proxy", href="https://asciimoo.github.io/searx/utils/filtron.sh.html#public-reverse-proxy"];
8− filtron [label="Filtron", href="https://github.com/asciimoo/filtron"];
8+ filtron [label="Filtron", href="https://asciimoo.github.io/searx/utils/filtron.sh.html"];
9− morty [label="Morty", href="https://github.com/asciimoo/morty"];
9+ morty [label="Morty", href="https://asciimoo.github.io/searx/utils/morty.sh.html"];
10 static [label="Static files", href="url to configure static files"];
10 static [label="Static files", href="url to configure static files"];
11− uwsgi [label="uwsgi", href="url to configure uwsgi"]
11+ uwsgi [label="uwsgi", href="https://asciimoo.github.io/searx/utils/searx.sh.html"]
12 searx1 [label="Searx #1"];
12 searx1 [label="Searx #1"];
13 searx2 [label="Searx #2"];
13 searx2 [label="Searx #2"];
14 searx3 [label="Searx #3"];
14 searx3 [label="Searx #3"];
docs/admin/architecture.rst+9−5
@@ -4,17 +4,21 @@
4Architecture
4Architecture
5============
5============
6
6
7−.. sidebar:: Needs work!
7+.. sidebar:: Further reading
8
8
9− This article needs some work / Searx is a collaborative effort. If you have
9+ - Reverse Proxy: :ref:`Apache <apache searx site>` & :ref:`nginx <nginx searx
10− any contribution, feel welcome to send us your :pull:`PR <../pulls>`, see
10+ site>`
11− :ref:`how to contribute`.
11+ - Filtron: :ref:`searx filtron`
12+ - Morty: :ref:`searx morty`
13+ - uWSGI: :ref:`searx uwsgi`
14+ - Searx: :ref:`installation basic`
12
15
13Herein you will find some hints and suggestions about typical architectures of
16Herein you will find some hints and suggestions about typical architectures of
14searx infrastructures.
17searx infrastructures.
15
18
16We start with a contribution from :pull:`@dalf <1776#issuecomment-567917320>`.
19We start with a contribution from :pull:`@dalf <1776#issuecomment-567917320>`.
17−It shows a *reference* setup for public searx instances.
20+It shows a *reference* setup for public searx instances which can build up and
21+maintained by the scripts from our :ref:`toolboxing`.
18
22
19.. _arch public:
23.. _arch public:
20
24
docs/admin/buildhosts.rst+63−11
@@ -9,8 +9,27 @@ Buildhosts
9 If you have any contribution send us your :pull:`PR <../pulls>`, see
9 If you have any contribution send us your :pull:`PR <../pulls>`, see
10 :ref:`how to contribute`.
10 :ref:`how to contribute`.
11
11
12+.. contents:: Contents
13+ :depth: 2
14+ :local:
15+ :backlinks: entry
16+
12To get best results from build, its recommend to install additional packages
17To get best results from build, its recommend to install additional packages
13−on build hosts.
18+on build hosts (see :ref:`searx.sh`).::
19+
20+ sudo -H ./utils/searx.sh install buildhost
21+
22+This will install packages needed by searx:
23+
24+.. kernel-include:: $DOCS_BUILD/includes/searx.rst
25+ :start-after: START distro-packages
26+ :end-before: END distro-packages
27+
28+and packages needed to build docuemtation and run tests:
29+
30+.. kernel-include:: $DOCS_BUILD/includes/searx.rst
31+ :start-after: START build-packages
32+ :end-before: END build-packages
14
33
15.. _docs build:
34.. _docs build:
16
35
@@ -35,8 +54,17 @@ processing additional packages are needed. The XeTeX_ needed not only for PDF
35creation, its also needed for :ref:`math` when HTML output is build.
54creation, its also needed for :ref:`math` when HTML output is build.
36
55
37To be able to do :ref:`sphinx:math-support` without CDNs, the math are rendered
56To be able to do :ref:`sphinx:math-support` without CDNs, the math are rendered
38−as images (``sphinx.ext.imgmath`` extension). If your docs build (``make
57+as images (``sphinx.ext.imgmath`` extension).
58+
59+Here is the extract from the :origin:`docs/conf.py` file, setting math renderer
60+to ``imgmath``:
61+
62+.. literalinclude:: ../conf.py
63+ :language: python
64+ :start-after: # sphinx.ext.imgmath setup
65+ :end-before: # sphinx.ext.imgmath setup END
66+
39−docs``) shows warnings like this::
67+If your docs build (``make docs``) shows warnings like this::
40
68
41 WARNING: dot(1) not found, for better output quality install \
69 WARNING: dot(1) not found, for better output quality install \
42 graphviz from http://www.graphviz.org
70 graphviz from http://www.graphviz.org
@@ -47,8 +75,6 @@ docs``) shows warnings like this::
47you need to install additional packages on your build host, to get better HTML
75you need to install additional packages on your build host, to get better HTML
48output.
76output.
49
77
50−.. _system requirements:
51−
52.. tabs::
78.. tabs::
53
79
54 .. group-tab:: Ubuntu / debian
80 .. group-tab:: Ubuntu / debian
@@ -92,12 +118,38 @@ For PDF output you also need:
92
118
93 $ sudo dnf install \
119 $ sudo dnf install \
94 texlive-collection-fontsrecommended texlive-collection-latex \
120 texlive-collection-fontsrecommended texlive-collection-latex \
95− dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts
121+ dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts \
122+ ImageMagick
96
123
97−.. _system requirements END:
124+.. _sh lint:
98
125
99−.. literalinclude:: ../conf.py
126+Lint shell scripts
100− :language: python
127+==================
101− :start-after: # sphinx.ext.imgmath setup
128+
102− :end-before: # sphinx.ext.imgmath setup END
129+.. _ShellCheck: https://github.com/koalaman/shellcheck
130+
131+To lint shell scripts, we use ShellCheck_ - A shell script static analysis tool.
132+
133+.. SNIP sh lint requirements
134+
135+.. tabs::
136+
137+ .. group-tab:: Ubuntu / debian
138+
139+ .. code-block:: sh
140+
141+ $ sudo apt install shellcheck
142+
143+ .. group-tab:: Arch Linux
144+
145+ .. code-block:: sh
146+
147+ $ sudo pacman -S shellcheck
148+
149+ .. group-tab:: Fedora / RHEL
150+
151+ .. code-block:: sh
152+
153+ $ sudo dnf install ShellCheck
103
154
155+.. SNAP sh lint requirements
docs/admin/filtron.rst+145−102
@@ -1,18 +1,51 @@
1+
2+.. _searx filtron:
3+
1==========================
4==========================
2How to protect an instance
5How to protect an instance
3==========================
6==========================
4
7
8+.. sidebar:: further reading
9+
10+ - :ref:`filtron.sh`
11+ - :ref:`nginx searx site`
12+
13+
14+.. contents:: Contents
15+ :depth: 2
16+ :local:
17+ :backlinks: entry
18+
19+.. _filtron: https://github.com/asciimoo/filtron
20+
5−Searx depens on external search services. To avoid the abuse of these services
21+Searx depends on external search services. To avoid the abuse of these services
6it is advised to limit the number of requests processed by searx.
22it is advised to limit the number of requests processed by searx.
7
23
8−An application firewall, ``filtron`` solves exactly this problem. Information
24+An application firewall, filtron_ solves exactly this problem. Filtron is just
9−on how to install it can be found at the `project page of filtron
25+a middleware between your web server (nginx, apache, ...) and searx, we describe
10−<https://github.com/asciimoo/filtron>`__.
26+such infratructures in chapter: :ref:`architecture`.
11
27
12
28
29+filtron & go
30+============
31+
32+.. _Go: https://golang.org/
33+.. _filtron README: https://github.com/asciimoo/filtron/blob/master/README.md
34+
35+Filtron needs Go_ installed. If Go_ is preinstalled, filtron_ is simply
36+installed by ``go get`` package management (see `filtron README`_). If you use
37+filtron as middleware, a more isolated setup is recommended. To simplify such
38+an installation and the maintenance of, use our script :ref:`filtron.sh`.
39+
40+.. _Sample configuration of filtron:
41+
13Sample configuration of filtron
42Sample configuration of filtron
14===============================
43===============================
15
44
45+.. sidebar:: Tooling box
46+
47+ - :origin:`/etc/filtron/rules.json <utils/templates/etc/filtron/rules.json>`
48+
16An example configuration can be find below. This configuration limits the access
49An example configuration can be find below. This configuration limits the access
17of:
50of:
18
51
@@ -24,105 +57,104 @@ of:
24
57
25.. code:: json
58.. code:: json
26
59
27− [{
60+ [
61+ {
28− "name":"search request",
62+ "name": "search request",
29− "filters":[
63+ "filters": [
30− "Param:q",
64+ "Param:q",
31− "Path=^(/|/search)$"
65+ "Path=^(/|/search)$"
32− ],
33− "interval":"<time-interval-in-sec (int)>",
34− "limit":"<max-request-number-in-interval (int)>",
35− "subrules":[
36− {
37− "name":"roboagent limit",
38− "interval":"<time-interval-in-sec (int)>",
39− "limit":"<max-request-number-in-interval (int)>",
40− "filters":[
41− "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"
42− ],
43− "actions":[
44− {
45− "name":"block",
46− "params":{
47− "message":"Rate limit exceeded"
48− }
49− }
50− ]
51− },
52− {
53− "name":"botlimit",
54− "limit":0,
55− "stop":true,
56− "filters":[
57− "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
58− ],
59− "actions":[
60− {
61− "name":"block",
62− "params":{
63− "message":"Rate limit exceeded"
64− }
65− }
66− ]
67− },
68− {
69− "name":"IP limit",
70− "interval":"<time-interval-in-sec (int)>",
71− "limit":"<max-request-number-in-interval (int)>",
72− "stop":true,
73− "aggregations":[
74− "Header:X-Forwarded-For"
75− ],
76− "actions":[
77− {
78− "name":"block",
79− "params":{
80− "message":"Rate limit exceeded"
81− }
82− }
83− ]
84− },
85− {
86− "name":"rss/json limit",
87− "interval":"<time-interval-in-sec (int)>",
88− "limit":"<max-request-number-in-interval (int)>",
89− "stop":true,
90− "filters":[
91− "Param:format=(csv|json|rss)"
92− ],
93− "actions":[
94− {
95− "name":"block",
96− "params":{
97− "message":"Rate limit exceeded"
98− }
99− }
100− ]
101− },
102− {
103− "name":"useragent limit",
104− "interval":"<time-interval-in-sec (int)>",
105− "limit":"<max-request-number-in-interval (int)>",
106− "aggregations":[
107− "Header:User-Agent"
108 ],
66 ],
67+ "interval": "<time-interval-in-sec (int)>"
68+ "limit": "<max-request-number-in-interval (int)>",
69+ "subrules": [
70+ {
71+ "name": "missing Accept-Language",
72+ "filters": ["!Header:Accept-Language"],
73+ "limit": "<max-request-number-in-interval (int)>",
74+ "stop": true,
109− "actions":[
75+ "actions": [
110− {
76+ {"name":"log"},
111− "name":"block",
77+ {"name": "block",
112− "params":{
78+ "params": {"message": "Rate limit exceeded"}}
79+ ]
80+ },
81+ {
82+ "name": "suspiciously Connection=close header",
83+ "filters": ["Header:Connection=close"],
84+ "limit": "<max-request-number-in-interval (int)>",
85+ "stop": true,
86+ "actions": [
87+ {"name":"log"},
88+ {"name": "block",
113− "message":"Rate limit exceeded"
89+ "params": {"message": "Rate limit exceeded"}}
90+ ]
114− }
91+ },
92+ {
93+ "name": "IP limit",
94+ "interval": "<time-interval-in-sec (int)>"
95+ "limit": "<max-request-number-in-interval (int)>",
96+ "stop": true,
97+ "aggregations": [
98+ "Header:X-Forwarded-For"
99+ ],
100+ "actions": [
101+ { "name": "log"},
102+ { "name": "block",
103+ "params": {
104+ "message": "Rate limit exceeded"
105+ }
106+ }
107+ ]
108+ },
109+ {
110+ "name": "rss/json limit",
111+ "filters": [
112+ "Param:format=(csv|json|rss)"
113+ ],
114+ "interval": "<time-interval-in-sec (int)>"
115+ "limit": "<max-request-number-in-interval (int)>",
116+ "stop": true,
117+ "actions": [
118+ { "name": "log"},
119+ { "name": "block",
120+ "params": {
121+ "message": "Rate limit exceeded"
122+ }
123+ }
124+ ]
125+ },
126+ {
127+ "name": "useragent limit",
128+ "interval": "<time-interval-in-sec (int)>"
129+ "limit": "<max-request-number-in-interval (int)>",
130+ "aggregations": [
131+ "Header:User-Agent"
132+ ],
133+ "actions": [
134+ { "name": "log"},
135+ { "name": "block",
136+ "params": {
137+ "message": "Rate limit exceeded"
138+ }
139+ }
140+ ]
115− }
141+ }
116 ]
142 ]
117− }
143+ }
118− ]
119− }]
144+ ]
120
145
121
146
147+.. _filtron route request:
122
148
123Route request through filtron
149Route request through filtron
124=============================
150=============================
125
151
152+.. sidebar:: further reading
153+
154+ - :ref:`filtron.sh overview`
155+ - :ref:`installation nginx`
156+ - :ref:`installation apache`
157+
126Filtron can be started using the following command:
158Filtron can be started using the following command:
127
159
128.. code:: sh
160.. code:: sh
@@ -136,13 +168,24 @@ Use it along with ``nginx`` with the following example configuration.
136
168
137.. code:: nginx
169.. code:: nginx
138
170
171+ # https://example.org/searx
172+
139− location / {
173+ location /searx {
174+ proxy_pass http://127.0.0.1:4004/;
175+
140− proxy_set_header Host $http_host;
176+ proxy_set_header Host $http_host;
177+ proxy_set_header Connection $http_connection;
141− proxy_set_header X-Real-IP $remote_addr;
178+ proxy_set_header X-Real-IP $remote_addr;
142− proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
179+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
143− proxy_set_header X-Scheme $scheme;
180+ proxy_set_header X-Scheme $scheme;
144− proxy_pass http://127.0.0.1:4004/;
181+ proxy_set_header X-Script-Name /searx;
145 }
182 }
146
183
184+ location /searx/static {
185+ /usr/local/searx/searx-src/searx/static;
186+ }
187+
188+
147Requests are coming from port 4004 going through filtron and then forwarded to
189Requests are coming from port 4004 going through filtron and then forwarded to
148−port 8888 where a searx is being run.
190+port 8888 where a searx is being run. For a complete setup see: :ref:`nginx
191+searx site`.
docs/admin/index.rst+8−1
@@ -3,9 +3,16 @@ Administrator documentation
3===========================
3===========================
4
4
5.. toctree::
5.. toctree::
6− :maxdepth: 1
6+ :maxdepth: 2
7+ :caption: Contents
7
8
8 installation
9 installation
10+ installation-searx
11+ installation-uwsgi
12+ installation-nginx
13+ installation-apache
14+ installation-docker
15+ update-searx
9 settings
16 settings
10 api
17 api
11 architecture
18 architecture
docs/admin/installation-apache.rstnew file+514
@@ -0,0 +1,514 @@
1+.. _installation apache:
2+
3+===================
4+Install with apache
5+===================
6+
7+.. _Apache: https://httpd.apache.org/
8+.. _Apache Debian:
9+ https://cwiki.apache.org/confluence/display/HTTPD/DistrosDefaultLayout#DistrosDefaultLayout-Debian,Ubuntu(Apachehttpd2.x):
10+.. _README.Debian:
11+ https://salsa.debian.org/apache-team/apache2/raw/master/debian/apache2.README.Debian
12+.. _Apache Arch Linux:
13+ https://wiki.archlinux.org/index.php/Apache_HTTP_Server
14+.. _Apache Fedora:
15+ https://docs.fedoraproject.org/en-US/quick-docs/getting-started-with-apache-http-server/index.html
16+.. _Apache directives:
17+ https://httpd.apache.org/docs/trunk/mod/directives.html
18+.. _Getting Started:
19+ https://httpd.apache.org/docs/current/en/getting-started.html
20+.. _Terms Used to Describe Directives:
21+ https://httpd.apache.org/docs/current/en/mod/directive-dict.html
22+.. _Configuration Files:
23+ https://httpd.apache.org/docs/current/en/configuring.html
24+.. _ProxyPreserveHost: https://httpd.apache.org/docs/trunk/mod/mod_proxy.html#proxypreservehost
25+.. _LoadModule:
26+ https://httpd.apache.org/docs/2.4/mod/mod_so.html#loadmodule
27+.. _DocumentRoot:
28+ https://httpd.apache.org/docs/trunk/mod/core.html#documentroot
29+.. _Location:
30+ https://httpd.apache.org/docs/trunk/mod/core.html#location
31+.. _uWSGI Apache support:
32+ https://uwsgi-docs.readthedocs.io/en/latest/Apache.html
33+.. _mod_proxy_uwsgi:
34+ https://uwsgi-docs.readthedocs.io/en/latest/Apache.html#mod-proxy-uwsgi
35+
36+.. sidebar:: further read
37+
38+ - `Apache Arch Linux`_
39+ - `Apache Debian`_ and `README.Debian`_
40+ - `Apache Fedora`_
41+ - `Apache directives`_
42+
43+.. contents:: Contents
44+ :depth: 2
45+ :local:
46+ :backlinks: entry
47+
48+----
49+
50+**Install** :ref:`apache searx site` using :ref:`filtron.sh <filtron.sh overview>`
51+
52+.. code:: bash
53+
54+ $ sudo -H ./utils/filtron.sh apache install
55+
56+**Install** :ref:`apache searx site` using :ref:`morty.sh <morty.sh overview>`
57+
58+.. code:: bash
59+
60+ $ sudo -H ./utils/morty.sh apache install
61+
62+----
63+
64+The apache HTTP server
65+======================
66+
67+If Apache_ is not installed, install it now. If apache_ is new to you, the
68+`Getting Started`_, `Configuration Files`_ and `Terms Used to Describe
69+Directives`_ documentation gives first orientation. There is also a list of
70+`Apache directives`_ *to keep in the pocket*.
71+
72+.. tabs::
73+
74+ .. group-tab:: Ubuntu / debian
75+
76+ .. code:: sh
77+
78+ sudo -H apt-get install apache2
79+
80+ .. group-tab:: Arch Linux
81+
82+ .. code:: sh
83+
84+ sudo -H pacman -S apache
85+ sudo -H systemctl enable httpd
86+ sudo -H systemctl start http
87+
88+ .. group-tab:: Fedora / RHEL
89+
90+ .. code:: sh
91+
92+ sudo -H dnf install httpd
93+ sudo -H systemctl enable httpd
94+ sudo -H systemctl start httpd
95+
96+Now at http://localhost you should see any kind of *Welcome* or *Test* page.
97+How this default intro site is configured, depends on the linux distribution
98+(compare `Apache directives`_).
99+
100+.. tabs::
101+
102+ .. group-tab:: Ubuntu / debian
103+
104+ .. code:: sh
105+
106+ less /etc/apache2/sites-enabled/000-default.conf
107+
108+ In this file, there is a line setting the `DocumentRoot`_ directive:
109+
110+ .. code:: apache
111+
112+ DocumentRoot /var/www/html
113+
114+ And the *welcome* page is the HTML file at ``/var/www/html/index.html``.
115+
116+ .. group-tab:: Arch Linux
117+
118+ .. code:: sh
119+
120+ less /etc/httpd/conf/httpd.conf
121+
122+ In this file, there is a line setting the `DocumentRoot`_ directive:
123+
124+ .. code:: apache
125+
126+ DocumentRoot "/srv/http"
127+ <Directory "/srv/http">
128+ Options Indexes FollowSymLinks
129+ AllowOverride None
130+ Require all granted
131+ </Directory>
132+
133+ The *welcome* page of Arch Linux is a page showing directory located at
134+ ``DocumentRoot``. This is *directory* page is generated by the Module
135+ `mod_autoindex <https://httpd.apache.org/docs/2.4/mod/mod_autoindex.html>`_:
136+
137+ .. code:: apache
138+
139+ LoadModule autoindex_module modules/mod_autoindex.so
140+ ...
141+ Include conf/extra/httpd-autoindex.conf
142+
143+ .. group-tab:: Fedora / RHEL
144+
145+ .. code:: sh
146+
147+ less /etc/httpd/conf/httpd.conf
148+
149+ In this file, there is a line setting the ``DocumentRoot`` directive:
150+
151+ .. code:: apache
152+
153+ DocumentRoot "/var/www/html"
154+ ...
155+ <Directory "/var/www">
156+ AllowOverride None
157+ # Allow open access:
158+ Require all granted
159+ </Directory>
160+
161+ On fresh installations, the ``/var/www`` is empty and the *default
162+ welcome page* is shown, the configuration is located at::
163+
164+ less /etc/httpd/conf.d/welcome.conf
165+
166+.. _apache searx site:
167+
168+Apache Reverse Proxy
169+====================
170+
171+.. sidebar:: public to the internet?
172+
173+ If your searx instance is public, stop here and first install :ref:`filtron
174+ reverse proxy <filtron.sh>` and :ref:`result proxy morty <morty.sh>`, see
175+ :ref:`installation scripts`. If already done, follow setup: *searx via
176+ filtron plus morty*.
177+
178+To setup a Apache revers proxy you have to enable the *headers* and *proxy*
179+modules and create a `Location`_ configuration for the searx site. In most
180+distributions you have to un-comment the lines in the main configuration file,
181+except in :ref:`The Debian Layout`.
182+
183+To pass the HTTP HOST header
184+With ProxyPreserveHost_ the incoming Host HTTP request header is passed to the
185+proxied host.
186+
187+.. tabs::
188+
189+ .. group-tab:: Ubuntu / debian
190+
191+ In the Apache setup, enable headers and proxy modules:
192+
193+ .. code:: sh
194+
195+ sudo -H a2enmod headers
196+ sudo -H a2enmod proxy
197+ sudo -H a2enmod proxy_http
198+
199+ In :ref:`The Debian Layout` you create a ``searx.conf`` with the
200+ ``<Location /searx >`` directive and save this file in the *sites
201+ available* folder at ``/etc/apache2/sites-available``. To enable the
202+ ``searx.conf`` use :man:`a2ensite`:
203+
204+ .. code:: sh
205+
206+ sudo -H a2ensite searx.conf
207+
208+ .. group-tab:: Arch Linux
209+
210+ In the ``/etc/httpd/conf/httpd.conf`` file, activate headers and proxy
211+ modules (LoadModule_):
212+
213+ .. code:: apache
214+
215+ FIXME needs test
216+
217+ LoadModule headers_module modules/mod_headers.so
218+ LoadModule proxy_module modules/mod_proxy.so
219+ LoadModule proxy_http_module modules/mod_proxy_http.so
220+
221+ .. group-tab:: Fedora / RHEL
222+
223+ In the ``/etc/httpd/conf/httpd.conf`` file, activate headers and proxy
224+ modules (LoadModule_):
225+
226+ .. code:: apache
227+
228+ FIXME needs test
229+
230+ LoadModule headers_module modules/mod_headers.so
231+ LoadModule proxy_module modules/mod_proxy.so
232+ LoadModule proxy_http_module modules/mod_proxy_http.so
233+
234+.. tabs::
235+
236+ .. group-tab:: searx via filtron plus morty
237+
238+ Use this setup, if your instance is public to the internet, compare
239+ figure: :ref:`architecture <arch public>` and :ref:`installation scripts`.
240+
241+ 1. Configure a reverse proxy for :ref:`filtron <filtron.sh>`, listening on
242+ *localhost 4004* (:ref:`filtron route request`):
243+
244+ .. code:: apache
245+
246+ <Location /searx >
247+
248+ # SetEnvIf Request_URI "/searx" dontlog
249+ # CustomLog /dev/null combined env=dontlog
250+
251+ Require all granted
252+
253+ Order deny,allow
254+ Deny from all
255+ #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
256+ Allow from all
257+
258+ ProxyPreserveHost On
259+ ProxyPass http://127.0.0.1:4004
260+ RequestHeader set X-Script-Name /searx
261+
262+ </Location>
263+
264+ 2. Configure reverse proxy for :ref:`morty <searx morty>`, listening on
265+ *localhost 3000*
266+
267+ .. code:: apache
268+
269+ ProxyPreserveHost On
270+
271+ <Location /morty >
272+
273+ # SetEnvIf Request_URI "/morty" dontlog
274+ # CustomLog /dev/null combined env=dontlog
275+
276+ Require all granted
277+
278+ Order deny,allow
279+ Deny from all
280+ #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
281+ Allow from all
282+
283+ ProxyPass http://127.0.0.1:3000
284+ RequestHeader set X-Script-Name /morty
285+
286+ </Location>
287+
288+ Note that reverse proxy advised to be used in case of single-user or
289+ low-traffic instances. For a fully result proxification add :ref:`morty's
290+ <searx morty>` **public URL** to your :origin:`searx/settings.yml`:
291+
292+ .. code:: yaml
293+
294+ result_proxy:
295+ # replace example.org with your server's public name
296+ url : https://example.org/morty
297+
298+ server:
299+ image_proxy : True
300+
301+uWSGI support
302+=============
303+
304+Be warned, with this setup, your instance isn't :ref:`protected <searx
305+filtron>`, nevertheless it is good enough for intranet usage. In modern Linux
306+distributions, the `mod_proxy_uwsgi`_ is compiled into the *normal* apache
307+package and you need to install only the :ref:`uWSGI <searx uwsgi>` package:
308+
309+.. tabs::
310+
311+ .. group-tab:: Ubuntu / debian
312+
313+ .. code:: sh
314+
315+ sudo -H apt-get install uwsgi
316+
317+ # Ubuntu =< 18.04
318+ sudo -H apt-get install libapache2-mod-proxy-uwsgi
319+
320+ .. group-tab:: Arch Linux
321+
322+ .. code:: sh
323+
324+ sudo -H pacman -S uwsgi
325+
326+ .. group-tab:: Fedora / RHEL
327+
328+ .. code:: sh
329+
330+ sudo -H dnf install uwsgi
331+
332+The next example shows a configuration using the `uWSGI Apache support`_ via
333+unix sockets and `mod_proxy_uwsgi`_.
334+
335+For socket communication, you have to activate ``socket =
336+/run/uwsgi/app/searx/socket`` and comment out the ``http = 127.0.0.1:8888``
337+configuration in your :ref:`uwsgi ini file <uwsgi configuration>`. If not
338+already exists, create a folder for the unix sockets, which can be used by the
339+searx account (see :ref:`create searx user`):
340+
341+.. code:: bash
342+
343+ sudo -H mkdir -p /run/uwsgi/app/searx/
344+ sudo -H chown -R searx:searx /run/uwsgi/app/searx/
345+
346+If the server is public; to limit access to your intranet replace ``Allow from
347+all`` directive and replace ``192.168.0.0/16`` with your subnet IP/class.
348+
349+.. tabs::
350+
351+ .. group-tab:: Ubuntu / debian
352+
353+ .. code:: apache
354+
355+ LoadModule headers_module /usr/lib/apache2/mod_headers.so
356+ LoadModule proxy_module /usr/lib/apache2/modules/mod_proxy.so
357+ LoadModule proxy_uwsgi_module /usr/lib/apache2/modules/mod_proxy_uwsgi.so
358+
359+ # SetEnvIf Request_URI /searx dontlog
360+ # CustomLog /dev/null combined env=dontlog
361+
362+ <Location /searx>
363+
364+ Require all granted
365+ Order deny,allow
366+ Deny from all
367+ # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
368+ Allow from all
369+
370+ ProxyPreserveHost On
371+ ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/
372+
373+ </Location>
374+
375+ .. group-tab:: Arch Linux
376+
377+ .. code:: apache
378+
379+ FIXME needs test
380+
381+ LoadModule proxy_module modules/mod_proxy.so
382+ LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so
383+
384+ # SetEnvIf Request_URI /searx dontlog
385+ # CustomLog /dev/null combined env=dontlog
386+
387+ <Location /searx>
388+
389+ Require all granted
390+ Order deny,allow
391+ Deny from all
392+ # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
393+ Allow from all
394+
395+ ProxyPreserveHost On
396+ ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/
397+
398+ </Location>
399+
400+ .. group-tab:: Fedora / RHEL
401+
402+ .. code:: apache
403+
404+ FIXME needs test
405+
406+ LoadModule proxy_module modules/mod_proxy.so
407+ LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so
408+ <IfModule proxy_uwsgi_module>
409+
410+ # SetEnvIf Request_URI /searx dontlog
411+ # CustomLog /dev/null combined env=dontlog
412+
413+ <Location /searx>
414+
415+ Require all granted
416+ Order deny,allow
417+ Deny from all
418+ # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
419+ Allow from all
420+
421+ ProxyPreserveHost On
422+ ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/
423+
424+ </Location>
425+
426+ </IfModule>
427+
428+ .. group-tab:: old mod_wsgi
429+
430+ We show this only for historical reasons, DON'T USE `mod_uwsgi
431+ <https://uwsgi-docs.readthedocs.io/en/latest/Apache.html#mod-uwsgi>`_.
432+ ANYMORE!
433+
434+ .. code:: apache
435+
436+ <IfModule mod_uwsgi.c>
437+
438+ # SetEnvIf Request_URI "/searx" dontlog
439+ # CustomLog /dev/null combined env=dontlog
440+
441+ <Location /searx >
442+
443+ Require all granted
444+
445+ Options FollowSymLinks Indexes
446+ SetHandler uwsgi-handler
447+ uWSGISocket /run/uwsgi/app/searx/socket
448+
449+ Order deny,allow
450+ Deny from all
451+ # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
452+ Allow from all
453+
454+ </Location>
455+
456+ </IfModule>
457+
458+.. _restart apache:
459+
460+Restart service
461+===============
462+
463+.. tabs::
464+
465+ .. group-tab:: Ubuntu / debian
466+
467+ .. code:: sh
468+
469+ sudo -H systemctl restart apache2
470+ sudo -H service uwsgi restart searx
471+
472+ .. group-tab:: Arch Linux
473+
474+ .. code:: sh
475+
476+ sudo -H systemctl restart httpd
477+ sudo -H systemctl restart uwsgi@searx
478+
479+ .. group-tab:: Fedora / RHEL
480+
481+ .. code:: sh
482+
483+ sudo -H systemctl restart httpd
484+ sudo -H touch /etc/uwsgi.d/searx.ini
485+
486+
487+disable logs
488+============
489+
490+For better privacy you can disable Apache logs. In the examples above activate
491+one of the lines and `restart apache`_::
492+
493+
494+ # SetEnvIf Request_URI "/searx" dontlog
495+ # CustomLog /dev/null combined env=dontlog
496+
497+The ``CustomLog`` directive disable logs for the whole (virtual) server, use it
498+when the URL of the service does not have a path component (``/searx``) / is
499+located at root (``/``).
500+
501+.. _The Debian Layout:
502+
503+The Debian Layout
504+=================
505+
506+Be aware that the Debian layout is quite different from the standard Apache
507+configuration. For details look at the README.Debian_
508+(``/usr/share/doc/apache2/README.Debian.gz``). Some commands you should know on
509+Debian:
510+
511+* :man:`apache2ctl`: Apache HTTP server control interface
512+* :man:`a2enmod`, :man:`a2dismod`: switch on/off modules
513+* :man:`a2enconf`, :man:`a2disconf`: switch on/off configurations
514+* :man:`a2ensite`, :man:`a2dissite`: switch on/off sites
docs/admin/installation-docker.rstnew file+28
@@ -0,0 +1,28 @@
1+.. _installation docker:
2+
3+===================
4+Docker installation
5+===================
6+
7+.. contents:: Contents
8+ :depth: 2
9+ :local:
10+ :backlinks: entry
11+
12+Make sure you have installed Docker. For instance, you can deploy searx like this:
13+
14+.. code:: sh
15+
16+ docker pull wonderfall/searx
17+ docker run -d --name searx -p $PORT:8888 wonderfall/searx
18+
19+Go to ``http://localhost:$PORT``.
20+
21+See https://hub.docker.com/r/wonderfall/searx/ for more informations. It's also
22+possible to build searx from the embedded Dockerfile.
23+
24+.. code:: sh
25+
26+ git clone https://github.com/asciimoo/searx.git
27+ cd searx
28+ docker build -t whatever/searx .
docs/admin/installation-nginx.rstnew file+381
@@ -0,0 +1,381 @@
1+.. _installation nginx:
2+
3+==================
4+Install with nginx
5+==================
6+
7+.. _nginx:
8+ https://docs.nginx.com/nginx/admin-guide/
9+.. _nginx server configuration:
10+ https://docs.nginx.com/nginx/admin-guide/web-server/web-server/#setting-up-virtual-servers
11+.. _nginx beginners guide:
12+ http://nginx.org/en/docs/beginners_guide.html
13+.. _Getting Started wiki:
14+ https://www.nginx.com/resources/wiki/start/
15+.. _uWSGI support from nginx:
16+ https://uwsgi-docs.readthedocs.io/en/latest/Nginx.html
17+.. _uwsgi_params:
18+ https://uwsgi-docs.readthedocs.io/en/latest/Nginx.html#configuring-nginx
19+.. _SCRIPT_NAME:
20+ https://werkzeug.palletsprojects.com/en/1.0.x/wsgi/#werkzeug.wsgi.get_script_name
21+
22+.. sidebar:: further reading
23+
24+ - nginx_
25+ - `nginx beginners guide`_
26+ - `nginx server configuration`_
27+ - `Getting Started wiki`_
28+ - `uWSGI support from nginx`_
29+
30+.. contents:: Contents
31+ :depth: 2
32+ :local:
33+ :backlinks: entry
34+
35+----
36+
37+**Install** :ref:`nginx searx site` using :ref:`filtron.sh <filtron.sh overview>`
38+
39+.. code:: bash
40+
41+ $ sudo -H ./utils/filtron.sh nginx install
42+
43+**Install** :ref:`nginx searx site` using :ref:`morty.sh <morty.sh overview>`
44+
45+.. code:: bash
46+
47+ $ sudo -H ./utils/morty.sh nginx install
48+
49+----
50+
51+
52+The nginx HTTP server
53+=====================
54+
55+If nginx_ is not installed (uwsgi will not work with the package nginx-light),
56+install it now.
57+
58+.. tabs::
59+
60+ .. group-tab:: Ubuntu / debian
61+
62+ .. code:: sh
63+
64+ sudo -H apt-get install nginx
65+
66+ .. group-tab:: Arch Linux
67+
68+ .. code-block:: sh
69+
70+ sudo -H pacman -S nginx-mainline
71+ sudo -H systemctl enable nginx
72+ sudo -H systemctl start nginx
73+
74+ .. group-tab:: Fedora / RHEL
75+
76+ .. code-block:: sh
77+
78+ sudo -H dnf install nginx
79+ sudo -H systemctl enable nginx
80+ sudo -H systemctl start nginx
81+
82+Now at http://localhost you should see a *Welcome to nginx!* page, on Fedora you
83+see a *Fedora Webserver - Test Page*. The test page comes from the default
84+`nginx server configuration`_. How this default intro site is configured,
85+depends on the linux distribution:
86+
87+.. tabs::
88+
89+ .. group-tab:: Ubuntu / debian
90+
91+ .. code:: sh
92+
93+ less /etc/nginx/nginx.conf
94+
95+ there is a line including site configurations from:
96+
97+ .. code:: nginx
98+
99+ include /etc/nginx/sites-enabled/*;
100+
101+ .. group-tab:: Arch Linux
102+
103+ .. code-block:: sh
104+
105+ less /etc/nginx/nginx.conf
106+
107+ in there is a configuration section named ``server``:
108+
109+ .. code-block:: nginx
110+
111+ server {
112+ listen 80;
113+ server_name localhost;
114+ # ...
115+ }
116+
117+ .. group-tab:: Fedora / RHEL
118+
119+ .. code-block:: sh
120+
121+ less /etc/nginx/nginx.conf
122+
123+ there is a line including site configurations from:
124+
125+ .. code:: nginx
126+
127+ include /etc/nginx/conf.d/*.conf;
128+
129+.. _nginx searx site:
130+
131+A nginx searx site
132+==================
133+
134+.. sidebar:: public to the internet?
135+
136+ If your searx instance is public, stop here and first install :ref:`filtron
137+ reverse proxy <filtron.sh>` and :ref:`result proxy morty <morty.sh>`, see
138+ :ref:`installation scripts`. If already done, follow setup: *searx via
139+ filtron plus morty*.
140+
141+Now you have to create a configuration for the searx site. If nginx_ is new to
142+you, the `nginx beginners guide`_ is a good starting point and the `Getting
143+Started wiki`_ is always a good resource *to keep in the pocket*.
144+
145+.. tabs::
146+
147+ .. group-tab:: Ubuntu / debian
148+
149+ Create configuration at ``/etc/nginx/sites-available/searx`` and place a
150+ symlink to sites-enabled:
151+
152+ .. code:: sh
153+
154+ sudo -H ln -s /etc/nginx/sites-available/searx /etc/nginx/sites-enabled/searx
155+
156+ .. group-tab:: Arch Linux
157+
158+ In the ``/etc/nginx/nginx.conf`` file, replace the configuration section
159+ named ``server``.
160+
161+ .. group-tab:: Fedora / RHEL
162+
163+ Create configuration at ``/etc/nginx/conf.d/searx`` and place a
164+ symlink to sites-enabled:
165+
166+.. tabs::
167+
168+ .. group-tab:: searx via filtron plus morty
169+
170+ Use this setup, if your instance is public to the internet, compare
171+ figure: :ref:`architecture <arch public>` and :ref:`installation scripts`.
172+
173+ 1. Configure a reverse proxy for :ref:`filtron <filtron.sh>`, listening on
174+ *localhost 4004* (:ref:`filtron route request`):
175+
176+ .. code:: nginx
177+
178+ # https://example.org/searx
179+
180+ location /searx {
181+ proxy_pass http://127.0.0.1:4004/;
182+
183+ proxy_set_header Host $http_host;
184+ proxy_set_header Connection $http_connection;
185+ proxy_set_header X-Real-IP $remote_addr;
186+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
187+ proxy_set_header X-Scheme $scheme;
188+ proxy_set_header X-Script-Name /searx;
189+ }
190+
191+ location /searx/static {
192+ /usr/local/searx/searx-src/searx/static;
193+ }
194+
195+
196+ 2. Configure reverse proxy for :ref:`morty <searx morty>`, listening on
197+ *localhost 3000*:
198+
199+ .. code:: nginx
200+
201+ # https://example.org/morty
202+
203+ location /morty {
204+ proxy_pass http://127.0.0.1:3000/;
205+
206+ proxy_set_header Host $http_host;
207+ proxy_set_header Connection $http_connection;
208+ proxy_set_header X-Real-IP $remote_addr;
209+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
210+ proxy_set_header X-Scheme $scheme;
211+ }
212+
213+ Note that reverse proxy advised to be used in case of single-user or
214+ low-traffic instances. For a fully result proxification add :ref:`morty's
215+ <searx morty>` **public URL** to your :origin:`searx/settings.yml`:
216+
217+ .. code:: yaml
218+
219+ result_proxy:
220+ # replace example.org with your server's public name
221+ url : https://example.org/morty
222+
223+ server:
224+ image_proxy : True
225+
226+
227+ .. group-tab:: proxy or uWSGI
228+
229+ Be warned, with this setup, your instance isn't :ref:`protected <searx
230+ filtron>`. Nevertheless it is good enough for intranet usage and it is a
231+ excellent example of; *how different services can be set up*. The next
232+ example shows a reverse proxy configuration wrapping the :ref:`searx-uWSGI
233+ application <uwsgi configuration>`, listening on ``http =
234+ 127.0.0.1:8888``.
235+
236+ .. code:: nginx
237+
238+ # https://hostname.local/
239+
240+ location / {
241+ proxy_pass http://127.0.0.1:8888;
242+
243+ proxy_set_header Host $host;
244+ proxy_set_header Connection $http_connection;
245+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
246+ proxy_set_header X-Scheme $scheme;
247+ proxy_buffering off;
248+ }
249+
250+ Alternatively you can use the `uWSGI support from nginx`_ via unix
251+ sockets. For socket communication, you have to activate ``socket =
252+ /run/uwsgi/app/searx/socket`` and comment out the ``http =
253+ 127.0.0.1:8888`` configuration in your :ref:`uwsgi ini file <uwsgi
254+ configuration>`.
255+
256+ The example shows a nginx virtual ``server`` configuration, listening on
257+ port 80 (IPv4 and IPv6 http://[::]:80). The uWSGI app is configured at
258+ location ``/`` by importing the `uwsgi_params`_ and passing requests to
259+ the uWSGI socket (``uwsgi_pass``). The ``server``\'s root points to the
260+ :ref:`searx-src clone <searx-src>` and wraps directly the
261+ :origin:`searx/static/` content at ``location /static``.
262+
263+ .. code:: nginx
264+
265+ server {
266+ # replace hostname.local with your server's name
267+ server_name hostname.local;
268+
269+ listen 80;
270+ listen [::]:80;
271+
272+ location / {
273+ include uwsgi_params;
274+ uwsgi_pass unix:/run/uwsgi/app/searx/socket;
275+ }
276+
277+ root /usr/local/searx/searx-src/searx;
278+ location /static { }
279+ }
280+
281+ If not already exists, create a folder for the unix sockets, which can be
282+ used by the searx account:
283+
284+ .. code:: bash
285+
286+ mkdir -p /run/uwsgi/app/searx/
287+ sudo -H chown -R searx:searx /run/uwsgi/app/searx/
288+
289+ .. group-tab:: \.\. at subdir URL
290+
291+ Be warned, with these setups, your instance isn't :ref:`protected <searx
292+ filtron>`. The examples are just here to demonstrate how to export the
293+ searx application from a subdirectory URL ``https://example.org/searx/``.
294+
295+ .. code:: nginx
296+
297+ # https://hostname.local/searx
298+
299+ location /searx {
300+ proxy_pass http://127.0.0.1:8888;
301+
302+ proxy_set_header Host $host;
303+ proxy_set_header Connection $http_connection;
304+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
305+ proxy_set_header X-Scheme $scheme;
306+ proxy_set_header X-Script-Name /searx;
307+ proxy_buffering off;
308+ }
309+
310+ location /searx/static {
311+ alias /usr/local/searx/searx-src/searx/static;
312+ }
313+
314+ The ``X-Script-Name /searx`` is needed by the searx implementation to
315+ calculate relative URLs correct. The next example shows a uWSGI
316+ configuration. Since there are no HTTP headers in a (u)WSGI protocol, the
317+ value is shipped via the SCRIPT_NAME_ in the WSGI environment.
318+
319+ .. code:: nginx
320+
321+ # https://hostname.local/searx
322+
323+ location /searx {
324+ uwsgi_param SCRIPT_NAME /searx;
325+ include uwsgi_params;
326+ uwsgi_pass unix:/run/uwsgi/app/searx/socket;
327+ }
328+
329+ location /searx/static {
330+ alias /usr/local/searx/searx-src/searx;
331+ }
332+
333+ For searx to work correctly the ``base_url`` must be set in the
334+ :origin:`searx/settings.yml`.
335+
336+ .. code:: yaml
337+
338+ server:
339+ # replace example.org with your server's public name
340+ base_url : https://example.org/searx/
341+
342+
343+Restart service:
344+
345+.. tabs::
346+
347+ .. group-tab:: Ubuntu / debian
348+
349+ .. code:: sh
350+
351+ sudo -H systemctl restart nginx
352+ sudo -H service uwsgi restart searx
353+
354+ .. group-tab:: Arch Linux
355+
356+ .. code:: sh
357+
358+ sudo -H systemctl restart nginx
359+ sudo -H systemctl restart uwsgi@searx
360+
361+ .. group-tab:: Fedora
362+
363+ .. code:: sh
364+
365+ sudo -H systemctl restart nginx
366+ sudo -H touch /etc/uwsgi.d/searx.ini
367+
368+
369+Disable logs
370+============
371+
372+For better privacy you can disable nginx logs in ``/etc/nginx/nginx.conf``.
373+
374+.. code:: nginx
375+
376+ http {
377+ # ...
378+ access_log /dev/null;
379+ error_log /dev/null;
380+ # ...
381+ }
docs/admin/installation-searx.rstnew file+92
@@ -0,0 +1,92 @@
1+.. _installation basic:
2+
3+=========================
4+Step by step installation
5+=========================
6+
7+.. contents:: Contents
8+ :depth: 2
9+ :local:
10+ :backlinks: entry
11+
12+Step by step installation with virtualenv. For Ubuntu, be sure to have enable
13+universe repository.
14+
15+.. _install packages:
16+
17+Install packages
18+================
19+
20+.. kernel-include:: $DOCS_BUILD/includes/searx.rst
21+ :start-after: START distro-packages
22+ :end-before: END distro-packages
23+
24+.. hint::
25+
26+ This installs also the packages needed by :ref:`searx uwsgi`
27+
28+.. _create searx user:
29+
30+Create user
31+===========
32+
33+.. kernel-include:: $DOCS_BUILD/includes/searx.rst
34+ :start-after: START create user
35+ :end-before: END create user
36+
37+.. _searx-src:
38+
39+install searx & dependencies
40+============================
41+
42+Start a interactive shell from new created user and clone searx:
43+
44+.. kernel-include:: $DOCS_BUILD/includes/searx.rst
45+ :start-after: START clone searx
46+ :end-before: END clone searx
47+
48+In the same shell create *virtualenv*:
49+
50+.. kernel-include:: $DOCS_BUILD/includes/searx.rst
51+ :start-after: START create virtualenv
52+ :end-before: END create virtualenv
53+
54+To install searx's dependencies, exit the searx *bash* session you opened above
55+and restart a new. Before install, first check if your *virualenv* was sourced
56+from the login (*~/.profile*):
57+
58+.. kernel-include:: $DOCS_BUILD/includes/searx.rst
59+ :start-after: START manage.sh update_packages
60+ :end-before: END manage.sh update_packages
61+
62+.. tip::
63+
64+ Open a second terminal for the configuration tasks and left the ``(searx)$``
65+ terminal open for the tasks below.
66+
67+Configuration
68+==============
69+
70+Create a copy of the :origin:`searx/settings.yml` configuration file in system's
71+*/etc* folder. Configure like shown below -- replace ``searx@\$(uname -n)`` with
72+a name of your choice -- *and/or* edit ``/etc/searx/settings.yml`` if necessary.
73+
74+.. kernel-include:: $DOCS_BUILD/includes/searx.rst
75+ :start-after: START searx config
76+ :end-before: END searx config
77+
78+Check
79+=====
80+
81+To check your searx setup, optional enable debugging and start the *webapp*.
82+Searx looks at the exported environment ``$SEARX_SETTINGS_PATH`` for a
83+configuration file.
84+
85+.. kernel-include:: $DOCS_BUILD/includes/searx.rst
86+ :start-after: START check searx installation
87+ :end-before: END check searx installation
88+
89+If everything works fine, hit ``[CTRL-C]`` to stop the *webapp* and disable the
90+debug option in ``settings.yml``. You can now exit searx user bash (enter exit
91+command twice). At this point searx is not demonized; uwsgi allows this.
92+
docs/admin/installation-uwsgi.rstnew file+149
@@ -0,0 +1,149 @@
1+.. _searx uwsgi:
2+
3+=====
4+uwsgi
5+=====
6+
7+.. sidebar:: further reading
8+
9+ - `systemd.unit`_
10+ - `uWSGI Emperor`_
11+
12+.. contents:: Contents
13+ :depth: 2
14+ :local:
15+ :backlinks: entry
16+
17+
18+.. _systemd.unit: https://www.freedesktop.org/software/systemd/man/systemd.unit.html
19+.. _One service per app in systemd:
20+ https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd
21+.. _uWSGI Emperor:
22+ https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html
23+.. _uwsgi ini file:
24+ https://uwsgi-docs.readthedocs.io/en/latest/Configuration.html#ini-files
25+.. _systemd unit template:
26+ http://0pointer.de/blog/projects/instances.html
27+
28+
29+Origin uWSGI
30+============
31+
32+How uWSGI is implemented by distributors is different. uWSGI itself
33+recommend two methods
34+
35+`systemd.unit`_ template files as described here `One service per app in systemd`_.
36+
37+ There is one `systemd unit template`_ and one `uwsgi ini file`_ per uWSGI-app
38+ placed at dedicated locations. Take archlinux and a searx.ini as example::
39+
40+ unit template --> /usr/lib/systemd/system/uwsgi@.service
41+ uwsgi ini files --> /etc/uwsgi/searx.ini
42+
43+ The searx app can be maintained as know from common systemd units::
44+
45+ systemctl enable uwsgi@searx
46+ systemctl start uwsgi@searx
47+ systemctl restart uwsgi@searx
48+ systemctl stop uwsgi@searx
49+
50+The `uWSGI Emperor`_ mode which fits for maintaining a large range of uwsgi apps.
51+
52+ The Emperor mode is a special uWSGI instance that will monitor specific
53+ events. The Emperor mode (service) is started by a (common, not template)
54+ systemd unit. The Emperor service will scan specific directories for `uwsgi
55+ ini file`_\s (also know as *vassals*). If a *vassal* is added, removed or the
56+ timestamp is modified, a corresponding action takes place: a new uWSGI
57+ instance is started, reload or stopped. Take Fedora and a searx.ini as
58+ example::
59+
60+ to start a new searx instance create --> /etc/uwsgi.d/searx.ini
61+ to reload the instance edit timestamp --> touch /etc/uwsgi.d/searx.ini
62+ to stop instance remove ini --> rm /etc/uwsgi.d/searx.ini
63+
64+Distributors
65+============
66+
67+The `uWSGI Emperor`_ mode and `systemd unit template`_ is what the distributors
68+mostly offer their users, even if they differ in the way they implement both
69+modes and their defaults. Another point they might differ is the packaging of
70+plugins (if so, compare :ref:`install packages`) and what the default python
71+interpreter is (python2 vs. python3).
72+
73+Fedora starts a Emperor by default, while archlinux does not start any uwsgi
74+service by default. Worth to know; debian (ubuntu) follow a complete different
75+approach. *debian*: your are familiar with the apache infrastructure? .. they
76+do similar for the uWSGI infrastructure (with less comfort), the folders are::
77+
78+ /etc/uwsgi/apps-available/
79+ /etc/uwsgi/apps-enabled/
80+
81+The `uwsgi ini file`_ is enabled by a symbolic link::
82+
83+ ln -s /etc/uwsgi/apps-available/searx.ini /etc/uwsgi/apps-enabled/
84+
85+From debian's documentation (``/usr/share/doc/uwsgi/README.Debian.gz``): You
86+could control specific instance(s) by issuing::
87+
88+ service uwsgi <command> <confname> <confname> ...
89+
90+ sudo -H service uwsgi start searx
91+ sudo -H service uwsgi stop searx
92+
93+My experience is, that this command is a bit buggy.
94+
95+.. _uwsgi configuration:
96+
97+Alltogether
98+===========
99+
100+Create the configuration ini-file according to your distribution (see below) and
101+restart the uwsgi application.
102+
103+.. tabs::
104+
105+ .. group-tab:: Ubuntu / debian
106+
107+ .. kernel-include:: $DOCS_BUILD/includes/searx.rst
108+ :start-after: START searx uwsgi-description ubuntu-20.04
109+ :end-before: END searx uwsgi-description ubuntu-20.04
110+
111+
112+ .. group-tab:: Arch Linux
113+
114+ .. kernel-include:: $DOCS_BUILD/includes/searx.rst
115+ :start-after: START searx uwsgi-description arch
116+ :end-before: END searx uwsgi-description arch
117+
118+
119+ .. group-tab:: Fedora / RHEL
120+
121+ .. kernel-include:: $DOCS_BUILD/includes/searx.rst
122+ :start-after: START searx uwsgi-description fedora
123+ :end-before: END searx uwsgi-description fedora
124+
125+
126+.. tabs::
127+
128+ .. group-tab:: Ubuntu / debian
129+
130+ .. kernel-include:: $DOCS_BUILD/includes/searx.rst
131+ :code: ini
132+ :start-after: START searx uwsgi-appini ubuntu-20.04
133+ :end-before: END searx uwsgi-appini ubuntu-20.04
134+
135+ .. group-tab:: Arch Linux
136+
137+ .. kernel-include:: $DOCS_BUILD/includes/searx.rst
138+ :code: ini
139+ :start-after: START searx uwsgi-appini arch
140+ :end-before: END searx uwsgi-appini arch
141+
142+ .. group-tab:: Fedora / RHEL
143+
144+ .. kernel-include:: $DOCS_BUILD/includes/searx.rst
145+ :code: ini
146+ :start-after: START searx uwsgi-appini fedora
147+ :end-before: END searx uwsgi-appini fedora
148+
149+
docs/admin/installation.rst+37−320
@@ -4,346 +4,63 @@
4Installation
4Installation
5============
5============
6
6
7−.. contents::
7+*You're spoilt for choice*, choose your preferred method of installation.
8− :depth: 3
9
8
10−Basic installation
9+- :ref:`installation docker`
11−==================
10+- :ref:`installation scripts`
11+- :ref:`installation basic`
12
12
13−Step by step installation for Debian/Ubuntu with virtualenv. For Ubuntu, be sure
13+The :ref:`installation basic` is good enough for intranet usage and it is a
14−to have enable universe repository.
14+excellent illustration of *how a searx instance is build up*. If you place your
15+instance public to the internet you should really consider to install a
16+:ref:`filtron reverse proxy <filtron.sh>` and for privacy a :ref:`result proxy
17+<morty.sh>` is mandatory.
15
18
16−Install packages:
19+Therefore, if you do not have any special preferences, its recommend to use the
20+:ref:`installation docker` or the `Installation scripts`_ from our :ref:`tooling
21+box <toolboxing>` as described below.
17
22
18−.. code:: sh
23+.. _installation scripts:
19
24
20− $ sudo -H apt-get install \
25+Installation scripts
21− git build-essential libxslt-dev \
26+====================
22− python-dev python-virtualenv python-babel \
23− zlib1g-dev libffi-dev libssl-dev
24
27
25−Install searx:
28+.. sidebar:: Update OS first!
26
29
27−.. code:: sh
30+ To avoid unwanted side effects, update your OS before installing searx.
28
31
29− cd /usr/local
32+The following will install a setup as shown in :ref:`architecture`. First you
30− sudo -H git clone https://github.com/asciimoo/searx.git
33+need to get a clone. The clone is only needed for the installation procedure
31− sudo -H useradd searx -d /usr/local/searx
34+and some maintenance tasks (alternatively you can create your own fork).
32− sudo -H chown searx:searx -R /usr/local/searx
33
35
34−Install dependencies in a virtualenv:
36+.. code:: bash
35
37
36−.. code:: sh
38+ $ cd ~/Downloads
39+ $ git clone https://github.com/asciimoo/searx searx
40+ $ cd searx
37
41
38− cd /usr/local/searx
42+**Install** :ref:`searx service <searx.sh>`
39− sudo -H -u searx -i
40
43
41−.. code:: sh
44+This installs searx as described in :ref:`installation basic`.
42
45
43− (searx)$ virtualenv searx-ve
46+.. code:: bash
44− (searx)$ . ./searx-ve/bin/activate
45− (searx)$ ./manage.sh update_packages
46
47
47−Configuration
48+ $ sudo -H ./utils/searx.sh install all
48−==============
49
49
50−.. code:: sh
50+**Install** :ref:`filtron reverse proxy <filtron.sh>`
51
51
52− sed -i -e "s/ultrasecretkey/`openssl rand -hex 16`/g" searx/settings.yml
52+.. code:: bash
53
53
54−Edit searx/settings.yml if necessary.
54+ $ sudo -H ./utils/filtron.sh install all
55
55
56−Check
56+**Install** :ref:`result proxy <morty.sh>`
57−=====
58
57
59−Start searx:
58+.. code:: bash
60
59
61−.. code:: sh
60+ $ sudo -H ./utils/morty.sh install all
62
61
63− python searx/webapp.py
62+If all services are running fine, you can add it to your HTTP server:
64
63
65−Go to http://localhost:8888
64+- :ref:`installation apache`
65+- :ref:`installation nginx`
66
66
67−If everything works fine, disable the debug option in settings.yml:
68−
69−.. code:: sh
70−
71− sed -i -e "s/debug : True/debug : False/g" searx/settings.yml
72−
73−At this point searx is not demonized ; uwsgi allows this.
74−
75−You can exit the virtualenv and the searx user bash (enter exit command
76−twice).
77−
78−uwsgi
79−=====
80−
81−Install packages:
82−
83−.. code:: sh
84−
85− sudo -H apt-get install \
86− uwsgi uwsgi-plugin-python
87−
88−Create the configuration file ``/etc/uwsgi/apps-available/searx.ini`` with this
89−content:
90−
91−.. code:: ini
92−
93− [uwsgi]
94− # Who will run the code
95− uid = searx
96− gid = searx
97−
98− # disable logging for privacy
99− disable-logging = true
100−
101− # Number of workers (usually CPU count)
102− workers = 4
103−
104− # The right granted on the created socket
105− chmod-socket = 666
106−
107− # Plugin to use and interpretor config
108− single-interpreter = true
109− master = true
110− plugin = python
111− lazy-apps = true
112− enable-threads = true
113−
114− # Module to import
115− module = searx.webapp
116−
117− # Support running the module from a webserver subdirectory.
118− route-run = fixpathinfo:
119−
120− # Virtualenv and python path
121− virtualenv = /usr/local/searx/searx-ve/
122− pythonpath = /usr/local/searx/
123− chdir = /usr/local/searx/searx/
124−
125−Activate the uwsgi application and restart:
126−
127−.. code:: sh
128−
129− cd /etc/uwsgi/apps-enabled
130− ln -s ../apps-available/searx.ini
131− /etc/init.d/uwsgi restart
132−
133−Web server
134−==========
135−
136−with nginx
137−----------
138−
139−If nginx is not installed (uwsgi will not work with the package
140−nginx-light):
141−
142−.. code:: sh
143−
144− sudo -H apt-get install nginx
145−
146−Hosted at /
147−~~~~~~~~~~~
148−
149−Create the configuration file ``/etc/nginx/sites-available/searx`` with this
150−content:
151−
152−.. code:: nginx
153−
154− server {
155− listen 80;
156− server_name searx.example.com;
157− root /usr/local/searx/searx;
158−
159− location /static {
160− }
161−
162− location / {
163− include uwsgi_params;
164− uwsgi_pass unix:/run/uwsgi/app/searx/socket;
165− }
166− }
167−
168−Create a symlink to sites-enabled:
169−
170−.. code:: sh
171−
172− sudo -H ln -s /etc/nginx/sites-available/searx /etc/nginx/sites-enabled/searx
173−
174−Restart service:
175−
176−.. code:: sh
177−
178− sudo -H service nginx restart
179− sudo -H service uwsgi restart
180−
181−from subdirectory URL (/searx)
182−~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
183−
184−Add this configuration in the server config file
185−``/etc/nginx/sites-enabled/default``:
186−
187−.. code:: nginx
188−
189− location /searx/static {
190− alias /usr/local/searx/searx/static;
191− }
192−
193− location /searx {
194− uwsgi_param SCRIPT_NAME /searx;
195− include uwsgi_params;
196− uwsgi_pass unix:/run/uwsgi/app/searx/socket;
197− }
198−
199−
200−**OR** using reverse proxy (Please, note that reverse proxy advised to be used
201−in case of single-user or low-traffic instances.)
202−
203−.. code:: nginx
204−
205− location /searx/static {
206− alias /usr/local/searx/searx/static;
207− }
208−
209− location /searx {
210− proxy_pass http://127.0.0.1:8888;
211− proxy_set_header Host $host;
212− proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
213− proxy_set_header X-Scheme $scheme;
214− proxy_set_header X-Script-Name /searx;
215− proxy_buffering off;
216− }
217−
218−
219−Enable ``base_url`` in ``searx/settings.yml``
220−
221−.. code:: yaml
222−
223− base_url : http://your.domain.tld/searx/
224−
225−Restart service:
226−
227−.. code:: sh
228−
229− sudo -H service nginx restart
230− sudo -H service uwsgi restart
231−
232−disable logs
233−^^^^^^^^^^^^
234−
235−for better privacy you can disable nginx logs about searx.
236−
237−how to proceed: below ``uwsgi_pass`` in ``/etc/nginx/sites-available/default``
238−add:
239−
240−.. code:: nginx
241−
242− access_log /dev/null;
243− error_log /dev/null;
244−
245−Restart service:
246−
247−.. code:: sh
248−
249− sudo -H service nginx restart
250−
251−with apache
252−-----------
253−
254−Add wsgi mod:
255−
256−.. code:: sh
257−
258− sudo -H apt-get install libapache2-mod-uwsgi
259− sudo -H a2enmod uwsgi
260−
261−Add this configuration in the file ``/etc/apache2/apache2.conf``:
262−
263−.. code:: apache
264−
265− <Location />
266− Options FollowSymLinks Indexes
267− SetHandler uwsgi-handler
268− uWSGISocket /run/uwsgi/app/searx/socket
269− </Location>
270−
271−Note that if your instance of searx is not at the root, you should change
272−``<Location />`` by the location of your instance, like ``<Location /searx>``.
273−
274−Restart Apache:
275−
276−.. code:: sh
277−
278− sudo -H /etc/init.d/apache2 restart
279−
280−disable logs
281−~~~~~~~~~~~~
282−
283−For better privacy you can disable Apache logs.
284−
285−.. warning::
286−
287− You can only disable logs for the whole (virtual) server not for a specific
288− path.
289−
290−Go back to ``/etc/apache2/apache2.conf`` and above ``<Location />`` add:
291−
292−.. code:: apache
293−
294− CustomLog /dev/null combined
295−
296−Restart Apache:
297−
298−.. code:: sh
299−
300− sudo -H /etc/init.d/apache2 restart
301−
302−How to update
303−=============
304−
305−.. code:: sh
306−
307− cd /usr/local/searx
308− sudo -H -u searx -i
309−
310−.. code:: sh
311−
312− (searx)$ . ./searx-ve/bin/activate
313− (searx)$ git stash
314− (searx)$ git pull origin master
315− (searx)$ git stash apply
316− (searx)$ ./manage.sh update_packages
317−
318−.. code:: sh
319−
320− sudo -H service uwsgi restart
321−
322−Docker
323−======
324−
325−Make sure you have installed Docker. For instance, you can deploy searx like this:
326−
327−.. code:: sh
328−
329− docker pull wonderfall/searx
330− docker run -d --name searx -p $PORT:8888 wonderfall/searx
331−
332−Go to ``http://localhost:$PORT``.
333−
334−See https://hub.docker.com/r/wonderfall/searx/ for more informations. It's also
335−possible to build searx from the embedded Dockerfile.
336−
337−.. code:: sh
338−
339− git clone https://github.com/asciimoo/searx.git
340− cd searx
341− docker build -t whatever/searx .
342−
343−References
344−==========
345−
346−* https://about.okhin.fr/posts/Searx/ with some additions
347−
348−* How to: `Setup searx in a couple of hours with a free SSL certificate
349− <https://www.reddit.com/r/privacytoolsIO/comments/366kvn/how_to_setup_your_own_privacy_respecting_search/>`__
docs/admin/morty.rst+7
@@ -1,7 +1,14 @@
1+
2+.. _searx morty:
3+
1=========================
4=========================
2How to setup result proxy
5How to setup result proxy
3=========================
6=========================
4
7
8+.. sidebar:: further reading
9+
10+ - :ref:`morty.sh`
11+
5.. _morty: https://github.com/asciimoo/morty
12.. _morty: https://github.com/asciimoo/morty
6.. _morty's README: https://github.com/asciimoo/morty
13.. _morty's README: https://github.com/asciimoo/morty
7
14
docs/admin/settings.rst+7−1
@@ -4,11 +4,17 @@
4``settings.yml``
4``settings.yml``
5================
5================
6
6
7+This page describe the options possibilities of the :origin:`searx/settings.yml`
8+file.
9+
7.. sidebar:: Further reading ..
10.. sidebar:: Further reading ..
8
11
9 - :ref:`search API`
12 - :ref:`search API`
10
13
11−This page describe the options possibilities of the settings.yml file.
14+.. contents:: Contents
15+ :depth: 2
16+ :local:
17+ :backlinks: entry
12
18
13.. _settings global:
19.. _settings global:
14
20
docs/admin/update-searx.rstnew file+23
@@ -0,0 +1,23 @@
1+.. _update searx:
2+
3+=============
4+How to update
5+=============
6+
7+.. code:: sh
8+
9+ sudo -H -u searx -i
10+ (searx)$ git stash
11+ (searx)$ git pull origin master
12+ (searx)$ git stash apply
13+ (searx)$ ./manage.sh update_packages
14+
15+Restart uwsgi:
16+
17+.. tabs::
18+
19+ .. group-tab:: Ubuntu / debian
20+
21+ .. code:: sh
22+
23+ sudo -H systemctl restart uwsgi
docs/blog/index.rst+2−1
@@ -3,7 +3,8 @@ Blog
3====
3====
4
4
5.. toctree::
5.. toctree::
6− :maxdepth: 1
6+ :maxdepth: 2
7+ :caption: Contents
7
8
8 python3
9 python3
9 admin
10 admin
docs/build-templates/filtron.rstnew file+52
@@ -0,0 +1,52 @@
1+.. START create user
2+
3+.. tabs::
4+
5+ .. group-tab:: bash
6+
7+ .. code-block:: sh
8+
9+ $ sudo -H useradd --shell /bin/bash --system \\
10+ --home-dir "$SERVICE_HOME" \\
11+ --comment "Privacy-respecting metasearch engine" $SERVICE_USER
12+
13+ $ sudo -H mkdir "$SERVICE_HOME"
14+ $ sudo -H chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
15+
16+.. END create user
17+
18+.. START install go
19+
20+.. tabs::
21+
22+ .. group-tab:: bash
23+
24+ .. code-block:: bash
25+
26+ $ cat > "$GO_ENV" <<EOF
27+ export GOPATH=${SERVICE_HOME}/go-apps
28+ export PATH=\$PATH:${SERVICE_HOME}/local/go/bin:\$GOPATH/bin
29+ EOF
30+ $ sudo -i -u "${SERVICE_USER}"
31+ (${SERVICE_USER}) $ echo 'source $GO_ENV' >> ~/.profile
32+ (${SERVICE_USER}) $ mkdir ${SERVICE_HOME}/local
33+ (${SERVICE_USER}) $ wget --progress=bar -O "${GO_TAR}" \\
34+ "${GO_PKG_URL}"
35+ (${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local/go -xzf "${GO_TAR}"
36+ (${SERVICE_USER}) $ which go
37+ ${SERVICE_HOME}/local/go/bin/go
38+
39+.. END install go
40+
41+.. START install filtron
42+
43+.. tabs::
44+
45+ .. group-tab:: bash
46+
47+ .. code-block:: bash
48+
49+ $ sudo -i -u "${SERVICE_USER}"
50+ (${SERVICE_USER}) $ go get -v -u github.com/asciimoo/filtron
51+
52+.. END install filtron
docs/build-templates/morty.rstnew file+52
@@ -0,0 +1,52 @@
1+.. START create user
2+
3+.. tabs::
4+
5+ .. group-tab:: bash
6+
7+ .. code-block:: sh
8+
9+ $ sudo -H useradd --shell /bin/bash --system \\
10+ --home-dir "$SERVICE_HOME" \\
11+ --comment "Privacy-respecting metasearch engine" $SERVICE_USER
12+
13+ $ sudo -H mkdir "$SERVICE_HOME"
14+ $ sudo -H chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
15+
16+.. END create user
17+
18+.. START install go
19+
20+.. tabs::
21+
22+ .. group-tab:: bash
23+
24+ .. code-block:: bash
25+
26+ $ cat > "$GO_ENV" <<EOF
27+ export GOPATH=${SERVICE_HOME}/go-apps
28+ export PATH=\$PATH:${SERVICE_HOME}/local/go/bin:\$GOPATH/bin
29+ EOF
30+ $ sudo -i -u "${SERVICE_USER}"
31+ (${SERVICE_USER}) $ echo 'source $GO_ENV' >> ~/.profile
32+ (${SERVICE_USER}) $ mkdir ${SERVICE_HOME}/local
33+ (${SERVICE_USER}) $ wget --progress=bar -O "${GO_TAR}" \\
34+ "${GO_PKG_URL}"
35+ (${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local/go -xzf "${GO_TAR}"
36+ (${SERVICE_USER}) $ which go
37+ ${SERVICE_HOME}/local/go/bin/go
38+
39+.. END install go
40+
41+.. START install morty
42+
43+.. tabs::
44+
45+ .. group-tab:: bash
46+
47+ .. code-block:: bash
48+
49+ $ sudo -i -u "${SERVICE_USER}"
50+ (${SERVICE_USER}) $ go get -v -u github.com/asciimoo/morty
51+
52+.. END install morty
docs/build-templates/searx.rstnew file+192
@@ -0,0 +1,192 @@
1+.. template evaluated by: ./utils/searx.sh docs
2+.. hint: all dollar-names are variables, dollar sign itself is quoted by: \\$
3+
4+.. START distro-packages
5+
6+.. tabs::
7+
8+ .. group-tab:: Ubuntu / debian
9+
10+ .. code-block:: sh
11+
12+ $ sudo -H apt-get install -y \\
13+${debian}
14+
15+ .. group-tab:: Arch Linux
16+
17+ .. code-block:: sh
18+
19+ $ sudo -H pacman -S --noconfirm \\
20+${arch}
21+
22+ .. group-tab:: Fedora / RHEL
23+
24+ .. code-block:: sh
25+
26+ $ sudo -H dnf install -y \\
27+${fedora}
28+
29+.. END distro-packages
30+
31+.. START build-packages
32+
33+.. tabs::
34+
35+ .. group-tab:: Ubuntu / debian
36+
37+ .. code-block:: sh
38+
39+ $ sudo -H apt-get install -y \\
40+${debian_build}
41+
42+ .. group-tab:: Arch Linux
43+
44+ .. code-block:: sh
45+
46+ $ sudo -H pacman -S --noconfirm \\
47+${arch_build}
48+
49+ .. group-tab:: Fedora / RHEL
50+
51+ .. code-block:: sh
52+
53+ $ sudo -H dnf install -y \\
54+${fedora_build}
55+
56+.. END build-packages
57+
58+.. START create user
59+
60+.. tabs::
61+
62+ .. group-tab:: bash
63+
64+ .. code-block:: sh
65+
66+ $ sudo -H useradd --shell /bin/bash --system \\
67+ --home-dir "$SERVICE_HOME" \\
68+ --comment "Privacy-respecting metasearch engine" $SERVICE_USER
69+
70+ $ sudo -H mkdir "$SERVICE_HOME"
71+ $ sudo -H chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
72+
73+.. END create user
74+
75+.. START clone searx
76+
77+.. tabs::
78+
79+ .. group-tab:: bash
80+
81+ .. code-block:: sh
82+
83+ $ sudo -H -u ${SERVICE_USER} -i
84+ (${SERVICE_USER})$ git clone "https://github.com/asciimoo/searx.git" "$SEARX_SRC"
85+
86+.. END clone searx
87+
88+.. START create virtualenv
89+
90+.. tabs::
91+
92+ .. group-tab:: bash
93+
94+ .. code-block:: sh
95+
96+ (${SERVICE_USER})$ python3 -m venv "${SEARX_PYENV}"
97+ (${SERVICE_USER})$ echo ". ${SEARX_PYENV}/bin/activate" >> "$SERVICE_HOME/.profile"
98+
99+.. END create virtualenv
100+
101+.. START manage.sh update_packages
102+
103+.. tabs::
104+
105+ .. group-tab:: bash
106+
107+ .. code-block:: sh
108+
109+ $ sudo -H -u ${SERVICE_USER} -i
110+
111+ (${SERVICE_USER})$ command -v python && python --version
112+ $SEARX_PYENV/bin/python
113+ Python 3.8.1
114+
115+ # update pip's boilerplate ..
116+ pip install -U pip
117+ pip install -U setuptools
118+ pip install -U wheel
119+
120+ # jump to searx's working tree and install searx into virtualenv
121+ (${SERVICE_USER})$ cd "$SEARX_SRC"
122+ (${SERVICE_USER})$ pip install -e .
123+
124+
125+.. END manage.sh update_packages
126+
127+.. START searx config
128+
129+.. tabs::
130+
131+ .. group-tab:: bash
132+
133+ .. code-block:: sh
134+
135+ $ sudo -H cp "$SEARX_SRC/searx/settings.yml" "${SEARX_SETTINGS_PATH}"
136+ $ sudo -H sed -i -e "s/ultrasecretkey/\\$(openssl rand -hex 16)/g" "$SEARX_SETTINGS_PATH"
137+ $ sudo -H sed -i -e "s/{instance_name}/searx@\\$(uname -n)/g" "$SEARX_SETTINGS_PATH"
138+
139+.. END searx config
140+
141+.. START check searx installation
142+
143+.. tabs::
144+
145+ .. group-tab:: bash
146+
147+ .. code-block:: sh
148+
149+ # enable debug ..
150+ $ sudo -H sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS_PATH"
151+
152+ # start webapp
153+ $ sudo -H -u ${SERVICE_USER} -i
154+ (${SERVICE_USER})$ cd ${SEARX_SRC}
155+ (${SERVICE_USER})$ export SEARX_SETTINGS_PATH="${SEARX_SETTINGS_PATH}"
156+ (${SERVICE_USER})$ python searx/webapp.py
157+
158+ # disable debug
159+ $ sudo -H sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS_PATH"
160+
161+Open WEB browser and visit http://$SEARX_INTERNAL_URL . If you are inside a
162+container or in a script, test with curl:
163+
164+.. tabs::
165+
166+ .. group-tab:: WEB browser
167+
168+ .. code-block:: sh
169+
170+ $ xgd-open http://$SEARX_INTERNAL_URL
171+
172+ .. group-tab:: curl
173+
174+ .. code-block:: none
175+
176+ $ curl --location --verbose --head --insecure $SEARX_INTERNAL_URL
177+
178+ * Trying 127.0.0.1:8888...
179+ * TCP_NODELAY set
180+ * Connected to 127.0.0.1 (127.0.0.1) port 8888 (#0)
181+ > HEAD / HTTP/1.1
182+ > Host: 127.0.0.1:8888
183+ > User-Agent: curl/7.68.0
184+ > Accept: */*
185+ >
186+ * Mark bundle as not supporting multiuse
187+ * HTTP 1.0, assume close after body
188+ < HTTP/1.0 200 OK
189+ HTTP/1.0 200 OK
190+ ...
191+
192+.. END check searx installation
docs/conf.py+13−1
@@ -1,10 +1,12 @@
1# -*- coding: utf-8 -*-
1# -*- coding: utf-8 -*-
2
2
3import sys, os
3import sys, os
4+from sphinx_build_tools import load_sphinx_config
4from searx.version import VERSION_STRING
5from searx.version import VERSION_STRING
5from pallets_sphinx_themes import ProjectLink
6from pallets_sphinx_themes import ProjectLink
6
7
7from searx.brand import GIT_URL
8from searx.brand import GIT_URL
9+GIT_BRANCH = os.environ.get("GIT_BRANCH", "master")
8from searx.brand import SEARX_URL
10from searx.brand import SEARX_URL
9from searx.brand import DOCS_URL
11from searx.brand import DOCS_URL
10
12
@@ -22,6 +24,8 @@ master_doc = "index"
22source_suffix = '.rst'
24source_suffix = '.rst'
23numfig = True
25numfig = True
24
26
27+exclude_patterns = ['build-templates/*.rst']
28+
25from searx import webapp
29from searx import webapp
26jinja_contexts = {
30jinja_contexts = {
27 'webapp': dict(**webapp.__dict__)
31 'webapp': dict(**webapp.__dict__)
@@ -35,7 +39,7 @@ extlinks['wiki'] = ('https://github.com/asciimoo/searx/wiki/%s', ' ')
35extlinks['pull'] = ('https://github.com/asciimoo/searx/pull/%s', 'PR ')
39extlinks['pull'] = ('https://github.com/asciimoo/searx/pull/%s', 'PR ')
36
40
37# links to custom brand
41# links to custom brand
38−extlinks['origin'] = (GIT_URL + '/blob/master/%s', 'git://')
42+extlinks['origin'] = (GIT_URL + '/blob/' + GIT_BRANCH + '/%s', 'git://')
39extlinks['patch'] = (GIT_URL + '/commit/%s', '#')
43extlinks['patch'] = (GIT_URL + '/commit/%s', '#')
40extlinks['search'] = (SEARX_URL + '/%s', '#')
44extlinks['search'] = (SEARX_URL + '/%s', '#')
41extlinks['docs'] = (DOCS_URL + '/%s', 'docs: ')
45extlinks['docs'] = (DOCS_URL + '/%s', 'docs: ')
@@ -61,6 +65,8 @@ extensions = [
61 "pallets_sphinx_themes",
65 "pallets_sphinx_themes",
62 "sphinx_issues", # https://github.com/sloria/sphinx-issues/blob/master/README.rst
66 "sphinx_issues", # https://github.com/sloria/sphinx-issues/blob/master/README.rst
63 "sphinxcontrib.jinja", # https://github.com/tardyp/sphinx-jinja
67 "sphinxcontrib.jinja", # https://github.com/tardyp/sphinx-jinja
68+ "sphinxcontrib.programoutput", # https://github.com/NextThought/sphinxcontrib-programoutput
69+ 'linuxdoc.kernel_include', # Implementation of the 'kernel-include' reST-directive.
64 'linuxdoc.rstFlatTable', # Implementation of the 'flat-table' reST-directive.
70 'linuxdoc.rstFlatTable', # Implementation of the 'flat-table' reST-directive.
65 'linuxdoc.kfigure', # Sphinx extension which implements scalable image handling.
71 'linuxdoc.kfigure', # Sphinx extension which implements scalable image handling.
66 "sphinx_tabs.tabs", # https://github.com/djungelorm/sphinx-tabs
72 "sphinx_tabs.tabs", # https://github.com/djungelorm/sphinx-tabs
@@ -112,3 +118,9 @@ html_show_sourcelink = False
112latex_documents = [
118latex_documents = [
113 (master_doc, "searx-{}.tex".format(VERSION_STRING), html_title, author, "manual")
119 (master_doc, "searx-{}.tex".format(VERSION_STRING), html_title, author, "manual")
114]
120]
121+
122+# ------------------------------------------------------------------------------
123+# Since loadConfig overwrites settings from the global namespace, it has to be
124+# the last statement in the conf.py file
125+# ------------------------------------------------------------------------------
126+load_sphinx_config(globals())
docs/dev/contribution_guide.rst+5
@@ -4,6 +4,11 @@
4How to contribute
4How to contribute
5=================
5=================
6
6
7+.. contents:: Contents
8+ :depth: 2
9+ :local:
10+ :backlinks: entry
11+
7Prime directives: Privacy, Hackability
12Prime directives: Privacy, Hackability
8======================================
13======================================
9
14
docs/dev/index.rst+2−1
@@ -3,7 +3,8 @@ Developer documentation
3=======================
3=======================
4
4
5.. toctree::
5.. toctree::
6− :maxdepth: 1
6+ :maxdepth: 2
7+ :caption: Contents
7
8
8 quickstart
9 quickstart
9 contribution_guide
10 contribution_guide
docs/dev/makefile.rst+24−24
@@ -11,23 +11,17 @@ Makefile Targets
11 Before looking deeper at the targets, first read about :ref:`makefile setup`
11 Before looking deeper at the targets, first read about :ref:`makefile setup`
12 and :ref:`make pyenv`.
12 and :ref:`make pyenv`.
13
13
14+ To install system requirements follow :ref:`buildhosts`.
15+
14With the aim to simplify development cycles, started with :pull:`1756` a
16With the aim to simplify development cycles, started with :pull:`1756` a
15``Makefile`` based boilerplate was added. If you are not familiar with
17``Makefile`` based boilerplate was added. If you are not familiar with
16Makefiles, we recommend to read gnu-make_ introduction.
18Makefiles, we recommend to read gnu-make_ introduction.
17
19
18The usage is simple, just type ``make {target-name}`` to *build* a target.
20The usage is simple, just type ``make {target-name}`` to *build* a target.
19−Calling the ``help`` target gives a first overview::
21+Calling the ``help`` target gives a first overview (``make help``):
20−
22+
21− $ make help
23+.. program-output:: bash -c "cd ..; make --no-print-directory help"
22− test - run developer tests
24+
23− docs - build documentation
24− docs-live - autobuild HTML documentation while editing
25− run - run developer instance
26− install - developer install (./local)
27− uninstall - uninstall (./local)
28− gh-pages - build docs & deploy on gh-pages branch
29− clean - drop builds and environments
30− ...
31
25
32.. contents:: Contents
26.. contents:: Contents
33 :depth: 2
27 :depth: 2
@@ -37,27 +31,33 @@ Calling the ``help`` target gives a first overview::
37
31
38.. _makefile setup:
32.. _makefile setup:
39
33
40−Setup
34+Makefile setup
41−=====
35+==============
42
36
43.. _git stash: https://git-scm.com/docs/git-stash
37.. _git stash: https://git-scm.com/docs/git-stash
44
38
45−The main setup is done in the :origin:`Makefile`::
46−
47− export GIT_URL=https://github.com/asciimoo/searx
48− export SEARX_URL=https://searx.me
49− export DOCS_URL=https://asciimoo.github.io/searx
50−
51.. sidebar:: fork & upstream
39.. sidebar:: fork & upstream
52
40
53 Commit changes in your (local) branch, fork or whatever, but do not push them
41 Commit changes in your (local) branch, fork or whatever, but do not push them
54 upstream / `git stash`_ is your friend.
42 upstream / `git stash`_ is your friend.
55
43
44+The main setup is done in the :origin:`Makefile`.
45+
46+.. literalinclude:: ../../Makefile
47+ :start-after: START Makefile setup
48+ :end-before: END Makefile setup
49+
56−:GIT_URL: Changes this, to point to your searx fork.
50+:GIT_URL: Changes this, to point to your searx fork.
51+:GIT_BRANCH: Changes this, to point to your searx branch.
52+:SEARX_URL: Changes this, to point to your searx instance.
53+:DOCS_URL: If you host your own (*brand*) documentation, change this URL.
57
54
58−:SEARX_URL: Changes this, to point to your searx instance.
55+If you change any of this build environment variables, you have to run ``make
56+buildenv``::
59
57
60−:DOCS_URL: If you host your own (branded) documentation, change this URL.
58+ $ make buildenv
59+ build searx/brand.py
60+ build utils/brand.env
61
61
62.. _make pyenv:
62.. _make pyenv:
63
63
@@ -170,7 +170,7 @@ e.g.:
170
170
171.. code:: sh
171.. code:: sh
172
172
173− $ make test.pep8 test.unit
173+ $ make test.pep8 test.unit test.sh
174 . ./local/py3/bin/activate; ./manage.sh pep8_check
174 . ./local/py3/bin/activate; ./manage.sh pep8_check
175 [!] Running pep8 check
175 [!] Running pep8 check
176 . ./local/py3/bin/activate; ./manage.sh unit_tests
176 . ./local/py3/bin/activate; ./manage.sh unit_tests
docs/dev/quickstart.rst+1−1
@@ -27,7 +27,7 @@ searx-ve virtualenv and install the required packages using ``manage.sh``.
27 cd ~/myprojects
27 cd ~/myprojects
28 git clone https://github.com/asciimoo/searx.git
28 git clone https://github.com/asciimoo/searx.git
29 cd searx
29 cd searx
30− virtualenv searx-ve
30+ python3 -m venv searx-ve
31 . ./searx-ve/bin/activate
31 . ./searx-ve/bin/activate
32 ./manage.sh update_dev_packages
32 ./manage.sh update_dev_packages
33
33
docs/dev/reST.rst+5−5
@@ -325,8 +325,9 @@ Literal blocks
325
325
326The simplest form of :duref:`literal-blocks` is a indented block introduced by
326The simplest form of :duref:`literal-blocks` is a indented block introduced by
327two colons (``::``). For highlighting use :dudir:`highlight` or :ref:`reST
327two colons (``::``). For highlighting use :dudir:`highlight` or :ref:`reST
328−code` directive. To include literals from external files use directive
328+code` directive. To include literals from external files use
329−:dudir:`literalinclude`.
329+:rst:dir:`literalinclude` or :ref:`kernel-include <kernel-include-directive>`
330+directive (latter one expands environment variables in the path name).
330
331
331.. _reST literal:
332.. _reST literal:
332
333
@@ -1312,9 +1313,8 @@ others are basic-tabs_ and code-tabs_. Below a *group-tab* example from
1312
1313
1313.. literalinclude:: ../admin/buildhosts.rst
1314.. literalinclude:: ../admin/buildhosts.rst
1314 :language: reST
1315 :language: reST
1315− :start-after: .. _system requirements:
1316+ :start-after: .. SNIP sh lint requirements
1316− :end-before: .. _system requirements END:
1317+ :end-before: .. SNAP sh lint requirements
1317−
1318
1318
1319.. _math:
1319.. _math:
1320
1320
docs/index.rst+10−8
@@ -2,7 +2,14 @@
2Welcome to searx
2Welcome to searx
3================
3================
4
4
5−Search without being tracked.
5+ *Search without being tracked.*
6+
7+Searx is a free internet metasearch engine which aggregates results from more
8+than 70 search services. Users are neither tracked nor profiled. Additionally,
9+searx can be used over Tor for online anonymity.
10+
11+Get started with searx by using one of the Searx-instances_. If you don't trust
12+anyone, you can set up your own, see :ref:`installation`.
6
13
7.. sidebar:: Features
14.. sidebar:: Features
8
15
@@ -16,19 +23,14 @@ Search without being tracked.
16 - Hosted by organizations, such as *La Quadrature du Net*, which promote
23 - Hosted by organizations, such as *La Quadrature du Net*, which promote
17 digital rights
24 digital rights
18
25
19−Searx is a free internet metasearch engine which aggregates results from more
20−than 70 search services. Users are neither tracked nor profiled. Additionally,
21−searx can be used over Tor for online anonymity.
22−
23−Get started with searx by using one of the Searx-instances_. If you don't trust
24−anyone, you can set up your own, see :ref:`installation`.
25−
26.. toctree::
26.. toctree::
27 :maxdepth: 2
27 :maxdepth: 2
28+ :caption: Contents
28
29
29 user/index
30 user/index
30 admin/index
31 admin/index
31 dev/index
32 dev/index
33+ utils/index
32 blog/index
34 blog/index
33
35
34.. _Searx-instances: https://searx.space
36.. _Searx-instances: https://searx.space
docs/user/conf.pynew file+19
@@ -0,0 +1,19 @@
1+# -*- coding: utf-8; mode: python -*-
2+"""Configuration for the Searx user handbook
3+"""
4+project = 'Searx User-HB'
5+version = release = VERSION_STRING
6+
7+# Grouping the document tree into LaTeX files. List of tuples
8+# (source start file, target name, title,
9+# author, documentclass [howto, manual, or own class]).
10+latex_documents = [
11+ ('index' # startdocname
12+ , 'searx-user-hb.tex' # targetname
13+ , '' # take title from .rst
14+ , author # author
15+ , 'howto' # documentclass
16+ , False # toctree_only
17+ ),
18+]
19+
docs/user/index.rst+2−1
@@ -3,7 +3,8 @@ User documentation
3==================
3==================
4
4
5.. toctree::
5.. toctree::
6− :maxdepth: 1
6+ :maxdepth: 2
7+ :caption: Contents
7
8
8 search_syntax
9 search_syntax
9 own-instance
10 own-instance
docs/user/own-instance.rst+4−2
@@ -2,8 +2,10 @@
2Why use a private instance?
2Why use a private instance?
3===========================
3===========================
4
4
5−"Is it worth to run my own instance?" is a common question among searx users.
5+ *"Is it worth to run my own instance?"*
6+
7+\.\. is a common question among searx users. Before answering this question,
6−Before answering this question, see what options a searx user has.
8+see what options a searx user has.
7
9
8Public instances are open to everyone who has access to its URL. Usually, these
10Public instances are open to everyone who has access to its URL. Usually, these
9are operated by unknown parties (from the users' point of view). Private
11are operated by unknown parties (from the users' point of view). Private
docs/user/public_instances.rstdeleted−3
@@ -1,3 +0,0 @@
1−:orphan:
2−
3−This page page has been moved to `searx.space <https://searx.space/>`__
docs/utils/filtron.sh.rstnew file+80
@@ -0,0 +1,80 @@
1+
2+.. _filtron.sh:
3+
4+====================
5+``utils/filtron.sh``
6+====================
7+
8+.. sidebar:: further reading
9+
10+ - :ref:`searx filtron`
11+ - :ref:`architecture`
12+ - :ref:`installation` (:ref:`nginx <installation nginx>` & :ref:`apache
13+ <installation apache>`)
14+
15+.. _Go: https://golang.org/
16+.. _filtron: https://github.com/asciimoo/filtron
17+.. _filtron README: https://github.com/asciimoo/filtron/blob/master/README.md
18+
19+To simplify installation and maintenance of a filtron instance you can use the
20+script :origin:`utils/filtron.sh`. In most cases you will install filtron_
21+simply by running the command:
22+
23+.. code:: bash
24+
25+ sudo -H ./utils/filtron.sh install all
26+
27+The script adds a ``${SERVICE_USER}`` (default:``filtron``) and installs filtron_
28+into this user account:
29+
30+#. Create a separated user account (``filtron``).
31+#. Download and install Go_ binary in user's $HOME (``~filtron``).
32+#. Install filtron with the package management from Go_ (``go get -v -u
33+ github.com/asciimoo/filtron``)
34+#. Setup a proper rule configuration :origin:`[ref]
35+ <utils/templates/etc/filtron/rules.json>` (``/etc/filtron/rules.json``).
36+#. Setup a systemd service unit :origin:`[ref]
37+ <utils/templates/lib/systemd/system/filtron.service>`
38+ (``/lib/systemd/system/filtron.service``).
39+
40+
41+Create user
42+===========
43+
44+.. kernel-include:: $DOCS_BUILD/includes/filtron.rst
45+ :start-after: START create user
46+ :end-before: END create user
47+
48+
49+Install go
50+==========
51+
52+.. kernel-include:: $DOCS_BUILD/includes/filtron.rst
53+ :start-after: START install go
54+ :end-before: END install go
55+
56+
57+Install filtron
58+===============
59+
60+Install :origin:`rules.json <utils/templates/etc/filtron/rules.json>` at
61+``/etc/filtron/rules.json`` (see :ref:`Sample configuration of filtron`) and
62+install filtron software and systemd unit:
63+
64+.. kernel-include:: $DOCS_BUILD/includes/filtron.rst
65+ :start-after: START install filtron
66+ :end-before: END install filtron
67+
68+.. kernel-include:: $DOCS_BUILD/includes/filtron.rst
69+ :start-after: START install systemd unit
70+ :end-before: END install systemd unit
71+
72+.. _filtron.sh overview:
73+
74+Overview
75+========
76+
77+The ``--help`` output of the script is largely self-explanatory
78+(:ref:`toolboxing common`):
79+
80+.. program-output:: ../utils/filtron.sh --help
docs/utils/index.rstnew file+53
@@ -0,0 +1,53 @@
1+.. _searx_utils:
2+.. _toolboxing:
3+
4+=======================
5+Tooling box ``utils/*``
6+=======================
7+
8+In the folder :origin:`utils/` we maintain some tools useful for admins and
9+developers.
10+
11+.. toctree::
12+ :maxdepth: 2
13+ :caption: Contents
14+
15+ searx.sh
16+ filtron.sh
17+ morty.sh
18+ lxc.sh
19+
20+.. _toolboxing common:
21+
22+Common commands & environment
23+=============================
24+
25+Scripts to maintain services often dispose of common commands and environments.
26+
27+``shell`` : command
28+ Opens a shell from the service user ``${SERVICE_USSR}``, very helpful for
29+ troubleshooting.
30+
31+``inspect service`` : command
32+ Shows status and log of the service, most often you have a option to enable
33+ more verbose debug logs. Very helpful for debugging, but be careful not to
34+ enable debugging in a production environment!
35+
36+``FORCE_TIMEOUT`` : environment
37+ Sets timeout for interactive prompts. If you want to run a script in batch
38+ job, with defaults choices, set ``FORCE_TIMEOUT=0``. By example; to install a
39+ reverse proxy for filtron on all containers of the :ref:`searx suite
40+ <lxc-searx.env>` use ::
41+
42+ sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh apache install
43+
44+.. _toolboxing setup:
45+
46+Tooling box setup
47+=================
48+
49+The main setup is done in the :origin:`.config.sh` (read also :ref:`makefile
50+setup`).
51+
52+.. literalinclude:: ../../.config.sh
53+ :language: bash
docs/utils/lxc.sh.rstnew file+148
@@ -0,0 +1,148 @@
1+
2+.. _snap: https://snapcraft.io
3+.. _snapcraft LXD: https://snapcraft.io/lxd
4+.. _LXC/LXD Image Server: https://uk.images.linuxcontainers.org/
5+.. _LXC: https://linuxcontainers.org/lxc/introduction/
6+.. _LXD: https://linuxcontainers.org/lxd/introduction/
7+.. _`LXD@github`: https://github.com/lxc/lxd
8+
9+.. _archlinux: https://www.archlinux.org/
10+
11+.. _lxc.sh:
12+
13+================
14+``utils/lxc.sh``
15+================
16+
17+.. sidebar:: further reading
18+
19+ - snap_, `snapcraft LXD`_
20+ - LXC_, LXD_
21+ - `LXC/LXD Image Server`_
22+ - `LXD@github`_
23+
24+With the use of *Linux Containers* (LXC_) we can scale our tasks over a stack of
25+containers, what we call the: *lxc suite*. The *searx suite*
26+(:origin:`lxc-searx.env <utils/lxc-searx.env>`) is loaded by default, every time
27+you start the ``lxc.sh`` script (*you do not need to care about*).
28+
29+Before you can start with containers, you need to install and initiate LXD_
30+once::
31+
32+ $ snap install lxd
33+ $ lxd init --auto
34+
35+To make use of the containers from the *searx suite*, you have to build the
36+:ref:`LXC suite containers <lxc.sh help>` initial. But be warned, **this might
37+take some time**::
38+
39+ $ sudo -H ./utils/lxc.sh build
40+
41+A cup of coffee later, your LXC suite is build up and you can run whatever task
42+you want / in a selected or even in all :ref:`LXC suite containers <lxc.sh
43+help>`. If you do not want to build all containers, **you can build just
44+one**::
45+
46+ $ sudo -H ./utils/lxc.sh build searx-ubu1804
47+
48+*Good to know ...*
49+
50+Eeach container shares the root folder of the repository and the
51+command ``utils/lxc.sh cmd`` **handles relative path names transparent**,
52+compare output of::
53+
54+ $ sudo -H ./utils/lxc.sh cmd -- ls -la Makefile
55+ ...
56+
57+In the containers, you can run what ever you want, e.g. to start a bash use::
58+
59+ $ sudo -H ./utils/lxc.sh cmd searx-ubu1804 bash
60+ INFO: [searx-ubu1804] bash
61+ root@searx-ubu1804:/share/searx#
62+
63+If there comes the time you want to **get rid off all** the containers and
64+**clean up local images** just type::
65+
66+ $ sudo -H ./utils/lxc.sh remove
67+ $ sudo -H ./utils/lxc.sh remove images
68+
69+
70+Install suite
71+=============
72+
73+To install the complete :ref:`searx suite (includes searx, morty & filtron)
74+<lxc-searx.env>` into all LXC_ use::
75+
76+ $ sudo -H ./utils/lxc.sh install suite
77+
78+The command above installs a searx suite (see :ref:`installation scripts`). To
79+get the IP (URL) of the filtron service in the containers use ``show suite``
80+command. To test instances from containers just open the URLs in your
81+WEB-Browser::
82+
83+ $ sudo ./utils/lxc.sh show suite | grep filtron
84+ [searx-ubu1604] INFO: (eth0) filtron: http://n.n.n.246:4004/ http://n.n.n.246/searx
85+ [searx-ubu1804] INFO: (eth0) filtron: http://n.n.n.147:4004/ http://n.n.n.147/searx
86+ [searx-ubu1910] INFO: (eth0) filtron: http://n.n.n.140:4004/ http://n.n.n.140/searx
87+ [searx-ubu2004] INFO: (eth0) filtron: http://n.n.n.18:4004/ http://n.n.n.18/searx
88+ [searx-fedora31] INFO: (eth0) filtron: http://n.n.n.46:4004/ http://n.n.n.46/searx
89+ [searx-archlinux] INFO: (eth0) filtron: http://n.n.n.32:4004/ http://n.n.n.32/searx
90+
91+To :ref:`install a nginx <installation nginx>` reverse proxy for filtron and
92+morty use (or alternatively use :ref:`apache <installation apache>`)::
93+
94+ sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh nginx install
95+ sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/morty.sh nginx install
96+
97+
98+Running commands
99+================
100+
101+**Inside containers, you can use make or run scripts** from the
102+:ref:`toolboxing`. By example: to setup a :ref:`buildhosts` and run the
103+Makefile target ``test`` in the archlinux_ container::
104+
105+ sudo -H ./utils/lxc.sh cmd searx-archlinux ./utils/searx.sh install buildhost
106+ sudo -H ./utils/lxc.sh cmd searx-archlinux make test
107+
108+
109+Setup searx buildhost
110+=====================
111+
112+You can **install the searx buildhost environment** into one or all containers.
113+The installation procedure to set up a :ref:`build host<buildhosts>` takes its
114+time. Installation in all containers will take more time (time for another cup
115+of coffee).::
116+
117+ sudo -H ./utils/lxc.sh cmd -- ./utils/searx.sh install buildhost
118+
119+To build (live) documentation inside a archlinux_ container::
120+
121+ sudo -H ./utils/lxc.sh cmd searx-archlinux make docs-clean docs-live
122+ ...
123+ [I 200331 15:00:42 server:296] Serving on http://0.0.0.0:8080
124+
125+To get IP of the container and the port number *live docs* is listening::
126+
127+ $ sudo ./utils/lxc.sh show suite | grep docs-live
128+ ...
129+ [searx-archlinux] INFO: (eth0) docs-live: http://n.n.n.12:8080/
130+
131+
132+.. _lxc.sh help:
133+
134+Overview
135+========
136+
137+The ``--help`` output of the script is largely self-explanatory:
138+
139+.. program-output:: ../utils/lxc.sh --help
140+
141+
142+.. _lxc-searx.env:
143+
144+searx suite
145+===========
146+
147+.. literalinclude:: ../../utils/lxc-searx.env
148+ :language: bash
docs/utils/morty.sh.rstnew file+80
@@ -0,0 +1,80 @@
1+
2+.. _morty: https://github.com/asciimoo/morty
3+.. _morty's README: https://github.com/asciimoo/morty
4+.. _Go: https://golang.org/
5+
6+.. _morty.sh:
7+
8+==================
9+``utils/morty.sh``
10+==================
11+
12+.. sidebar:: further reading
13+
14+ - :ref:`architecture`
15+ - :ref:`installation` (:ref:`nginx <installation nginx>` & :ref:`apache
16+ <installation apache>`)
17+ - :ref:`searx morty`
18+
19+To simplify installation and maintenance of a morty_ instance you can use the
20+script :origin:`utils/morty.sh`. In most cases you will install morty_ simply by
21+running the command:
22+
23+.. code:: bash
24+
25+ sudo -H ./utils/morty.sh install all
26+
27+The script adds a ``${SERVICE_USER}`` (default:``morty``) and installs morty_
28+into this user account:
29+
30+#. Create a separated user account (``morty``).
31+#. Download and install Go_ binary in user's $HOME (``~morty``).
32+#. Install morty_ with the package management from Go_ (``go get -v -u
33+ github.com/asciimoo/morty``)
34+#. Setup a systemd service unit :origin:`[ref]
35+ <utils/templates/lib/systemd/system/morty.service>`
36+ (``/lib/systemd/system/morty.service``).
37+
38+.. hint::
39+
40+ To add morty to your searx instance read chapter :ref:`searx morty`.
41+
42+Create user
43+===========
44+
45+.. kernel-include:: $DOCS_BUILD/includes/morty.rst
46+ :start-after: START create user
47+ :end-before: END create user
48+
49+
50+Install go
51+==========
52+
53+.. kernel-include:: $DOCS_BUILD/includes/morty.rst
54+ :start-after: START install go
55+ :end-before: END install go
56+
57+
58+Install morty
59+=============
60+
61+Install morty software and systemd unit:
62+
63+.. kernel-include:: $DOCS_BUILD/includes/morty.rst
64+ :start-after: START install morty
65+ :end-before: END install morty
66+
67+.. kernel-include:: $DOCS_BUILD/includes/morty.rst
68+ :start-after: START install systemd unit
69+ :end-before: END install systemd unit
70+
71+.. _morty.sh overview:
72+
73+Overview
74+========
75+
76+The ``--help`` output of the script is largely self-explanatory
77+(:ref:`toolboxing common`):
78+
79+.. program-output:: ../utils/morty.sh --help
80+
docs/utils/searx.sh.rstnew file+39
@@ -0,0 +1,39 @@
1+
2+.. _searx.sh:
3+
4+==================
5+``utils/searx.sh``
6+==================
7+
8+.. sidebar:: further reading
9+
10+ - :ref:`architecture`
11+ - :ref:`installation`
12+ - :ref:`installation nginx`
13+ - :ref:`installation apache`
14+
15+To simplify installation and maintenance of a searx instance you can use the
16+script :origin:`utils/searx.sh`.
17+
18+Install
19+=======
20+
21+In most cases you will install searx simply by running the command:
22+
23+.. code:: bash
24+
25+ sudo -H ./utils/searx.sh install all
26+
27+The script adds a ``${SERVICE_USER}`` (default:``searx``) and installs searx
28+into this user account. The installation is described in chapter
29+:ref:`installation basic`.
30+
31+.. _intranet reverse proxy:
32+
33+Overview
34+========
35+
36+The ``--help`` output of the script is largely self-explanatory
37+(:ref:`toolboxing common`):
38+
39+.. program-output:: ../utils/searx.sh --help
requirements-dev.txt+2
@@ -15,3 +15,5 @@ selenium==3.141.0
15linuxdoc @ git+http://github.com/return42/linuxdoc.git
15linuxdoc @ git+http://github.com/return42/linuxdoc.git
16sphinx-jinja
16sphinx-jinja
17sphinx-tabs
17sphinx-tabs
18+sphinxcontrib-programoutput
19+twine
searx/brand.py+1
@@ -1,4 +1,5 @@
1GIT_URL = 'https://github.com/asciimoo/searx'
1GIT_URL = 'https://github.com/asciimoo/searx'
2+GIT_BRANCH = 'master'
2ISSUE_URL = 'https://github.com/asciimoo/searx/issues'
3ISSUE_URL = 'https://github.com/asciimoo/searx/issues'
3SEARX_URL = 'https://searx.me'
4SEARX_URL = 'https://searx.me'
4DOCS_URL = 'https://asciimoo.github.io/searx'
5DOCS_URL = 'https://asciimoo.github.io/searx'
searx/static/plugins/js/search_on_category_select.js+18
@@ -6,19 +6,37 @@ $(document).ready(function() {
6 });
6 });
7 $(document.getElementById($(this).attr("for"))).prop('checked', true);
7 $(document.getElementById($(this).attr("for"))).prop('checked', true);
8 if($('#q').val()) {
8 if($('#q').val()) {
9+ if (getHttpRequest() == "GET") {
10+ $('#search_form').attr('action', $('#search_form').serialize());
11+ }
9 $('#search_form').submit();
12 $('#search_form').submit();
10 }
13 }
11 return false;
14 return false;
12 });
15 });
13 $('#time-range').change(function(e) {
16 $('#time-range').change(function(e) {
14 if($('#q').val()) {
17 if($('#q').val()) {
18+ if (getHttpRequest() == "GET") {
19+ $('#search_form').attr('action', $('#search_form').serialize());
20+ }
15 $('#search_form').submit();
21 $('#search_form').submit();
16 }
22 }
17 });
23 });
18 $('#language').change(function(e) {
24 $('#language').change(function(e) {
19 if($('#q').val()) {
25 if($('#q').val()) {
26+ if (getHttpRequest() == "GET") {
27+ $('#search_form').attr('action', $('#search_form').serialize());
28+ }
20 $('#search_form').submit();
29 $('#search_form').submit();
21 }
30 }
22 });
31 });
23 }
32 }
24});
33});
34+
35+function getHttpRequest() {
36+ httpRequest = "POST";
37+ urlParams = new URLSearchParams(window.location.search);
38+ if (urlParams.has('method')) {
39+ httpRequest = urlParams.get('method');
40+ }
41+ return httpRequest;
42+}
searx/templates/__common__/opensearch.xml+5−15
@@ -6,23 +6,13 @@
6 <Image>{{ urljoin(host, url_for('static', filename='img/favicon.png')) }}</Image>
6 <Image>{{ urljoin(host, url_for('static', filename='img/favicon.png')) }}</Image>
7 <LongName>searx metasearch</LongName>
7 <LongName>searx metasearch</LongName>
8 {% if opensearch_method == 'get' %}
8 {% if opensearch_method == 'get' %}
9− <Url type="text/html" method="get" template="{{ host }}search?q={searchTerms}"/>
9+ <Url rel="results" type="text/html" method="get" template="{{ host }}search?q={searchTerms}"/>
10− {% if autocomplete %}
11− <Url type="application/x-suggestions+json" method="get" template="{{ host }}autocompleter">
12− <Param name="format" value="x-suggestions" />
13− <Param name="q" value="{searchTerms}" />
14− </Url>
15− {% endif %}
16 {% else %}
10 {% else %}
17− <Url type="text/html" method="post" template="{{ host }}">
11+ <Url rel="results" type="text/html" method="post" template="{{ host }}">
18 <Param name="q" value="{searchTerms}" />
12 <Param name="q" value="{searchTerms}" />
19 </Url>
13 </Url>
14+ {% endif %}
20− {% if autocomplete %}
15+ {% if autocomplete %}
21− <!-- TODO, POST REQUEST doesn't work -->
22− <Url type="application/x-suggestions+json" method="get" template="{{ host }}autocompleter">
16+ <Url rel="suggestions" type="application/json" template="{{ host }}autocompleter"/>
23− <Param name="format" value="x-suggestions" />
24− <Param name="q" value="{searchTerms}" />
25− </Url>
26− {% endif %}
27 {% endif %}
17 {% endif %}
28</OpenSearchDescription>
18</OpenSearchDescription>
searx/webapp.py+1−1
@@ -949,7 +949,7 @@ def opensearch():
949
949
950 resp = Response(response=ret,
950 resp = Response(response=ret,
951 status=200,
951 status=200,
952− mimetype="text/xml")
952+ mimetype="application/opensearchdescription+xml")
953 return resp
953 return resp
954
954
955
955
utils/brand.env+1
@@ -1,4 +1,5 @@
1export GIT_URL='https://github.com/asciimoo/searx'
1export GIT_URL='https://github.com/asciimoo/searx'
2+export GIT_BRANCH='master'
2export ISSUE_URL='https://github.com/asciimoo/searx/issues'
3export ISSUE_URL='https://github.com/asciimoo/searx/issues'
3export SEARX_URL='https://searx.me'
4export SEARX_URL='https://searx.me'
4export DOCS_URL='https://asciimoo.github.io/searx'
5export DOCS_URL='https://asciimoo.github.io/searx'
utils/filtron.shnew file+561
@@ -0,0 +1,561 @@
1+#!/usr/bin/env bash
2+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
3+# SPDX-License-Identifier: AGPL-3.0-or-later
4+# shellcheck disable=SC2119,SC2001
5+
6+# shellcheck source=utils/lib.sh
7+source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
8+# shellcheck source=utils/brand.env
9+source "${REPO_ROOT}/utils/brand.env"
10+source_dot_config
11+source "${REPO_ROOT}/utils/lxc-searx.env"
12+in_container && lxc_set_suite_env
13+
14+# ----------------------------------------------------------------------------
15+# config
16+# ----------------------------------------------------------------------------
17+
18+PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
19+PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}"
20+
21+FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \
22+| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}"
23+[[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/
24+
25+FILTRON_ETC="/etc/filtron"
26+FILTRON_RULES="$FILTRON_ETC/rules.json"
27+
28+FILTRON_API="${FILTRON_API:-127.0.0.1:4005}"
29+FILTRON_LISTEN="${FILTRON_LISTEN:-127.0.0.1:4004}"
30+FILTRON_TARGET="${FILTRON_TARGET:-127.0.0.1:8888}"
31+
32+SERVICE_NAME="filtron"
33+SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}"
34+SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}"
35+SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}"
36+SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service"
37+# shellcheck disable=SC2034
38+SERVICE_GROUP="${SERVICE_USER}"
39+
40+# shellcheck disable=SC2034
41+SERVICE_GROUP="${SERVICE_USER}"
42+
43+GO_ENV="${SERVICE_HOME}/.go_env"
44+GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz"
45+GO_TAR=$(basename "$GO_PKG_URL")
46+
47+APACHE_FILTRON_SITE="searx.conf"
48+NGINX_FILTRON_SITE="searx.conf"
49+
50+# shellcheck disable=SC2034
51+CONFIG_FILES=(
52+ "${FILTRON_RULES}"
53+ "${SERVICE_SYSTEMD_UNIT}"
54+)
55+
56+# ----------------------------------------------------------------------------
57+usage() {
58+# ----------------------------------------------------------------------------
59+
60+ # shellcheck disable=SC1117
61+ cat <<EOF
62+usage::
63+ $(basename "$0") shell
64+ $(basename "$0") install [all|user|rules]
65+ $(basename "$0") update [filtron]
66+ $(basename "$0") remove [all]
67+ $(basename "$0") activate [service]
68+ $(basename "$0") deactivate [service]
69+ $(basename "$0") inspect [service]
70+ $(basename "$0") option [debug-on|debug-off]
71+ $(basename "$0") apache [install|remove]
72+ $(basename "$0") nginx [install|remove]
73+
74+shell
75+ start interactive shell from user ${SERVICE_USER}
76+install / remove
77+ :all: complete setup of filtron service
78+ :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME)
79+ :rules: reinstall filtron rules $FILTRON_RULES
80+update filtron
81+ Update filtron installation ($SERVICE_HOME)
82+activate service
83+ activate and start service daemon (systemd unit)
84+deactivate service
85+ stop and deactivate service daemon (systemd unit)
86+inspect service
87+ show service status and log
88+option
89+ set one of the available options
90+apache (${PUBLIC_URL})
91+ :install: apache site with a reverse proxy (ProxyPass)
92+ :remove: apache site ${APACHE_FILTRON_SITE}
93+nginx (${PUBLIC_URL})
94+ :install: nginx site with a reverse proxy (ProxyPass)
95+ :remove: nginx site ${NGINX_FILTRON_SITE}
96+
97+filtron rules: ${FILTRON_RULES}
98+
99+If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file::
100+ PUBLIC_URL : ${PUBLIC_URL}
101+ PUBLIC_HOST : ${PUBLIC_HOST}
102+ SERVICE_USER : ${SERVICE_USER}
103+ FILTRON_TARGET : ${FILTRON_TARGET}
104+ FILTRON_API : ${FILTRON_API}
105+ FILTRON_LISTEN : ${FILTRON_LISTEN}
106+EOF
107+ if in_container; then
108+ # in containers the service is listening on 0.0.0.0 (see lxc-searx.env)
109+ for ip in $(global_IPs) ; do
110+ if [[ $ip =~ .*:.* ]]; then
111+ echo " container URL (IPv6): http://[${ip#*|}]:4005/"
112+ else
113+ # IPv4:
114+ echo " container URL (IPv4): http://${ip#*|}:4005/"
115+ fi
116+ done
117+ fi
118+ [[ -n ${1} ]] && err_msg "$1"
119+}
120+
121+main() {
122+ required_commands \
123+ sudo install git wget curl \
124+ || exit
125+
126+ local _usage="unknown or missing $1 command $2"
127+
128+ case $1 in
129+ --getenv) var="$2"; echo "${!var}"; exit 0;;
130+ -h|--help) usage; exit 0;;
131+
132+ shell)
133+ sudo_or_exit
134+ interactive_shell "${SERVICE_USER}"
135+ ;;
136+ inspect)
137+ case $2 in
138+ service)
139+ sudo_or_exit
140+ inspect_service
141+ ;;
142+ *) usage "$_usage"; exit 42;;
143+ esac ;;
144+ install)
145+ rst_title "$SERVICE_NAME" part
146+ sudo_or_exit
147+ case $2 in
148+ all) install_all ;;
149+ user) assert_user ;;
150+ rules)
151+ rst_title "Re-Install filtron rules"
152+ echo
153+ install_template --no-eval "$FILTRON_RULES" root root 644
154+ systemd_restart_service "${SERVICE_NAME}"
155+ ;;
156+ *) usage "$_usage"; exit 42;;
157+ esac ;;
158+ update)
159+ sudo_or_exit
160+ case $2 in
161+ filtron) update_filtron ;;
162+ *) usage "$_usage"; exit 42;;
163+ esac ;;
164+ remove)
165+ sudo_or_exit
166+ case $2 in
167+ all) remove_all;;
168+ user) drop_service_account "${SERVICE_USER}" ;;
169+ *) usage "$_usage"; exit 42;;
170+ esac ;;
171+ activate)
172+ sudo_or_exit
173+ case $2 in
174+ service) systemd_activate_service "${SERVICE_NAME}" ;;
175+ *) usage "$_usage"; exit 42;;
176+ esac ;;
177+ deactivate)
178+ sudo_or_exit
179+ case $2 in
180+ service) systemd_deactivate_service "${SERVICE_NAME}" ;;
181+ *) usage "$_usage"; exit 42;;
182+ esac ;;
183+ apache)
184+ sudo_or_exit
185+ case $2 in
186+ install) install_apache_site ;;
187+ remove) remove_apache_site ;;
188+ *) usage "$_usage"; exit 42;;
189+ esac ;;
190+ nginx)
191+ sudo_or_exit
192+ case $2 in
193+ install) install_nginx_site ;;
194+ remove) remove_nginx_site ;;
195+ *) usage "$_usage"; exit 42;;
196+ esac ;;
197+ option)
198+ sudo_or_exit
199+ case $2 in
200+ debug-on) echo; enable_debug ;;
201+ debug-off) echo; disable_debug ;;
202+ *) usage "$_usage"; exit 42;;
203+ esac ;;
204+ doc) rst-doc ;;
205+ *) usage "unknown or missing command $1"; exit 42;;
206+ esac
207+}
208+
209+install_all() {
210+ rst_title "Install $SERVICE_NAME (service)"
211+ assert_user
212+ wait_key
213+ install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}"
214+ wait_key
215+ install_filtron
216+ wait_key
217+ systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
218+ wait_key
219+ echo
220+ if ! service_is_available "http://${FILTRON_LISTEN}" ; then
221+ err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}"
222+ fi
223+ if apache_is_installed; then
224+ info_msg "Apache is installed on this host."
225+ if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
226+ install_apache_site
227+ fi
228+ elif nginx_is_installed; then
229+ info_msg "nginx is installed on this host."
230+ if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
231+ install_nginx_site
232+ fi
233+ fi
234+ if ask_yn "Do you want to inspect the installation?" Ny; then
235+ inspect_service
236+ fi
237+
238+}
239+
240+remove_all() {
241+ rst_title "De-Install $SERVICE_NAME (service)"
242+
243+ rst_para "\
244+It goes without saying that this script can only be used to remove
245+installations that were installed with this script."
246+
247+ if ! systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then
248+ return 42
249+ fi
250+ drop_service_account "${SERVICE_USER}"
251+ rm -r "$FILTRON_ETC" 2>&1 | prefix_stdout
252+ if service_is_available "${PUBLIC_URL}"; then
253+ MSG="** Don't forget to remove your public site! (${PUBLIC_URL}) **" wait_key 10
254+ fi
255+}
256+
257+assert_user() {
258+ rst_title "user $SERVICE_USER" section
259+ echo
260+ tee_stderr 1 <<EOF | bash | prefix_stdout
261+useradd --shell /bin/bash --system \
262+ --home-dir "$SERVICE_HOME" \
263+ --comment 'Reverse HTTP proxy to filter requests' $SERVICE_USER
264+mkdir "$SERVICE_HOME"
265+chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
266+groups $SERVICE_USER
267+EOF
268+ SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)"
269+ export SERVICE_HOME
270+ echo "export SERVICE_HOME=$SERVICE_HOME"
271+
272+ cat > "$GO_ENV" <<EOF
273+export GOPATH=\$HOME/go-apps
274+export PATH=\$PATH:\$HOME/local/go/bin:\$GOPATH/bin
275+EOF
276+ echo "Environment $GO_ENV has been setup."
277+
278+ tee_stderr <<EOF | sudo -i -u "$SERVICE_USER"
279+grep -qFs -- 'source $GO_ENV' ~/.profile || echo 'source $GO_ENV' >> ~/.profile
280+EOF
281+}
282+
283+filtron_is_installed() {
284+ [[ -f $SERVICE_HOME/go-apps/bin/filtron ]]
285+}
286+
287+_svcpr=" ${_Yellow}|${SERVICE_USER}|${_creset} "
288+
289+install_filtron() {
290+ rst_title "Install filtron in user's ~/go-apps" section
291+ echo
292+ tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
293+go get -v -u github.com/asciimoo/filtron
294+EOF
295+ install_template --no-eval "$FILTRON_RULES" root root 644
296+}
297+
298+update_filtron() {
299+ rst_title "Update filtron" section
300+ echo
301+ tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
302+go get -v -u github.com/asciimoo/filtron
303+EOF
304+}
305+
306+inspect_service() {
307+
308+ rst_title "service status & log"
309+
310+ cat <<EOF
311+
312+sourced ${DOT_CONFIG#"$REPO_ROOT/"} :
313+
314+ PUBLIC_URL : ${PUBLIC_URL}
315+ PUBLIC_HOST : ${PUBLIC_HOST}
316+ FILTRON_URL_PATH : ${FILTRON_URL_PATH}
317+ FILTRON_API : ${FILTRON_API}
318+ FILTRON_LISTEN : ${FILTRON_LISTEN}
319+ FILTRON_TARGET : ${FILTRON_TARGET}
320+
321+EOF
322+
323+ if service_account_is_available "$SERVICE_USER"; then
324+ info_msg "service account $SERVICE_USER available."
325+ else
326+ err_msg "service account $SERVICE_USER not available!"
327+ fi
328+ if go_is_available "$SERVICE_USER"; then
329+ info_msg "~$SERVICE_USER: go is installed"
330+ else
331+ err_msg "~$SERVICE_USER: go is not installed"
332+ fi
333+ if filtron_is_installed; then
334+ info_msg "~$SERVICE_USER: filtron app is installed"
335+ else
336+ err_msg "~$SERVICE_USER: filtron app is not installed!"
337+ fi
338+
339+ if ! service_is_available "http://${FILTRON_API}"; then
340+ err_msg "API not available at: http://${FILTRON_API}"
341+ fi
342+
343+ if ! service_is_available "http://${FILTRON_LISTEN}" ; then
344+ err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}"
345+ fi
346+
347+ if service_is_available "http://${FILTRON_TARGET}" ; then
348+ info_msg "Filtron's target is available at: http://${FILTRON_TARGET}"
349+ fi
350+
351+ if ! service_is_available "${PUBLIC_URL}"; then
352+ warn_msg "Public service at ${PUBLIC_URL} is not available!"
353+ if ! in_container; then
354+ warn_msg "Check if public name is correct and routed or use the public IP from above."
355+ fi
356+ fi
357+
358+ if in_container; then
359+ lxc_suite_info
360+ else
361+ info_msg "public URL --> ${PUBLIC_URL}"
362+ info_msg "internal URL --> http://${FILTRON_LISTEN}"
363+ fi
364+
365+
366+ local _debug_on
367+ if ask_yn "Enable filtron debug mode?"; then
368+ enable_debug
369+ _debug_on=1
370+ fi
371+ echo
372+ systemctl --no-pager -l status "${SERVICE_NAME}"
373+ echo
374+
375+ info_msg "public URL --> ${PUBLIC_URL}"
376+ # shellcheck disable=SC2059
377+ printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log"
378+ read -r -s -n1 -t 5
379+ echo
380+ while true; do
381+ trap break 2
382+ journalctl -f -u "${SERVICE_NAME}"
383+ done
384+
385+ if [[ $_debug_on == 1 ]]; then
386+ disable_debug
387+ fi
388+ return 0
389+}
390+
391+
392+enable_debug() {
393+ info_msg "try to enable debug mode ..."
394+ python <<EOF
395+import sys, json
396+
397+debug = {
398+ u'name': u'debug request'
399+ , u'filters': []
400+ , u'interval': 0
401+ , u'limit': 0
402+ , u'actions': [{u'name': u'log'}]
403+}
404+
405+with open('$FILTRON_RULES') as rules:
406+ j = json.load(rules)
407+
408+pos = None
409+for i in range(len(j)):
410+ if j[i].get('name') == 'debug request':
411+ pos = i
412+ break
413+if pos is not None:
414+ j[pos] = debug
415+else:
416+ j.append(debug)
417+with open('$FILTRON_RULES', 'w') as rules:
418+ json.dump(j, rules, indent=2, sort_keys=True)
419+
420+EOF
421+ systemctl restart "${SERVICE_NAME}.service"
422+}
423+
424+disable_debug() {
425+ info_msg "try to disable debug mode ..."
426+ python <<EOF
427+import sys, json
428+with open('$FILTRON_RULES') as rules:
429+ j = json.load(rules)
430+
431+pos = None
432+for i in range(len(j)):
433+ if j[i].get('name') == 'debug request':
434+ pos = i
435+ break
436+if pos is not None:
437+ del j[pos]
438+ with open('$FILTRON_RULES', 'w') as rules:
439+ json.dump(j, rules, indent=2, sort_keys=True)
440+EOF
441+ systemctl restart "${SERVICE_NAME}.service"
442+}
443+
444+install_apache_site() {
445+
446+ rst_title "Install Apache site $APACHE_FILTRON_SITE"
447+
448+ rst_para "\
449+This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SITE})"
450+
451+ ! apache_is_installed && info_msg "Apache is not installed."
452+
453+ if ! ask_yn "Do you really want to continue?" Yn; then
454+ return
455+ else
456+ install_apache
457+ fi
458+
459+ "${REPO_ROOT}/utils/searx.sh" install uwsgi
460+
461+ apache_install_site --variant=filtron "${APACHE_FILTRON_SITE}"
462+
463+ info_msg "testing public url .."
464+ if ! service_is_available "${PUBLIC_URL}"; then
465+ err_msg "Public service at ${PUBLIC_URL} is not available!"
466+ fi
467+}
468+
469+remove_apache_site() {
470+
471+ rst_title "Remove Apache site $APACHE_FILTRON_SITE"
472+
473+ rst_para "\
474+This removes apache site ${APACHE_FILTRON_SITE}."
475+
476+ ! apache_is_installed && err_msg "Apache is not installed."
477+
478+ if ! ask_yn "Do you really want to continue?" Yn; then
479+ return
480+ fi
481+
482+ apache_remove_site "$APACHE_FILTRON_SITE"
483+
484+}
485+
486+install_nginx_site() {
487+
488+ rst_title "Install nginx site $NGINX_FILTRON_SITE"
489+
490+ rst_para "\
491+This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_FILTRON_SITE})"
492+
493+ ! nginx_is_installed && info_msg "nginx is not installed."
494+
495+ if ! ask_yn "Do you really want to continue?" Yn; then
496+ return
497+ else
498+ install_nginx
499+ fi
500+
501+ "${REPO_ROOT}/utils/searx.sh" install uwsgi
502+
503+ # shellcheck disable=SC2034
504+ SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC)
505+ # shellcheck disable=SC2034
506+ SEARX_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_URL_PATH)
507+ nginx_install_app --variant=filtron "${NGINX_FILTRON_SITE}"
508+
509+ info_msg "testing public url .."
510+ if ! service_is_available "${PUBLIC_URL}"; then
511+ err_msg "Public service at ${PUBLIC_URL} is not available!"
512+ fi
513+}
514+
515+remove_nginx_site() {
516+
517+ rst_title "Remove nginx site $NGINX_FILTRON_SITE"
518+
519+ rst_para "\
520+This removes nginx site ${NGINX_FILTRON_SITE}."
521+
522+ ! nginx_is_installed && err_msg "nginx is not installed."
523+
524+ if ! ask_yn "Do you really want to continue?" Yn; then
525+ return
526+ fi
527+
528+ nginx_remove_site "$FILTRON_FILTRON_SITE"
529+
530+}
531+
532+
533+rst-doc() {
534+
535+ eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/filtron.rst")\""
536+
537+ echo -e "\n.. START install systemd unit"
538+ cat <<EOF
539+.. tabs::
540+
541+ .. group-tab:: systemd
542+
543+ .. code:: bash
544+
545+EOF
546+ eval "echo \"$(< "${TEMPLATES}/${SERVICE_SYSTEMD_UNIT}")\"" | prefix_stdout " "
547+ echo -e "\n.. END install systemd unit"
548+
549+ # for DIST_NAME in ubuntu-20.04 arch fedora; do
550+ # (
551+ # DIST_ID=${DIST_NAME%-*}
552+ # DIST_VERS=${DIST_NAME#*-}
553+ # [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS=
554+ # # ...
555+ # )
556+ # done
557+}
558+
559+# ----------------------------------------------------------------------------
560+main "$@"
561+# ----------------------------------------------------------------------------
utils/lib.shnew file+1519
@@ -0,0 +1,1519 @@
1+#!/usr/bin/env bash
2+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
3+# SPDX-License-Identifier: AGPL-3.0-or-later
4+# shellcheck disable=SC2059,SC1117
5+
6+# ubuntu, debian, arch, fedora ...
7+DIST_ID=$(source /etc/os-release; echo "$ID");
8+# shellcheck disable=SC2034
9+DIST_VERS=$(source /etc/os-release; echo "$VERSION_ID");
10+
11+ADMIN_NAME="${ADMIN_NAME:-$(git config user.name)}"
12+ADMIN_NAME="${ADMIN_NAME:-$USER}"
13+
14+ADMIN_EMAIL="${ADMIN_EMAIL:-$(git config user.email)}"
15+ADMIN_EMAIL="${ADMIN_EMAIL:-$USER@$(hostname)}"
16+
17+if [[ -z "${REPO_ROOT}" ]]; then
18+ REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")
19+ while [ -h "${REPO_ROOT}" ] ; do
20+ REPO_ROOT=$(readlink "${REPO_ROOT}")
21+ done
22+ REPO_ROOT=$(cd "${REPO_ROOT}/.." && pwd -P )
23+fi
24+
25+if [[ -z ${TEMPLATES} ]]; then
26+ TEMPLATES="${REPO_ROOT}/utils/templates"
27+fi
28+
29+if [[ -z "$CACHE" ]]; then
30+ CACHE="${REPO_ROOT}/cache"
31+fi
32+
33+if [[ -z ${DIFF_CMD} ]]; then
34+ DIFF_CMD="diff -u"
35+ if command -v colordiff >/dev/null; then
36+ DIFF_CMD="colordiff -u"
37+ fi
38+fi
39+
40+DOT_CONFIG="${DOT_CONFIG:-${REPO_ROOT}/.config.sh}"
41+
42+source_dot_config() {
43+ if [[ ! -e "${DOT_CONFIG}" ]]; then
44+ err_msg "configuration does not extsts at: ${DOT_CONFIG}"
45+ return 42
46+ fi
47+ # shellcheck disable=SC1090
48+ source "${DOT_CONFIG}"
49+}
50+
51+sudo_or_exit() {
52+ # usage: sudo_or_exit
53+
54+ if [ ! "$(id -u)" -eq 0 ]; then
55+ err_msg "this command requires root (sudo) privilege!" >&2
56+ exit 42
57+ fi
58+}
59+
60+required_commands() {
61+
62+ # usage: required_commands [cmd1 ...]
63+
64+ local exit_val=0
65+ while [ -n "$1" ]; do
66+
67+ if ! command -v "$1" &>/dev/null; then
68+ err_msg "missing command $1"
69+ exit_val=42
70+ fi
71+ shift
72+ done
73+ return $exit_val
74+}
75+
76+# colors
77+# ------
78+
79+# shellcheck disable=SC2034
80+set_terminal_colors() {
81+ _colors=8
82+ _creset='\e[0m' # reset all attributes
83+
84+ _Black='\e[0;30m'
85+ _White='\e[1;37m'
86+ _Red='\e[0;31m'
87+ _Green='\e[0;32m'
88+ _Yellow='\e[0;33m'
89+ _Blue='\e[0;34m'
90+ _Violet='\e[0;35m'
91+ _Cyan='\e[0;36m'
92+
93+ _BBlack='\e[1;30m'
94+ _BWhite='\e[1;37m'
95+ _BRed='\e[1;31m'
96+ _BGreen='\e[1;32m'
97+ _BYellow='\e[1;33m'
98+ _BBlue='\e[1;34m'
99+ _BPurple='\e[1;35m'
100+ _BCyan='\e[1;36m'
101+}
102+
103+if [ ! -p /dev/stdout ]; then
104+ set_terminal_colors
105+fi
106+
107+# reST
108+# ----
109+
110+if command -v fmt >/dev/null; then
111+ export FMT="fmt -u"
112+else
113+ export FMT="cat"
114+fi
115+
116+rst_title() {
117+ # usage: rst_title <header-text> [part|chapter|section]
118+
119+ case ${2-chapter} in
120+ part) printf "\n${_BGreen}${1//?/=}${_creset}\n${_BCyan}${1}${_creset}\n${_BGreen}${1//?/=}${_creset}\n";;
121+ chapter) printf "\n${_BCyan}${1}${_creset}\n${_BGreen}${1//?/=}${_creset}\n";;
122+ section) printf "\n${_BCyan}${1}${_creset}\n${_BGreen}${1//?/-}${_creset}\n";;
123+ *)
124+ err_msg "invalid argument '${2}' in line $(caller)"
125+ return 42
126+ ;;
127+ esac
128+}
129+
130+rst_para() {
131+ # usage: RST_INDENT=1 rst_para "lorem ipsum ..."
132+ local prefix=''
133+ if [[ -n $RST_INDENT ]] && [[ $RST_INDENT -gt 0 ]]; then
134+ prefix="$(for i in $(seq 1 "$RST_INDENT"); do printf " "; done)"
135+ echo -en "\n$*\n" | $FMT | prefix_stdout "$prefix"
136+ else
137+ echo -en "\n$*\n" | $FMT
138+ fi
139+}
140+
141+die() {
142+ echo -e "${_BRed}ERROR:${_creset} ${BASH_SOURCE[1]}: line ${BASH_LINENO[0]}: ${2-died ${1-1}}" >&2;
143+ exit "${1-1}"
144+}
145+
146+die_caller() {
147+ echo -e "${_BRed}ERROR:${_creset} ${BASH_SOURCE[2]}: line ${BASH_LINENO[1]}: ${FUNCNAME[1]}(): ${2-died ${1-1}}" >&2;
148+ exit "${1-1}"
149+}
150+
151+err_msg() { echo -e "${_BRed}ERROR:${_creset} $*" >&2; }
152+warn_msg() { echo -e "${_BBlue}WARN:${_creset} $*" >&2; }
153+info_msg() { echo -e "${_BYellow}INFO:${_creset} $*" >&2; }
154+
155+clean_stdin() {
156+ if [[ $(uname -s) != 'Darwin' ]]; then
157+ while read -r -n1 -t 0.1; do : ; done
158+ fi
159+}
160+
161+wait_key(){
162+ # usage: waitKEY [<timeout in sec>]
163+
164+ clean_stdin
165+ local _t=$1
166+ local msg="${MSG}"
167+ [[ -z "$msg" ]] && msg="${_Green}** press any [${_BCyan}KEY${_Green}] to continue **${_creset}"
168+
169+ [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT
170+ [[ -n $_t ]] && _t="-t $_t"
171+ printf "$msg"
172+ # shellcheck disable=SC2086
173+ read -r -s -n1 $_t
174+ echo
175+ clean_stdin
176+}
177+
178+ask_yn() {
179+ # usage: ask_yn <prompt-text> [Ny|Yn] [<timeout in sec>]
180+
181+ local EXIT_YES=0 # exit status 0 --> successful
182+ local EXIT_NO=1 # exit status 1 --> error code
183+
184+ local _t=$3
185+ [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT
186+ [[ -n $_t ]] && _t="-t $_t"
187+ case "${FORCE_SELECTION:-${2}}" in
188+ Y) return ${EXIT_YES} ;;
189+ N) return ${EXIT_NO} ;;
190+ Yn)
191+ local exit_val=${EXIT_YES}
192+ local choice="[${_BGreen}YES${_creset}/no]"
193+ local default="Yes"
194+ ;;
195+ *)
196+ local exit_val=${EXIT_NO}
197+ local choice="[${_BGreen}NO${_creset}/yes]"
198+ local default="No"
199+ ;;
200+ esac
201+ echo
202+ while true; do
203+ clean_stdin
204+ printf "$1 ${choice} "
205+ # shellcheck disable=SC2086
206+ read -r -n1 $_t
207+ if [[ -z $REPLY ]]; then
208+ printf "$default\n"; break
209+ elif [[ $REPLY =~ ^[Yy]$ ]]; then
210+ exit_val=${EXIT_YES}
211+ printf "\n"
212+ break
213+ elif [[ $REPLY =~ ^[Nn]$ ]]; then
214+ exit_val=${EXIT_NO}
215+ printf "\n"
216+ break
217+ fi
218+ _t=""
219+ err_msg "invalid choice"
220+ done
221+ clean_stdin
222+ return $exit_val
223+}
224+
225+tee_stderr () {
226+
227+ # usage::
228+ # tee_stderr 1 <<EOF | python -i
229+ # print("hello")
230+ # EOF
231+ # ...
232+ # >>> print("hello")
233+ # hello
234+
235+ local _t="0";
236+ if [[ -n $1 ]] ; then _t="$1"; fi
237+
238+ (while read -r line; do
239+ # shellcheck disable=SC2086
240+ sleep $_t
241+ echo -e "$line" >&2
242+ echo "$line"
243+ done)
244+}
245+
246+prefix_stdout () {
247+ # usage: <cmd> | prefix_stdout [prefix]
248+
249+ local prefix="${_BYellow}-->|${_creset}"
250+
251+ if [[ -n $1 ]] ; then prefix="$1"; fi
252+
253+ # shellcheck disable=SC2162
254+ (while IFS= read line; do
255+ echo -e "${prefix}$line"
256+ done)
257+}
258+
259+append_line() {
260+
261+ # usage: append_line <line> <file>
262+ #
263+ # Append line if not exists, create file if not exists. E.g::
264+ #
265+ # append_line 'source ~/.foo' ~/bashrc
266+
267+ local LINE=$1
268+ local FILE=$2
269+ grep -qFs -- "$LINE" "$FILE" || echo "$LINE" >> "$FILE"
270+}
271+
272+cache_download() {
273+
274+ # usage: cache_download <url> <local-filename>
275+
276+ local exit_value=0
277+
278+ if [[ -n ${SUDO_USER} ]]; then
279+ sudo -u "${SUDO_USER}" mkdir -p "${CACHE}"
280+ else
281+ mkdir -p "${CACHE}"
282+ fi
283+
284+ if [[ -f "${CACHE}/$2" ]] ; then
285+ info_msg "already cached: $1"
286+ info_msg " --> ${CACHE}/$2"
287+ fi
288+
289+ if [[ ! -f "${CACHE}/$2" ]]; then
290+ info_msg "caching: $1"
291+ info_msg " --> ${CACHE}/$2"
292+ if [[ -n ${SUDO_USER} ]]; then
293+ sudo -u "${SUDO_USER}" wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$?
294+ else
295+ wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$?
296+ fi
297+ if [[ ! $exit_value = 0 ]]; then
298+ err_msg "failed to download: $1"
299+ fi
300+ fi
301+}
302+
303+backup_file() {
304+
305+ # usage: backup_file /path/to/file.foo
306+
307+ local stamp
308+ stamp=$(date +"_%Y%m%d_%H%M%S")
309+ info_msg "create backup: ${1}${stamp}"
310+ cp -a "${1}" "${1}${stamp}"
311+}
312+
313+choose_one() {
314+
315+ # usage:
316+ #
317+ # DEFAULT_SELECT= 2 \
318+ # choose_one <name> "your selection?" "Coffee" "Coffee with milk"
319+
320+ local default=${DEFAULT_SELECT-1}
321+ local REPLY
322+ local env_name=$1 && shift
323+ local choice=$1;
324+ local max="${#@}"
325+ local _t
326+ [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT
327+ [[ -n $_t ]] && _t="-t $_t"
328+
329+ list=("$@")
330+ echo -e "${_BGreen}Menu::${_creset}"
331+ for ((i=1; i<= $((max -1)); i++)); do
332+ if [[ "$i" == "$default" ]]; then
333+ echo -e " ${_BGreen}$i.${_creset}) ${list[$i]} [default]"
334+ else
335+ echo -e " $i.) ${list[$i]}"
336+ fi
337+ done
338+ while true; do
339+ clean_stdin
340+ printf "$1 [${_BGreen}$default${_creset}] "
341+
342+ if (( 10 > max )); then
343+ # shellcheck disable=SC2086
344+ read -r -n1 $_t
345+ else
346+ # shellcheck disable=SC2086,SC2229
347+ read -r $_t
348+ fi
349+ # selection fits
350+ [[ $REPLY =~ ^-?[0-9]+$ ]] && (( REPLY > 0 )) && (( REPLY < max )) && break
351+
352+ # take default
353+ [[ -z $REPLY ]] && REPLY=$default && break
354+
355+ _t=""
356+ err_msg "invalid choice"
357+ done
358+ eval "$env_name"='${list[${REPLY}]}'
359+ echo
360+ clean_stdin
361+}
362+
363+install_template() {
364+
365+ # usage:
366+ #
367+ # install_template [--no-eval] [--variant=<name>] \
368+ # {file} [{owner} [{group} [{chmod}]]]
369+ #
370+ # E.g. the origin of variant 'raw' of /etc/updatedb.conf is::
371+ #
372+ # ${TEMPLATES}/etc/updatedb.conf:raw
373+ #
374+ # To install variant 'raw' of /etc/updatedb.conf without evaluated
375+ # replacements you can use::
376+ #
377+ # install_template --variant=raw --no-eval \
378+ # /etc/updatedb.conf root root 644
379+
380+ local _reply=""
381+ local do_eval=1
382+ local variant=""
383+ local pos_args=("$0")
384+
385+ for i in "$@"; do
386+ case $i in
387+ --no-eval) do_eval=0; shift ;;
388+ --variant=*) variant=":${i#*=}"; shift ;;
389+ *) pos_args+=("$i") ;;
390+ esac
391+ done
392+
393+ local dst="${pos_args[1]}"
394+ local template_origin="${TEMPLATES}${dst}${variant}"
395+ local template_file="${TEMPLATES}${dst}"
396+
397+ local owner="${pos_args[2]-$(id -un)}"
398+ local group="${pos_args[3]-$(id -gn)}"
399+ local chmod="${pos_args[4]-644}"
400+
401+ info_msg "install (eval=$do_eval): ${dst}"
402+ [[ -n $variant ]] && info_msg "variant --> ${variant}"
403+
404+ if [[ ! -f "${template_origin}" ]] ; then
405+ err_msg "${template_origin} does not exists"
406+ err_msg "... can't install $dst"
407+ wait_key 30
408+ return 42
409+ fi
410+
411+ if [[ "$do_eval" == "1" ]]; then
412+ template_file="${CACHE}${dst}${variant}"
413+ info_msg "BUILD template ${template_file}"
414+ if [[ -n ${SUDO_USER} ]]; then
415+ sudo -u "${SUDO_USER}" mkdir -p "$(dirname "${template_file}")"
416+ else
417+ mkdir -p "$(dirname "${template_file}")"
418+ fi
419+ # shellcheck disable=SC2086
420+ eval "echo \"$(cat ${template_origin})\"" > "${template_file}"
421+ if [[ -n ${SUDO_USER} ]]; then
422+ chown "${SUDO_USER}:${SUDO_USER}" "${template_file}"
423+ fi
424+ else
425+ template_file=$template_origin
426+ fi
427+
428+ mkdir -p "$(dirname "${dst}")"
429+
430+ if [[ ! -f "${dst}" ]]; then
431+ info_msg "install: ${template_file}"
432+ sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \
433+ "${template_file}" "${dst}" | prefix_stdout
434+ return $?
435+ fi
436+
437+ if [[ -f "${dst}" ]] && cmp --silent "${template_file}" "${dst}" ; then
438+ info_msg "file ${dst} allready installed"
439+ return 0
440+ fi
441+
442+ info_msg "diffrent file ${dst} allready exists on this host"
443+
444+ while true; do
445+ choose_one _reply "choose next step with file $dst" \
446+ "replace file" \
447+ "leave file unchanged" \
448+ "interactiv shell" \
449+ "diff files"
450+
451+ case $_reply in
452+ "replace file")
453+ info_msg "install: ${template_file}"
454+ sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \
455+ "${template_file}" "${dst}" | prefix_stdout
456+ break
457+ ;;
458+ "leave file unchanged")
459+ break
460+ ;;
461+ "interactiv shell")
462+ echo -e "// edit ${_Red}${dst}${_creset} to your needs"
463+ echo -e "// exit with [${_BCyan}CTRL-D${_creset}]"
464+ sudo -H -u "${owner}" -i
465+ $DIFF_CMD "${dst}" "${template_file}"
466+ echo
467+ echo -e "// ${_BBlack}did you edit file ...${_creset}"
468+ echo -en "// ${_Red}${dst}${_creset}"
469+ if ask_yn "//${_BBlack}... to your needs?${_creset}"; then
470+ break
471+ fi
472+ ;;
473+ "diff files")
474+ $DIFF_CMD "${dst}" "${template_file}" | prefix_stdout
475+ esac
476+ done
477+}
478+
479+
480+service_is_available() {
481+
482+ # usage: service_is_available <URL>
483+
484+ [[ -z $1 ]] && die_caller 42 "missing argument <URL>"
485+ local URL="$1"
486+ http_code=$(curl -H 'Cache-Control: no-cache' \
487+ --silent -o /dev/null --head --write-out '%{http_code}' --insecure \
488+ "${URL}")
489+ exit_val=$?
490+ if [[ $exit_val = 0 ]]; then
491+ info_msg "got $http_code from ${URL}"
492+ fi
493+ case "$http_code" in
494+ 404|410|423) exit_val=$http_code;;
495+ esac
496+ return "$exit_val"
497+}
498+
499+# golang
500+# ------
501+
502+go_is_available() {
503+
504+ # usage: go_is_available $SERVICE_USER && echo "go is installed!"
505+
506+ sudo -i -u "${1}" which go &>/dev/null
507+}
508+
509+install_go() {
510+
511+ # usage: install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}"
512+
513+ local _svcpr=" ${_Yellow}|${3}|${_creset} "
514+
515+ rst_title "Install Go in user's HOME" section
516+
517+ rst_para "download and install go binary .."
518+ cache_download "${1}" "${2}"
519+
520+ tee_stderr 0.1 <<EOF | sudo -i -u "${3}" | prefix_stdout "$_svcpr"
521+echo \$PATH
522+echo \$GOPATH
523+mkdir -p \$HOME/local
524+rm -rf \$HOME/local/go
525+tar -C \$HOME/local -xzf ${CACHE}/${2}
526+EOF
527+ sudo -i -u "${3}" <<EOF | prefix_stdout
528+! which go >/dev/null && echo "ERROR - Go Installation not found in PATH!?!"
529+which go >/dev/null && go version && echo "congratulations -- Go installation OK :)"
530+EOF
531+}
532+
533+# system accounts
534+# ---------------
535+
536+service_account_is_available() {
537+
538+ # usage: service_account_is_available "$SERVICE_USER" && echo "OK"
539+
540+ sudo -i -u "$1" echo \$HOME &>/dev/null
541+}
542+
543+drop_service_account() {
544+
545+ # usage: drop_service_account "${SERVICE_USER}"
546+
547+ rst_title "Drop ${1} HOME" section
548+ if ask_yn "Do you really want to drop ${1} home folder?"; then
549+ userdel -r -f "${1}" 2>&1 | prefix_stdout
550+ else
551+ rst_para "Leave HOME folder $(du -sh "${1}") unchanged."
552+ fi
553+}
554+
555+interactive_shell(){
556+
557+ # usage: interactive_shell "${SERVICE_USER}"
558+
559+ echo -e "// exit with [${_BCyan}CTRL-D${_creset}]"
560+ sudo -H -u "${1}" -i
561+}
562+
563+
564+# systemd
565+# -------
566+
567+SYSTEMD_UNITS="${SYSTEMD_UNITS:-/lib/systemd/system}"
568+
569+systemd_install_service() {
570+
571+ # usage: systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
572+
573+ rst_title "Install System-D Unit ${1}" section
574+ echo
575+ install_template "${2}" root root 644
576+ wait_key
577+ systemd_activate_service "${1}"
578+}
579+
580+systemd_remove_service() {
581+
582+ # usage: systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
583+
584+ if ! ask_yn "Do you really want to deinstall systemd unit ${1}?"; then
585+ return 42
586+ fi
587+ systemd_deactivate_service "${1}"
588+ rm "${2}" 2>&1 | prefix_stdout
589+}
590+
591+systemd_activate_service() {
592+
593+ # usage: systemd_activate_service "${SERVICE_NAME}"
594+
595+ rst_title "Activate ${1} (service)" section
596+ echo
597+ tee_stderr <<EOF | bash 2>&1
598+systemctl enable ${1}.service
599+systemctl restart ${1}.service
600+EOF
601+ tee_stderr <<EOF | bash 2>&1
602+systemctl status --no-pager ${1}.service
603+EOF
604+}
605+
606+systemd_deactivate_service() {
607+
608+ # usage: systemd_deactivate_service "${SERVICE_NAME}"
609+
610+ rst_title "De-Activate ${1} (service)" section
611+ echo
612+ tee_stderr <<EOF | bash 2>&1 | prefix_stdout
613+systemctl stop ${1}.service
614+systemctl disable ${1}.service
615+EOF
616+}
617+
618+systemd_restart_service() {
619+
620+ # usage: systemd_restart_service "${SERVICE_NAME}"
621+
622+ rst_title "Restart ${1} (service)" section
623+ echo
624+ tee_stderr <<EOF | bash 2>&1
625+systemctl restart ${1}.service
626+EOF
627+ tee_stderr <<EOF | bash 2>&1
628+systemctl status --no-pager ${1}.service
629+EOF
630+}
631+
632+
633+# nginx
634+# -----
635+
636+nginx_distro_setup() {
637+ # shellcheck disable=SC2034
638+
639+ NGINX_DEFAULT_SERVER=/etc/nginx/nginx.conf
640+
641+ # Including *location* directives from a dedicated config-folder into the
642+ # server directive is, what what fedora (already) does.
643+ NGINX_APPS_ENABLED="/etc/nginx/default.d"
644+
645+ # We add a apps-available folder and linking configurations into the
646+ # NGINX_APPS_ENABLED folder. See also nginx_include_apps_enabled().
647+ NGINX_APPS_AVAILABLE="/etc/nginx/default.apps-available"
648+
649+ case $DIST_ID-$DIST_VERS in
650+ ubuntu-*|debian-*)
651+ NGINX_PACKAGES="nginx"
652+ NGINX_DEFAULT_SERVER=/etc/nginx/sites-available/default
653+ ;;
654+ arch-*)
655+ NGINX_PACKAGES="nginx-mainline"
656+ ;;
657+ fedora-*)
658+ NGINX_PACKAGES="nginx"
659+ ;;
660+ *)
661+ err_msg "$DIST_ID-$DIST_VERS: nginx not yet implemented"
662+ ;;
663+ esac
664+}
665+nginx_distro_setup
666+
667+install_nginx(){
668+ info_msg "installing nginx ..."
669+ pkg_install "${NGINX_PACKAGES}"
670+ case $DIST_ID-$DIST_VERS in
671+ arch-*|fedora-*)
672+ systemctl enable nginx
673+ systemctl start nginx
674+ ;;
675+ esac
676+}
677+
678+nginx_is_installed() {
679+ command -v nginx &>/dev/null
680+}
681+
682+nginx_reload() {
683+
684+ info_msg "reload nginx .."
685+ echo
686+ if ! nginx -t; then
687+ err_msg "testing nginx configuration failed"
688+ return 42
689+ fi
690+ systemctl restart nginx
691+}
692+
693+nginx_install_app() {
694+
695+ # usage: nginx_install_app [<template option> ...] <myapp>
696+ #
697+ # <template option>: see install_template
698+
699+ local template_opts=()
700+ local pos_args=("$0")
701+
702+ for i in "$@"; do
703+ case $i in
704+ -*) template_opts+=("$i");;
705+ *) pos_args+=("$i");;
706+ esac
707+ done
708+
709+ nginx_include_apps_enabled "${NGINX_DEFAULT_SERVER}"
710+
711+ install_template "${template_opts[@]}" \
712+ "${NGINX_APPS_AVAILABLE}/${pos_args[1]}" \
713+ root root 644
714+ nginx_enable_app "${pos_args[1]}"
715+ info_msg "installed nginx app: ${pos_args[1]}"
716+}
717+
718+nginx_include_apps_enabled() {
719+
720+ # Add the *NGINX_APPS_ENABLED* infrastruture to a nginx server block. Such
721+ # infrastruture is already known from fedora, including apps (location
722+ # directives) from the /etc/nginx/default.d folder into the *default* nginx
723+ # server.
724+
725+ # usage: nginx_include_apps_enabled <config-file>
726+ #
727+ # config-file: Config file with server directive in.
728+
729+ [[ -z $1 ]] && die_caller 42 "missing argument <config-file>"
730+ local server_conf="$1"
731+
732+ # include /etc/nginx/default.d/*.conf;
733+ local include_directive="include ${NGINX_APPS_ENABLED}/*.conf;"
734+ local include_directive_re="^\s*include ${NGINX_APPS_ENABLED}/\*\.conf;"
735+
736+ info_msg "checking existence: '${include_directive}' in file ${server_conf}"
737+ if grep "${include_directive_re}" "${server_conf}"; then
738+ info_msg "OK, already exists."
739+ return
740+ fi
741+
742+ info_msg "add missing directive: '${include_directive}'"
743+ cp "${server_conf}" "${server_conf}.bak"
744+
745+ (
746+ local line
747+ local stage=0
748+ while IFS= read -r line
749+ do
750+ echo "$line"
751+ if [[ $stage = 0 ]]; then
752+ if [[ $line =~ ^[[:space:]]*server*[[:space:]]*\{ ]]; then
753+ stage=1
754+ fi
755+ fi
756+
757+ if [[ $stage = 1 ]]; then
758+ echo " # Load configuration files for the default server block."
759+ echo " $include_directive"
760+ echo ""
761+ stage=2
762+ fi
763+ done < "${server_conf}.bak"
764+ ) > "${server_conf}"
765+
766+}
767+
768+nginx_remove_app() {
769+
770+ # usage: nginx_remove_app <myapp.conf>
771+
772+ info_msg "remove nginx app: $1"
773+ nginx_dissable_app "$1"
774+ rm -f "${NGINX_APPS_AVAILABLE}/$1"
775+}
776+
777+nginx_enable_app() {
778+
779+ # usage: nginx_enable_app <myapp.conf>
780+
781+ local CONF="$1"
782+
783+ info_msg "enable nginx app: ${CONF}"
784+ mkdir -p "${NGINX_APPS_ENABLED}"
785+ rm -f "${NGINX_APPS_ENABLED}/${CONF}"
786+ ln -s "${NGINX_APPS_AVAILABLE}/${CONF}" "${NGINX_APPS_ENABLED}/${CONF}"
787+ nginx_reload
788+}
789+
790+nginx_dissable_app() {
791+
792+ # usage: nginx_disable_app <myapp.conf>
793+
794+ local CONF="$1"
795+
796+ info_msg "disable nginx app: ${CONF}"
797+ rm -f "${NGINX_APPS_ENABLED}/${CONF}"
798+ nginx_reload
799+}
800+
801+
802+# Apache
803+# ------
804+
805+apache_distro_setup() {
806+ # shellcheck disable=SC2034
807+ case $DIST_ID-$DIST_VERS in
808+ ubuntu-*|debian-*)
809+ # debian uses the /etc/apache2 path, while other distros use
810+ # the apache default at /etc/httpd
811+ APACHE_SITES_AVAILABLE="/etc/apache2/sites-available"
812+ APACHE_SITES_ENABLED="/etc/apache2/sites-enabled"
813+ APACHE_MODULES="/usr/lib/apache2/modules"
814+ APACHE_PACKAGES="apache2"
815+ ;;
816+ arch-*)
817+ APACHE_SITES_AVAILABLE="/etc/httpd/sites-available"
818+ APACHE_SITES_ENABLED="/etc/httpd/sites-enabled"
819+ APACHE_MODULES="modules"
820+ APACHE_PACKAGES="apache"
821+ ;;
822+ fedora-*)
823+ APACHE_SITES_AVAILABLE="/etc/httpd/sites-available"
824+ APACHE_SITES_ENABLED="/etc/httpd/sites-enabled"
825+ APACHE_MODULES="modules"
826+ APACHE_PACKAGES="httpd"
827+ ;;
828+ *)
829+ err_msg "$DIST_ID-$DIST_VERS: apache not yet implemented"
830+ ;;
831+ esac
832+}
833+
834+apache_distro_setup
835+
836+install_apache(){
837+ info_msg "installing apache ..."
838+ pkg_install "$APACHE_PACKAGES"
839+ case $DIST_ID-$DIST_VERS in
840+ arch-*|fedora-*)
841+ if ! grep "IncludeOptional sites-enabled" "/etc/httpd/conf/httpd.conf"; then
842+ echo "IncludeOptional sites-enabled/*.conf" >> "/etc/httpd/conf/httpd.conf"
843+ fi
844+ systemctl enable httpd
845+ systemctl start httpd
846+ ;;
847+ esac
848+}
849+
850+apache_is_installed() {
851+ case $DIST_ID-$DIST_VERS in
852+ ubuntu-*|debian-*) (command -v apachectl) &>/dev/null;;
853+ arch-*) (command -v httpd) &>/dev/null;;
854+ fedora-*) (command -v httpd) &>/dev/null;;
855+ esac
856+}
857+
858+apache_reload() {
859+
860+ info_msg "reload apache .."
861+ echo
862+ case $DIST_ID-$DIST_VERS in
863+ ubuntu-*|debian-*)
864+ sudo -H apachectl configtest
865+ sudo -H systemctl force-reload apache2
866+ ;;
867+ arch-*| fedora-*)
868+ sudo -H httpd -t
869+ sudo -H systemctl force-reload httpd
870+ ;;
871+ esac
872+}
873+
874+apache_install_site() {
875+
876+ # usage: apache_install_site [<template option> ...] <mysite.conf>
877+ #
878+ # <template option>: see install_template
879+
880+ local template_opts=()
881+ local pos_args=("$0")
882+
883+ for i in "$@"; do
884+ case $i in
885+ -*) template_opts+=("$i");;
886+ *) pos_args+=("$i");;
887+ esac
888+ done
889+
890+ install_template "${template_opts[@]}" \
891+ "${APACHE_SITES_AVAILABLE}/${pos_args[1]}" \
892+ root root 644
893+ apache_enable_site "${pos_args[1]}"
894+ info_msg "installed apache site: ${pos_args[1]}"
895+}
896+
897+apache_remove_site() {
898+
899+ # usage: apache_remove_site <mysite.conf>
900+
901+ info_msg "remove apache site: $1"
902+ apache_dissable_site "$1"
903+ rm -f "${APACHE_SITES_AVAILABLE}/$1"
904+}
905+
906+apache_enable_site() {
907+
908+ # usage: apache_enable_site <mysite.conf>
909+
910+ local CONF="$1"
911+
912+ info_msg "enable apache site: ${CONF}"
913+
914+ case $DIST_ID-$DIST_VERS in
915+ ubuntu-*|debian-*)
916+ sudo -H a2ensite -q "${CONF}"
917+ ;;
918+ arch-*)
919+ mkdir -p "${APACHE_SITES_ENABLED}"
920+ rm -f "${APACHE_SITES_ENABLED}/${CONF}"
921+ ln -s "${APACHE_SITES_AVAILABLE}/${CONF}" "${APACHE_SITES_ENABLED}/${CONF}"
922+ ;;
923+ fedora-*)
924+ mkdir -p "${APACHE_SITES_ENABLED}"
925+ rm -f "${APACHE_SITES_ENABLED}/${CONF}"
926+ ln -s "${APACHE_SITES_AVAILABLE}/${CONF}" "${APACHE_SITES_ENABLED}/${CONF}"
927+ ;;
928+ esac
929+ apache_reload
930+}
931+
932+apache_dissable_site() {
933+
934+ # usage: apache_disable_site <mysite.conf>
935+
936+ local CONF="$1"
937+
938+ info_msg "disable apache site: ${CONF}"
939+
940+ case $DIST_ID-$DIST_VERS in
941+ ubuntu-*|debian-*)
942+ sudo -H a2dissite -q "${CONF}"
943+ ;;
944+ arch-*)
945+ rm -f "${APACHE_SITES_ENABLED}/${CONF}"
946+ ;;
947+ fedora-*)
948+ rm -f "${APACHE_SITES_ENABLED}/${CONF}"
949+ ;;
950+ esac
951+ apache_reload
952+}
953+
954+# uWSGI
955+# -----
956+
957+uWSGI_SETUP="${uWSGI_SETUP:=/etc/uwsgi}"
958+uWSGI_USER=
959+uWSGI_GROUP=
960+
961+# How distros manage uWSGI apps is very different. From uWSGI POV read:
962+# - https://uwsgi-docs.readthedocs.io/en/latest/Management.html
963+
964+uWSGI_distro_setup() {
965+ case $DIST_ID-$DIST_VERS in
966+ ubuntu-*|debian-*)
967+ # init.d --> /usr/share/doc/uwsgi/README.Debian.gz
968+ # For uWSGI debian uses the LSB init process, this might be changed
969+ # one day, see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=833067
970+ uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-available"
971+ uWSGI_APPS_ENABLED="${uWSGI_SETUP}/apps-enabled"
972+ uWSGI_PACKAGES="uwsgi"
973+ ;;
974+ arch-*)
975+ # systemd --> /usr/lib/systemd/system/uwsgi@.service
976+ # For uWSGI archlinux uses systemd template units, see
977+ # - http://0pointer.de/blog/projects/instances.html
978+ # - https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd
979+ uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-archlinux"
980+ uWSGI_APPS_ENABLED="${uWSGI_SETUP}"
981+ uWSGI_PACKAGES="uwsgi"
982+ ;;
983+ fedora-*)
984+ # systemd --> /usr/lib/systemd/system/uwsgi.service
985+ # The unit file starts uWSGI in emperor mode (/etc/uwsgi.ini), see
986+ # - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html
987+ uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-available"
988+ uWSGI_APPS_ENABLED="${uWSGI_SETUP}.d"
989+ uWSGI_PACKAGES="uwsgi"
990+ uWSGI_USER="uwsgi"
991+ uWSGI_GROUP="uwsgi"
992+ ;;
993+ *)
994+ err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented"
995+ ;;
996+esac
997+}
998+
999+uWSGI_distro_setup
1000+
1001+install_uwsgi(){
1002+ info_msg "installing uwsgi ..."
1003+ pkg_install "$uWSGI_PACKAGES"
1004+ case $DIST_ID-$DIST_VERS in
1005+ fedora-*)
1006+ # enable & start should be called once at uWSGI installation time
1007+ systemctl enable uwsgi
1008+ systemctl restart uwsgi
1009+ ;;
1010+ esac
1011+}
1012+
1013+uWSGI_restart() {
1014+
1015+ # usage: uWSGI_restart() <myapp.ini>
1016+
1017+ local CONF="$1"
1018+
1019+ [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
1020+ info_msg "restart uWSGI service"
1021+ case $DIST_ID-$DIST_VERS in
1022+ ubuntu-*|debian-*)
1023+ # the 'service' method seems broken in that way, that it (re-)starts
1024+ # the whole uwsgi process.
1025+ service uwsgi restart "${CONF%.*}"
1026+ ;;
1027+ arch-*)
1028+ # restart systemd template instance
1029+ if uWSGI_app_available "${CONF}"; then
1030+ systemctl restart "uwsgi@${CONF%.*}"
1031+ else
1032+ info_msg "[uWSGI:systemd-template] ${CONF} not installed (no need to restart)"
1033+ fi
1034+ ;;
1035+ fedora-*)
1036+ # in emperor mode, just touch the file to restart
1037+ if uWSGI_app_enabled "${CONF}"; then
1038+ touch "${uWSGI_APPS_ENABLED}/${CONF}"
1039+ # it seems, there is a polling time in between touch and restart
1040+ # of the service.
1041+ sleep 3
1042+ else
1043+ info_msg "[uWSGI:emperor] ${CONF} not installed (no need to restart)"
1044+ fi
1045+ ;;
1046+ *)
1047+ err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented"
1048+ return 42
1049+ ;;
1050+ esac
1051+}
1052+
1053+uWSGI_prepare_app() {
1054+
1055+ # usage: uWSGI_prepare_app <myapp.ini>
1056+
1057+ [[ -z $1 ]] && die_caller 42 "missing argument <myapp.ini>"
1058+
1059+ local APP="${1%.*}"
1060+
1061+ case $DIST_ID-$DIST_VERS in
1062+ fedora-*)
1063+ # in emperor mode, the uwsgi user is the owner of the sockets
1064+ info_msg "prepare (uwsgi:uwsgi) /run/uwsgi/app/${APP}"
1065+ mkdir -p "/run/uwsgi/app/${APP}"
1066+ chown -R "uwsgi:uwsgi" "/run/uwsgi/app/${APP}"
1067+ ;;
1068+ *)
1069+ info_msg "prepare (${SERVICE_USER}:${SERVICE_GROUP}) /run/uwsgi/app/${APP}"
1070+ mkdir -p "/run/uwsgi/app/${APP}"
1071+ chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "/run/uwsgi/app/${APP}"
1072+ ;;
1073+ esac
1074+}
1075+
1076+
1077+uWSGI_app_available() {
1078+ # usage: uWSGI_app_available <myapp.ini>
1079+ local CONF="$1"
1080+
1081+ [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
1082+ [[ -f "${uWSGI_APPS_AVAILABLE}/${CONF}" ]]
1083+}
1084+
1085+uWSGI_install_app() {
1086+
1087+ # usage: uWSGI_install_app [<template option> ...] <myapp.ini>
1088+ #
1089+ # <template option>: see install_template
1090+
1091+ local pos_args=("$0")
1092+
1093+ for i in "$@"; do
1094+ case $i in
1095+ -*) template_opts+=("$i");;
1096+ *) pos_args+=("$i");;
1097+ esac
1098+ done
1099+ uWSGI_prepare_app "${pos_args[1]}"
1100+ mkdir -p "${uWSGI_APPS_AVAILABLE}"
1101+ install_template "${template_opts[@]}" \
1102+ "${uWSGI_APPS_AVAILABLE}/${pos_args[1]}" \
1103+ root root 644
1104+ uWSGI_enable_app "${pos_args[1]}"
1105+ uWSGI_restart "${pos_args[1]}"
1106+ info_msg "uWSGI app: ${pos_args[1]} is installed"
1107+}
1108+
1109+uWSGI_remove_app() {
1110+
1111+ # usage: uWSGI_remove_app <myapp.ini>
1112+
1113+ local CONF="$1"
1114+
1115+ [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
1116+ info_msg "remove uWSGI app: ${CONF}"
1117+ uWSGI_disable_app "${CONF}"
1118+ uWSGI_restart "${CONF}"
1119+ rm -f "${uWSGI_APPS_AVAILABLE}/${CONF}"
1120+}
1121+
1122+uWSGI_app_enabled() {
1123+ # usage: uWSGI_app_enabled <myapp.ini>
1124+
1125+ local exit_val=0
1126+ local CONF="$1"
1127+
1128+ [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
1129+ case $DIST_ID-$DIST_VERS in
1130+ ubuntu-*|debian-*)
1131+ [[ -f "${uWSGI_APPS_ENABLED}/${CONF}" ]]
1132+ exit_val=$?
1133+ ;;
1134+ arch-*)
1135+ systemctl -q is-enabled "uwsgi@${CONF%.*}"
1136+ exit_val=$?
1137+ ;;
1138+ fedora-*)
1139+ [[ -f "${uWSGI_APPS_ENABLED}/${CONF}" ]]
1140+ exit_val=$?
1141+ ;;
1142+ *)
1143+ # FIXME
1144+ err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented"
1145+ exit_val=1
1146+ ;;
1147+ esac
1148+ return $exit_val
1149+}
1150+
1151+# shellcheck disable=SC2164
1152+uWSGI_enable_app() {
1153+
1154+ # usage: uWSGI_enable_app <myapp.ini>
1155+
1156+ local CONF="$1"
1157+
1158+ [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
1159+ case $DIST_ID-$DIST_VERS in
1160+ ubuntu-*|debian-*)
1161+ mkdir -p "${uWSGI_APPS_ENABLED}"
1162+ rm -f "${uWSGI_APPS_ENABLED}/${CONF}"
1163+ ln -s "${uWSGI_APPS_AVAILABLE}/${CONF}" "${uWSGI_APPS_ENABLED}/${CONF}"
1164+ info_msg "enabled uWSGI app: ${CONF} (restart required)"
1165+ ;;
1166+ arch-*)
1167+ mkdir -p "${uWSGI_APPS_ENABLED}"
1168+ rm -f "${uWSGI_APPS_ENABLED}/${CONF}"
1169+ ln -s "${uWSGI_APPS_AVAILABLE}/${CONF}" "${uWSGI_APPS_ENABLED}/${CONF}"
1170+ systemctl enable "uwsgi@${CONF%.*}"
1171+ info_msg "enabled uWSGI app: ${CONF} (restart required)"
1172+ ;;
1173+ fedora-*)
1174+ mkdir -p "${uWSGI_APPS_ENABLED}"
1175+ rm -f "${uWSGI_APPS_ENABLED}/${CONF}"
1176+ ln -s "${uWSGI_APPS_AVAILABLE}/${CONF}" "${uWSGI_APPS_ENABLED}/${CONF}"
1177+ chown "${uWSGI_USER}:${uWSGI_GROUP}" "${uWSGI_APPS_ENABLED}/${CONF}"
1178+ info_msg "enabled uWSGI app: ${CONF}"
1179+ ;;
1180+ *)
1181+ # FIXME
1182+ err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented"
1183+ ;;
1184+ esac
1185+}
1186+
1187+uWSGI_disable_app() {
1188+
1189+ # usage: uWSGI_disable_app <myapp.ini>
1190+
1191+ local CONF="$1"
1192+
1193+ [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
1194+ case $DIST_ID-$DIST_VERS in
1195+ ubuntu-*|debian-*)
1196+ service uwsgi stop "${CONF%.*}"
1197+ rm -f "${uWSGI_APPS_ENABLED}/${CONF}"
1198+ info_msg "disabled uWSGI app: ${CONF} (restart uWSGI required)"
1199+ ;;
1200+ arch-*)
1201+ systemctl stop "uwsgi@${CONF%.*}"
1202+ systemctl disable "uwsgi@${CONF%.*}"
1203+ rm -f "${uWSGI_APPS_ENABLED}/${CONF}"
1204+ ;;
1205+ fedora-*)
1206+ # in emperor mode, just remove the app.ini file
1207+ rm -f "${uWSGI_APPS_ENABLED}/${CONF}"
1208+ ;;
1209+ *)
1210+ # FIXME
1211+ err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented"
1212+ ;;
1213+ esac
1214+}
1215+
1216+# distro's package manager
1217+# ------------------------
1218+
1219+_apt_pkg_info_is_updated=0
1220+
1221+pkg_install() {
1222+
1223+ # usage: TITEL='install foobar' pkg_install foopkg barpkg
1224+
1225+ rst_title "${TITLE:-installation of packages}" section
1226+ echo -e "\npackage(s)::\n"
1227+ # shellcheck disable=SC2068
1228+ echo " " $@ | $FMT
1229+
1230+ if ! ask_yn "Should packages be installed?" Yn 30; then
1231+ return 42
1232+ fi
1233+ case $DIST_ID in
1234+ ubuntu|debian)
1235+ if [[ $_apt_pkg_info_is_updated == 0 ]]; then
1236+ export _apt_pkg_info_is_updated=1
1237+ apt update
1238+ fi
1239+ # shellcheck disable=SC2068
1240+ apt-get install -m -y $@
1241+ ;;
1242+ arch)
1243+ # shellcheck disable=SC2068
1244+ pacman -Sy --noconfirm $@
1245+ ;;
1246+ fedora)
1247+ # shellcheck disable=SC2068
1248+ dnf install -y $@
1249+ ;;
1250+ esac
1251+}
1252+
1253+pkg_remove() {
1254+
1255+ # usage: TITEL='remove foobar' pkg_remove foopkg barpkg
1256+
1257+ rst_title "${TITLE:-remove packages}" section
1258+ echo -e "\npackage(s)::\n"
1259+ # shellcheck disable=SC2068
1260+ echo " " $@ | $FMT
1261+
1262+ if ! ask_yn "Should packages be removed (purge)?" Yn 30; then
1263+ return 42
1264+ fi
1265+ case $DIST_ID in
1266+ ubuntu|debian)
1267+ # shellcheck disable=SC2068
1268+ apt-get purge --autoremove --ignore-missing -y $@
1269+ ;;
1270+ arch)
1271+ # shellcheck disable=SC2068
1272+ pacman -R --noconfirm $@
1273+ ;;
1274+ fedora)
1275+ # shellcheck disable=SC2068
1276+ dnf remove -y $@
1277+ ;;
1278+ esac
1279+}
1280+
1281+pkg_is_installed() {
1282+
1283+ # usage: pkg_is_install foopkg || pkg_install foopkg
1284+
1285+ case $DIST_ID in
1286+ ubuntu|debian)
1287+ dpkg -l "$1" &> /dev/null
1288+ return $?
1289+ ;;
1290+ arch)
1291+ pacman -Qsq "$1" &> /dev/null
1292+ return $?
1293+ ;;
1294+ fedora)
1295+ dnf list -q --installed "$1" &> /dev/null
1296+ return $?
1297+ ;;
1298+ esac
1299+}
1300+
1301+# git tooling
1302+# -----------
1303+
1304+# shellcheck disable=SC2164
1305+git_clone() {
1306+
1307+ # usage:
1308+ #
1309+ # git_clone <url> <name> [<branch> [<user>]]
1310+ # git_clone <url> <path> [<branch> [<user>]]
1311+ #
1312+ # First form uses $CACHE/<name> as destination folder, second form clones
1313+ # into <path>. If repository is allready cloned, pull from <branch> and
1314+ # update working tree (if needed, the caller has to stash local changes).
1315+ #
1316+ # git clone https://github.com/asciimoo/searx searx-src origin/master searxlogin
1317+ #
1318+
1319+ local url="$1"
1320+ local dest="$2"
1321+ local branch="$3"
1322+ local user="$4"
1323+ local bash_cmd="bash"
1324+ local remote="origin"
1325+
1326+ if [[ ! "${dest:0:1}" = "/" ]]; then
1327+ dest="$CACHE/$dest"
1328+ fi
1329+
1330+ [[ -z $branch ]] && branch=master
1331+ [[ -z $user ]] && [[ -n "${SUDO_USER}" ]] && user="${SUDO_USER}"
1332+ [[ -n $user ]] && bash_cmd="sudo -H -u $user -i"
1333+
1334+ if [[ -d "${dest}" ]] ; then
1335+ info_msg "already cloned: $dest"
1336+ tee_stderr 0.1 <<EOF | $bash_cmd 2>&1 | prefix_stdout " ${_Yellow}|$user|${_creset} "
1337+cd "${dest}"
1338+git checkout -m -B "$branch" --track "$remote/$branch"
1339+git pull --all
1340+EOF
1341+ else
1342+ info_msg "clone into: $dest"
1343+ tee_stderr 0.1 <<EOF | $bash_cmd 2>&1 | prefix_stdout " ${_Yellow}|$user|${_creset} "
1344+mkdir -p "$(dirname "$dest")"
1345+cd "$(dirname "$dest")"
1346+git clone --branch "$branch" --origin "$remote" "$url" "$(basename "$dest")"
1347+EOF
1348+ fi
1349+}
1350+
1351+# containers
1352+# ----------
1353+
1354+in_container() {
1355+ # Test if shell runs in a container.
1356+ #
1357+ # usage: in_container && echo "process running inside a LXC container"
1358+ # in_container || echo "process is not running inside a LXC container"
1359+ #
1360+ # sudo_or_exit
1361+ # hint: Reads init process environment, therefore root access is required!
1362+ # to be safe, take a look at the environment of process 1 (/sbin/init)
1363+ # grep -qa 'container=lxc' /proc/1/environ
1364+
1365+ # see lxc_init_container_env
1366+ [[ -f /.lxcenv ]]
1367+}
1368+
1369+LXC_ENV_FOLDER=
1370+if in_container; then
1371+ # shellcheck disable=SC2034
1372+ LXC_ENV_FOLDER="lxc/$(hostname)/"
1373+fi
1374+
1375+lxc_init_container_env() {
1376+
1377+ # usage: lxc_init_container_env <name>
1378+
1379+ # Create a /.lxcenv file in the root folder. Call this once after the
1380+ # container is inital started and before installing any boilerplate stuff.
1381+
1382+ info_msg "create /.lxcenv in container $1"
1383+ cat <<EOF | lxc exec "${1}" -- bash | prefix_stdout "[${_BBlue}${1}${_creset}] "
1384+touch "/.lxcenv"
1385+ls -l "/.lxcenv"
1386+EOF
1387+}
1388+
1389+# apt packages
1390+LXC_BASE_PACKAGES_debian="bash git build-essential python3 virtualenv"
1391+
1392+# pacman packages
1393+LXC_BASE_PACKAGES_arch="bash git base-devel python python-virtualenv"
1394+
1395+# dnf packages
1396+LXC_BASE_PACKAGES_fedora="bash git @development-tools python virtualenv"
1397+
1398+case $DIST_ID in
1399+ ubuntu|debian) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_debian}" ;;
1400+ arch) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_arch}" ;;
1401+ fedora) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_fedora}" ;;
1402+ *) err_msg "$DIST_ID-$DIST_VERS: pkg_install LXC_BASE_PACKAGES not yet implemented" ;;
1403+esac
1404+
1405+lxc_install_base_packages() {
1406+ info_msg "install LXC_BASE_PACKAGES in container $1"
1407+ pkg_install "${LXC_BASE_PACKAGES}"
1408+}
1409+
1410+
1411+lxc_image_copy() {
1412+
1413+ # usage: lxc_copy_image <remote image> <local image>
1414+ #
1415+ # lxc_copy_image "images:ubuntu/19.10" "ubu1910"
1416+
1417+ if lxc_image_exists "local:${LXC_SUITE[i+1]}"; then
1418+ info_msg "image ${LXC_SUITE[i]} already copied --> ${LXC_SUITE[i+1]}"
1419+ else
1420+ info_msg "copy image locally ${LXC_SUITE[i]} --> ${LXC_SUITE[i+1]}"
1421+ lxc image copy "${LXC_SUITE[i]}" local: \
1422+ --alias "${LXC_SUITE[i+1]}" | prefix_stdout
1423+ fi
1424+}
1425+
1426+lxc_init_container() {
1427+
1428+ # usage: lxc_init_container <image name> <container name>
1429+
1430+ local image_name="$1"
1431+ local container_name="$2"
1432+
1433+ if lxc info "${container_name}" &>/dev/null; then
1434+ info_msg "container '${container_name}' already exists"
1435+ else
1436+ info_msg "create container instance: ${container_name}"
1437+ lxc init "local:${image_name}" "${container_name}"
1438+ fi
1439+}
1440+
1441+lxc_exists(){
1442+
1443+ # usage: lxc_exists <name> || echo "container <name> does not exists"
1444+
1445+ lxc info "$1" &>/dev/null
1446+}
1447+
1448+lxc_image_exists(){
1449+ # usage: lxc_image_exists <alias> || echo "image <alias> does locally not exists"
1450+
1451+ lxc image info "local:$1" &>/dev/null
1452+
1453+}
1454+
1455+lxc_delete_container() {
1456+
1457+ # usage: lxc_delete_container <container-name>
1458+
1459+ if lxc info "$1" &>/dev/null; then
1460+ info_msg "stop & delete instance ${_BBlue}${1}${_creset}"
1461+ lxc stop "$1" &>/dev/null
1462+ lxc delete "$1" | prefix_stdout
1463+ else
1464+ warn_msg "instance '$1' does not exist / can't delete :o"
1465+ fi
1466+}
1467+
1468+lxc_delete_local_image() {
1469+
1470+ # usage: lxc_delete_local_image <container-name>
1471+
1472+ info_msg "delete image 'local:$i'"
1473+ lxc image delete "local:$i"
1474+}
1475+
1476+
1477+# IP
1478+# --
1479+
1480+global_IPs(){
1481+ # usage: global_IPS
1482+ #
1483+ # print list of host's SCOPE global addresses and adapters e.g::
1484+ #
1485+ # $ global_IPs
1486+ # enp4s0|192.168.1.127
1487+ # lxdbr0|10.246.86.1
1488+ # lxdbr0|fd42:8c58:2cd:b73f::1
1489+
1490+ ip -o addr show | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\1|\2/p'
1491+}
1492+
1493+primary_ip() {
1494+
1495+ case $DIST_ID in
1496+ arch)
1497+ ip -o addr show \
1498+ | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\2/p' \
1499+ | head -n 1
1500+ ;;
1501+ *) hostname -I | cut -d' ' -f1 ;;
1502+ esac
1503+}
1504+
1505+# URL
1506+# ---
1507+
1508+url_replace_hostname(){
1509+
1510+ # usage: url_replace_hostname <url> <new hostname>
1511+
1512+ # to replace hostname by primary IP::
1513+ #
1514+ # url_replace_hostname http://searx-ubu1604/morty $(primary_ip)
1515+ # http://10.246.86.250/morty
1516+
1517+ # shellcheck disable=SC2001
1518+ echo "$1" | sed "s|\(http[s]*://\)[^/]*\(.*\)|\1$2\2|"
1519+}
utils/lxc-searx.envnew file+95
@@ -0,0 +1,95 @@
1+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
2+# SPDX-License-Identifier: AGPL-3.0-or-later
3+# shellcheck shell=bash
4+
5+# This file is a setup of a LXC suite. It is sourced from different context, do
6+# not manipulate the environment directly, implement functions and manipulate
7+# environment only is subshells!
8+
9+# ----------------------------------------------------------------------------
10+# config
11+# ----------------------------------------------------------------------------
12+
13+# shellcheck disable=SC2034
14+LXC_SUITE_NAME="searx"
15+lxc_set_suite_env() {
16+ # name of https://images.linuxcontainers.org
17+ export LINUXCONTAINERS_ORG_NAME="${LINUXCONTAINERS_ORG_NAME:-images}"
18+ export LXC_HOST_PREFIX="${LXC_SUITE_NAME:-searx}"
19+ export LXC_SUITE=(
20+
21+ # to disable containers, comment out lines ..
22+
23+ # end of standard support see https://wiki.ubuntu.com/Releases
24+ "$LINUXCONTAINERS_ORG_NAME:ubuntu/16.04" "ubu1604" # April 2021
25+ "$LINUXCONTAINERS_ORG_NAME:ubuntu/18.04" "ubu1804" # April 2023
26+ "$LINUXCONTAINERS_ORG_NAME:ubuntu/19.10" "ubu1910" # July 2020
27+ "$LINUXCONTAINERS_ORG_NAME:ubuntu/20.04" "ubu2004" # future (EOL 2030)
28+
29+ # EOL see https://fedoraproject.org/wiki/Releases
30+ "$LINUXCONTAINERS_ORG_NAME:fedora/31" "fedora31"
31+
32+ # rolling releases see https://www.archlinux.org/releng/releases/
33+ "$LINUXCONTAINERS_ORG_NAME:archlinux" "archlinux"
34+ )
35+
36+ PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
37+ if in_container; then
38+ # container hostnames do not have a DNS entry: use primary IP!
39+ PUBLIC_URL="http://$(primary_ip)/searx"
40+
41+ # make GUEST's services public to the HOST
42+ FILTRON_API="0.0.0.0:4005"
43+ FILTRON_LISTEN="0.0.0.0:4004"
44+ MORTY_LISTEN="0.0.0.0:3000"
45+
46+ # export LXC specific environment
47+ export PUBLIC_URL FILTRON_API FILTRON_LISTEN MORTY_LISTEN
48+ fi
49+}
50+
51+lxc_suite_install_info() {
52+ (
53+ lxc_set_suite_env
54+ cat <<EOF
55+LXC suite: ${LXC_SUITE_NAME} --> ${PUBLIC_URL}
56+ suite includes searx, morty & filtron
57+suite images:
58+$(echo " ${LOCAL_IMAGES[*]}" | $FMT)
59+suite containers:
60+$(echo " ${CONTAINERS[*]}" | $FMT)
61+EOF
62+ )
63+ }
64+
65+lxc_suite_install() {
66+ (
67+ lxc_set_suite_env
68+ FORCE_TIMEOUT=0
69+ export FORCE_TIMEOUT
70+ "${LXC_REPO_ROOT}/utils/searx.sh" install all
71+ "${LXC_REPO_ROOT}/utils/morty.sh" install all
72+ "${LXC_REPO_ROOT}/utils/filtron.sh" install all
73+
74+ rst_title "suite installation finished ($(hostname))" part
75+ lxc_suite_info
76+ echo
77+ )
78+}
79+
80+lxc_suite_info() {
81+ (
82+ lxc_set_suite_env
83+ for ip in $(global_IPs) ; do
84+ if [[ $ip =~ .*:.* ]]; then
85+ info_msg "(${ip%|*}) IPv6: http://[${ip#*|}]"
86+ else
87+ # IPv4:
88+ # shellcheck disable=SC2034,SC2031
89+ info_msg "(${ip%|*}) filtron: http://${ip#*|}:4004/ $PUBLIC_URL"
90+ info_msg "(${ip%|*}) morty: http://${ip#*|}:3000/ $PUBLIC_URL_MORTY"
91+ info_msg "(${ip%|*}) docs-live: http://${ip#*|}:8080/"
92+ fi
93+ done
94+ )
95+}
utils/lxc.shnew file+552
@@ -0,0 +1,552 @@
1+#!/usr/bin/env bash
2+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
3+# SPDX-License-Identifier: AGPL-3.0-or-later
4+
5+# shellcheck source=utils/lib.sh
6+source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
7+source_dot_config
8+
9+# load environment of the LXC suite
10+LXC_ENV="${LXC_ENV:-${REPO_ROOT}/utils/lxc-searx.env}"
11+source "$LXC_ENV"
12+lxc_set_suite_env
13+
14+# ----------------------------------------------------------------------------
15+# config
16+# ----------------------------------------------------------------------------
17+#
18+# read also:
19+# - https://lxd.readthedocs.io/en/latest/
20+
21+LXC_HOST_PREFIX="${LXC_HOST_PREFIX:-test}"
22+
23+# where all folders from HOST are mounted
24+LXC_SHARE_FOLDER="/share"
25+LXC_REPO_ROOT="${LXC_SHARE_FOLDER}/$(basename "${REPO_ROOT}")"
26+
27+ubu1604_boilerplate="
28+export DEBIAN_FRONTEND=noninteractive
29+apt-get update -y
30+apt-get upgrade -y
31+apt-get install -y git curl wget
32+"
33+ubu1804_boilerplate="$ubu1604_boilerplate"
34+ubu1904_boilerplate="$ubu1804_boilerplate"
35+ubu1910_boilerplate="$ubu1904_boilerplate"
36+
37+# shellcheck disable=SC2034
38+ubu2004_boilerplate="
39+$ubu1910_boilerplate
40+echo 'Set disable_coredump false' >> /etc/sudo.conf
41+"
42+
43+# shellcheck disable=SC2034
44+archlinux_boilerplate="
45+pacman -Syu --noconfirm
46+pacman -S --noconfirm inetutils git curl wget sudo
47+echo 'Set disable_coredump false' >> /etc/sudo.conf
48+"
49+
50+# shellcheck disable=SC2034
51+fedora31_boilerplate="
52+dnf update -y
53+dnf install -y git curl wget hostname
54+echo 'Set disable_coredump false' >> /etc/sudo.conf
55+"
56+
57+REMOTE_IMAGES=()
58+CONTAINERS=()
59+LOCAL_IMAGES=()
60+
61+for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do
62+ REMOTE_IMAGES=("${REMOTE_IMAGES[@]}" "${LXC_SUITE[i]}")
63+ CONTAINERS=("${CONTAINERS[@]}" "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}")
64+ LOCAL_IMAGES=("${LOCAL_IMAGES[@]}" "${LXC_SUITE[i+1]}")
65+done
66+
67+HOST_USER="${SUDO_USER:-$USER}"
68+HOST_USER_ID=$(id -u "${HOST_USER}")
69+HOST_GROUP_ID=$(id -g "${HOST_USER}")
70+
71+# ----------------------------------------------------------------------------
72+usage() {
73+# ----------------------------------------------------------------------------
74+ _cmd="$(basename "$0")"
75+ cat <<EOF
76+usage::
77+ $_cmd build [containers|<name>]
78+ $_cmd copy [images]
79+ $_cmd remove [containers|<name>|images]
80+ $_cmd [start|stop] [containers|<name>]
81+ $_cmd show [images|suite|info|config [<name>]]
82+ $_cmd cmd [--|<name>] '...'
83+ $_cmd install [suite|base [<name>]]
84+
85+build
86+ :containers: build, launch all containers and 'install base' packages
87+ :<name>: build, launch container <name> and 'install base' packages
88+copy:
89+ :images: copy remote images of the suite into local storage
90+remove
91+ :containers: delete all 'containers' or only <container-name>
92+ :images: delete local images of the suite
93+start/stop
94+ :containers: start/stop all 'containers' from the suite
95+ :<name>: start/stop container <name> from suite
96+show
97+ :info: show info of all (or <name>) containers from LXC suite
98+ :config: show config of all (or <name>) containers from the LXC suite
99+ :suite: show services of all (or <name>) containers from the LXC suite
100+ :images: show information of local images
101+cmd
102+ use single qoutes to evaluate in container's bash, e.g.: 'echo \$(hostname)'
103+ -- run command '...' in all containers of the LXC suite
104+ :<name>: run command '...' in container <name>
105+install
106+ :base: prepare LXC; install basic packages
107+ :suite: install LXC ${LXC_SUITE_NAME} suite into all (or <name>) containers
108+
109+EOF
110+ usage_containers
111+ [ -n "${1+x}" ] && err_msg "$1"
112+}
113+
114+usage_containers() {
115+ lxc_suite_install_info
116+ [ -n "${1+x}" ] && err_msg "$1"
117+}
118+
119+lxd_info() {
120+
121+ cat <<EOF
122+
123+LXD is needed, to install run::
124+
125+ snap install lxd
126+ lxd init --auto
127+
128+EOF
129+}
130+
131+main() {
132+
133+ local exit_val
134+ local _usage="unknown or missing $1 command $2"
135+
136+ # don't check prerequisite when in recursion
137+ if [[ ! $1 == __* ]]; then
138+ if ! in_container; then
139+ ! required_commands lxc && lxd_info && exit 42
140+ fi
141+ [[ -z $LXC_SUITE ]] && err_msg "missing LXC_SUITE" && exit 42
142+ fi
143+
144+ case $1 in
145+ --getenv) var="$2"; echo "${!var}"; exit 0;;
146+ -h|--help) usage; exit 0;;
147+
148+ build)
149+ sudo_or_exit
150+ case $2 in
151+ ${LXC_HOST_PREFIX}-*) build_container "$2" ;;
152+ ''|--|containers) build_all_containers ;;
153+ *) usage "$_usage"; exit 42;;
154+ esac
155+ ;;
156+ copy)
157+ case $2 in
158+ ''|images) lxc_copy_images_localy;;
159+ *) usage "$_usage"; exit 42;;
160+ esac
161+ ;;
162+ remove)
163+ sudo_or_exit
164+ case $2 in
165+ ''|--|containers) remove_containers ;;
166+ images) lxc_delete_images_localy ;;
167+ ${LXC_HOST_PREFIX}-*)
168+ ! lxc_exists "$2" && warn_msg "container not yet exists: $2" && exit 0
169+ if ask_yn "Do you really want to delete container $2"; then
170+ lxc_delete_container "$2"
171+ fi
172+ ;;
173+ *) usage "uknown or missing container <name> $2"; exit 42;;
174+ esac
175+ ;;
176+ start|stop)
177+ sudo_or_exit
178+ case $2 in
179+ ''|--|containers) lxc_cmd "$1" ;;
180+ ${LXC_HOST_PREFIX}-*)
181+ ! lxc_exists "$2" && usage_containers "unknown container: $2" && exit 42
182+ info_msg "lxc $1 $2"
183+ lxc "$1" "$2" | prefix_stdout "[${_BBlue}${i}${_creset}] "
184+ ;;
185+ *) usage "uknown or missing container <name> $2"; exit 42;;
186+ esac
187+ ;;
188+ show)
189+ sudo_or_exit
190+ case $2 in
191+ suite)
192+ case $3 in
193+ ${LXC_HOST_PREFIX}-*)
194+ lxc exec -t "$3" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __show suite \
195+ | prefix_stdout "[${_BBlue}$3${_creset}] "
196+ ;;
197+ *) show_suite;;
198+ esac
199+ ;;
200+ images) show_images ;;
201+ config)
202+ case $3 in
203+ ${LXC_HOST_PREFIX}-*)
204+ ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42
205+ lxc config show "$3" | prefix_stdout "[${_BBlue}${3}${_creset}] "
206+ ;;
207+ *)
208+ rst_title "container configurations"
209+ echo
210+ lxc list "$LXC_HOST_PREFIX-"
211+ echo
212+ lxc_cmd config show
213+ ;;
214+ esac
215+ ;;
216+ info)
217+ case $3 in
218+ ${LXC_HOST_PREFIX}-*)
219+ ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42
220+ lxc info "$3" | prefix_stdout "[${_BBlue}${3}${_creset}] "
221+ ;;
222+ *)
223+ rst_title "container info"
224+ echo
225+ lxc_cmd info
226+ ;;
227+ esac
228+ ;;
229+ *) usage "$_usage"; exit 42;;
230+ esac
231+ ;;
232+ __show)
233+ # wrapped show commands, called once in each container
234+ case $2 in
235+ suite) lxc_suite_info ;;
236+ esac
237+ ;;
238+ cmd)
239+ sudo_or_exit
240+ shift
241+ case $1 in
242+ --) shift; lxc_exec "$@" ;;
243+ ${LXC_HOST_PREFIX}-*)
244+ ! lxc_exists "$1" && usage_containers "unknown container: $1" && exit 42
245+ local name=$1
246+ shift
247+ lxc_exec_cmd "${name}" "$@"
248+ ;;
249+ *) usage_containers "unknown container: $1" && exit 42
250+ esac
251+ ;;
252+ install)
253+ sudo_or_exit
254+ case $2 in
255+ suite|base)
256+ case $3 in
257+ ${LXC_HOST_PREFIX}-*)
258+ ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42
259+ lxc_exec_cmd "$3" "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2"
260+ ;;
261+ ''|--) lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2" ;;
262+ *) usage_containers "unknown container: $3" && exit 42
263+ esac
264+ ;;
265+ *) usage "$_usage"; exit 42 ;;
266+ esac
267+ ;;
268+ __install)
269+ # wrapped install commands, called once in each container
270+ # shellcheck disable=SC2119
271+ case $2 in
272+ suite) lxc_suite_install ;;
273+ base) FORCE_TIMEOUT=0 lxc_install_base_packages ;;
274+ esac
275+ ;;
276+ doc)
277+ echo
278+ echo ".. generic utils/lxc.sh documentation"
279+ ;;
280+ -*) usage "unknown option $1"; exit 42;;
281+ *) usage "unknown or missing command $1"; exit 42;;
282+ esac
283+}
284+
285+
286+build_all_containers() {
287+ rst_title "Build all LXC containers of suite"
288+ echo
289+ usage_containers
290+ lxc_copy_images_localy
291+ lxc_init_all_containers
292+ lxc_config_all_containers
293+ lxc_boilerplate_all_containers
294+ rst_title "install LXC base packages" section
295+ echo
296+ lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install base
297+ echo
298+ lxc list "$LXC_HOST_PREFIX"
299+}
300+
301+build_container() {
302+ rst_title "Build container $1"
303+
304+ local remote_image
305+ local container
306+ local image
307+ local boilerplate_script
308+
309+ for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do
310+ if [ "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}" = "$1" ]; then
311+ remote_image="${LXC_SUITE[i]}"
312+ container="${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}"
313+ image="${LXC_SUITE[i+1]}"
314+ boilerplate_script="${image}_boilerplate"
315+ boilerplate_script="${!boilerplate_script}"
316+ break
317+ fi
318+ done
319+ echo
320+ if [ -z "$container" ]; then
321+ err_msg "container $1 unknown"
322+ usage_containers
323+ return 42
324+ fi
325+ lxc_image_copy "${remote_image}" "${image}"
326+ rst_title "init container" section
327+ lxc_init_container "${image}" "${container}"
328+ rst_title "configure container" section
329+ lxc_config_container "${container}"
330+ rst_title "run LXC boilerplate scripts" section
331+ lxc_install_boilerplate "${container}" "$boilerplate_script"
332+ echo
333+ rst_title "install LXC base packages" section
334+ lxc_exec_cmd "${container}" "${LXC_REPO_ROOT}/utils/lxc.sh" __install base \
335+ | prefix_stdout "[${_BBlue}${container}${_creset}] "
336+ echo
337+ lxc list "$container"
338+}
339+
340+remove_containers() {
341+ rst_title "Remove all LXC containers of suite"
342+ rst_para "existing containers matching ${_BGreen}$LXC_HOST_PREFIX-*${_creset}"
343+ echo
344+ lxc list "$LXC_HOST_PREFIX-"
345+ echo -en "\\n${_BRed}LXC containers to delete::${_creset}\\n\\n ${CONTAINERS[*]}\\n" | $FMT
346+ local default=Ny
347+ [[ $FORCE_TIMEOUT = 0 ]] && default=Yn
348+ if ask_yn "Do you really want to delete these containers" $default; then
349+ for i in "${CONTAINERS[@]}"; do
350+ lxc_delete_container "$i"
351+ done
352+ fi
353+ echo
354+ lxc list "$LXC_HOST_PREFIX-"
355+}
356+
357+# images
358+# ------
359+
360+lxc_copy_images_localy() {
361+ rst_title "copy images" section
362+ for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do
363+ lxc_image_copy "${LXC_SUITE[i]}" "${LXC_SUITE[i+1]}"
364+ done
365+ # lxc image list local: && wait_key
366+}
367+
368+lxc_delete_images_localy() {
369+ rst_title "Delete LXC images"
370+ rst_para "local existing images"
371+ echo
372+ lxc image list local:
373+ echo -en "\\n${_BRed}LXC images to delete::${_creset}\\n\\n ${LOCAL_IMAGES[*]}\\n"
374+ if ask_yn "Do you really want to delete these images"; then
375+ for i in "${LOCAL_IMAGES[@]}"; do
376+ lxc_delete_local_image "$i"
377+ done
378+ fi
379+
380+ for i in $(lxc image list --format csv | grep '^,' | sed 's/,\([^,]*\).*$/\1/'); do
381+ if ask_yn "Image $i has no alias, do you want to delete the image?" Yn; then
382+ lxc_delete_local_image "$i"
383+ fi
384+ done
385+
386+ echo
387+ lxc image list local:
388+}
389+
390+show_images(){
391+ rst_title "local images"
392+ echo
393+ lxc image list local:
394+ echo -en "\\n${_Green}LXC suite images::${_creset}\\n\\n ${LOCAL_IMAGES[*]}\\n"
395+ wait_key
396+ for i in "${LOCAL_IMAGES[@]}"; do
397+ if lxc_image_exists "$i"; then
398+ info_msg "lxc image info ${_BBlue}${i}${_creset}"
399+ lxc image info "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] "
400+ else
401+ warn_msg "image ${_BBlue}$i${_creset} does not yet exists"
402+ fi
403+ done
404+
405+}
406+
407+
408+# container
409+# ---------
410+
411+show_suite(){
412+ rst_title "LXC suite ($LXC_HOST_PREFIX-*)"
413+ echo
414+ lxc list "$LXC_HOST_PREFIX-"
415+ echo
416+ for i in "${CONTAINERS[@]}"; do
417+ if ! lxc_exists "$i"; then
418+ warn_msg "container ${_BBlue}$i${_creset} does not yet exists"
419+ else
420+ lxc exec -t "${i}" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __show suite \
421+ | prefix_stdout "[${_BBlue}${i}${_creset}] "
422+ echo
423+ fi
424+ done
425+}
426+
427+lxc_cmd() {
428+ for i in "${CONTAINERS[@]}"; do
429+ if ! lxc_exists "$i"; then
430+ warn_msg "container ${_BBlue}$i${_creset} does not yet exists"
431+ else
432+ info_msg "lxc $* $i"
433+ lxc "$@" "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] "
434+ fi
435+ done
436+}
437+
438+lxc_exec_cmd() {
439+ local name="$1"
440+ shift
441+ exit_val=
442+ info_msg "[${_BBlue}${name}${_creset}] ${_BGreen}${*}${_creset}"
443+ lxc exec -t --cwd "${LXC_REPO_ROOT}" "${name}" -- bash -c "$*"
444+ exit_val=$?
445+ if [[ $exit_val -ne 0 ]]; then
446+ warn_msg "[${_BBlue}${name}${_creset}] exit code (${_BRed}${exit_val}${_creset}) from ${_BGreen}${*}${_creset}"
447+ else
448+ info_msg "[${_BBlue}${name}${_creset}] exit code (${exit_val}) from ${_BGreen}${*}${_creset}"
449+ fi
450+}
451+
452+lxc_exec() {
453+ for i in "${CONTAINERS[@]}"; do
454+ if ! lxc_exists "$i"; then
455+ warn_msg "container ${_BBlue}$i${_creset} does not yet exists"
456+ else
457+ lxc_exec_cmd "${i}" "$@" | prefix_stdout "[${_BBlue}${i}${_creset}] "
458+ fi
459+ done
460+}
461+
462+lxc_init_all_containers() {
463+ rst_title "init all containers" section
464+
465+ local image_name
466+ local container_name
467+
468+ for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do
469+ lxc_init_container "${LXC_SUITE[i+1]}" "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}"
470+ done
471+}
472+
473+lxc_config_all_containers() {
474+ rst_title "configure all containers" section
475+
476+ for i in "${CONTAINERS[@]}"; do
477+ lxc_config_container "${i}"
478+ done
479+}
480+
481+lxc_config_container() {
482+ info_msg "[${_BBlue}$1${_creset}] configure container ..."
483+
484+ info_msg "[${_BBlue}$1${_creset}] map uid/gid from host to container"
485+ # https://lxd.readthedocs.io/en/latest/userns-idmap/#custom-idmaps
486+ echo -e -n "uid $HOST_USER_ID 0\\ngid $HOST_GROUP_ID 0"\
487+ | lxc config set "$1" raw.idmap -
488+
489+ info_msg "[${_BBlue}$1${_creset}] share ${REPO_ROOT} (repo_share) from HOST into container"
490+ # https://lxd.readthedocs.io/en/latest/instances/#type-disk
491+ lxc config device add "$1" repo_share disk \
492+ source="${REPO_ROOT}" \
493+ path="${LXC_REPO_ROOT}" &>/dev/null
494+ # lxc config show "$1" && wait_key
495+}
496+
497+lxc_boilerplate_all_containers() {
498+ rst_title "run LXC boilerplate scripts" section
499+
500+ local boilerplate_script
501+ local image_name
502+
503+ for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do
504+
505+ image_name="${LXC_SUITE[i+1]}"
506+ boilerplate_script="${image_name}_boilerplate"
507+ boilerplate_script="${!boilerplate_script}"
508+
509+ lxc_install_boilerplate "${LXC_HOST_PREFIX}-${image_name}" "$boilerplate_script"
510+
511+ if [[ -z "${boilerplate_script}" ]]; then
512+ err_msg "[${_BBlue}${container_name}${_creset}] no boilerplate for image '${image_name}'"
513+ fi
514+ done
515+}
516+
517+lxc_install_boilerplate() {
518+
519+ # usage: lxc_install_boilerplate <container-name> <string: shell commands ..>
520+ #
521+ # usage: lxc_install_boilerplate searx-archlinux "${archlinux_boilerplate}"
522+
523+ local container_name="$1"
524+ local boilerplate_script="$2"
525+
526+ info_msg "[${_BBlue}${container_name}${_creset}] init .."
527+ if lxc start -q "${container_name}" &>/dev/null; then
528+ sleep 5 # guest needs some time to come up and get an IP
529+ fi
530+ lxc_init_container_env "${container_name}"
531+ info_msg "[${_BBlue}${container_name}${_creset}] install /.lxcenv.mk .."
532+ cat <<EOF | lxc exec "${container_name}" -- bash | prefix_stdout "[${_BBlue}${container_name}${_creset}] "
533+rm -f "/.lxcenv.mk"
534+ln -s "${LXC_REPO_ROOT}/utils/makefile.lxc" "/.lxcenv.mk"
535+ls -l "/.lxcenv.mk"
536+EOF
537+
538+ info_msg "[${_BBlue}${container_name}${_creset}] run LXC boilerplate scripts .."
539+ if lxc start -q "${container_name}" &>/dev/null; then
540+ sleep 5 # guest needs some time to come up and get an IP
541+ fi
542+ if [[ -n "${boilerplate_script}" ]]; then
543+ echo "${boilerplate_script}" \
544+ | lxc exec "${container_name}" -- bash \
545+ | prefix_stdout "[${_BBlue}${container_name}${_creset}] "
546+ fi
547+}
548+
549+
550+# ----------------------------------------------------------------------------
551+main "$@"
552+# ----------------------------------------------------------------------------
utils/makefile.include+15−1
@@ -1,12 +1,25 @@
1# -*- coding: utf-8; mode: makefile-gmake -*-
1# -*- coding: utf-8; mode: makefile-gmake -*-
2
2
3+ifeq (,$(wildcard /.lxcenv.mk))
4+PHONY += lxc-activate lxc-purge
5+lxc-activate:
6+ @$(MAKE) -s -f /share/searx/utils/makefile.lxc lxc-activate
7+lxc-purge:
8+ $(Q)rm -rf ./lxc
9+else
10+ include /.lxcenv.mk
11+endif
12+
13+ifeq (,$(wildcard /.lxcenv.mk))
3make-help:
14make-help:
15+else
16+make-help: lxc-help
17+endif
4 @echo ' make V=0|1 [targets] 0 => quiet build (default), 1 => verbose build'
18 @echo ' make V=0|1 [targets] 0 => quiet build (default), 1 => verbose build'
5 @echo ' make V=2 [targets] 2 => give reason for rebuild of target'
19 @echo ' make V=2 [targets] 2 => give reason for rebuild of target'
6
20
7quiet_cmd_common_clean = CLEAN $@
21quiet_cmd_common_clean = CLEAN $@
8 cmd_common_clean = \
22 cmd_common_clean = \
9− rm -rf tests/build ;\
10 find . -name '*.orig' -exec rm -f {} + ;\
23 find . -name '*.orig' -exec rm -f {} + ;\
11 find . -name '*.rej' -exec rm -f {} + ;\
24 find . -name '*.rej' -exec rm -f {} + ;\
12 find . -name '*~' -exec rm -f {} + ;\
25 find . -name '*~' -exec rm -f {} + ;\
@@ -126,3 +139,4 @@ echo-cmd = $(if $($(quiet)cmd_$(1)),echo '$(call escsq,$($(quiet)cmd_$(1)))$(ech
126# printing commands
139# printing commands
127cmd = @$(echo-cmd) $(cmd_$(1))
140cmd = @$(echo-cmd) $(cmd_$(1))
128
141
142+.PHONY: $(PHONY)
utils/makefile.lxcnew file+29
@@ -0,0 +1,29 @@
1+# -*- coding: utf-8; mode: makefile-gmake -*-
2+#
3+# LXC environment
4+# ===============
5+#
6+# To activate/deactivate LXC makefile environment in a container, set/unset link
7+# from root '/.lxcenv.mk' to *this* file::
8+#
9+# sudo make ./utils/makefile.lxc lxc-activate
10+# sudo make ./utils/makefile.lxc lxc-deactivate
11+
12+LXC_ENV_FOLDER=lxc/$(shell hostname)/
13+
14+lxc-help::
15+ @echo 'LXC: running in container LXC_ENV_FOLDER=$(LXC_ENV_FOLDER)'
16+
17+# If not activated, serve target 'lxc-activate' ..
18+ifeq (,$(wildcard /.lxcenv.mk))
19+PHONY += lxc-activate
20+lxc-activate:
21+ ln -s "$(abspath $(lastword $(MAKEFILE_LIST)))" "/.lxcenv.mk"
22+else
23+# .. and if activated, serve target 'lxc-deactivate'.
24+PHONY += lxc-deactivate
25+lxc-deactivate:
26+ rm /.lxcenv.mk
27+endif
28+
29+.PHONY: $(PHONY)
utils/makefile.python+4−4
@@ -8,9 +8,9 @@ export PYTHONPATH := $(SITE_PYTHON):$$PYTHONPATH
8export PY_ENV PYDIST PYBUILD
8export PY_ENV PYDIST PYBUILD
9
9
10# folder where the python distribution takes place
10# folder where the python distribution takes place
11−PYDIST ?= ./py_dist
11+PYDIST = ./$(LXC_ENV_FOLDER)dist
12# folder where the python intermediate build files take place
12# folder where the python intermediate build files take place
13−PYBUILD ?= ./py_build
13+PYBUILD = ./$(LXC_ENV_FOLDER)build
14# python version to use
14# python version to use
15PY ?=3
15PY ?=3
16# $(PYTHON) points to the python interpreter from the OS! The python from the
16# $(PYTHON) points to the python interpreter from the OS! The python from the
@@ -30,8 +30,7 @@ PYLINT_RC ?= .pylintrc
30TEST_FOLDER ?= ./tests
30TEST_FOLDER ?= ./tests
31TEST ?= .
31TEST ?= .
32
32
33−VTENV_OPTS = "--no-site-packages"
34−PY_ENV = ./local/py$(PY)
33+PY_ENV = ./$(LXC_ENV_FOLDER)local/py$(PY)
35PY_ENV_BIN = $(PY_ENV)/bin
34PY_ENV_BIN = $(PY_ENV)/bin
36PY_ENV_ACT = . $(PY_ENV_BIN)/activate
35PY_ENV_ACT = . $(PY_ENV_BIN)/activate
37
36
@@ -41,6 +40,7 @@ ifeq ($(OS),Windows_NT)
41 PY_ENV_ACT = $(PY_ENV_BIN)/activate
40 PY_ENV_ACT = $(PY_ENV_BIN)/activate
42endif
41endif
43
42
43+VTENV_OPTS ?=
44ifeq ($(PYTHON),python)
44ifeq ($(PYTHON),python)
45 VIRTUALENV = virtualenv
45 VIRTUALENV = virtualenv
46else
46else
utils/makefile.sphinx+15−11
@@ -1,17 +1,19 @@
1# -*- coding: utf-8; mode: makefile-gmake -*-
1# -*- coding: utf-8; mode: makefile-gmake -*-
2
2
3+export DOCS_FOLDER DOCS_BUILD DOCS_DIST BOOKS_FOLDER BOOKS_DIST
4+
3# You can set these variables from the command line.
5# You can set these variables from the command line.
4SPHINXOPTS ?=
6SPHINXOPTS ?=
5SPHINXBUILD ?= $(PY_ENV_BIN)/sphinx-build
7SPHINXBUILD ?= $(PY_ENV_BIN)/sphinx-build
6SPHINX_CONF ?= conf.py
8SPHINX_CONF ?= conf.py
7
9
8−DOCS_FOLDER ?= docs
10+DOCS_FOLDER = ./docs
9−DOCS_BUILD ?= build/docs
11+DOCS_BUILD = ./$(LXC_ENV_FOLDER)build/docs
10−DOCS_DIST ?= dist/docs
12+DOCS_DIST = ./$(LXC_ENV_FOLDER)dist/docs
11GH_PAGES ?= gh-pages
13GH_PAGES ?= gh-pages
12
14
13−BOOKS_FOLDER ?= docs
15+BOOKS_FOLDER = ./docs
14−BOOKS_DIST ?= dist/books
16+BOOKS_DIST = ./$(LXC_ENV_FOLDER)dist/books
15
17
16ifeq ($(KBUILD_VERBOSE),1)
18ifeq ($(KBUILD_VERBOSE),1)
17 SPHINX_VERBOSE = "-v"
19 SPHINX_VERBOSE = "-v"
@@ -54,11 +56,13 @@ docs-help:
54# requirements
56# requirements
55# ------------------------------------------------------------------------------
57# ------------------------------------------------------------------------------
56
58
57−sphinx-doc: $(PY_ENV)
59+sphinx-doc-prebuilds:: $(PY_ENV)
60+
61+sphinx-doc: sphinx-doc-prebuilds
58 @echo "PYENV installing Sphinx$(SPHINXVERS)"
62 @echo "PYENV installing Sphinx$(SPHINXVERS)"
59 $(Q)$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) 'Sphinx$(SPHINXVERS)'
63 $(Q)$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) 'Sphinx$(SPHINXVERS)'
60
64
61−sphinx-live: $(PY_ENV)
65+sphinx-live: sphinx-doc-prebuilds
62 @echo "PYENV installing Sphinx$(SPHINXVERS)"
66 @echo "PYENV installing Sphinx$(SPHINXVERS)"
63 $(Q)$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) 'Sphinx$(SPHINXVERS)' sphinx-autobuild
67 $(Q)$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) 'Sphinx$(SPHINXVERS)' sphinx-autobuild
64
68
@@ -113,7 +117,7 @@ quiet_cmd_sphinx_clean = CLEAN $@
113# targets
117# targets
114# ------------------------------------------------------------------------------
118# ------------------------------------------------------------------------------
115
119
116−# build PDF of whole documentation in: $(DOCS_DIST)/pdf
120+# build PDF of whole documentation in: $(DOCS_DIST)/pdf
117
121
118PHONY += sphinx-pdf
122PHONY += sphinx-pdf
119sphinx-pdf: sphinx-latex
123sphinx-pdf: sphinx-latex
@@ -154,7 +158,7 @@ $(BOOKS_HTML): sphinx-doc | $(BOOKS_DIST)
154 -b html \
158 -b html \
155 -c $(DOCS_FOLDER) \
159 -c $(DOCS_FOLDER) \
156 -d $(DOCS_BUILD)/books/$(patsubst books/%.html,%,$@)/.doctrees \
160 -d $(DOCS_BUILD)/books/$(patsubst books/%.html,%,$@)/.doctrees \
157− $(patsubst books/%.html,%,$@) \
161+ $(BOOKS_FOLDER)/$(patsubst books/%.html,%,$@) \
158 $(BOOKS_DIST)/$(patsubst books/%.html,%,$@)
162 $(BOOKS_DIST)/$(patsubst books/%.html,%,$@)
159 @echo "SPHINX $@ --> file://$(abspath $(BOOKS_DIST)/$(patsubst books/%.html,%,$@))"
163 @echo "SPHINX $@ --> file://$(abspath $(BOOKS_DIST)/$(patsubst books/%.html,%,$@))"
160
164
@@ -166,7 +170,7 @@ $(BOOKS_LIVE): sphinx-live | $(BOOKS_DIST)
166 -b html \
170 -b html \
167 -c $(DOCS_FOLDER) \
171 -c $(DOCS_FOLDER) \
168 -d $(DOCS_BUILD)/books/$(patsubst books/%.live,%,$@)/.doctrees \
172 -d $(DOCS_BUILD)/books/$(patsubst books/%.live,%,$@)/.doctrees \
169− $(patsubst books/%.live,%,$@) \
173+ $(BOOKS_FOLDER)/$(patsubst books/%.live,%,$@) \
170 $(BOOKS_DIST)/$(patsubst books/%.live,%,$@)
174 $(BOOKS_DIST)/$(patsubst books/%.live,%,$@)
171
175
172$(BOOKS_PDF): %.pdf : %.latex
176$(BOOKS_PDF): %.pdf : %.latex
@@ -182,7 +186,7 @@ $(BOOKS_LATEX): sphinx-doc | $(BOOKS_DIST)
182 -b latex \
186 -b latex \
183 -c $(DOCS_FOLDER) \
187 -c $(DOCS_FOLDER) \
184 -d $(DOCS_BUILD)/books/$(patsubst books/%.latex,%,$@)/.doctrees \
188 -d $(DOCS_BUILD)/books/$(patsubst books/%.latex,%,$@)/.doctrees \
185− $(patsubst books/%.latex,%,$@) \
189+ $(BOOKS_FOLDER)/$(patsubst books/%.latex,%,$@) \
186 $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@)
190 $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@)
187 @echo "SPHINX $@ --> file://$(abspath $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@))"
191 @echo "SPHINX $@ --> file://$(abspath $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@))"
188
192
utils/morty.shnew file+546
@@ -0,0 +1,546 @@
1+#!/usr/bin/env bash
2+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
3+# SPDX-License-Identifier: AGPL-3.0-or-later
4+
5+# shellcheck source=utils/lib.sh
6+source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
7+# shellcheck source=utils/brand.env
8+source "${REPO_ROOT}/utils/brand.env"
9+source_dot_config
10+SEARX_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
11+source "${REPO_ROOT}/utils/lxc-searx.env"
12+in_container && lxc_set_suite_env
13+
14+# ----------------------------------------------------------------------------
15+# config
16+# ----------------------------------------------------------------------------
17+
18+MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}"
19+PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty/}"
20+
21+PUBLIC_URL_MORTY="${PUBLIC_URL_MORTY:-$(echo "$SEARX_URL" | sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}}"
22+
23+# shellcheck disable=SC2034
24+MORTY_TIMEOUT=5
25+
26+SERVICE_NAME="morty"
27+SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}"
28+SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}"
29+SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}"
30+SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service"
31+# shellcheck disable=SC2034
32+SERVICE_GROUP="${SERVICE_USER}"
33+# shellcheck disable=SC2034
34+SERVICE_ENV_DEBUG=false
35+
36+GO_ENV="${SERVICE_HOME}/.go_env"
37+GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz"
38+GO_TAR=$(basename "$GO_PKG_URL")
39+
40+# shellcheck disable=SC2034
41+CONFIG_FILES=()
42+
43+# Apache Settings
44+
45+APACHE_MORTY_SITE="morty.conf"
46+NGINX_MORTY_SITE="morty.conf"
47+
48+# ----------------------------------------------------------------------------
49+usage() {
50+# ----------------------------------------------------------------------------
51+
52+ # shellcheck disable=SC1117
53+ cat <<EOF
54+usage::
55+ $(basename "$0") shell
56+ $(basename "$0") install [all|user]
57+ $(basename "$0") update [morty]
58+ $(basename "$0") remove [all]
59+ $(basename "$0") activate [service]
60+ $(basename "$0") deactivate [service]
61+ $(basename "$0") inspect [service]
62+ $(basename "$0") option [debug-on|debug-off|new-key]
63+ $(basename "$0") apache [install|remove]
64+ $(basename "$0") nginx [install|remove]
65+ $(basename "$0") info [searx]
66+
67+shell
68+ start interactive shell from user ${SERVICE_USER}
69+install / remove
70+ all: complete setup of morty service
71+ user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME)
72+update morty
73+ Update morty installation ($SERVICE_HOME)
74+activate service
75+ activate and start service daemon (systemd unit)
76+deactivate service
77+ stop and deactivate service daemon (systemd unit)
78+inspect service
79+ show service status and log
80+option
81+ set one of the available options
82+ :new-key: set new morty key
83+apache : ${PUBLIC_URL_MORTY}
84+ :install: apache site with a reverse proxy (ProxyPass)
85+ :remove: apache site ${APACHE_MORTY_SITE}
86+nginx (${PUBLIC_URL_MORTY})
87+ :install: nginx site with a reverse proxy (ProxyPass)
88+ :remove: nginx site ${NGINX_MORTY_SITE}
89+
90+If needed, set the environment variables in the '${DOT_CONFIG#"$REPO_ROOT/"}' file::
91+ PUBLIC_URL_MORTY: ${PUBLIC_URL_MORTY}
92+ MORTY_LISTEN: ${MORTY_LISTEN}
93+ SERVICE_USER: ${SERVICE_USER}
94+EOF
95+ if in_container; then
96+ # in containers the service is listening on 0.0.0.0 (see lxc-searx.env)
97+ for ip in $(global_IPs) ; do
98+ if [[ $ip =~ .*:.* ]]; then
99+ echo " container URL (IPv6): http://[${ip#*|}]:3000/"
100+ else
101+ # IPv4:
102+ echo " container URL (IPv4): http://${ip#*|}:3000/"
103+ fi
104+ done
105+ fi
106+ echo
107+ info_searx
108+
109+ [[ -n ${1} ]] && err_msg "$1"
110+}
111+
112+info_searx() {
113+ # shellcheck disable=SC1117
114+ cat <<EOF
115+To activate result and image proxy in searx, edit settings.yml (read:
116+${DOCS_URL}/admin/morty.html)::
117+ result_proxy:
118+ url : ${PUBLIC_URL_MORTY}
119+ server:
120+ image_proxy : True
121+EOF
122+}
123+
124+main() {
125+ required_commands \
126+ sudo install git wget curl \
127+ || exit
128+
129+ local _usage="ERROR: unknown or missing $1 command $2"
130+
131+ case $1 in
132+ --getenv) var="$2"; echo "${!var}"; exit 0;;
133+ -h|--help) usage; exit 0;;
134+
135+ shell)
136+ sudo_or_exit
137+ interactive_shell "${SERVICE_USER}"
138+ ;;
139+ inspect)
140+ case $2 in
141+ service)
142+ sudo_or_exit
143+ inspect_service
144+ ;;
145+ *) usage "$_usage"; exit 42;;
146+ esac ;;
147+ install)
148+ rst_title "$SERVICE_NAME" part
149+ sudo_or_exit
150+ case $2 in
151+ all) install_all ;;
152+ user) assert_user ;;
153+ *) usage "$_usage"; exit 42;;
154+ esac ;;
155+ update)
156+ sudo_or_exit
157+ case $2 in
158+ morty) update_morty ;;
159+ *) usage "$_usage"; exit 42;;
160+ esac ;;
161+ remove)
162+ sudo_or_exit
163+ case $2 in
164+ all) remove_all;;
165+ user) drop_service_account "${SERVICE_USER}" ;;
166+ *) usage "$_usage"; exit 42;;
167+ esac ;;
168+ activate)
169+ sudo_or_exit
170+ case $2 in
171+ service) systemd_activate_service "${SERVICE_NAME}" ;;
172+ *) usage "$_usage"; exit 42;;
173+ esac ;;
174+ deactivate)
175+ sudo_or_exit
176+ case $2 in
177+ service) systemd_deactivate_service "${SERVICE_NAME}" ;;
178+ *) usage "$_usage"; exit 42;;
179+ esac ;;
180+ apache)
181+ sudo_or_exit
182+ case $2 in
183+ install) install_apache_site ;;
184+ remove) remove_apache_site ;;
185+ *) usage "$_usage"; exit 42;;
186+ esac ;;
187+ nginx)
188+ sudo_or_exit
189+ case $2 in
190+ install) install_nginx_site ;;
191+ remove) remove_nginx_site ;;
192+ *) usage "$_usage"; exit 42;;
193+ esac ;;
194+ info)
195+ case $2 in
196+ searx) info_searx ;;
197+ *) usage "$_usage"; exit 42;;
198+ esac ;;
199+ option)
200+ sudo_or_exit
201+ case $2 in
202+ new-key) set_new_key ;;
203+ debug-on) enable_debug ;;
204+ debug-off) disable_debug ;;
205+ *) usage "$_usage"; exit 42;;
206+ esac ;;
207+ doc) rst-doc ;;
208+ *) usage "ERROR: unknown or missing command $1"; exit 42;;
209+ esac
210+}
211+
212+install_all() {
213+
214+ MORTY_KEY="$(head -c 32 /dev/urandom | base64)"
215+
216+ rst_title "Install $SERVICE_NAME (service)"
217+ assert_user
218+ wait_key
219+ install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}"
220+ wait_key
221+ install_morty
222+ wait_key
223+ systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
224+ wait_key
225+ if ! service_is_available "http://${MORTY_LISTEN}" ; then
226+ err_msg "Morty does not listening on: http://${MORTY_LISTEN}"
227+ fi
228+ if apache_is_installed; then
229+ info_msg "Apache is installed on this host."
230+ if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
231+ install_apache_site
232+ fi
233+ elif nginx_is_installed; then
234+ info_msg "nginx is installed on this host."
235+ if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
236+ install_nginx_site
237+ fi
238+ fi
239+ info_searx
240+ if ask_yn "Add image and result proxy to searx settings.yml?" Yn; then
241+ "${REPO_ROOT}/utils/searx.sh" option result-proxy "${PUBLIC_URL_MORTY}" "${MORTY_KEY}"
242+ "${REPO_ROOT}/utils/searx.sh" option image-proxy-on
243+ fi
244+
245+ if ask_yn "Do you want to inspect the installation?" Ny; then
246+ inspect_service
247+ fi
248+
249+}
250+
251+remove_all() {
252+ rst_title "De-Install $SERVICE_NAME (service)"
253+
254+ rst_para "\
255+It goes without saying that this script can only be used to remove
256+installations that were installed with this script."
257+
258+ if systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then
259+ drop_service_account "${SERVICE_USER}"
260+ fi
261+}
262+
263+assert_user() {
264+ rst_title "user $SERVICE_USER" section
265+ echo
266+ tee_stderr 1 <<EOF | bash | prefix_stdout
267+useradd --shell /bin/bash --system \
268+ --home-dir "$SERVICE_HOME" \
269+ --comment 'Web content sanitizer proxy' $SERVICE_USER
270+mkdir "$SERVICE_HOME"
271+chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
272+groups $SERVICE_USER
273+EOF
274+ SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)"
275+ export SERVICE_HOME
276+ echo "export SERVICE_HOME=$SERVICE_HOME"
277+
278+ cat > "$GO_ENV" <<EOF
279+export GOPATH=\$HOME/go-apps
280+export PATH=\$PATH:\$HOME/local/go/bin:\$GOPATH/bin
281+EOF
282+ echo "Environment $GO_ENV has been setup."
283+
284+ tee_stderr <<EOF | sudo -i -u "$SERVICE_USER"
285+grep -qFs -- 'source $GO_ENV' ~/.profile || echo 'source $GO_ENV' >> ~/.profile
286+EOF
287+}
288+
289+morty_is_installed() {
290+ [[ -f $SERVICE_HOME/go-apps/bin/morty ]]
291+}
292+
293+_svcpr=" ${_Yellow}|${SERVICE_USER}|${_creset} "
294+
295+install_morty() {
296+ rst_title "Install morty in user's ~/go-apps" section
297+ echo
298+ tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
299+go get -v -u github.com/asciimoo/morty
300+EOF
301+ tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
302+cd \$GOPATH/src/github.com/asciimoo/morty
303+go test
304+go test -benchmem -bench .
305+EOF
306+}
307+
308+update_morty() {
309+ rst_title "Update morty" section
310+ echo
311+ tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
312+go get -v -u github.com/asciimoo/morty
313+EOF
314+ tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
315+cd \$GOPATH/src/github.com/asciimoo/morty
316+go test
317+go test -benchmem -bench .
318+EOF
319+}
320+
321+set_service_env_debug() {
322+
323+ # usage: set_service_env_debug [false|true]
324+
325+ # shellcheck disable=SC2034
326+ local SERVICE_ENV_DEBUG="${1:-false}"
327+ if systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then
328+ systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
329+ fi
330+}
331+
332+inspect_service() {
333+
334+ rst_title "service status & log"
335+
336+ cat <<EOF
337+
338+sourced ${DOT_CONFIG#"$REPO_ROOT/"} :
339+
340+ MORTY_LISTEN : ${MORTY_LISTEN}
341+
342+EOF
343+
344+ if service_account_is_available "$SERVICE_USER"; then
345+ info_msg "service account $SERVICE_USER available."
346+ else
347+ err_msg "service account $SERVICE_USER not available!"
348+ fi
349+ if go_is_available "$SERVICE_USER"; then
350+ info_msg "~$SERVICE_USER: go is installed"
351+ else
352+ err_msg "~$SERVICE_USER: go is not installed"
353+ fi
354+ if morty_is_installed; then
355+ info_msg "~$SERVICE_USER: morty app is installed"
356+ else
357+ err_msg "~$SERVICE_USER: morty app is not installed!"
358+ fi
359+
360+ if ! service_is_available "http://${MORTY_LISTEN}" ; then
361+ err_msg "Morty does not listening on: http://${MORTY_LISTEN}"
362+ echo -e "${_Green}stop with [${_BCyan}CTRL-C${_Green}] or .."
363+ wait_key
364+ fi
365+
366+ if ! service_is_available "${PUBLIC_URL_MORTY}"; then
367+ warn_msg "Public service at ${PUBLIC_URL_MORTY} is not available!"
368+ if ! in_container; then
369+ warn_msg "Check if public name is correct and routed or use the public IP from above."
370+ fi
371+ fi
372+
373+ if in_container; then
374+ lxc_suite_info
375+ else
376+ info_msg "public URL --> ${PUBLIC_URL_MORTY}"
377+ info_msg "morty URL --> http://${MORTY_LISTEN}"
378+ fi
379+
380+ local _debug_on
381+ if ask_yn "Enable morty debug mode (needs reinstall of systemd service)?"; then
382+ enable_debug
383+ _debug_on=1
384+ else
385+ systemctl --no-pager -l status "${SERVICE_NAME}"
386+ fi
387+ echo
388+
389+ # shellcheck disable=SC2059
390+ printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log"
391+ read -r -s -n1 -t 5
392+ echo
393+ while true; do
394+ trap break 2
395+ journalctl -f -u "${SERVICE_NAME}"
396+ done
397+
398+ if [[ $_debug_on == 1 ]]; then
399+ FORCE_SELECTION=Y disable_debug
400+ fi
401+ return 0
402+}
403+
404+enable_debug() {
405+ warn_msg "Do not enable debug in production enviroments!!"
406+ info_msg "Enabling debug option needs to reinstall systemd service!"
407+ set_service_env_debug true
408+}
409+
410+disable_debug() {
411+ info_msg "Disabling debug option needs to reinstall systemd service!"
412+ set_service_env_debug false
413+}
414+
415+
416+set_new_key() {
417+ rst_title "Set morty key"
418+ echo
419+
420+ MORTY_KEY="$(head -c 32 /dev/urandom | base64)"
421+ info_msg "morty key: '${MORTY_KEY}'"
422+
423+ warn_msg "this will need to reinstall services .."
424+ MSG="${_Green}press any [${_BCyan}KEY${_Green}] to continue // stop with [${_BCyan}CTRL-C${_creset}]" wait_key
425+
426+ systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
427+ "${REPO_ROOT}/utils/searx.sh" option result-proxy "${PUBLIC_URL_MORTY}" "${MORTY_KEY}"
428+ "${REPO_ROOT}/utils/searx.sh" option image-proxy-on
429+}
430+
431+
432+install_apache_site() {
433+
434+ rst_title "Install Apache site $APACHE_MORTY_SITE"
435+
436+ rst_para "\
437+This installs a reverse proxy (ProxyPass) into apache site (${APACHE_MORTY_SITE})"
438+
439+ ! apache_is_installed && err_msg "Apache is not installed."
440+
441+ if ! ask_yn "Do you really want to continue?" Yn; then
442+ return
443+ else
444+ install_apache
445+ fi
446+
447+ apache_install_site "${APACHE_MORTY_SITE}"
448+
449+ info_msg "testing public url .."
450+ if ! service_is_available "${PUBLIC_URL_MORTY}"; then
451+ err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!"
452+ fi
453+}
454+
455+remove_apache_site() {
456+
457+ rst_title "Remove Apache site $APACHE_MORTY_SITE"
458+
459+ rst_para "\
460+This removes apache site ${APACHE_MORTY_SITE}."
461+
462+ ! apache_is_installed && err_msg "Apache is not installed."
463+
464+ if ! ask_yn "Do you really want to continue?" Yn; then
465+ return
466+ fi
467+
468+ apache_remove_site "$APACHE_MORTY_SITE"
469+}
470+
471+install_nginx_site() {
472+
473+ rst_title "Install nginx site $NGINX_MORTY_SITE"
474+
475+ rst_para "\
476+This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_MORTY_SITE})"
477+
478+ ! nginx_is_installed && err_msg "nginx is not installed."
479+
480+ if ! ask_yn "Do you really want to continue?" Yn; then
481+ return
482+ else
483+ install_nginx
484+ fi
485+
486+ "${REPO_ROOT}/utils/searx.sh" install uwsgi
487+
488+ # shellcheck disable=SC2034
489+ SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC)
490+ # shellcheck disable=SC2034
491+ SEARX_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_URL_PATH)
492+ nginx_install_app "${NGINX_MORTY_SITE}"
493+
494+ info_msg "testing public url .."
495+ if ! service_is_available "${PUBLIC_URL_MORTY}"; then
496+ err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!"
497+ fi
498+}
499+
500+remove_nginx_site() {
501+
502+ rst_title "Remove nginx site $NGINX_MORTY_SITE"
503+
504+ rst_para "\
505+This removes nginx site ${NGINX_MORTY_SITE}."
506+
507+ ! nginx_is_installed && err_msg "nginx is not installed."
508+
509+ if ! ask_yn "Do you really want to continue?" Yn; then
510+ return
511+ fi
512+
513+ nginx_remove_site "$NGINX_MORTY_SITE"
514+
515+}
516+
517+rst-doc() {
518+
519+ eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/morty.rst")\""
520+
521+ echo -e "\n.. START install systemd unit"
522+ cat <<EOF
523+.. tabs::
524+
525+ .. group-tab:: systemd
526+
527+ .. code:: bash
528+
529+EOF
530+ eval "echo \"$(< "${TEMPLATES}/${SERVICE_SYSTEMD_UNIT}")\"" | prefix_stdout " "
531+ echo -e "\n.. END install systemd unit"
532+
533+ # for DIST_NAME in ubuntu-20.04 arch fedora; do
534+ # (
535+ # DIST_ID=${DIST_NAME%-*}
536+ # DIST_VERS=${DIST_NAME#*-}
537+ # [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS=
538+ # # ...
539+ # )
540+ # done
541+}
542+
543+
544+# ----------------------------------------------------------------------------
545+main "$@"
546+# ----------------------------------------------------------------------------
utils/searx.shnew file+869
@@ -0,0 +1,869 @@
1+#!/usr/bin/env bash
2+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
3+# SPDX-License-Identifier: AGPL-3.0-or-later
4+# shellcheck disable=SC2001
5+
6+# shellcheck source=utils/lib.sh
7+source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
8+# shellcheck source=utils/brand.env
9+source "${REPO_ROOT}/utils/brand.env"
10+source_dot_config
11+source "${REPO_ROOT}/utils/lxc-searx.env"
12+in_container && lxc_set_suite_env
13+
14+# ----------------------------------------------------------------------------
15+# config
16+# ----------------------------------------------------------------------------
17+
18+PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
19+
20+SEARX_INTERNAL_HTTP="${SEARX_INTERNAL_HTTP:-127.0.0.1:8888}"
21+
22+SEARX_URL_PATH="${SEARX_URL_PATH:-$(echo "${PUBLIC_URL}" \
23+| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}"
24+[[ "${SEARX_URL_PATH}" == "${PUBLIC_URL}" ]] && SEARX_URL_PATH=/
25+SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(echo "$PUBLIC_URL" \
26+| sed -e 's,^.*://\([^\:/]*\).*,\1,g') }"
27+
28+SERVICE_NAME="searx"
29+SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}"
30+SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}"
31+SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}"
32+# shellcheck disable=SC2034
33+SERVICE_GROUP="${SERVICE_USER}"
34+
35+GIT_BRANCH="${GIT_BRANCH:-master}"
36+SEARX_PYENV="${SERVICE_HOME}/searx-pyenv"
37+SEARX_SRC="${SERVICE_HOME}/searx-src"
38+SEARX_SETTINGS_PATH="/etc/searx/settings.yml"
39+SEARX_UWSGI_APP="searx.ini"
40+# shellcheck disable=SC2034
41+SEARX_UWSGI_SOCKET="/run/uwsgi/app/searx/socket"
42+
43+# apt packages
44+SEARX_PACKAGES_debian="\
45+virtualenv python3-dev python3-babel python3-venv
46+uwsgi uwsgi-plugin-python3
47+git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev
48+shellcheck"
49+
50+BUILD_PACKAGES_debian="\
51+firefox graphviz imagemagick texlive-xetex librsvg2-bin
52+texlive-latex-recommended texlive-extra-utils ttf-dejavu
53+latexmk"
54+
55+# pacman packages
56+SEARX_PACKAGES_arch="\
57+python-virtualenv python python-pip python-lxml python-babel
58+uwsgi uwsgi-plugin-python
59+git base-devel libxml2
60+shellcheck"
61+
62+BUILD_PACKAGES_arch="\
63+firefox graphviz imagemagick texlive-bin extra/librsvg
64+texlive-core texlive-latexextra ttf-dejavu"
65+
66+# dnf packages
67+SEARX_PACKAGES_fedora="\
68+virtualenv python python-pip python-lxml python-babel
69+uwsgi uwsgi-plugin-python3
70+git @development-tools libxml2
71+ShellCheck"
72+
73+BUILD_PACKAGES_fedora="\
74+firefox graphviz graphviz-gd ImageMagick librsvg2-tools
75+texlive-xetex-bin texlive-collection-fontsrecommended
76+texlive-collection-latex dejavu-sans-fonts dejavu-serif-fonts
77+dejavu-sans-mono-fonts"
78+
79+case $DIST_ID-$DIST_VERS in
80+ ubuntu-16.04|ubuntu-18.04)
81+ SEARX_PACKAGES="${SEARX_PACKAGES_debian}"
82+ BUILD_PACKAGES="${BUILD_PACKAGES_debian}"
83+ APACHE_PACKAGES="$APACHE_PACKAGES libapache2-mod-proxy-uwsgi"
84+ ;;
85+ ubuntu-20.04)
86+ # https://askubuntu.com/a/1224710
87+ SEARX_PACKAGES="${SEARX_PACKAGES_debian} python-is-python3"
88+ BUILD_PACKAGES="${BUILD_PACKAGES_debian}"
89+ ;;
90+ ubuntu-*|debian-*)
91+ SEARX_PACKAGES="${SEARX_PACKAGES_debian}"
92+ BUILD_PACKAGES="${BUILD_PACKAGES_debian}"
93+ ;;
94+ arch-*)
95+ SEARX_PACKAGES="${SEARX_PACKAGES_arch}"
96+ BUILD_PACKAGES="${BUILD_PACKAGES_arch}"
97+ ;;
98+ fedora-*)
99+ SEARX_PACKAGES="${SEARX_PACKAGES_fedora}"
100+ BUILD_PACKAGES="${BUILD_PACKAGES_fedora}"
101+ ;;
102+esac
103+
104+# Apache Settings
105+APACHE_SEARX_SITE="searx.conf"
106+
107+# shellcheck disable=SC2034
108+CONFIG_FILES=(
109+ "${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP}"
110+)
111+
112+# shellcheck disable=SC2034
113+CONFIG_BACKUP_ENCRYPTED=(
114+ "${SEARX_SETTINGS_PATH}"
115+)
116+
117+# ----------------------------------------------------------------------------
118+usage() {
119+# ----------------------------------------------------------------------------
120+
121+ # shellcheck disable=SC1117
122+ cat <<EOF
123+usage::
124+ $(basename "$0") shell
125+ $(basename "$0") install [all|user|searx-src|pyenv|uwsgi|packages|buildhost]
126+ $(basename "$0") update [searx]
127+ $(basename "$0") remove [all|user|pyenv|searx-src]
128+ $(basename "$0") activate [service]
129+ $(basename "$0") deactivate [service]
130+ $(basename "$0") inspect [service]
131+ $(basename "$0") option [debug-[on|off]|image-proxy-[on|off]|result-proxy <url> <key>]
132+ $(basename "$0") apache [install|remove]
133+
134+shell
135+ start interactive shell from user ${SERVICE_USER}
136+install / remove
137+ :all: complete (de-) installation of searx service
138+ :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME)
139+ :searx-src: clone $GIT_URL
140+ :pyenv: create/remove virtualenv (python) in $SEARX_PYENV
141+ :uwsgi: install searx uWSGI application
142+ :settings: reinstall settings from ${REPO_ROOT}/searx/settings.yml
143+ :packages: install needed packages from OS package manager
144+ :buildhost: install packages from OS package manager needed by buildhosts
145+update searx
146+ Update searx installation ($SERVICE_HOME)
147+activate service
148+ activate and start service daemon (systemd unit)
149+deactivate service
150+ stop and deactivate service daemon (systemd unit)
151+inspect service
152+ run some small tests and inspect service's status and log
153+option
154+ set one of the available options
155+apache
156+ :install: apache site with the searx uwsgi app
157+ :remove: apache site ${APACHE_FILTRON_SITE}
158+
159+searx settings: ${SEARX_SETTINGS_PATH}
160+
161+If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file::
162+ PUBLIC_URL : ${PUBLIC_URL}
163+ SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME}
164+ SERVICE_USER : ${SERVICE_USER}
165+ SEARX_INTERNAL_HTTP : http://${SEARX_INTERNAL_HTTP}
166+EOF
167+ if in_container; then
168+ # searx is listening on 127.0.0.1 and not available from outside container
169+ # in containers the service is listening on 0.0.0.0 (see lxc-searx.env)
170+ echo -e "${_BBlack}HINT:${_creset} searx only listen on loopback device" \
171+ "${_BBlack}inside${_creset} the container."
172+ for ip in $(global_IPs) ; do
173+ if [[ $ip =~ .*:.* ]]; then
174+ echo " container (IPv6): [${ip#*|}]"
175+ else
176+ # IPv4:
177+ echo " container (IPv4): ${ip#*|}"
178+ fi
179+ done
180+ fi
181+ [[ -n ${1} ]] && err_msg "$1"
182+}
183+
184+main() {
185+ required_commands \
186+ sudo systemctl install git wget curl \
187+ || exit
188+
189+ local _usage="unknown or missing $1 command $2"
190+
191+ case $1 in
192+ --getenv) var="$2"; echo "${!var}"; exit 0;;
193+ -h|--help) usage; exit 0;;
194+ shell)
195+ sudo_or_exit
196+ interactive_shell "${SERVICE_USER}"
197+ ;;
198+ inspect)
199+ case $2 in
200+ service)
201+ sudo_or_exit
202+ inspect_service
203+ ;;
204+ *) usage "$_usage"; exit 42;;
205+ esac ;;
206+ install)
207+ rst_title "$SEARX_INSTANCE_NAME" part
208+ sudo_or_exit
209+ case $2 in
210+ all) install_all ;;
211+ user) assert_user ;;
212+ pyenv) create_pyenv ;;
213+ searx-src) clone_searx ;;
214+ settings) install_settings ;;
215+ uwsgi)
216+ install_searx_uwsgi
217+ if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then
218+ err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check searx & uwsgi setup!"
219+ fi
220+ ;;
221+ packages)
222+ pkg_install "$SEARX_PACKAGES"
223+ ;;
224+ buildhost)
225+ pkg_install "$SEARX_PACKAGES"
226+ pkg_install "$BUILD_PACKAGES"
227+ ;;
228+ *) usage "$_usage"; exit 42;;
229+ esac ;;
230+ update)
231+ sudo_or_exit
232+ case $2 in
233+ searx) update_searx;;
234+ *) usage "$_usage"; exit 42;;
235+ esac ;;
236+ remove)
237+ sudo_or_exit
238+ case $2 in
239+ all) remove_all;;
240+ user) drop_service_account "${SERVICE_USER}";;
241+ pyenv) remove_pyenv ;;
242+ searx-src) remove_searx ;;
243+ *) usage "$_usage"; exit 42;;
244+ esac ;;
245+ activate)
246+ sudo_or_exit
247+ case $2 in
248+ service)
249+ activate_service ;;
250+ *) usage "$_usage"; exit 42;;
251+ esac ;;
252+ deactivate)
253+ sudo_or_exit
254+ case $2 in
255+ service) deactivate_service ;;
256+ *) usage "$_usage"; exit 42;;
257+ esac ;;
258+ option)
259+ sudo_or_exit
260+ case $2 in
261+ debug-on) echo; enable_debug ;;
262+ debug-off) echo; disable_debug ;;
263+ result-proxy) set_result_proxy "$3" "$4" ;;
264+ image-proxy-on) enable_image_proxy ;;
265+ image-proxy-off) disable_image_proxy ;;
266+ *) usage "$_usage"; exit 42;;
267+ esac ;;
268+ apache)
269+ sudo_or_exit
270+ case $2 in
271+ install) install_apache_site ;;
272+ remove) remove_apache_site ;;
273+ *) usage "$_usage"; exit 42;;
274+ esac ;;
275+ doc) rst-doc;;
276+ *) usage "unknown or missing command $1"; exit 42;;
277+ esac
278+}
279+
280+_service_prefix=" ${_Yellow}|$SERVICE_USER|${_creset} "
281+
282+install_all() {
283+ rst_title "Install $SEARX_INSTANCE_NAME (service)"
284+ pkg_install "$SEARX_PACKAGES"
285+ wait_key
286+ assert_user
287+ wait_key
288+ clone_searx
289+ wait_key
290+ create_pyenv
291+ wait_key
292+ install_settings
293+ wait_key
294+ test_local_searx
295+ wait_key
296+ install_searx_uwsgi
297+ if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then
298+ err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check searx & uwsgi setup!"
299+ fi
300+ if ask_yn "Do you want to inspect the installation?" Ny; then
301+ inspect_service
302+ fi
303+}
304+
305+update_searx() {
306+ rst_title "Update searx instance"
307+
308+ echo
309+ tee_stderr 0.3 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
310+cd ${SEARX_SRC}
311+git checkout -B "$GIT_BRANCH"
312+git pull
313+pip install -U pip
314+pip install -U setuptools
315+pip install -U wheel
316+pip install -U -e .
317+EOF
318+ install_settings
319+ uWSGI_restart "$SEARX_UWSGI_APP"
320+}
321+
322+remove_all() {
323+ rst_title "De-Install $SEARX_INSTANCE_NAME (service)"
324+
325+ rst_para "\
326+It goes without saying that this script can only be used to remove
327+installations that were installed with this script."
328+
329+ if ! ask_yn "Do you really want to deinstall $SEARX_INSTANCE_NAME?"; then
330+ return
331+ fi
332+ remove_searx_uwsgi
333+ drop_service_account "${SERVICE_USER}"
334+ remove_settings
335+ wait_key
336+ if service_is_available "${PUBLIC_URL}"; then
337+ MSG="** Don't forgett to remove your public site! (${PUBLIC_URL}) **" wait_key 10
338+ fi
339+}
340+
341+assert_user() {
342+ rst_title "user $SERVICE_USER" section
343+ echo
344+ tee_stderr 1 <<EOF | bash | prefix_stdout
345+useradd --shell /bin/bash --system \
346+ --home-dir "$SERVICE_HOME" \
347+ --comment 'Privacy-respecting metasearch engine' $SERVICE_USER
348+mkdir "$SERVICE_HOME"
349+chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
350+groups $SERVICE_USER
351+EOF
352+ #SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)"
353+ #export SERVICE_HOME
354+ #echo "export SERVICE_HOME=$SERVICE_HOME"
355+}
356+
357+clone_is_available() {
358+ [[ -f "$SEARX_SRC/.git/config" ]]
359+}
360+
361+# shellcheck disable=SC2164
362+clone_searx() {
363+ rst_title "Clone searx sources" section
364+ echo
365+ if ! sudo -i -u "$SERVICE_USER" ls -d "$REPO_ROOT" > /dev/null; then
366+ die 42 "user '$SERVICE_USER' missed read permission: $REPO_ROOT"
367+ fi
368+ SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME 2>/dev/null)"
369+ if [[ ! "${SERVICE_HOME}" ]]; then
370+ err_msg "to clone searx sources, user $SERVICE_USER hast to be created first"
371+ return 42
372+ fi
373+ export SERVICE_HOME
374+ git_clone "$REPO_ROOT" "$SEARX_SRC" \
375+ "$GIT_BRANCH" "$SERVICE_USER"
376+
377+ pushd "${SEARX_SRC}" > /dev/null
378+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
379+cd "${SEARX_SRC}"
380+git remote set-url origin ${GIT_URL}
381+git config user.email "$ADMIN_EMAIL"
382+git config user.name "$ADMIN_NAME"
383+git config --list
384+EOF
385+ popd > /dev/null
386+}
387+
388+install_settings() {
389+ rst_title "${SEARX_SETTINGS_PATH}" section
390+ if ! clone_is_available; then
391+ err_msg "you have to install searx first"
392+ exit 42
393+ fi
394+ mkdir -p "$(dirname ${SEARX_SETTINGS_PATH})"
395+
396+ if [[ ! -f ${SEARX_SETTINGS_PATH} ]]; then
397+ info_msg "install settings ${REPO_ROOT}/searx/settings.yml"
398+ info_msg " --> ${SEARX_SETTINGS_PATH}"
399+ cp "${REPO_ROOT}/searx/settings.yml" "${SEARX_SETTINGS_PATH}"
400+ configure_searx
401+ return
402+ fi
403+
404+ rst_para "Diff between origin's setting file (+) and current (-):"
405+ echo
406+ $DIFF_CMD "${SEARX_SETTINGS_PATH}" "${SEARX_SRC}/searx/settings.yml"
407+
408+ local action
409+ choose_one action "What should happen to the settings file? " \
410+ "keep configuration unchanged" \
411+ "use origin settings" \
412+ "start interactiv shell"
413+ case $action in
414+ "keep configuration unchanged")
415+ info_msg "leave settings file unchanged"
416+ ;;
417+ "use origin settings")
418+ backup_file "${SEARX_SETTINGS_PATH}"
419+ info_msg "install origin settings"
420+ cp "${SEARX_SRC}/searx/settings.yml" "${SEARX_SETTINGS_PATH}"
421+ ;;
422+ "start interactiv shell")
423+ backup_file "${SEARX_SETTINGS_PATH}"
424+ echo -e "// exit with [${_BCyan}CTRL-D${_creset}]"
425+ sudo -H -i
426+ rst_para 'Diff between new setting file (-) and current (+):'
427+ echo
428+ $DIFF_CMD "${SEARX_SRC}/searx/settings.yml" "${SEARX_SETTINGS_PATH}"
429+ wait_key
430+ ;;
431+ esac
432+}
433+
434+remove_settings() {
435+ rst_title "remove searx settings" section
436+ echo
437+ info_msg "delete ${SEARX_SETTINGS_PATH}"
438+ rm -f "${SEARX_SETTINGS_PATH}"
439+}
440+
441+remove_searx() {
442+ rst_title "Drop searx sources" section
443+ if ask_yn "Do you really want to drop searx sources ($SEARX_SRC)?"; then
444+ rm -rf "$SEARX_SRC"
445+ else
446+ rst_para "Leave searx sources unchanged."
447+ fi
448+}
449+
450+pyenv_is_available() {
451+ [[ -f "${SEARX_PYENV}/bin/activate" ]]
452+}
453+
454+create_pyenv() {
455+ rst_title "Create virtualenv (python)" section
456+ echo
457+ if [[ ! -f "${SEARX_SRC}/manage.sh" ]]; then
458+ err_msg "to create pyenv for searx, searx has to be cloned first"
459+ return 42
460+ fi
461+ info_msg "create pyenv in ${SEARX_PYENV}"
462+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
463+rm -rf "${SEARX_PYENV}"
464+python3 -m venv "${SEARX_PYENV}"
465+grep -qFs -- 'source ${SEARX_PYENV}/bin/activate' ~/.profile \
466+ || echo 'source ${SEARX_PYENV}/bin/activate' >> ~/.profile
467+EOF
468+ info_msg "inspect python's virtual environment"
469+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
470+command -v python && python --version
471+EOF
472+ wait_key
473+ info_msg "install needed python packages"
474+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
475+pip install -U pip
476+pip install -U setuptools
477+pip install -U wheel
478+pip install -U -e .
479+cd ${SEARX_SRC}
480+pip install -e .
481+EOF
482+}
483+
484+remove_pyenv() {
485+ rst_title "Remove virtualenv (python)" section
486+ if ! ask_yn "Do you really want to drop ${SEARX_PYENV} ?"; then
487+ return
488+ fi
489+ info_msg "remove pyenv activation from ~/.profile"
490+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
491+grep -v 'source ${SEARX_PYENV}/bin/activate' ~/.profile > ~/.profile.##
492+mv ~/.profile.## ~/.profile
493+EOF
494+ rm -rf "${SEARX_PYENV}"
495+}
496+
497+configure_searx() {
498+ rst_title "Configure searx" section
499+ rst_para "Setup searx config located at $SEARX_SETTINGS_PATH"
500+ echo
501+ tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix"
502+cd ${SEARX_SRC}
503+sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "$SEARX_SETTINGS_PATH"
504+sed -i -e "s/{instance_name}/${SEARX_INSTANCE_NAME}/g" "$SEARX_SETTINGS_PATH"
505+EOF
506+}
507+
508+test_local_searx() {
509+ rst_title "Testing searx instance localy" section
510+ echo
511+
512+ if service_is_available "http://${SEARX_INTERNAL_HTTP}" &>/dev/null; then
513+ err_msg "URL/port http://${SEARX_INTERNAL_HTTP} is already in use, you"
514+ err_msg "should stop that service before starting local tests!"
515+ if ! ask_yn "Continue with local tests?"; then
516+ return
517+ fi
518+ fi
519+ sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS_PATH"
520+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
521+export SEARX_SETTINGS_PATH="${SEARX_SETTINGS_PATH}"
522+cd ${SEARX_SRC}
523+timeout 10 python searx/webapp.py &
524+sleep 3
525+curl --location --verbose --head --insecure $SEARX_INTERNAL_HTTP
526+EOF
527+ sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS_PATH"
528+}
529+
530+install_searx_uwsgi() {
531+ rst_title "Install searx's uWSGI app (searx.ini)" section
532+ echo
533+ install_uwsgi
534+ uWSGI_install_app "$SEARX_UWSGI_APP"
535+}
536+
537+remove_searx_uwsgi() {
538+ rst_title "Remove searx's uWSGI app (searx.ini)" section
539+ echo
540+ uWSGI_remove_app "$SEARX_UWSGI_APP"
541+}
542+
543+activate_service() {
544+ rst_title "Activate $SEARX_INSTANCE_NAME (service)" section
545+ echo
546+ uWSGI_enable_app "$SEARX_UWSGI_APP"
547+ uWSGI_restart "$SEARX_UWSGI_APP"
548+}
549+
550+deactivate_service() {
551+ rst_title "De-Activate $SEARX_INSTANCE_NAME (service)" section
552+ echo
553+ uWSGI_disable_app "$SEARX_UWSGI_APP"
554+ uWSGI_restart "$SEARX_UWSGI_APP"
555+}
556+
557+enable_image_proxy() {
558+ info_msg "try to enable image_proxy ..."
559+ tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix"
560+cd ${SEARX_SRC}
561+sed -i -e "s/image_proxy : False/image_proxy : True/g" "$SEARX_SETTINGS_PATH"
562+EOF
563+ uWSGI_restart "$SEARX_UWSGI_APP"
564+}
565+
566+disable_image_proxy() {
567+ info_msg "try to enable image_proxy ..."
568+ tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix"
569+cd ${SEARX_SRC}
570+sed -i -e "s/image_proxy : True/image_proxy : False/g" "$SEARX_SETTINGS_PATH"
571+EOF
572+ uWSGI_restart "$SEARX_UWSGI_APP"
573+}
574+
575+enable_debug() {
576+ warn_msg "Do not enable debug in production enviroments!!"
577+ info_msg "try to enable debug mode ..."
578+ tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix"
579+cd ${SEARX_SRC}
580+sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS_PATH"
581+EOF
582+ uWSGI_restart "$SEARX_UWSGI_APP"
583+}
584+
585+disable_debug() {
586+ info_msg "try to disable debug mode ..."
587+ tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix"
588+cd ${SEARX_SRC}
589+sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS_PATH"
590+EOF
591+ uWSGI_restart "$SEARX_UWSGI_APP"
592+}
593+
594+set_result_proxy() {
595+
596+ # usage: set_result_proxy <URL> [<key>]
597+
598+ info_msg "try to set result proxy: '$1' ($2)"
599+ cp "${SEARX_SETTINGS_PATH}" "${SEARX_SETTINGS_PATH}.bak"
600+ _set_result_proxy "$1" "$2" > "${SEARX_SETTINGS_PATH}"
601+}
602+
603+_set_result_proxy() {
604+ local line
605+ local stage=0
606+ local url=" url: $1"
607+ local key=" key: !!binary \"$2\""
608+ if [[ -z $2 ]]; then
609+ key=
610+ fi
611+
612+ while IFS= read -r line
613+ do
614+ if [[ $stage = 0 ]] || [[ $stage = 2 ]] ; then
615+ if [[ $line =~ ^[[:space:]]*#*[[:space:]]*result_proxy[[:space:]]*:[[:space:]]*$ ]]; then
616+ if [[ $stage = 0 ]]; then
617+ stage=1
618+ echo "result_proxy:"
619+ continue
620+ elif [[ $stage = 2 ]]; then
621+ continue
622+ fi
623+ fi
624+ fi
625+ if [[ $stage = 1 ]] || [[ $stage = 2 ]] ; then
626+ if [[ $line =~ ^[[:space:]]*#*[[:space:]]*url[[:space:]]*:[[:space:]] ]]; then
627+ [[ $stage = 1 ]] && echo "$url"
628+ continue
629+ elif [[ $line =~ ^[[:space:]]*#*[[:space:]]*key[[:space:]]*:[[:space:]] ]]; then
630+ [[ $stage = 1 ]] && [[ -n $key ]] && echo "$key"
631+ continue
632+ elif [[ $line =~ ^[[:space:]]*$ ]]; then
633+ stage=2
634+ fi
635+ fi
636+ echo "$line"
637+ done < "${SEARX_SETTINGS_PATH}.bak"
638+}
639+
640+function has_substring() {
641+ [[ "$1" != "${2/$1/}" ]]
642+}
643+inspect_service() {
644+ rst_title "service status & log"
645+ cat <<EOF
646+
647+sourced ${DOT_CONFIG#"$REPO_ROOT/"} :
648+
649+ PUBLIC_URL : ${PUBLIC_URL}
650+ SEARX_URL_PATH : ${SEARX_URL_PATH}
651+ SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME}
652+ SEARX_INTERNAL_HTTP : ${SEARX_INTERNAL_HTTP}
653+
654+EOF
655+
656+ if service_account_is_available "$SERVICE_USER"; then
657+ info_msg "Service account $SERVICE_USER exists."
658+ else
659+ err_msg "Service account $SERVICE_USER does not exists!"
660+ fi
661+
662+ if pyenv_is_available; then
663+ info_msg "~$SERVICE_USER: python environment is available."
664+ else
665+ err_msg "~$SERVICE_USER: python environment is not available!"
666+ fi
667+
668+ if clone_is_available; then
669+ info_msg "~$SERVICE_USER: Searx software is installed."
670+ else
671+ err_msg "~$SERVICE_USER: Missing searx software!"
672+ fi
673+
674+ if uWSGI_app_enabled "$SEARX_UWSGI_APP"; then
675+ info_msg "uWSGI app $SEARX_UWSGI_APP is enabled."
676+ else
677+ err_msg "uWSGI app $SEARX_UWSGI_APP not enabled!"
678+ fi
679+
680+ uWSGI_app_available "$SEARX_UWSGI_APP" \
681+ || err_msg "uWSGI app $SEARX_UWSGI_APP not available!"
682+
683+ if in_container; then
684+ lxc_suite_info
685+ else
686+ info_msg "public URL --> ${PUBLIC_URL}"
687+ info_msg "internal URL --> http://${SEARX_INTERNAL_HTTP}"
688+ fi
689+
690+ if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then
691+ err_msg "uWSGI app (service) at http://${SEARX_INTERNAL_HTTP} is not available!"
692+ MSG="${_Green}[${_BCyan}CTRL-C${_Green}] to stop or [${_BCyan}KEY${_Green}] to continue"\
693+ wait_key
694+ fi
695+
696+ if ! service_is_available "${PUBLIC_URL}"; then
697+ warn_msg "Public service at ${PUBLIC_URL} is not available!"
698+ if ! in_container; then
699+ warn_msg "Check if public name is correct and routed or use the public IP from above."
700+ fi
701+ fi
702+
703+ local _debug_on
704+ if ask_yn "Enable searx debug mode?"; then
705+ enable_debug
706+ _debug_on=1
707+ fi
708+ echo
709+
710+ case $DIST_ID-$DIST_VERS in
711+ ubuntu-*|debian-*)
712+ systemctl --no-pager -l status "${SERVICE_NAME}"
713+ ;;
714+ arch-*)
715+ systemctl --no-pager -l status "uwsgi@${SERVICE_NAME%.*}"
716+ ;;
717+ fedora-*)
718+ systemctl --no-pager -l status uwsgi
719+ ;;
720+ esac
721+
722+ # shellcheck disable=SC2059
723+ printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log"
724+ read -r -s -n1 -t 5
725+ echo
726+
727+ while true; do
728+ trap break 2
729+ case $DIST_ID-$DIST_VERS in
730+ ubuntu-*|debian-*) tail -f /var/log/uwsgi/app/searx.log ;;
731+ arch-*) journalctl -f -u "uwsgi@${SERVICE_NAME%.*}" ;;
732+ fedora-*) journalctl -f -u uwsgi ;;
733+ esac
734+ done
735+
736+ if [[ $_debug_on == 1 ]]; then
737+ disable_debug
738+ fi
739+ return 0
740+}
741+
742+install_apache_site() {
743+ rst_title "Install Apache site $APACHE_SEARX_SITE"
744+
745+ rst_para "\
746+This installs the searx uwsgi app as apache site. If your server is public to
747+the internet, you should instead use a reverse proxy (filtron) to block
748+excessively bot queries."
749+
750+ ! apache_is_installed && err_msg "Apache is not installed."
751+
752+ if ! ask_yn "Do you really want to continue?" Yn; then
753+ return
754+ else
755+ install_apache
756+ fi
757+
758+ apache_install_site --variant=uwsgi "${APACHE_SEARX_SITE}"
759+
760+ rst_title "Install searx's uWSGI app (searx.ini)" section
761+ echo
762+ uWSGI_install_app --variant=socket "$SEARX_UWSGI_APP"
763+
764+ if ! service_is_available "${PUBLIC_URL}"; then
765+ err_msg "Public service at ${PUBLIC_URL} is not available!"
766+ fi
767+}
768+
769+remove_apache_site() {
770+
771+ rst_title "Remove Apache site ${APACHE_SEARX_SITE}"
772+
773+ rst_para "\
774+This removes apache site ${APACHE_SEARX_SITE}."
775+
776+ ! apache_is_installed && err_msg "Apache is not installed."
777+
778+ if ! ask_yn "Do you really want to continue?" Yn; then
779+ return
780+ fi
781+
782+ apache_remove_site "${APACHE_SEARX_SITE}"
783+
784+ rst_title "Remove searx's uWSGI app (searx.ini)" section
785+ echo
786+ uWSGI_remove_app "$SEARX_UWSGI_APP"
787+}
788+
789+rst-doc() {
790+ local debian="${SEARX_PACKAGES_debian}"
791+ local arch="${SEARX_PACKAGES_arch}"
792+ local fedora="${SEARX_PACKAGES_fedora}"
793+ local debian_build="${BUILD_PACKAGES_debian}"
794+ local arch_build="${BUILD_PACKAGES_arch}"
795+ local fedora_build="${BUILD_PACKAGES_fedora}"
796+ debian="$(echo "${debian}" | sed 's/.*/ & \\/' | sed '$ s/.$//')"
797+ arch="$(echo "${arch}" | sed 's/.*/ & \\/' | sed '$ s/.$//')"
798+ fedora="$(echo "${fedora}" | sed 's/.*/ & \\/' | sed '$ s/.$//')"
799+ debian_build="$(echo "${debian_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')"
800+ arch_build="$(echo "${arch_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')"
801+ fedora_build="$(echo "${fedora_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')"
802+
803+ eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/searx.rst")\""
804+
805+ # I use ubuntu-20.04 here to demonstrate that versions are also suported,
806+ # normaly debian-* and ubuntu-* are most the same.
807+
808+ for DIST_NAME in ubuntu-20.04 arch fedora; do
809+ (
810+ DIST_ID=${DIST_NAME%-*}
811+ DIST_VERS=${DIST_NAME#*-}
812+ [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS=
813+ uWSGI_distro_setup
814+
815+ echo -e "\n.. START searx uwsgi-description $DIST_NAME"
816+
817+ case $DIST_ID-$DIST_VERS in
818+ ubuntu-*|debian-*) cat <<EOF
819+# init.d --> /usr/share/doc/uwsgi/README.Debian.gz
820+# For uWSGI debian uses the LSB init process, this might be changed
821+# one day, see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=833067
822+
823+create ${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP}
824+enable: sudo -H ln -s ${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP} ${uWSGI_APPS_ENABLED}/
825+start: sudo -H service uwsgi start ${SEARX_UWSGI_APP%.*}
826+restart: sudo -H service uwsgi restart ${SEARX_UWSGI_APP%.*}
827+stop: sudo -H service uwsgi stop ${SEARX_UWSGI_APP%.*}
828+disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP}
829+EOF
830+ ;;
831+ arch-*) cat <<EOF
832+# systemd --> /usr/lib/systemd/system/uwsgi@.service
833+# For uWSGI archlinux uses systemd template units, see
834+# - http://0pointer.de/blog/projects/instances.html
835+# - https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd
836+
837+create: ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP}
838+enable: sudo -H systemctl enable uwsgi@${SEARX_UWSGI_APP%.*}
839+start: sudo -H systemctl start uwsgi@${SEARX_UWSGI_APP%.*}
840+restart: sudo -H systemctl restart uwsgi@${SEARX_UWSGI_APP%.*}
841+stop: sudo -H systemctl stop uwsgi@${SEARX_UWSGI_APP%.*}
842+disable: sudo -H systemctl disable uwsgi@${SEARX_UWSGI_APP%.*}
843+EOF
844+ ;;
845+ fedora-*) cat <<EOF
846+# systemd --> /usr/lib/systemd/system/uwsgi.service
847+# The unit file starts uWSGI in emperor mode (/etc/uwsgi.ini), see
848+# - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html
849+
850+create: ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP}
851+restart: sudo -H touch ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP}
852+disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP}
853+EOF
854+ ;;
855+ esac
856+ echo -e ".. END searx uwsgi-description $DIST_NAME"
857+
858+ echo -e "\n.. START searx uwsgi-appini $DIST_NAME"
859+ eval "echo \"$(< "${TEMPLATES}/${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP}")\""
860+ echo -e "\n.. END searx uwsgi-appini $DIST_NAME"
861+
862+ )
863+ done
864+
865+}
866+
867+# ----------------------------------------------------------------------------
868+main "$@"
869+# ----------------------------------------------------------------------------
utils/site-python/sphinx_build_tools.pynew file+48
@@ -0,0 +1,48 @@
1+# -*- coding: utf-8; mode: python -*-
2+"""Implement some sphinx-build tools.
3+
4+"""
5+
6+import os
7+import sys
8+from sphinx.util.pycompat import execfile_
9+
10+# ------------------------------------------------------------------------------
11+def load_sphinx_config(namespace):
12+# ------------------------------------------------------------------------------
13+
14+ u"""Load an additional configuration file into *namespace*.
15+
16+ The name of the configuration file is taken from the environment
17+ ``SPHINX_CONF``. The external configuration file extends (or overwrites) the
18+ configuration values from the origin ``conf.py``. With this you are able to
19+ maintain *build themes*. To your docs/conf.py add::
20+
21+ from sphinx_build_tools import load_sphinx_config
22+ ...
23+
24+ # Since loadConfig overwrites settings from the global namespace, it has to be
25+ # the last statement in the conf.py file
26+
27+ load_sphinx_config(globals())
28+
29+ """
30+
31+ config_file = os.environ.get("SPHINX_CONF", None)
32+ if (config_file is not None
33+ and os.path.normpath(namespace["__file__"]) != os.path.normpath(config_file) ):
34+ config_file = os.path.abspath(config_file)
35+
36+ if os.path.isfile(config_file):
37+ sys.stdout.write(
38+ "load additional sphinx-config: %s\n"
39+ % config_file)
40+ config = namespace.copy()
41+ config['__file__'] = config_file
42+ execfile_(config_file, config)
43+ del config['__file__']
44+ namespace.update(config)
45+ else:
46+ sys.stderr.write(
47+ "WARNING: additional sphinx-config not found: %s\n"
48+ % config_file)
utils/templates/etc/apache2new file+1
@@ -0,0 +1 @@
1+httpd
utils/templates/etc/filtron/rules.jsonnew file+129
@@ -0,0 +1,129 @@
1+[
2+ {
3+ "name": "roboagent limit",
4+ "filters": [
5+ "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client|Ruby|UniversalFeedParser)"
6+ ],
7+ "limit": 0,
8+ "stop": true,
9+ "actions": [
10+ { "name": "log"},
11+ { "name": "block",
12+ "params": {
13+ "message": "Rate limit exceeded"
14+ }
15+ }
16+ ]
17+ },
18+ {
19+ "name": "botlimit",
20+ "filters": [
21+ "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
22+ ],
23+ "limit": 0,
24+ "stop": true,
25+ "actions": [
26+ { "name": "log"},
27+ { "name": "block",
28+ "params": {
29+ "message": "Rate limit exceeded"
30+ }
31+ }
32+ ]
33+ },
34+ {
35+ "name": "suspiciously frequent IP",
36+ "filters": [],
37+ "interval": 600,
38+ "limit": 30,
39+ "aggregations": [
40+ "Header:X-Forwarded-For"
41+ ],
42+ "actions":[
43+ {"name":"log"}
44+ ]
45+ },
46+ {
47+ "name": "search request",
48+ "filters": [
49+ "Param:q",
50+ "Path=^(/|/search)$"
51+ ],
52+ "interval": 61,
53+ "limit": 999,
54+ "subrules": [
55+ {
56+ "name": "missing Accept-Language",
57+ "filters": ["!Header:Accept-Language"],
58+ "limit": 0,
59+ "stop": true,
60+ "actions": [
61+ {"name":"log"},
62+ {"name": "block",
63+ "params": {"message": "Rate limit exceeded"}}
64+ ]
65+ },
66+ {
67+ "name": "suspiciously Connection=close header",
68+ "filters": ["Header:Connection=close"],
69+ "limit": 0,
70+ "stop": true,
71+ "actions": [
72+ {"name":"log"},
73+ {"name": "block",
74+ "params": {"message": "Rate limit exceeded"}}
75+ ]
76+ },
77+ {
78+ "name": "IP limit",
79+ "interval": 61,
80+ "limit": 9,
81+ "stop": true,
82+ "aggregations": [
83+ "Header:X-Forwarded-For"
84+ ],
85+ "actions": [
86+ { "name": "log"},
87+ { "name": "block",
88+ "params": {
89+ "message": "Rate limit exceeded"
90+ }
91+ }
92+ ]
93+ },
94+ {
95+ "name": "rss/json limit",
96+ "filters": [
97+ "Param:format=(csv|json|rss)"
98+ ],
99+ "interval": 121,
100+ "limit": 2,
101+ "stop": true,
102+ "actions": [
103+ { "name": "log"},
104+ { "name": "block",
105+ "params": {
106+ "message": "Rate limit exceeded"
107+ }
108+ }
109+ ]
110+ },
111+ {
112+ "name": "useragent limit",
113+ "interval": 61,
114+ "limit": 199,
115+ "aggregations": [
116+ "Header:User-Agent"
117+ ],
118+ "actions": [
119+ { "name": "log"},
120+ { "name": "block",
121+ "params": {
122+ "message": "Rate limit exceeded"
123+ }
124+ }
125+ ]
126+ }
127+ ]
128+ }
129+]
utils/templates/etc/httpd/sites-available/morty.confnew file+28
@@ -0,0 +1,28 @@
1+# -*- coding: utf-8; mode: apache -*-
2+
3+LoadModule headers_module ${APACHE_MODULES}/mod_headers.so
4+LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so
5+LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so
6+#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so
7+
8+# SetEnvIf Request_URI "${PUBLIC_URL_PATH_MORTY}" dontlog
9+# CustomLog /dev/null combined env=dontlog
10+
11+<Location ${PUBLIC_URL_PATH_MORTY} >
12+
13+ <IfModule mod_security2.c>
14+ SecRuleEngine Off
15+ </IfModule>
16+
17+ Require all granted
18+
19+ Order deny,allow
20+ Deny from all
21+ #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
22+ Allow from all
23+
24+ ProxyPreserveHost On
25+ ProxyPass http://${MORTY_LISTEN}
26+ RequestHeader set X-Script-Name ${PUBLIC_URL_PATH_MORTY}
27+
28+</Location>
utils/templates/etc/httpd/sites-available/searx.conf:filtronnew file+33
@@ -0,0 +1,33 @@
1+# -*- coding: utf-8; mode: apache -*-
2+
3+LoadModule headers_module ${APACHE_MODULES}/mod_headers.so
4+LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so
5+LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so
6+#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so
7+
8+# SetEnvIf Request_URI "${FILTRON_URL_PATH}" dontlog
9+# CustomLog /dev/null combined env=dontlog
10+
11+# SecRuleRemoveById 981054
12+# SecRuleRemoveById 981059
13+# SecRuleRemoveById 981060
14+# SecRuleRemoveById 950907
15+
16+<Location ${FILTRON_URL_PATH} >
17+
18+ <IfModule mod_security2.c>
19+ SecRuleEngine Off
20+ </IfModule>
21+
22+ Require all granted
23+
24+ Order deny,allow
25+ Deny from all
26+ #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
27+ Allow from all
28+
29+ ProxyPreserveHost On
30+ ProxyPass http://${FILTRON_LISTEN}
31+ RequestHeader set X-Script-Name ${FILTRON_URL_PATH}
32+
33+</Location>
utils/templates/etc/httpd/sites-available/searx.conf:uwsginew file+27
@@ -0,0 +1,27 @@
1+# -*- coding: utf-8; mode: apache -*-
2+
3+LoadModule headers_module ${APACHE_MODULES}/mod_headers.so
4+LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so
5+LoadModule proxy_uwsgi_module ${APACHE_MODULES}/mod_proxy_uwsgi.so
6+# LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so
7+
8+# SetEnvIf Request_URI "${SEARX_URL_PATH}" dontlog
9+# CustomLog /dev/null combined env=dontlog
10+
11+<Location ${SEARX_URL_PATH}>
12+
13+ <IfModule mod_security2.c>
14+ SecRuleEngine Off
15+ </IfModule>
16+
17+ Require all granted
18+
19+ Order deny,allow
20+ Deny from all
21+ # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
22+ Allow from all
23+
24+ ProxyPreserveHost On
25+ ProxyPass unix:${SEARX_UWSGI_SOCKET}|uwsgi://uwsgi-uds-searx/
26+
27+</Location>
utils/templates/etc/nginx/default.apps-available/morty.confnew file+11
@@ -0,0 +1,11 @@
1+# https://example.org/morty
2+
3+location /morty {
4+ proxy_pass http://127.0.0.1:3000/;
5+
6+ proxy_set_header Host \$http_host;
7+ proxy_set_header Connection \$http_connection;
8+ proxy_set_header X-Real-IP \$remote_addr;
9+ proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
10+ proxy_set_header X-Scheme \$scheme;
11+}
utils/templates/etc/nginx/default.apps-available/searx.conf:filtronnew file+16
@@ -0,0 +1,16 @@
1+# https://example.org/searx
2+
3+location ${SEARX_URL_PATH} {
4+ proxy_pass http://127.0.0.1:4004/;
5+
6+ proxy_set_header Host \$http_host;
7+ proxy_set_header Connection \$http_connection;
8+ proxy_set_header X-Real-IP \$remote_addr;
9+ proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
10+ proxy_set_header X-Scheme \$scheme;
11+ proxy_set_header X-Script-Name ${SEARX_URL_PATH};
12+}
13+
14+location ${SEARX_URL_PATH}/static {
15+ alias ${SEARX_SRC}/searx/static;
16+}
utils/templates/etc/uwsgi/apps-archlinux/searx.ininew file+80
@@ -0,0 +1,80 @@
1+[uwsgi]
2+
3+# uWSGI core
4+# ----------
5+#
6+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
7+
8+# Who will run the code
9+uid = ${SERVICE_USER}
10+gid = ${SERVICE_GROUP}
11+
12+# chdir to specified directory before apps loading
13+chdir = ${SEARX_SRC}/searx
14+
15+# searx configuration (settings.yml)
16+env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH}
17+
18+# disable logging for privacy
19+logger = systemd
20+disable-logging = true
21+
22+# The right granted on the created socket
23+chmod-socket = 666
24+
25+# Plugin to use and interpretor config
26+single-interpreter = true
27+
28+# enable master process
29+master = true
30+
31+# load apps in each worker instead of the master
32+lazy-apps = true
33+
34+# load uWSGI plugins
35+plugin = python
36+
37+# By default the Python plugin does not initialize the GIL. This means your
38+# app-generated threads will not run. If you need threads, remember to enable
39+# them with enable-threads. Running uWSGI in multithreading mode (with the
40+# threads options) will automatically enable threading support. This *strange*
41+# default behaviour is for performance reasons.
42+enable-threads = true
43+
44+
45+# plugin: python
46+# --------------
47+#
48+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python
49+
50+# load a WSGI module
51+module = searx.webapp
52+
53+# set PYTHONHOME/virtualenv
54+virtualenv = ${SEARX_PYENV}
55+
56+# add directory (or glob) to pythonpath
57+pythonpath = ${SEARX_SRC}
58+
59+
60+# speak to upstream
61+# -----------------
62+#
63+# Activate the 'http' configuration for filtron or activate the 'socket'
64+# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
65+
66+# using IP:
67+#
68+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
69+# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
70+
71+http = ${SEARX_INTERNAL_HTTP}
72+
73+# using unix-sockets:
74+#
75+# On some distributions you need to create the app folder for the sockets::
76+#
77+# mkdir -p /run/uwsgi/app/searx
78+# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
79+#
80+# socket = /run/uwsgi/app/searx/socket
utils/templates/etc/uwsgi/apps-archlinux/searx.ini:socketnew file+80
@@ -0,0 +1,80 @@
1+[uwsgi]
2+
3+# uWSGI core
4+# ----------
5+#
6+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
7+
8+# Who will run the code
9+uid = ${SERVICE_USER}
10+gid = ${SERVICE_GROUP}
11+
12+# chdir to specified directory before apps loading
13+chdir = ${SEARX_SRC}/searx
14+
15+# searx configuration (settings.yml)
16+env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH}
17+
18+# disable logging for privacy
19+logger = systemd
20+disable-logging = true
21+
22+# The right granted on the created socket
23+chmod-socket = 666
24+
25+# Plugin to use and interpretor config
26+single-interpreter = true
27+
28+# enable master process
29+master = true
30+
31+# load apps in each worker instead of the master
32+lazy-apps = true
33+
34+# load uWSGI plugins
35+plugin = python
36+
37+# By default the Python plugin does not initialize the GIL. This means your
38+# app-generated threads will not run. If you need threads, remember to enable
39+# them with enable-threads. Running uWSGI in multithreading mode (with the
40+# threads options) will automatically enable threading support. This *strange*
41+# default behaviour is for performance reasons.
42+enable-threads = true
43+
44+
45+# plugin: python
46+# --------------
47+#
48+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python
49+
50+# load a WSGI module
51+module = searx.webapp
52+
53+# set PYTHONHOME/virtualenv
54+virtualenv = ${SEARX_PYENV}
55+
56+# add directory (or glob) to pythonpath
57+pythonpath = ${SEARX_SRC}
58+
59+
60+# speak to upstream
61+# -----------------
62+#
63+# Activate the 'http' configuration for filtron or activate the 'socket'
64+# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
65+
66+# using IP:
67+#
68+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
69+# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
70+
71+# http = ${SEARX_INTERNAL_HTTP}
72+
73+# using unix-sockets:
74+#
75+# On some distributions you need to create the app folder for the sockets::
76+#
77+# mkdir -p /run/uwsgi/app/searx
78+# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
79+#
80+socket = /run/uwsgi/app/searx/socket
utils/templates/etc/uwsgi/apps-available/searx.ininew file+79
@@ -0,0 +1,79 @@
1+[uwsgi]
2+
3+# uWSGI core
4+# ----------
5+#
6+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
7+
8+# Who will run the code
9+uid = ${SERVICE_USER}
10+gid = ${SERVICE_GROUP}
11+
12+# chdir to specified directory before apps loading
13+chdir = ${SEARX_SRC}/searx
14+
15+# searx configuration (settings.yml)
16+env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH}
17+
18+# disable logging for privacy
19+disable-logging = true
20+
21+# The right granted on the created socket
22+chmod-socket = 666
23+
24+# Plugin to use and interpretor config
25+single-interpreter = true
26+
27+# enable master process
28+master = true
29+
30+# load apps in each worker instead of the master
31+lazy-apps = true
32+
33+# load uWSGI plugins
34+plugin = python3,http
35+
36+# By default the Python plugin does not initialize the GIL. This means your
37+# app-generated threads will not run. If you need threads, remember to enable
38+# them with enable-threads. Running uWSGI in multithreading mode (with the
39+# threads options) will automatically enable threading support. This *strange*
40+# default behaviour is for performance reasons.
41+enable-threads = true
42+
43+
44+# plugin: python
45+# --------------
46+#
47+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python
48+
49+# load a WSGI module
50+module = searx.webapp
51+
52+# set PYTHONHOME/virtualenv
53+virtualenv = ${SEARX_PYENV}
54+
55+# add directory (or glob) to pythonpath
56+pythonpath = ${SEARX_SRC}
57+
58+
59+# speak to upstream
60+# -----------------
61+#
62+# Activate the 'http' configuration for filtron or activate the 'socket'
63+# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
64+
65+# using IP:
66+#
67+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
68+# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
69+
70+http = ${SEARX_INTERNAL_HTTP}
71+
72+# using unix-sockets:
73+#
74+# On some distributions you need to create the app folder for the sockets::
75+#
76+# mkdir -p /run/uwsgi/app/searx
77+# chmod -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
78+#
79+# socket = /run/uwsgi/app/searx/socket
utils/templates/etc/uwsgi/apps-available/searx.ini:socketnew file+79
@@ -0,0 +1,79 @@
1+[uwsgi]
2+
3+# uWSGI core
4+# ----------
5+#
6+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
7+
8+# Who will run the code
9+uid = ${SERVICE_USER}
10+gid = ${SERVICE_GROUP}
11+
12+# chdir to specified directory before apps loading
13+chdir = ${SEARX_SRC}/searx
14+
15+# searx configuration (settings.yml)
16+env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH}
17+
18+# disable logging for privacy
19+disable-logging = true
20+
21+# The right granted on the created socket
22+chmod-socket = 666
23+
24+# Plugin to use and interpretor config
25+single-interpreter = true
26+
27+# enable master process
28+master = true
29+
30+# load apps in each worker instead of the master
31+lazy-apps = true
32+
33+# load uWSGI plugins
34+plugin = python3,http
35+
36+# By default the Python plugin does not initialize the GIL. This means your
37+# app-generated threads will not run. If you need threads, remember to enable
38+# them with enable-threads. Running uWSGI in multithreading mode (with the
39+# threads options) will automatically enable threading support. This *strange*
40+# default behaviour is for performance reasons.
41+enable-threads = true
42+
43+
44+# plugin: python
45+# --------------
46+#
47+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python
48+
49+# load a WSGI module
50+module = searx.webapp
51+
52+# set PYTHONHOME/virtualenv
53+virtualenv = ${SEARX_PYENV}
54+
55+# add directory (or glob) to pythonpath
56+pythonpath = ${SEARX_SRC}
57+
58+
59+# speak to upstream
60+# -----------------
61+#
62+# Activate the 'http' configuration for filtron or activate the 'socket'
63+# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
64+
65+# using IP:
66+#
67+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
68+# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
69+
70+# http = ${SEARX_INTERNAL_HTTP}
71+
72+# using unix-sockets:
73+#
74+# On some distributions you need to create the app folder for the sockets::
75+#
76+# mkdir -p /run/uwsgi/app/searx
77+# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
78+#
79+socket = /run/uwsgi/app/searx/socket
utils/templates/lib/systemd/system/filtron.servicenew file+29
@@ -0,0 +1,29 @@
1+[Unit]
2+
3+Description=${SERVICE_NAME}
4+After=syslog.target
5+After=network.target
6+
7+[Service]
8+
9+Type=simple
10+User=${SERVICE_USER}
11+Group=${SERVICE_GROUP}
12+WorkingDirectory=${SERVICE_HOME}
13+ExecStart=${SERVICE_HOME}/go-apps/bin/filtron -api '${FILTRON_API}' -listen '${FILTRON_LISTEN}' -rules '${FILTRON_RULES}' -target '${FILTRON_TARGET}'
14+
15+Restart=always
16+Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME}
17+
18+# Some distributions may not support these hardening directives. If you cannot
19+# start the service due to an unknown option, comment out the ones not supported
20+# by your version of systemd.
21+
22+ProtectSystem=full
23+PrivateDevices=yes
24+PrivateTmp=yes
25+NoNewPrivileges=true
26+
27+[Install]
28+
29+WantedBy=multi-user.target
utils/templates/lib/systemd/system/morty.servicenew file+29
@@ -0,0 +1,29 @@
1+[Unit]
2+
3+Description=${SERVICE_NAME}
4+After=syslog.target
5+After=network.target
6+
7+[Service]
8+
9+Type=simple
10+User=${SERVICE_USER}
11+Group=${SERVICE_GROUP}
12+WorkingDirectory=${SERVICE_HOME}
13+ExecStart=${SERVICE_HOME}/go-apps/bin/morty -key '${MORTY_KEY}' -listen '${MORTY_LISTEN}' -timeout ${MORTY_TIMEOUT}
14+
15+Restart=always
16+Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME} DEBUG=${SERVICE_ENV_DEBUG}
17+
18+# Some distributions may not support these hardening directives. If you cannot
19+# start the service due to an unknown option, comment out the ones not supported
20+# by your version of systemd.
21+
22+ProtectSystem=full
23+PrivateDevices=yes
24+PrivateTmp=yes
25+NoNewPrivileges=true
26+
27+[Install]
28+
29+WantedBy=multi-user.target