on master help sign in
Merge branch 'master' into csp-oscar-theme
commitca1c3bd15d60faf235894cb4f6f3a15150e3a6c3 authorMarkus Heiser <markus.heiser@darmarIT.de> dateFri 19 Jun 2020, 04:51 (6 years ago) parentcc721b5c83007a
68 files changed+7300−537
.config.sh+55 .gitignore+1 Makefile+52−13 docs/_themes/searx/static/searx.css+40−1 docs/admin/arch_public.dot+4−4 docs/admin/architecture.rst+9−5 docs/admin/buildhosts.rst+63−11 docs/admin/filtron.rst+145−102 docs/admin/index.rst+8−1 docs/admin/installation-apache.rst+514 docs/admin/installation-docker.rst+28 docs/admin/installation-nginx.rst+381 docs/admin/installation-searx.rst+92 docs/admin/installation-uwsgi.rst+149 docs/admin/installation.rst+37−320 docs/admin/morty.rst+7 docs/admin/settings.rst+7−1 docs/admin/update-searx.rst+23 docs/blog/index.rst+2−1 docs/build-templates/filtron.rst+52 docs/build-templates/morty.rst+52 docs/build-templates/searx.rst+192 docs/conf.py+13−1 docs/dev/contribution_guide.rst+5 docs/dev/index.rst+2−1 docs/dev/makefile.rst+24−24 docs/dev/quickstart.rst+1−1 docs/dev/reST.rst+5−5 docs/index.rst+10−8 docs/user/conf.py+19 docs/user/index.rst+2−1 docs/user/own-instance.rst+4−2 docs/user/public_instances.rst−3 docs/utils/filtron.sh.rst+80 docs/utils/index.rst+53 docs/utils/lxc.sh.rst+148 docs/utils/morty.sh.rst+80 docs/utils/searx.sh.rst+39 requirements-dev.txt+2 searx/brand.py+1 searx/static/plugins/js/search_on_category_select.js+18 searx/templates/__common__/opensearch.xml+5−15 searx/webapp.py+1−1 utils/brand.env+1 utils/filtron.sh+561 utils/lib.sh+1519 utils/lxc-searx.env+95 utils/lxc.sh+552 utils/makefile.include+15−1 utils/makefile.lxc+29 utils/makefile.python+4−4 utils/makefile.sphinx+15−11 utils/morty.sh+546 utils/searx.sh+869 utils/site-python/sphinx_build_tools.py+48 utils/templates/etc/apache2+1 utils/templates/etc/filtron/rules.json+129 utils/templates/etc/httpd/sites-available/morty.conf+28 utils/templates/etc/httpd/sites-available/searx.conf:filtron+33 utils/templates/etc/httpd/sites-available/searx.conf:uwsgi+27 utils/templates/etc/nginx/default.apps-available/morty.conf+11 utils/templates/etc/nginx/default.apps-available/searx.conf:filtron+16 utils/templates/etc/uwsgi/apps-archlinux/searx.ini+80 utils/templates/etc/uwsgi/apps-archlinux/searx.ini:socket+80 utils/templates/etc/uwsgi/apps-available/searx.ini+79 utils/templates/etc/uwsgi/apps-available/searx.ini:socket+79 utils/templates/lib/systemd/system/filtron.service+29 utils/templates/lib/systemd/system/morty.service+29
.config.shnew file+55
@@ -0,0 +1,55 @@
1+# -*- coding: utf-8; mode: sh -*-
2+# SPDX-License-Identifier: AGPL-3.0-or-later
3+# shellcheck shell=bash disable=SC2034
4+#
5+# This environment is used by ./utils scripts like filtron.sh or searx.sh. The
6+# default values are *most flexible* and *best maintained*, you normally not
7+# need to change the defaults (except PUBLIC_URL).
8+#
9+# Before you change any value here you have to uninstall any previous
10+# installation. Further is it recommended to backup your changes simply by
11+# adding them to you local brand (git branch)::
12+#
13+# git add .config
14+
15+# The public URL of the searx instance: PUBLIC_URL="https://mydomain.xy/searx"
16+# The default is taken from ./utils/brand.env.
17+
18+PUBLIC_URL="${SEARX_URL}"
19+
20+if [[ ${PUBLIC_URL} == "https://searx.me" ]]; then
21+ # hint: Linux containers do not have DNS entries, lets use IPs
22+ PUBLIC_URL="http://$(primary_ip)/searx"
23+fi
24+
25+# searx.sh
26+# ---------
27+
28+# SEARX_INTERNAL_URL="127.0.0.1:8888"
29+
30+# Only change, if you maintain a searx brand in your searx fork.
31+# GIT_BRANCH="${GIT_BRANCH:-master}"
32+
33+# filtron.sh
34+# ----------
35+
36+# FILTRON_API="127.0.0.1:4005"
37+# FILTRON_LISTEN="127.0.0.1:4004"
38+# FILTRON_TARGET="127.0.0.1:8888"
39+
40+# morty.sh
41+# --------
42+
43+# morty listen address
44+# MORTY_LISTEN="127.0.0.1:3000"
45+# PUBLIC_URL_PATH_MORTY="/morty/"
46+
47+# system services
48+# ---------------
49+
50+# Common $HOME folder of the service accounts
51+# SERVICE_HOME_BASE="/usr/local"
52+
53+# **experimental**: Set SERVICE_USER to run all services by one account, but be
54+# aware that removing discrete components might conflict!
55+# SERVICE_USER=searx
.gitignore+1
@@ -1,6 +1,7 @@
11# to sync with .dockerignore
22.coverage
33coverage/
4+cache/
45.installed.cfg
56engines.cfg
67env
Makefile+52−13
@@ -1,24 +1,31 @@
11# -*- coding: utf-8; mode: makefile-gmake -*-
2+.DEFAULT_GOAL=help
23
4+# START Makefile setup
35export GIT_URL=https://github.com/asciimoo/searx
6+export GIT_BRANCH=master
47export SEARX_URL=https://searx.me
58export DOCS_URL=https://asciimoo.github.io/searx
9+# END Makefile setup
10+
11+include utils/makefile.include
612
713PYOBJECTS = searx
814DOC = docs
915PY_SETUP_EXTRAS ?= \[test\]
1016
11−PYDIST=./dist/py
12−PYBUILD=./build/py
13−
14−include utils/makefile.include
1517include utils/makefile.python
1618include utils/makefile.sphinx
1719
1820all: clean install
1921
20−PHONY += help
21−help:
22+PHONY += help-min help-all help
23+
24+help: help-min
25+ @echo ''
26+ @echo 'to get more help: make help-all'
27+
28+help-min:
2229 @echo ' test - run developer tests'
2330 @echo ' docs - build documentation'
2431 @echo ' docs-live - autobuild HTML documentation while editing'
@@ -33,9 +40,18 @@ help:
3340 @echo ' docker - build Docker image'
3441 @echo ' node.env - download & install npm dependencies locally'
3542 @echo ''
36− @$(MAKE) -s -f utils/makefile.include make-help
43+ @echo 'environment'
44+ @echo ' SEARX_URL = $(SEARX_URL)'
45+ @echo ' GIT_URL = $(GIT_URL)'
46+ @echo ' DOCS_URL = $(DOCS_URL)'
47+ @echo ''
48+ @$(MAKE) -e -s make-help
49+
50+help-all: help-min
51+ @echo ''
52+ @$(MAKE) -e -s python-help
3753 @echo ''
38− @$(MAKE) -s -f utils/makefile.python python-help
54+ @$(MAKE) -e -s docs-help
3955
4056PHONY += install
4157install: buildenv pyenvinstall
@@ -44,7 +60,7 @@ PHONY += uninstall
4460uninstall: pyenvuninstall
4561
4662PHONY += clean
47−clean: pyclean node.clean test.clean
63+clean: pyclean docs-clean node.clean test.clean
4864 $(call cmd,common_clean)
4965
5066PHONY += run
@@ -61,14 +77,24 @@ run: buildenv pyenvinstall
6177# docs
6278# ----
6379
80+sphinx-doc-prebuilds:: buildenv pyenvinstall prebuild-includes
81+
6482PHONY += docs
65−docs: buildenv pyenvinstall sphinx-doc
83+docs: sphinx-doc-prebuilds sphinx-doc
6684 $(call cmd,sphinx,html,docs,docs)
6785
6886PHONY += docs-live
69−docs-live: buildenv pyenvinstall sphinx-live
87+docs-live: sphinx-doc-prebuilds sphinx-live
7088 $(call cmd,sphinx_autobuild,html,docs,docs)
7189
90+PHONY += prebuild-includes
91+prebuild-includes:
92+ $(Q)mkdir -p $(DOCS_BUILD)/includes
93+ $(Q)./utils/searx.sh doc | cat > $(DOCS_BUILD)/includes/searx.rst
94+ $(Q)./utils/filtron.sh doc | cat > $(DOCS_BUILD)/includes/filtron.rst
95+ $(Q)./utils/morty.sh doc | cat > $(DOCS_BUILD)/includes/morty.rst
96+
97+
7298$(GH_PAGES)::
7399 @echo "doc available at --> $(DOCS_URL)"
74100
@@ -94,12 +120,14 @@ useragents.update: pyenvinstall
94120buildenv:
95121 $(Q)echo "build searx/brand.py"
96122 $(Q)echo "GIT_URL = '$(GIT_URL)'" > searx/brand.py
123+ $(Q)echo "GIT_BRANCH = '$(GIT_BRANCH)'" >> searx/brand.py
97124 $(Q)echo "ISSUE_URL = 'https://github.com/asciimoo/searx/issues'" >> searx/brand.py
98125 $(Q)echo "SEARX_URL = '$(SEARX_URL)'" >> searx/brand.py
99126 $(Q)echo "DOCS_URL = '$(DOCS_URL)'" >> searx/brand.py
100127 $(Q)echo "PUBLIC_INSTANCES = 'https://searx.space'" >> searx/brand.py
101128 $(Q)echo "build utils/brand.env"
102129 $(Q)echo "export GIT_URL='$(GIT_URL)'" > utils/brand.env
130+ $(Q)echo "export GIT_BRANCH='$(GIT_BRANCH)'" >> utils/brand.env
103131 $(Q)echo "export ISSUE_URL='https://github.com/asciimoo/searx/issues'" >> utils/brand.env
104132 $(Q)echo "export SEARX_URL='$(SEARX_URL)'" >> utils/brand.env
105133 $(Q)echo "export DOCS_URL='$(DOCS_URL)'" >> utils/brand.env
@@ -182,8 +210,7 @@ gecko.driver:
182210# test
183211# ----
184212
185−PHONY += test test.pylint test.pep8 test.unit test.coverage test.robot
186−
213+PHONY += test test.sh test.pylint test.pep8 test.unit test.coverage test.robot
187214test: buildenv test.pylint test.pep8 test.unit gecko.driver test.robot
188215
189216ifeq ($(PY),2)
@@ -191,6 +218,7 @@ test.pylint:
191218 @echo "LINT skip liniting py2"
192219else
193220# TODO: balance linting with pylint
221+
194222test.pylint: pyenvinstall
195223 $(call cmd,pylint,\
196224 searx/preferences.py \
@@ -202,6 +230,17 @@ endif
202230# E402 module level import not at top of file
203231# W503 line break before binary operator
204232
233+# ubu1604: uses shellcheck v0.3.7 (from 04/2015), no longer supported!
234+test.sh:
235+ shellcheck -x -s bash utils/brand.env
236+ shellcheck -x utils/lib.sh
237+ shellcheck -x utils/filtron.sh
238+ shellcheck -x utils/searx.sh
239+ shellcheck -x utils/morty.sh
240+ shellcheck -x utils/lxc.sh
241+ shellcheck -x utils/lxc-searx.env
242+ shellcheck -x .config.sh
243+
205244test.pep8: pyenvinstall
206245 @echo "TEST pep8"
207246 $(Q)$(PY_ENV_ACT); pep8 --exclude=searx/static --max-line-length=120 --ignore "E402,W503" searx tests
docs/_themes/searx/static/searx.css+40−1
@@ -33,7 +33,7 @@ p.sidebar-title, .sidebar p {
3333/* admonitions
3434*/
3535
36−div.admonition, div.topic {
36+div.admonition, div.topic, div.toctree-wrapper {
3737 background-color: #fafafa;
3838 margin: 8px 0px;
3939 padding: 1em;
@@ -42,6 +42,16 @@ div.admonition, div.topic {
4242 border-right: none;
4343 border-bottom: none;
4444 border-left: 5pt solid #ccc;
45+ list-style-type: disclosure-closed;
46+}
47+
48+div.toctree-wrapper p.caption {
49+ font-weight: normal;
50+ font-size: 24px;
51+ margin: 0 0 10px 0;
52+ padding: 0;
53+ line-height: 1;
54+ display: inline;
4555}
4656
4757p.admonition-title:after {
@@ -128,3 +138,32 @@ caption {
128138 caption-side: top;
129139 text-align: left;
130140}
141+
142+/* bugs since sphinx 3.1
143+
144+See sphinx-doc project, PR 7838 & 7484 with elementary patch to the basic CSS:
145+
146+- https://github.com/sphinx-doc/sphinx/issues/7838#issuecomment-646009605
147+- https://github.com/sphinx-doc/sphinx/pull/7484#issuecomment-646058972
148+
149+*/
150+
151+li > p:first-child {
152+ margin-top: 0;
153+}
154+
155+li > p:last-child {
156+ margin-bottom: 0;
157+}
158+
159+div.admonition dl {
160+ margin-bottom: 0;
161+}
162+
163+div.sidebar {
164+ clear: none;
165+}
166+
167+div.admonition, div.topic, pre {
168+ clear: none;
169+}
docs/admin/arch_public.dot+4−4
@@ -4,11 +4,11 @@ digraph G {
44 edge [fontname="Sans"];
55
66 browser [label="Browser", shape=Mdiamond];
7− rp [label="Reverse Proxy", href="url to configure reverse proxy"];
8− filtron [label="Filtron", href="https://github.com/asciimoo/filtron"];
9− morty [label="Morty", href="https://github.com/asciimoo/morty"];
7+ rp [label="Reverse Proxy", href="https://asciimoo.github.io/searx/utils/filtron.sh.html#public-reverse-proxy"];
8+ filtron [label="Filtron", href="https://asciimoo.github.io/searx/utils/filtron.sh.html"];
9+ morty [label="Morty", href="https://asciimoo.github.io/searx/utils/morty.sh.html"];
1010 static [label="Static files", href="url to configure static files"];
11− uwsgi [label="uwsgi", href="url to configure uwsgi"]
11+ uwsgi [label="uwsgi", href="https://asciimoo.github.io/searx/utils/searx.sh.html"]
1212 searx1 [label="Searx #1"];
1313 searx2 [label="Searx #2"];
1414 searx3 [label="Searx #3"];
docs/admin/architecture.rst+9−5
@@ -4,17 +4,21 @@
44Architecture
55============
66
7−.. sidebar:: Needs work!
7+.. sidebar:: Further reading
88
9− This article needs some work / Searx is a collaborative effort. If you have
10− any contribution, feel welcome to send us your :pull:`PR <../pulls>`, see
11− :ref:`how to contribute`.
9+ - Reverse Proxy: :ref:`Apache <apache searx site>` & :ref:`nginx <nginx searx
10+ site>`
11+ - Filtron: :ref:`searx filtron`
12+ - Morty: :ref:`searx morty`
13+ - uWSGI: :ref:`searx uwsgi`
14+ - Searx: :ref:`installation basic`
1215
1316Herein you will find some hints and suggestions about typical architectures of
1417searx infrastructures.
1518
1619We start with a contribution from :pull:`@dalf <1776#issuecomment-567917320>`.
17−It shows a *reference* setup for public searx instances.
20+It shows a *reference* setup for public searx instances which can build up and
21+maintained by the scripts from our :ref:`toolboxing`.
1822
1923.. _arch public:
2024
docs/admin/buildhosts.rst+63−11
@@ -9,8 +9,27 @@ Buildhosts
99 If you have any contribution send us your :pull:`PR <../pulls>`, see
1010 :ref:`how to contribute`.
1111
12+.. contents:: Contents
13+ :depth: 2
14+ :local:
15+ :backlinks: entry
16+
1217To get best results from build, its recommend to install additional packages
13−on build hosts.
18+on build hosts (see :ref:`searx.sh`).::
19+
20+ sudo -H ./utils/searx.sh install buildhost
21+
22+This will install packages needed by searx:
23+
24+.. kernel-include:: $DOCS_BUILD/includes/searx.rst
25+ :start-after: START distro-packages
26+ :end-before: END distro-packages
27+
28+and packages needed to build docuemtation and run tests:
29+
30+.. kernel-include:: $DOCS_BUILD/includes/searx.rst
31+ :start-after: START build-packages
32+ :end-before: END build-packages
1433
1534.. _docs build:
1635
@@ -35,8 +54,17 @@ processing additional packages are needed. The XeTeX_ needed not only for PDF
3554creation, its also needed for :ref:`math` when HTML output is build.
3655
3756To be able to do :ref:`sphinx:math-support` without CDNs, the math are rendered
38−as images (``sphinx.ext.imgmath`` extension). If your docs build (``make
39−docs``) shows warnings like this::
57+as images (``sphinx.ext.imgmath`` extension).
58+
59+Here is the extract from the :origin:`docs/conf.py` file, setting math renderer
60+to ``imgmath``:
61+
62+.. literalinclude:: ../conf.py
63+ :language: python
64+ :start-after: # sphinx.ext.imgmath setup
65+ :end-before: # sphinx.ext.imgmath setup END
66+
67+If your docs build (``make docs``) shows warnings like this::
4068
4169 WARNING: dot(1) not found, for better output quality install \
4270 graphviz from http://www.graphviz.org
@@ -47,8 +75,6 @@ docs``) shows warnings like this::
4775you need to install additional packages on your build host, to get better HTML
4876output.
4977
50−.. _system requirements:
51−
5278.. tabs::
5379
5480 .. group-tab:: Ubuntu / debian
@@ -92,12 +118,38 @@ For PDF output you also need:
92118
93119 $ sudo dnf install \
94120 texlive-collection-fontsrecommended texlive-collection-latex \
95− dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts
121+ dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts \
122+ ImageMagick
96123
97−.. _system requirements END:
124+.. _sh lint:
98125
99−.. literalinclude:: ../conf.py
100− :language: python
101− :start-after: # sphinx.ext.imgmath setup
102− :end-before: # sphinx.ext.imgmath setup END
126+Lint shell scripts
127+==================
128+
129+.. _ShellCheck: https://github.com/koalaman/shellcheck
130+
131+To lint shell scripts, we use ShellCheck_ - A shell script static analysis tool.
132+
133+.. SNIP sh lint requirements
134+
135+.. tabs::
136+
137+ .. group-tab:: Ubuntu / debian
138+
139+ .. code-block:: sh
140+
141+ $ sudo apt install shellcheck
142+
143+ .. group-tab:: Arch Linux
144+
145+ .. code-block:: sh
146+
147+ $ sudo pacman -S shellcheck
148+
149+ .. group-tab:: Fedora / RHEL
150+
151+ .. code-block:: sh
152+
153+ $ sudo dnf install ShellCheck
103154
155+.. SNAP sh lint requirements
docs/admin/filtron.rst+145−102
@@ -1,18 +1,51 @@
1+
2+.. _searx filtron:
3+
14==========================
25How to protect an instance
36==========================
47
5−Searx depens on external search services. To avoid the abuse of these services
8+.. sidebar:: further reading
9+
10+ - :ref:`filtron.sh`
11+ - :ref:`nginx searx site`
12+
13+
14+.. contents:: Contents
15+ :depth: 2
16+ :local:
17+ :backlinks: entry
18+
19+.. _filtron: https://github.com/asciimoo/filtron
20+
21+Searx depends on external search services. To avoid the abuse of these services
622it is advised to limit the number of requests processed by searx.
723
8−An application firewall, ``filtron`` solves exactly this problem. Information
9−on how to install it can be found at the `project page of filtron
10−<https://github.com/asciimoo/filtron>`__.
24+An application firewall, filtron_ solves exactly this problem. Filtron is just
25+a middleware between your web server (nginx, apache, ...) and searx, we describe
26+such infratructures in chapter: :ref:`architecture`.
1127
1228
29+filtron & go
30+============
31+
32+.. _Go: https://golang.org/
33+.. _filtron README: https://github.com/asciimoo/filtron/blob/master/README.md
34+
35+Filtron needs Go_ installed. If Go_ is preinstalled, filtron_ is simply
36+installed by ``go get`` package management (see `filtron README`_). If you use
37+filtron as middleware, a more isolated setup is recommended. To simplify such
38+an installation and the maintenance of, use our script :ref:`filtron.sh`.
39+
40+.. _Sample configuration of filtron:
41+
1342Sample configuration of filtron
1443===============================
1544
45+.. sidebar:: Tooling box
46+
47+ - :origin:`/etc/filtron/rules.json <utils/templates/etc/filtron/rules.json>`
48+
1649An example configuration can be find below. This configuration limits the access
1750of:
1851
@@ -24,105 +57,104 @@ of:
2457
2558.. code:: json
2659
27− [{
28− "name":"search request",
29− "filters":[
30− "Param:q",
31− "Path=^(/|/search)$"
32− ],
33− "interval":"<time-interval-in-sec (int)>",
34− "limit":"<max-request-number-in-interval (int)>",
35− "subrules":[
36− {
37− "name":"roboagent limit",
38− "interval":"<time-interval-in-sec (int)>",
39− "limit":"<max-request-number-in-interval (int)>",
40− "filters":[
41− "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"
42− ],
43− "actions":[
44− {
45− "name":"block",
46− "params":{
47− "message":"Rate limit exceeded"
48− }
49− }
50− ]
51− },
52− {
53− "name":"botlimit",
54− "limit":0,
55− "stop":true,
56− "filters":[
57− "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
58− ],
59− "actions":[
60− {
61− "name":"block",
62− "params":{
63− "message":"Rate limit exceeded"
64− }
65− }
66− ]
67− },
68− {
69− "name":"IP limit",
70− "interval":"<time-interval-in-sec (int)>",
71− "limit":"<max-request-number-in-interval (int)>",
72− "stop":true,
73− "aggregations":[
74− "Header:X-Forwarded-For"
75− ],
76− "actions":[
77− {
78− "name":"block",
79− "params":{
80− "message":"Rate limit exceeded"
81− }
82− }
83− ]
84− },
85− {
86− "name":"rss/json limit",
87− "interval":"<time-interval-in-sec (int)>",
88− "limit":"<max-request-number-in-interval (int)>",
89− "stop":true,
90− "filters":[
91− "Param:format=(csv|json|rss)"
92− ],
93− "actions":[
94− {
95− "name":"block",
96− "params":{
97− "message":"Rate limit exceeded"
98− }
99− }
100− ]
101− },
102− {
103− "name":"useragent limit",
104− "interval":"<time-interval-in-sec (int)>",
105− "limit":"<max-request-number-in-interval (int)>",
106− "aggregations":[
107− "Header:User-Agent"
60+ [
61+ {
62+ "name": "search request",
63+ "filters": [
64+ "Param:q",
65+ "Path=^(/|/search)$"
10866 ],
109− "actions":[
110− {
111− "name":"block",
112− "params":{
113− "message":"Rate limit exceeded"
114− }
115− }
67+ "interval": "<time-interval-in-sec (int)>"
68+ "limit": "<max-request-number-in-interval (int)>",
69+ "subrules": [
70+ {
71+ "name": "missing Accept-Language",
72+ "filters": ["!Header:Accept-Language"],
73+ "limit": "<max-request-number-in-interval (int)>",
74+ "stop": true,
75+ "actions": [
76+ {"name":"log"},
77+ {"name": "block",
78+ "params": {"message": "Rate limit exceeded"}}
79+ ]
80+ },
81+ {
82+ "name": "suspiciously Connection=close header",
83+ "filters": ["Header:Connection=close"],
84+ "limit": "<max-request-number-in-interval (int)>",
85+ "stop": true,
86+ "actions": [
87+ {"name":"log"},
88+ {"name": "block",
89+ "params": {"message": "Rate limit exceeded"}}
90+ ]
91+ },
92+ {
93+ "name": "IP limit",
94+ "interval": "<time-interval-in-sec (int)>"
95+ "limit": "<max-request-number-in-interval (int)>",
96+ "stop": true,
97+ "aggregations": [
98+ "Header:X-Forwarded-For"
99+ ],
100+ "actions": [
101+ { "name": "log"},
102+ { "name": "block",
103+ "params": {
104+ "message": "Rate limit exceeded"
105+ }
106+ }
107+ ]
108+ },
109+ {
110+ "name": "rss/json limit",
111+ "filters": [
112+ "Param:format=(csv|json|rss)"
113+ ],
114+ "interval": "<time-interval-in-sec (int)>"
115+ "limit": "<max-request-number-in-interval (int)>",
116+ "stop": true,
117+ "actions": [
118+ { "name": "log"},
119+ { "name": "block",
120+ "params": {
121+ "message": "Rate limit exceeded"
122+ }
123+ }
124+ ]
125+ },
126+ {
127+ "name": "useragent limit",
128+ "interval": "<time-interval-in-sec (int)>"
129+ "limit": "<max-request-number-in-interval (int)>",
130+ "aggregations": [
131+ "Header:User-Agent"
132+ ],
133+ "actions": [
134+ { "name": "log"},
135+ { "name": "block",
136+ "params": {
137+ "message": "Rate limit exceeded"
138+ }
139+ }
140+ ]
141+ }
116142 ]
117− }
118− ]
119− }]
143+ }
144+ ]
120145
121146
147+.. _filtron route request:
122148
123149Route request through filtron
124150=============================
125151
152+.. sidebar:: further reading
153+
154+ - :ref:`filtron.sh overview`
155+ - :ref:`installation nginx`
156+ - :ref:`installation apache`
157+
126158Filtron can be started using the following command:
127159
128160.. code:: sh
@@ -136,13 +168,24 @@ Use it along with ``nginx`` with the following example configuration.
136168
137169.. code:: nginx
138170
139− location / {
140− proxy_set_header Host $http_host;
141− proxy_set_header X-Real-IP $remote_addr;
142− proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
143− proxy_set_header X-Scheme $scheme;
144− proxy_pass http://127.0.0.1:4004/;
171+ # https://example.org/searx
172+
173+ location /searx {
174+ proxy_pass http://127.0.0.1:4004/;
175+
176+ proxy_set_header Host $http_host;
177+ proxy_set_header Connection $http_connection;
178+ proxy_set_header X-Real-IP $remote_addr;
179+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
180+ proxy_set_header X-Scheme $scheme;
181+ proxy_set_header X-Script-Name /searx;
145182 }
146183
184+ location /searx/static {
185+ /usr/local/searx/searx-src/searx/static;
186+ }
187+
188+
147189Requests are coming from port 4004 going through filtron and then forwarded to
148−port 8888 where a searx is being run.
190+port 8888 where a searx is being run. For a complete setup see: :ref:`nginx
191+searx site`.
docs/admin/index.rst+8−1
@@ -3,9 +3,16 @@ Administrator documentation
33===========================
44
55.. toctree::
6− :maxdepth: 1
6+ :maxdepth: 2
7+ :caption: Contents
78
89 installation
10+ installation-searx
11+ installation-uwsgi
12+ installation-nginx
13+ installation-apache
14+ installation-docker
15+ update-searx
916 settings
1017 api
1118 architecture
docs/admin/installation-apache.rstnew file+514
@@ -0,0 +1,514 @@
1+.. _installation apache:
2+
3+===================
4+Install with apache
5+===================
6+
7+.. _Apache: https://httpd.apache.org/
8+.. _Apache Debian:
9+ https://cwiki.apache.org/confluence/display/HTTPD/DistrosDefaultLayout#DistrosDefaultLayout-Debian,Ubuntu(Apachehttpd2.x):
10+.. _README.Debian:
11+ https://salsa.debian.org/apache-team/apache2/raw/master/debian/apache2.README.Debian
12+.. _Apache Arch Linux:
13+ https://wiki.archlinux.org/index.php/Apache_HTTP_Server
14+.. _Apache Fedora:
15+ https://docs.fedoraproject.org/en-US/quick-docs/getting-started-with-apache-http-server/index.html
16+.. _Apache directives:
17+ https://httpd.apache.org/docs/trunk/mod/directives.html
18+.. _Getting Started:
19+ https://httpd.apache.org/docs/current/en/getting-started.html
20+.. _Terms Used to Describe Directives:
21+ https://httpd.apache.org/docs/current/en/mod/directive-dict.html
22+.. _Configuration Files:
23+ https://httpd.apache.org/docs/current/en/configuring.html
24+.. _ProxyPreserveHost: https://httpd.apache.org/docs/trunk/mod/mod_proxy.html#proxypreservehost
25+.. _LoadModule:
26+ https://httpd.apache.org/docs/2.4/mod/mod_so.html#loadmodule
27+.. _DocumentRoot:
28+ https://httpd.apache.org/docs/trunk/mod/core.html#documentroot
29+.. _Location:
30+ https://httpd.apache.org/docs/trunk/mod/core.html#location
31+.. _uWSGI Apache support:
32+ https://uwsgi-docs.readthedocs.io/en/latest/Apache.html
33+.. _mod_proxy_uwsgi:
34+ https://uwsgi-docs.readthedocs.io/en/latest/Apache.html#mod-proxy-uwsgi
35+
36+.. sidebar:: further read
37+
38+ - `Apache Arch Linux`_
39+ - `Apache Debian`_ and `README.Debian`_
40+ - `Apache Fedora`_
41+ - `Apache directives`_
42+
43+.. contents:: Contents
44+ :depth: 2
45+ :local:
46+ :backlinks: entry
47+
48+----
49+
50+**Install** :ref:`apache searx site` using :ref:`filtron.sh <filtron.sh overview>`
51+
52+.. code:: bash
53+
54+ $ sudo -H ./utils/filtron.sh apache install
55+
56+**Install** :ref:`apache searx site` using :ref:`morty.sh <morty.sh overview>`
57+
58+.. code:: bash
59+
60+ $ sudo -H ./utils/morty.sh apache install
61+
62+----
63+
64+The apache HTTP server
65+======================
66+
67+If Apache_ is not installed, install it now. If apache_ is new to you, the
68+`Getting Started`_, `Configuration Files`_ and `Terms Used to Describe
69+Directives`_ documentation gives first orientation. There is also a list of
70+`Apache directives`_ *to keep in the pocket*.
71+
72+.. tabs::
73+
74+ .. group-tab:: Ubuntu / debian
75+
76+ .. code:: sh
77+
78+ sudo -H apt-get install apache2
79+
80+ .. group-tab:: Arch Linux
81+
82+ .. code:: sh
83+
84+ sudo -H pacman -S apache
85+ sudo -H systemctl enable httpd
86+ sudo -H systemctl start http
87+
88+ .. group-tab:: Fedora / RHEL
89+
90+ .. code:: sh
91+
92+ sudo -H dnf install httpd
93+ sudo -H systemctl enable httpd
94+ sudo -H systemctl start httpd
95+
96+Now at http://localhost you should see any kind of *Welcome* or *Test* page.
97+How this default intro site is configured, depends on the linux distribution
98+(compare `Apache directives`_).
99+
100+.. tabs::
101+
102+ .. group-tab:: Ubuntu / debian
103+
104+ .. code:: sh
105+
106+ less /etc/apache2/sites-enabled/000-default.conf
107+
108+ In this file, there is a line setting the `DocumentRoot`_ directive:
109+
110+ .. code:: apache
111+
112+ DocumentRoot /var/www/html
113+
114+ And the *welcome* page is the HTML file at ``/var/www/html/index.html``.
115+
116+ .. group-tab:: Arch Linux
117+
118+ .. code:: sh
119+
120+ less /etc/httpd/conf/httpd.conf
121+
122+ In this file, there is a line setting the `DocumentRoot`_ directive:
123+
124+ .. code:: apache
125+
126+ DocumentRoot "/srv/http"
127+ <Directory "/srv/http">
128+ Options Indexes FollowSymLinks
129+ AllowOverride None
130+ Require all granted
131+ </Directory>
132+
133+ The *welcome* page of Arch Linux is a page showing directory located at
134+ ``DocumentRoot``. This is *directory* page is generated by the Module
135+ `mod_autoindex <https://httpd.apache.org/docs/2.4/mod/mod_autoindex.html>`_:
136+
137+ .. code:: apache
138+
139+ LoadModule autoindex_module modules/mod_autoindex.so
140+ ...
141+ Include conf/extra/httpd-autoindex.conf
142+
143+ .. group-tab:: Fedora / RHEL
144+
145+ .. code:: sh
146+
147+ less /etc/httpd/conf/httpd.conf
148+
149+ In this file, there is a line setting the ``DocumentRoot`` directive:
150+
151+ .. code:: apache
152+
153+ DocumentRoot "/var/www/html"
154+ ...
155+ <Directory "/var/www">
156+ AllowOverride None
157+ # Allow open access:
158+ Require all granted
159+ </Directory>
160+
161+ On fresh installations, the ``/var/www`` is empty and the *default
162+ welcome page* is shown, the configuration is located at::
163+
164+ less /etc/httpd/conf.d/welcome.conf
165+
166+.. _apache searx site:
167+
168+Apache Reverse Proxy
169+====================
170+
171+.. sidebar:: public to the internet?
172+
173+ If your searx instance is public, stop here and first install :ref:`filtron
174+ reverse proxy <filtron.sh>` and :ref:`result proxy morty <morty.sh>`, see
175+ :ref:`installation scripts`. If already done, follow setup: *searx via
176+ filtron plus morty*.
177+
178+To setup a Apache revers proxy you have to enable the *headers* and *proxy*
179+modules and create a `Location`_ configuration for the searx site. In most
180+distributions you have to un-comment the lines in the main configuration file,
181+except in :ref:`The Debian Layout`.
182+
183+To pass the HTTP HOST header
184+With ProxyPreserveHost_ the incoming Host HTTP request header is passed to the
185+proxied host.
186+
187+.. tabs::
188+
189+ .. group-tab:: Ubuntu / debian
190+
191+ In the Apache setup, enable headers and proxy modules:
192+
193+ .. code:: sh
194+
195+ sudo -H a2enmod headers
196+ sudo -H a2enmod proxy
197+ sudo -H a2enmod proxy_http
198+
199+ In :ref:`The Debian Layout` you create a ``searx.conf`` with the
200+ ``<Location /searx >`` directive and save this file in the *sites
201+ available* folder at ``/etc/apache2/sites-available``. To enable the
202+ ``searx.conf`` use :man:`a2ensite`:
203+
204+ .. code:: sh
205+
206+ sudo -H a2ensite searx.conf
207+
208+ .. group-tab:: Arch Linux
209+
210+ In the ``/etc/httpd/conf/httpd.conf`` file, activate headers and proxy
211+ modules (LoadModule_):
212+
213+ .. code:: apache
214+
215+ FIXME needs test
216+
217+ LoadModule headers_module modules/mod_headers.so
218+ LoadModule proxy_module modules/mod_proxy.so
219+ LoadModule proxy_http_module modules/mod_proxy_http.so
220+
221+ .. group-tab:: Fedora / RHEL
222+
223+ In the ``/etc/httpd/conf/httpd.conf`` file, activate headers and proxy
224+ modules (LoadModule_):
225+
226+ .. code:: apache
227+
228+ FIXME needs test
229+
230+ LoadModule headers_module modules/mod_headers.so
231+ LoadModule proxy_module modules/mod_proxy.so
232+ LoadModule proxy_http_module modules/mod_proxy_http.so
233+
234+.. tabs::
235+
236+ .. group-tab:: searx via filtron plus morty
237+
238+ Use this setup, if your instance is public to the internet, compare
239+ figure: :ref:`architecture <arch public>` and :ref:`installation scripts`.
240+
241+ 1. Configure a reverse proxy for :ref:`filtron <filtron.sh>`, listening on
242+ *localhost 4004* (:ref:`filtron route request`):
243+
244+ .. code:: apache
245+
246+ <Location /searx >
247+
248+ # SetEnvIf Request_URI "/searx" dontlog
249+ # CustomLog /dev/null combined env=dontlog
250+
251+ Require all granted
252+
253+ Order deny,allow
254+ Deny from all
255+ #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
256+ Allow from all
257+
258+ ProxyPreserveHost On
259+ ProxyPass http://127.0.0.1:4004
260+ RequestHeader set X-Script-Name /searx
261+
262+ </Location>
263+
264+ 2. Configure reverse proxy for :ref:`morty <searx morty>`, listening on
265+ *localhost 3000*
266+
267+ .. code:: apache
268+
269+ ProxyPreserveHost On
270+
271+ <Location /morty >
272+
273+ # SetEnvIf Request_URI "/morty" dontlog
274+ # CustomLog /dev/null combined env=dontlog
275+
276+ Require all granted
277+
278+ Order deny,allow
279+ Deny from all
280+ #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
281+ Allow from all
282+
283+ ProxyPass http://127.0.0.1:3000
284+ RequestHeader set X-Script-Name /morty
285+
286+ </Location>
287+
288+ Note that reverse proxy advised to be used in case of single-user or
289+ low-traffic instances. For a fully result proxification add :ref:`morty's
290+ <searx morty>` **public URL** to your :origin:`searx/settings.yml`:
291+
292+ .. code:: yaml
293+
294+ result_proxy:
295+ # replace example.org with your server's public name
296+ url : https://example.org/morty
297+
298+ server:
299+ image_proxy : True
300+
301+uWSGI support
302+=============
303+
304+Be warned, with this setup, your instance isn't :ref:`protected <searx
305+filtron>`, nevertheless it is good enough for intranet usage. In modern Linux
306+distributions, the `mod_proxy_uwsgi`_ is compiled into the *normal* apache
307+package and you need to install only the :ref:`uWSGI <searx uwsgi>` package:
308+
309+.. tabs::
310+
311+ .. group-tab:: Ubuntu / debian
312+
313+ .. code:: sh
314+
315+ sudo -H apt-get install uwsgi
316+
317+ # Ubuntu =< 18.04
318+ sudo -H apt-get install libapache2-mod-proxy-uwsgi
319+
320+ .. group-tab:: Arch Linux
321+
322+ .. code:: sh
323+
324+ sudo -H pacman -S uwsgi
325+
326+ .. group-tab:: Fedora / RHEL
327+
328+ .. code:: sh
329+
330+ sudo -H dnf install uwsgi
331+
332+The next example shows a configuration using the `uWSGI Apache support`_ via
333+unix sockets and `mod_proxy_uwsgi`_.
334+
335+For socket communication, you have to activate ``socket =
336+/run/uwsgi/app/searx/socket`` and comment out the ``http = 127.0.0.1:8888``
337+configuration in your :ref:`uwsgi ini file <uwsgi configuration>`. If not
338+already exists, create a folder for the unix sockets, which can be used by the
339+searx account (see :ref:`create searx user`):
340+
341+.. code:: bash
342+
343+ sudo -H mkdir -p /run/uwsgi/app/searx/
344+ sudo -H chown -R searx:searx /run/uwsgi/app/searx/
345+
346+If the server is public; to limit access to your intranet replace ``Allow from
347+all`` directive and replace ``192.168.0.0/16`` with your subnet IP/class.
348+
349+.. tabs::
350+
351+ .. group-tab:: Ubuntu / debian
352+
353+ .. code:: apache
354+
355+ LoadModule headers_module /usr/lib/apache2/mod_headers.so
356+ LoadModule proxy_module /usr/lib/apache2/modules/mod_proxy.so
357+ LoadModule proxy_uwsgi_module /usr/lib/apache2/modules/mod_proxy_uwsgi.so
358+
359+ # SetEnvIf Request_URI /searx dontlog
360+ # CustomLog /dev/null combined env=dontlog
361+
362+ <Location /searx>
363+
364+ Require all granted
365+ Order deny,allow
366+ Deny from all
367+ # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
368+ Allow from all
369+
370+ ProxyPreserveHost On
371+ ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/
372+
373+ </Location>
374+
375+ .. group-tab:: Arch Linux
376+
377+ .. code:: apache
378+
379+ FIXME needs test
380+
381+ LoadModule proxy_module modules/mod_proxy.so
382+ LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so
383+
384+ # SetEnvIf Request_URI /searx dontlog
385+ # CustomLog /dev/null combined env=dontlog
386+
387+ <Location /searx>
388+
389+ Require all granted
390+ Order deny,allow
391+ Deny from all
392+ # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
393+ Allow from all
394+
395+ ProxyPreserveHost On
396+ ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/
397+
398+ </Location>
399+
400+ .. group-tab:: Fedora / RHEL
401+
402+ .. code:: apache
403+
404+ FIXME needs test
405+
406+ LoadModule proxy_module modules/mod_proxy.so
407+ LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so
408+ <IfModule proxy_uwsgi_module>
409+
410+ # SetEnvIf Request_URI /searx dontlog
411+ # CustomLog /dev/null combined env=dontlog
412+
413+ <Location /searx>
414+
415+ Require all granted
416+ Order deny,allow
417+ Deny from all
418+ # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
419+ Allow from all
420+
421+ ProxyPreserveHost On
422+ ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/
423+
424+ </Location>
425+
426+ </IfModule>
427+
428+ .. group-tab:: old mod_wsgi
429+
430+ We show this only for historical reasons, DON'T USE `mod_uwsgi
431+ <https://uwsgi-docs.readthedocs.io/en/latest/Apache.html#mod-uwsgi>`_.
432+ ANYMORE!
433+
434+ .. code:: apache
435+
436+ <IfModule mod_uwsgi.c>
437+
438+ # SetEnvIf Request_URI "/searx" dontlog
439+ # CustomLog /dev/null combined env=dontlog
440+
441+ <Location /searx >
442+
443+ Require all granted
444+
445+ Options FollowSymLinks Indexes
446+ SetHandler uwsgi-handler
447+ uWSGISocket /run/uwsgi/app/searx/socket
448+
449+ Order deny,allow
450+ Deny from all
451+ # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
452+ Allow from all
453+
454+ </Location>
455+
456+ </IfModule>
457+
458+.. _restart apache:
459+
460+Restart service
461+===============
462+
463+.. tabs::
464+
465+ .. group-tab:: Ubuntu / debian
466+
467+ .. code:: sh
468+
469+ sudo -H systemctl restart apache2
470+ sudo -H service uwsgi restart searx
471+
472+ .. group-tab:: Arch Linux
473+
474+ .. code:: sh
475+
476+ sudo -H systemctl restart httpd
477+ sudo -H systemctl restart uwsgi@searx
478+
479+ .. group-tab:: Fedora / RHEL
480+
481+ .. code:: sh
482+
483+ sudo -H systemctl restart httpd
484+ sudo -H touch /etc/uwsgi.d/searx.ini
485+
486+
487+disable logs
488+============
489+
490+For better privacy you can disable Apache logs. In the examples above activate
491+one of the lines and `restart apache`_::
492+
493+
494+ # SetEnvIf Request_URI "/searx" dontlog
495+ # CustomLog /dev/null combined env=dontlog
496+
497+The ``CustomLog`` directive disable logs for the whole (virtual) server, use it
498+when the URL of the service does not have a path component (``/searx``) / is
499+located at root (``/``).
500+
501+.. _The Debian Layout:
502+
503+The Debian Layout
504+=================
505+
506+Be aware that the Debian layout is quite different from the standard Apache
507+configuration. For details look at the README.Debian_
508+(``/usr/share/doc/apache2/README.Debian.gz``). Some commands you should know on
509+Debian:
510+
511+* :man:`apache2ctl`: Apache HTTP server control interface
512+* :man:`a2enmod`, :man:`a2dismod`: switch on/off modules
513+* :man:`a2enconf`, :man:`a2disconf`: switch on/off configurations
514+* :man:`a2ensite`, :man:`a2dissite`: switch on/off sites
docs/admin/installation-docker.rstnew file+28
@@ -0,0 +1,28 @@
1+.. _installation docker:
2+
3+===================
4+Docker installation
5+===================
6+
7+.. contents:: Contents
8+ :depth: 2
9+ :local:
10+ :backlinks: entry
11+
12+Make sure you have installed Docker. For instance, you can deploy searx like this:
13+
14+.. code:: sh
15+
16+ docker pull wonderfall/searx
17+ docker run -d --name searx -p $PORT:8888 wonderfall/searx
18+
19+Go to ``http://localhost:$PORT``.
20+
21+See https://hub.docker.com/r/wonderfall/searx/ for more informations. It's also
22+possible to build searx from the embedded Dockerfile.
23+
24+.. code:: sh
25+
26+ git clone https://github.com/asciimoo/searx.git
27+ cd searx
28+ docker build -t whatever/searx .
docs/admin/installation-nginx.rstnew file+381
@@ -0,0 +1,381 @@
1+.. _installation nginx:
2+
3+==================
4+Install with nginx
5+==================
6+
7+.. _nginx:
8+ https://docs.nginx.com/nginx/admin-guide/
9+.. _nginx server configuration:
10+ https://docs.nginx.com/nginx/admin-guide/web-server/web-server/#setting-up-virtual-servers
11+.. _nginx beginners guide:
12+ http://nginx.org/en/docs/beginners_guide.html
13+.. _Getting Started wiki:
14+ https://www.nginx.com/resources/wiki/start/
15+.. _uWSGI support from nginx:
16+ https://uwsgi-docs.readthedocs.io/en/latest/Nginx.html
17+.. _uwsgi_params:
18+ https://uwsgi-docs.readthedocs.io/en/latest/Nginx.html#configuring-nginx
19+.. _SCRIPT_NAME:
20+ https://werkzeug.palletsprojects.com/en/1.0.x/wsgi/#werkzeug.wsgi.get_script_name
21+
22+.. sidebar:: further reading
23+
24+ - nginx_
25+ - `nginx beginners guide`_
26+ - `nginx server configuration`_
27+ - `Getting Started wiki`_
28+ - `uWSGI support from nginx`_
29+
30+.. contents:: Contents
31+ :depth: 2
32+ :local:
33+ :backlinks: entry
34+
35+----
36+
37+**Install** :ref:`nginx searx site` using :ref:`filtron.sh <filtron.sh overview>`
38+
39+.. code:: bash
40+
41+ $ sudo -H ./utils/filtron.sh nginx install
42+
43+**Install** :ref:`nginx searx site` using :ref:`morty.sh <morty.sh overview>`
44+
45+.. code:: bash
46+
47+ $ sudo -H ./utils/morty.sh nginx install
48+
49+----
50+
51+
52+The nginx HTTP server
53+=====================
54+
55+If nginx_ is not installed (uwsgi will not work with the package nginx-light),
56+install it now.
57+
58+.. tabs::
59+
60+ .. group-tab:: Ubuntu / debian
61+
62+ .. code:: sh
63+
64+ sudo -H apt-get install nginx
65+
66+ .. group-tab:: Arch Linux
67+
68+ .. code-block:: sh
69+
70+ sudo -H pacman -S nginx-mainline
71+ sudo -H systemctl enable nginx
72+ sudo -H systemctl start nginx
73+
74+ .. group-tab:: Fedora / RHEL
75+
76+ .. code-block:: sh
77+
78+ sudo -H dnf install nginx
79+ sudo -H systemctl enable nginx
80+ sudo -H systemctl start nginx
81+
82+Now at http://localhost you should see a *Welcome to nginx!* page, on Fedora you
83+see a *Fedora Webserver - Test Page*. The test page comes from the default
84+`nginx server configuration`_. How this default intro site is configured,
85+depends on the linux distribution:
86+
87+.. tabs::
88+
89+ .. group-tab:: Ubuntu / debian
90+
91+ .. code:: sh
92+
93+ less /etc/nginx/nginx.conf
94+
95+ there is a line including site configurations from:
96+
97+ .. code:: nginx
98+
99+ include /etc/nginx/sites-enabled/*;
100+
101+ .. group-tab:: Arch Linux
102+
103+ .. code-block:: sh
104+
105+ less /etc/nginx/nginx.conf
106+
107+ in there is a configuration section named ``server``:
108+
109+ .. code-block:: nginx
110+
111+ server {
112+ listen 80;
113+ server_name localhost;
114+ # ...
115+ }
116+
117+ .. group-tab:: Fedora / RHEL
118+
119+ .. code-block:: sh
120+
121+ less /etc/nginx/nginx.conf
122+
123+ there is a line including site configurations from:
124+
125+ .. code:: nginx
126+
127+ include /etc/nginx/conf.d/*.conf;
128+
129+.. _nginx searx site:
130+
131+A nginx searx site
132+==================
133+
134+.. sidebar:: public to the internet?
135+
136+ If your searx instance is public, stop here and first install :ref:`filtron
137+ reverse proxy <filtron.sh>` and :ref:`result proxy morty <morty.sh>`, see
138+ :ref:`installation scripts`. If already done, follow setup: *searx via
139+ filtron plus morty*.
140+
141+Now you have to create a configuration for the searx site. If nginx_ is new to
142+you, the `nginx beginners guide`_ is a good starting point and the `Getting
143+Started wiki`_ is always a good resource *to keep in the pocket*.
144+
145+.. tabs::
146+
147+ .. group-tab:: Ubuntu / debian
148+
149+ Create configuration at ``/etc/nginx/sites-available/searx`` and place a
150+ symlink to sites-enabled:
151+
152+ .. code:: sh
153+
154+ sudo -H ln -s /etc/nginx/sites-available/searx /etc/nginx/sites-enabled/searx
155+
156+ .. group-tab:: Arch Linux
157+
158+ In the ``/etc/nginx/nginx.conf`` file, replace the configuration section
159+ named ``server``.
160+
161+ .. group-tab:: Fedora / RHEL
162+
163+ Create configuration at ``/etc/nginx/conf.d/searx`` and place a
164+ symlink to sites-enabled:
165+
166+.. tabs::
167+
168+ .. group-tab:: searx via filtron plus morty
169+
170+ Use this setup, if your instance is public to the internet, compare
171+ figure: :ref:`architecture <arch public>` and :ref:`installation scripts`.
172+
173+ 1. Configure a reverse proxy for :ref:`filtron <filtron.sh>`, listening on
174+ *localhost 4004* (:ref:`filtron route request`):
175+
176+ .. code:: nginx
177+
178+ # https://example.org/searx
179+
180+ location /searx {
181+ proxy_pass http://127.0.0.1:4004/;
182+
183+ proxy_set_header Host $http_host;
184+ proxy_set_header Connection $http_connection;
185+ proxy_set_header X-Real-IP $remote_addr;
186+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
187+ proxy_set_header X-Scheme $scheme;
188+ proxy_set_header X-Script-Name /searx;
189+ }
190+
191+ location /searx/static {
192+ /usr/local/searx/searx-src/searx/static;
193+ }
194+
195+
196+ 2. Configure reverse proxy for :ref:`morty <searx morty>`, listening on
197+ *localhost 3000*:
198+
199+ .. code:: nginx
200+
201+ # https://example.org/morty
202+
203+ location /morty {
204+ proxy_pass http://127.0.0.1:3000/;
205+
206+ proxy_set_header Host $http_host;
207+ proxy_set_header Connection $http_connection;
208+ proxy_set_header X-Real-IP $remote_addr;
209+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
210+ proxy_set_header X-Scheme $scheme;
211+ }
212+
213+ Note that reverse proxy advised to be used in case of single-user or
214+ low-traffic instances. For a fully result proxification add :ref:`morty's
215+ <searx morty>` **public URL** to your :origin:`searx/settings.yml`:
216+
217+ .. code:: yaml
218+
219+ result_proxy:
220+ # replace example.org with your server's public name
221+ url : https://example.org/morty
222+
223+ server:
224+ image_proxy : True
225+
226+
227+ .. group-tab:: proxy or uWSGI
228+
229+ Be warned, with this setup, your instance isn't :ref:`protected <searx
230+ filtron>`. Nevertheless it is good enough for intranet usage and it is a
231+ excellent example of; *how different services can be set up*. The next
232+ example shows a reverse proxy configuration wrapping the :ref:`searx-uWSGI
233+ application <uwsgi configuration>`, listening on ``http =
234+ 127.0.0.1:8888``.
235+
236+ .. code:: nginx
237+
238+ # https://hostname.local/
239+
240+ location / {
241+ proxy_pass http://127.0.0.1:8888;
242+
243+ proxy_set_header Host $host;
244+ proxy_set_header Connection $http_connection;
245+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
246+ proxy_set_header X-Scheme $scheme;
247+ proxy_buffering off;
248+ }
249+
250+ Alternatively you can use the `uWSGI support from nginx`_ via unix
251+ sockets. For socket communication, you have to activate ``socket =
252+ /run/uwsgi/app/searx/socket`` and comment out the ``http =
253+ 127.0.0.1:8888`` configuration in your :ref:`uwsgi ini file <uwsgi
254+ configuration>`.
255+
256+ The example shows a nginx virtual ``server`` configuration, listening on
257+ port 80 (IPv4 and IPv6 http://[::]:80). The uWSGI app is configured at
258+ location ``/`` by importing the `uwsgi_params`_ and passing requests to
259+ the uWSGI socket (``uwsgi_pass``). The ``server``\'s root points to the
260+ :ref:`searx-src clone <searx-src>` and wraps directly the
261+ :origin:`searx/static/` content at ``location /static``.
262+
263+ .. code:: nginx
264+
265+ server {
266+ # replace hostname.local with your server's name
267+ server_name hostname.local;
268+
269+ listen 80;
270+ listen [::]:80;
271+
272+ location / {
273+ include uwsgi_params;
274+ uwsgi_pass unix:/run/uwsgi/app/searx/socket;
275+ }
276+
277+ root /usr/local/searx/searx-src/searx;
278+ location /static { }
279+ }
280+
281+ If not already exists, create a folder for the unix sockets, which can be
282+ used by the searx account:
283+
284+ .. code:: bash
285+
286+ mkdir -p /run/uwsgi/app/searx/
287+ sudo -H chown -R searx:searx /run/uwsgi/app/searx/
288+
289+ .. group-tab:: \.\. at subdir URL
290+
291+ Be warned, with these setups, your instance isn't :ref:`protected <searx
292+ filtron>`. The examples are just here to demonstrate how to export the
293+ searx application from a subdirectory URL ``https://example.org/searx/``.
294+
295+ .. code:: nginx
296+
297+ # https://hostname.local/searx
298+
299+ location /searx {
300+ proxy_pass http://127.0.0.1:8888;
301+
302+ proxy_set_header Host $host;
303+ proxy_set_header Connection $http_connection;
304+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
305+ proxy_set_header X-Scheme $scheme;
306+ proxy_set_header X-Script-Name /searx;
307+ proxy_buffering off;
308+ }
309+
310+ location /searx/static {
311+ alias /usr/local/searx/searx-src/searx/static;
312+ }
313+
314+ The ``X-Script-Name /searx`` is needed by the searx implementation to
315+ calculate relative URLs correct. The next example shows a uWSGI
316+ configuration. Since there are no HTTP headers in a (u)WSGI protocol, the
317+ value is shipped via the SCRIPT_NAME_ in the WSGI environment.
318+
319+ .. code:: nginx
320+
321+ # https://hostname.local/searx
322+
323+ location /searx {
324+ uwsgi_param SCRIPT_NAME /searx;
325+ include uwsgi_params;
326+ uwsgi_pass unix:/run/uwsgi/app/searx/socket;
327+ }
328+
329+ location /searx/static {
330+ alias /usr/local/searx/searx-src/searx;
331+ }
332+
333+ For searx to work correctly the ``base_url`` must be set in the
334+ :origin:`searx/settings.yml`.
335+
336+ .. code:: yaml
337+
338+ server:
339+ # replace example.org with your server's public name
340+ base_url : https://example.org/searx/
341+
342+
343+Restart service:
344+
345+.. tabs::
346+
347+ .. group-tab:: Ubuntu / debian
348+
349+ .. code:: sh
350+
351+ sudo -H systemctl restart nginx
352+ sudo -H service uwsgi restart searx
353+
354+ .. group-tab:: Arch Linux
355+
356+ .. code:: sh
357+
358+ sudo -H systemctl restart nginx
359+ sudo -H systemctl restart uwsgi@searx
360+
361+ .. group-tab:: Fedora
362+
363+ .. code:: sh
364+
365+ sudo -H systemctl restart nginx
366+ sudo -H touch /etc/uwsgi.d/searx.ini
367+
368+
369+Disable logs
370+============
371+
372+For better privacy you can disable nginx logs in ``/etc/nginx/nginx.conf``.
373+
374+.. code:: nginx
375+
376+ http {
377+ # ...
378+ access_log /dev/null;
379+ error_log /dev/null;
380+ # ...
381+ }
docs/admin/installation-searx.rstnew file+92
@@ -0,0 +1,92 @@
1+.. _installation basic:
2+
3+=========================
4+Step by step installation
5+=========================
6+
7+.. contents:: Contents
8+ :depth: 2
9+ :local:
10+ :backlinks: entry
11+
12+Step by step installation with virtualenv. For Ubuntu, be sure to have enable
13+universe repository.
14+
15+.. _install packages:
16+
17+Install packages
18+================
19+
20+.. kernel-include:: $DOCS_BUILD/includes/searx.rst
21+ :start-after: START distro-packages
22+ :end-before: END distro-packages
23+
24+.. hint::
25+
26+ This installs also the packages needed by :ref:`searx uwsgi`
27+
28+.. _create searx user:
29+
30+Create user
31+===========
32+
33+.. kernel-include:: $DOCS_BUILD/includes/searx.rst
34+ :start-after: START create user
35+ :end-before: END create user
36+
37+.. _searx-src:
38+
39+install searx & dependencies
40+============================
41+
42+Start a interactive shell from new created user and clone searx:
43+
44+.. kernel-include:: $DOCS_BUILD/includes/searx.rst
45+ :start-after: START clone searx
46+ :end-before: END clone searx
47+
48+In the same shell create *virtualenv*:
49+
50+.. kernel-include:: $DOCS_BUILD/includes/searx.rst
51+ :start-after: START create virtualenv
52+ :end-before: END create virtualenv
53+
54+To install searx's dependencies, exit the searx *bash* session you opened above
55+and restart a new. Before install, first check if your *virualenv* was sourced
56+from the login (*~/.profile*):
57+
58+.. kernel-include:: $DOCS_BUILD/includes/searx.rst
59+ :start-after: START manage.sh update_packages
60+ :end-before: END manage.sh update_packages
61+
62+.. tip::
63+
64+ Open a second terminal for the configuration tasks and left the ``(searx)$``
65+ terminal open for the tasks below.
66+
67+Configuration
68+==============
69+
70+Create a copy of the :origin:`searx/settings.yml` configuration file in system's
71+*/etc* folder. Configure like shown below -- replace ``searx@\$(uname -n)`` with
72+a name of your choice -- *and/or* edit ``/etc/searx/settings.yml`` if necessary.
73+
74+.. kernel-include:: $DOCS_BUILD/includes/searx.rst
75+ :start-after: START searx config
76+ :end-before: END searx config
77+
78+Check
79+=====
80+
81+To check your searx setup, optional enable debugging and start the *webapp*.
82+Searx looks at the exported environment ``$SEARX_SETTINGS_PATH`` for a
83+configuration file.
84+
85+.. kernel-include:: $DOCS_BUILD/includes/searx.rst
86+ :start-after: START check searx installation
87+ :end-before: END check searx installation
88+
89+If everything works fine, hit ``[CTRL-C]`` to stop the *webapp* and disable the
90+debug option in ``settings.yml``. You can now exit searx user bash (enter exit
91+command twice). At this point searx is not demonized; uwsgi allows this.
92+
docs/admin/installation-uwsgi.rstnew file+149
@@ -0,0 +1,149 @@
1+.. _searx uwsgi:
2+
3+=====
4+uwsgi
5+=====
6+
7+.. sidebar:: further reading
8+
9+ - `systemd.unit`_
10+ - `uWSGI Emperor`_
11+
12+.. contents:: Contents
13+ :depth: 2
14+ :local:
15+ :backlinks: entry
16+
17+
18+.. _systemd.unit: https://www.freedesktop.org/software/systemd/man/systemd.unit.html
19+.. _One service per app in systemd:
20+ https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd
21+.. _uWSGI Emperor:
22+ https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html
23+.. _uwsgi ini file:
24+ https://uwsgi-docs.readthedocs.io/en/latest/Configuration.html#ini-files
25+.. _systemd unit template:
26+ http://0pointer.de/blog/projects/instances.html
27+
28+
29+Origin uWSGI
30+============
31+
32+How uWSGI is implemented by distributors is different. uWSGI itself
33+recommend two methods
34+
35+`systemd.unit`_ template files as described here `One service per app in systemd`_.
36+
37+ There is one `systemd unit template`_ and one `uwsgi ini file`_ per uWSGI-app
38+ placed at dedicated locations. Take archlinux and a searx.ini as example::
39+
40+ unit template --> /usr/lib/systemd/system/uwsgi@.service
41+ uwsgi ini files --> /etc/uwsgi/searx.ini
42+
43+ The searx app can be maintained as know from common systemd units::
44+
45+ systemctl enable uwsgi@searx
46+ systemctl start uwsgi@searx
47+ systemctl restart uwsgi@searx
48+ systemctl stop uwsgi@searx
49+
50+The `uWSGI Emperor`_ mode which fits for maintaining a large range of uwsgi apps.
51+
52+ The Emperor mode is a special uWSGI instance that will monitor specific
53+ events. The Emperor mode (service) is started by a (common, not template)
54+ systemd unit. The Emperor service will scan specific directories for `uwsgi
55+ ini file`_\s (also know as *vassals*). If a *vassal* is added, removed or the
56+ timestamp is modified, a corresponding action takes place: a new uWSGI
57+ instance is started, reload or stopped. Take Fedora and a searx.ini as
58+ example::
59+
60+ to start a new searx instance create --> /etc/uwsgi.d/searx.ini
61+ to reload the instance edit timestamp --> touch /etc/uwsgi.d/searx.ini
62+ to stop instance remove ini --> rm /etc/uwsgi.d/searx.ini
63+
64+Distributors
65+============
66+
67+The `uWSGI Emperor`_ mode and `systemd unit template`_ is what the distributors
68+mostly offer their users, even if they differ in the way they implement both
69+modes and their defaults. Another point they might differ is the packaging of
70+plugins (if so, compare :ref:`install packages`) and what the default python
71+interpreter is (python2 vs. python3).
72+
73+Fedora starts a Emperor by default, while archlinux does not start any uwsgi
74+service by default. Worth to know; debian (ubuntu) follow a complete different
75+approach. *debian*: your are familiar with the apache infrastructure? .. they
76+do similar for the uWSGI infrastructure (with less comfort), the folders are::
77+
78+ /etc/uwsgi/apps-available/
79+ /etc/uwsgi/apps-enabled/
80+
81+The `uwsgi ini file`_ is enabled by a symbolic link::
82+
83+ ln -s /etc/uwsgi/apps-available/searx.ini /etc/uwsgi/apps-enabled/
84+
85+From debian's documentation (``/usr/share/doc/uwsgi/README.Debian.gz``): You
86+could control specific instance(s) by issuing::
87+
88+ service uwsgi <command> <confname> <confname> ...
89+
90+ sudo -H service uwsgi start searx
91+ sudo -H service uwsgi stop searx
92+
93+My experience is, that this command is a bit buggy.
94+
95+.. _uwsgi configuration:
96+
97+Alltogether
98+===========
99+
100+Create the configuration ini-file according to your distribution (see below) and
101+restart the uwsgi application.
102+
103+.. tabs::
104+
105+ .. group-tab:: Ubuntu / debian
106+
107+ .. kernel-include:: $DOCS_BUILD/includes/searx.rst
108+ :start-after: START searx uwsgi-description ubuntu-20.04
109+ :end-before: END searx uwsgi-description ubuntu-20.04
110+
111+
112+ .. group-tab:: Arch Linux
113+
114+ .. kernel-include:: $DOCS_BUILD/includes/searx.rst
115+ :start-after: START searx uwsgi-description arch
116+ :end-before: END searx uwsgi-description arch
117+
118+
119+ .. group-tab:: Fedora / RHEL
120+
121+ .. kernel-include:: $DOCS_BUILD/includes/searx.rst
122+ :start-after: START searx uwsgi-description fedora
123+ :end-before: END searx uwsgi-description fedora
124+
125+
126+.. tabs::
127+
128+ .. group-tab:: Ubuntu / debian
129+
130+ .. kernel-include:: $DOCS_BUILD/includes/searx.rst
131+ :code: ini
132+ :start-after: START searx uwsgi-appini ubuntu-20.04
133+ :end-before: END searx uwsgi-appini ubuntu-20.04
134+
135+ .. group-tab:: Arch Linux
136+
137+ .. kernel-include:: $DOCS_BUILD/includes/searx.rst
138+ :code: ini
139+ :start-after: START searx uwsgi-appini arch
140+ :end-before: END searx uwsgi-appini arch
141+
142+ .. group-tab:: Fedora / RHEL
143+
144+ .. kernel-include:: $DOCS_BUILD/includes/searx.rst
145+ :code: ini
146+ :start-after: START searx uwsgi-appini fedora
147+ :end-before: END searx uwsgi-appini fedora
148+
149+
docs/admin/installation.rst+37−320
@@ -4,346 +4,63 @@
44Installation
55============
66
7−.. contents::
8− :depth: 3
7+*You're spoilt for choice*, choose your preferred method of installation.
98
10−Basic installation
11−==================
9+- :ref:`installation docker`
10+- :ref:`installation scripts`
11+- :ref:`installation basic`
1212
13−Step by step installation for Debian/Ubuntu with virtualenv. For Ubuntu, be sure
14−to have enable universe repository.
13+The :ref:`installation basic` is good enough for intranet usage and it is a
14+excellent illustration of *how a searx instance is build up*. If you place your
15+instance public to the internet you should really consider to install a
16+:ref:`filtron reverse proxy <filtron.sh>` and for privacy a :ref:`result proxy
17+<morty.sh>` is mandatory.
1518
16−Install packages:
19+Therefore, if you do not have any special preferences, its recommend to use the
20+:ref:`installation docker` or the `Installation scripts`_ from our :ref:`tooling
21+box <toolboxing>` as described below.
1722
18−.. code:: sh
23+.. _installation scripts:
1924
20− $ sudo -H apt-get install \
21− git build-essential libxslt-dev \
22− python-dev python-virtualenv python-babel \
23− zlib1g-dev libffi-dev libssl-dev
25+Installation scripts
26+====================
2427
25−Install searx:
28+.. sidebar:: Update OS first!
2629
27−.. code:: sh
30+ To avoid unwanted side effects, update your OS before installing searx.
2831
29− cd /usr/local
30− sudo -H git clone https://github.com/asciimoo/searx.git
31− sudo -H useradd searx -d /usr/local/searx
32− sudo -H chown searx:searx -R /usr/local/searx
32+The following will install a setup as shown in :ref:`architecture`. First you
33+need to get a clone. The clone is only needed for the installation procedure
34+and some maintenance tasks (alternatively you can create your own fork).
3335
34−Install dependencies in a virtualenv:
36+.. code:: bash
3537
36−.. code:: sh
38+ $ cd ~/Downloads
39+ $ git clone https://github.com/asciimoo/searx searx
40+ $ cd searx
3741
38− cd /usr/local/searx
39− sudo -H -u searx -i
42+**Install** :ref:`searx service <searx.sh>`
4043
41−.. code:: sh
44+This installs searx as described in :ref:`installation basic`.
4245
43− (searx)$ virtualenv searx-ve
44− (searx)$ . ./searx-ve/bin/activate
45− (searx)$ ./manage.sh update_packages
46+.. code:: bash
4647
47−Configuration
48−==============
48+ $ sudo -H ./utils/searx.sh install all
4949
50−.. code:: sh
50+**Install** :ref:`filtron reverse proxy <filtron.sh>`
5151
52− sed -i -e "s/ultrasecretkey/`openssl rand -hex 16`/g" searx/settings.yml
52+.. code:: bash
5353
54−Edit searx/settings.yml if necessary.
54+ $ sudo -H ./utils/filtron.sh install all
5555
56−Check
57−=====
56+**Install** :ref:`result proxy <morty.sh>`
5857
59−Start searx:
58+.. code:: bash
6059
61−.. code:: sh
60+ $ sudo -H ./utils/morty.sh install all
6261
63− python searx/webapp.py
62+If all services are running fine, you can add it to your HTTP server:
6463
65−Go to http://localhost:8888
64+- :ref:`installation apache`
65+- :ref:`installation nginx`
6666
67−If everything works fine, disable the debug option in settings.yml:
68−
69−.. code:: sh
70−
71− sed -i -e "s/debug : True/debug : False/g" searx/settings.yml
72−
73−At this point searx is not demonized ; uwsgi allows this.
74−
75−You can exit the virtualenv and the searx user bash (enter exit command
76−twice).
77−
78−uwsgi
79−=====
80−
81−Install packages:
82−
83−.. code:: sh
84−
85− sudo -H apt-get install \
86− uwsgi uwsgi-plugin-python
87−
88−Create the configuration file ``/etc/uwsgi/apps-available/searx.ini`` with this
89−content:
90−
91−.. code:: ini
92−
93− [uwsgi]
94− # Who will run the code
95− uid = searx
96− gid = searx
97−
98− # disable logging for privacy
99− disable-logging = true
100−
101− # Number of workers (usually CPU count)
102− workers = 4
103−
104− # The right granted on the created socket
105− chmod-socket = 666
106−
107− # Plugin to use and interpretor config
108− single-interpreter = true
109− master = true
110− plugin = python
111− lazy-apps = true
112− enable-threads = true
113−
114− # Module to import
115− module = searx.webapp
116−
117− # Support running the module from a webserver subdirectory.
118− route-run = fixpathinfo:
119−
120− # Virtualenv and python path
121− virtualenv = /usr/local/searx/searx-ve/
122− pythonpath = /usr/local/searx/
123− chdir = /usr/local/searx/searx/
124−
125−Activate the uwsgi application and restart:
126−
127−.. code:: sh
128−
129− cd /etc/uwsgi/apps-enabled
130− ln -s ../apps-available/searx.ini
131− /etc/init.d/uwsgi restart
132−
133−Web server
134−==========
135−
136−with nginx
137−----------
138−
139−If nginx is not installed (uwsgi will not work with the package
140−nginx-light):
141−
142−.. code:: sh
143−
144− sudo -H apt-get install nginx
145−
146−Hosted at /
147−~~~~~~~~~~~
148−
149−Create the configuration file ``/etc/nginx/sites-available/searx`` with this
150−content:
151−
152−.. code:: nginx
153−
154− server {
155− listen 80;
156− server_name searx.example.com;
157− root /usr/local/searx/searx;
158−
159− location /static {
160− }
161−
162− location / {
163− include uwsgi_params;
164− uwsgi_pass unix:/run/uwsgi/app/searx/socket;
165− }
166− }
167−
168−Create a symlink to sites-enabled:
169−
170−.. code:: sh
171−
172− sudo -H ln -s /etc/nginx/sites-available/searx /etc/nginx/sites-enabled/searx
173−
174−Restart service:
175−
176−.. code:: sh
177−
178− sudo -H service nginx restart
179− sudo -H service uwsgi restart
180−
181−from subdirectory URL (/searx)
182−~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
183−
184−Add this configuration in the server config file
185−``/etc/nginx/sites-enabled/default``:
186−
187−.. code:: nginx
188−
189− location /searx/static {
190− alias /usr/local/searx/searx/static;
191− }
192−
193− location /searx {
194− uwsgi_param SCRIPT_NAME /searx;
195− include uwsgi_params;
196− uwsgi_pass unix:/run/uwsgi/app/searx/socket;
197− }
198−
199−
200−**OR** using reverse proxy (Please, note that reverse proxy advised to be used
201−in case of single-user or low-traffic instances.)
202−
203−.. code:: nginx
204−
205− location /searx/static {
206− alias /usr/local/searx/searx/static;
207− }
208−
209− location /searx {
210− proxy_pass http://127.0.0.1:8888;
211− proxy_set_header Host $host;
212− proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
213− proxy_set_header X-Scheme $scheme;
214− proxy_set_header X-Script-Name /searx;
215− proxy_buffering off;
216− }
217−
218−
219−Enable ``base_url`` in ``searx/settings.yml``
220−
221−.. code:: yaml
222−
223− base_url : http://your.domain.tld/searx/
224−
225−Restart service:
226−
227−.. code:: sh
228−
229− sudo -H service nginx restart
230− sudo -H service uwsgi restart
231−
232−disable logs
233−^^^^^^^^^^^^
234−
235−for better privacy you can disable nginx logs about searx.
236−
237−how to proceed: below ``uwsgi_pass`` in ``/etc/nginx/sites-available/default``
238−add:
239−
240−.. code:: nginx
241−
242− access_log /dev/null;
243− error_log /dev/null;
244−
245−Restart service:
246−
247−.. code:: sh
248−
249− sudo -H service nginx restart
250−
251−with apache
252−-----------
253−
254−Add wsgi mod:
255−
256−.. code:: sh
257−
258− sudo -H apt-get install libapache2-mod-uwsgi
259− sudo -H a2enmod uwsgi
260−
261−Add this configuration in the file ``/etc/apache2/apache2.conf``:
262−
263−.. code:: apache
264−
265− <Location />
266− Options FollowSymLinks Indexes
267− SetHandler uwsgi-handler
268− uWSGISocket /run/uwsgi/app/searx/socket
269− </Location>
270−
271−Note that if your instance of searx is not at the root, you should change
272−``<Location />`` by the location of your instance, like ``<Location /searx>``.
273−
274−Restart Apache:
275−
276−.. code:: sh
277−
278− sudo -H /etc/init.d/apache2 restart
279−
280−disable logs
281−~~~~~~~~~~~~
282−
283−For better privacy you can disable Apache logs.
284−
285−.. warning::
286−
287− You can only disable logs for the whole (virtual) server not for a specific
288− path.
289−
290−Go back to ``/etc/apache2/apache2.conf`` and above ``<Location />`` add:
291−
292−.. code:: apache
293−
294− CustomLog /dev/null combined
295−
296−Restart Apache:
297−
298−.. code:: sh
299−
300− sudo -H /etc/init.d/apache2 restart
301−
302−How to update
303−=============
304−
305−.. code:: sh
306−
307− cd /usr/local/searx
308− sudo -H -u searx -i
309−
310−.. code:: sh
311−
312− (searx)$ . ./searx-ve/bin/activate
313− (searx)$ git stash
314− (searx)$ git pull origin master
315− (searx)$ git stash apply
316− (searx)$ ./manage.sh update_packages
317−
318−.. code:: sh
319−
320− sudo -H service uwsgi restart
321−
322−Docker
323−======
324−
325−Make sure you have installed Docker. For instance, you can deploy searx like this:
326−
327−.. code:: sh
328−
329− docker pull wonderfall/searx
330− docker run -d --name searx -p $PORT:8888 wonderfall/searx
331−
332−Go to ``http://localhost:$PORT``.
333−
334−See https://hub.docker.com/r/wonderfall/searx/ for more informations. It's also
335−possible to build searx from the embedded Dockerfile.
336−
337−.. code:: sh
338−
339− git clone https://github.com/asciimoo/searx.git
340− cd searx
341− docker build -t whatever/searx .
342−
343−References
344−==========
345−
346−* https://about.okhin.fr/posts/Searx/ with some additions
347−
348−* How to: `Setup searx in a couple of hours with a free SSL certificate
349− <https://www.reddit.com/r/privacytoolsIO/comments/366kvn/how_to_setup_your_own_privacy_respecting_search/>`__
docs/admin/morty.rst+7
@@ -1,7 +1,14 @@
1+
2+.. _searx morty:
3+
14=========================
25How to setup result proxy
36=========================
47
8+.. sidebar:: further reading
9+
10+ - :ref:`morty.sh`
11+
512.. _morty: https://github.com/asciimoo/morty
613.. _morty's README: https://github.com/asciimoo/morty
714
docs/admin/settings.rst+7−1
@@ -4,11 +4,17 @@
44``settings.yml``
55================
66
7+This page describe the options possibilities of the :origin:`searx/settings.yml`
8+file.
9+
710.. sidebar:: Further reading ..
811
912 - :ref:`search API`
1013
11−This page describe the options possibilities of the settings.yml file.
14+.. contents:: Contents
15+ :depth: 2
16+ :local:
17+ :backlinks: entry
1218
1319.. _settings global:
1420
docs/admin/update-searx.rstnew file+23
@@ -0,0 +1,23 @@
1+.. _update searx:
2+
3+=============
4+How to update
5+=============
6+
7+.. code:: sh
8+
9+ sudo -H -u searx -i
10+ (searx)$ git stash
11+ (searx)$ git pull origin master
12+ (searx)$ git stash apply
13+ (searx)$ ./manage.sh update_packages
14+
15+Restart uwsgi:
16+
17+.. tabs::
18+
19+ .. group-tab:: Ubuntu / debian
20+
21+ .. code:: sh
22+
23+ sudo -H systemctl restart uwsgi
docs/blog/index.rst+2−1
@@ -3,7 +3,8 @@ Blog
33====
44
55.. toctree::
6− :maxdepth: 1
6+ :maxdepth: 2
7+ :caption: Contents
78
89 python3
910 admin
docs/build-templates/filtron.rstnew file+52
@@ -0,0 +1,52 @@
1+.. START create user
2+
3+.. tabs::
4+
5+ .. group-tab:: bash
6+
7+ .. code-block:: sh
8+
9+ $ sudo -H useradd --shell /bin/bash --system \\
10+ --home-dir "$SERVICE_HOME" \\
11+ --comment "Privacy-respecting metasearch engine" $SERVICE_USER
12+
13+ $ sudo -H mkdir "$SERVICE_HOME"
14+ $ sudo -H chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
15+
16+.. END create user
17+
18+.. START install go
19+
20+.. tabs::
21+
22+ .. group-tab:: bash
23+
24+ .. code-block:: bash
25+
26+ $ cat > "$GO_ENV" <<EOF
27+ export GOPATH=${SERVICE_HOME}/go-apps
28+ export PATH=\$PATH:${SERVICE_HOME}/local/go/bin:\$GOPATH/bin
29+ EOF
30+ $ sudo -i -u "${SERVICE_USER}"
31+ (${SERVICE_USER}) $ echo 'source $GO_ENV' >> ~/.profile
32+ (${SERVICE_USER}) $ mkdir ${SERVICE_HOME}/local
33+ (${SERVICE_USER}) $ wget --progress=bar -O "${GO_TAR}" \\
34+ "${GO_PKG_URL}"
35+ (${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local/go -xzf "${GO_TAR}"
36+ (${SERVICE_USER}) $ which go
37+ ${SERVICE_HOME}/local/go/bin/go
38+
39+.. END install go
40+
41+.. START install filtron
42+
43+.. tabs::
44+
45+ .. group-tab:: bash
46+
47+ .. code-block:: bash
48+
49+ $ sudo -i -u "${SERVICE_USER}"
50+ (${SERVICE_USER}) $ go get -v -u github.com/asciimoo/filtron
51+
52+.. END install filtron
docs/build-templates/morty.rstnew file+52
@@ -0,0 +1,52 @@
1+.. START create user
2+
3+.. tabs::
4+
5+ .. group-tab:: bash
6+
7+ .. code-block:: sh
8+
9+ $ sudo -H useradd --shell /bin/bash --system \\
10+ --home-dir "$SERVICE_HOME" \\
11+ --comment "Privacy-respecting metasearch engine" $SERVICE_USER
12+
13+ $ sudo -H mkdir "$SERVICE_HOME"
14+ $ sudo -H chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
15+
16+.. END create user
17+
18+.. START install go
19+
20+.. tabs::
21+
22+ .. group-tab:: bash
23+
24+ .. code-block:: bash
25+
26+ $ cat > "$GO_ENV" <<EOF
27+ export GOPATH=${SERVICE_HOME}/go-apps
28+ export PATH=\$PATH:${SERVICE_HOME}/local/go/bin:\$GOPATH/bin
29+ EOF
30+ $ sudo -i -u "${SERVICE_USER}"
31+ (${SERVICE_USER}) $ echo 'source $GO_ENV' >> ~/.profile
32+ (${SERVICE_USER}) $ mkdir ${SERVICE_HOME}/local
33+ (${SERVICE_USER}) $ wget --progress=bar -O "${GO_TAR}" \\
34+ "${GO_PKG_URL}"
35+ (${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local/go -xzf "${GO_TAR}"
36+ (${SERVICE_USER}) $ which go
37+ ${SERVICE_HOME}/local/go/bin/go
38+
39+.. END install go
40+
41+.. START install morty
42+
43+.. tabs::
44+
45+ .. group-tab:: bash
46+
47+ .. code-block:: bash
48+
49+ $ sudo -i -u "${SERVICE_USER}"
50+ (${SERVICE_USER}) $ go get -v -u github.com/asciimoo/morty
51+
52+.. END install morty
docs/build-templates/searx.rstnew file+192
@@ -0,0 +1,192 @@
1+.. template evaluated by: ./utils/searx.sh docs
2+.. hint: all dollar-names are variables, dollar sign itself is quoted by: \\$
3+
4+.. START distro-packages
5+
6+.. tabs::
7+
8+ .. group-tab:: Ubuntu / debian
9+
10+ .. code-block:: sh
11+
12+ $ sudo -H apt-get install -y \\
13+${debian}
14+
15+ .. group-tab:: Arch Linux
16+
17+ .. code-block:: sh
18+
19+ $ sudo -H pacman -S --noconfirm \\
20+${arch}
21+
22+ .. group-tab:: Fedora / RHEL
23+
24+ .. code-block:: sh
25+
26+ $ sudo -H dnf install -y \\
27+${fedora}
28+
29+.. END distro-packages
30+
31+.. START build-packages
32+
33+.. tabs::
34+
35+ .. group-tab:: Ubuntu / debian
36+
37+ .. code-block:: sh
38+
39+ $ sudo -H apt-get install -y \\
40+${debian_build}
41+
42+ .. group-tab:: Arch Linux
43+
44+ .. code-block:: sh
45+
46+ $ sudo -H pacman -S --noconfirm \\
47+${arch_build}
48+
49+ .. group-tab:: Fedora / RHEL
50+
51+ .. code-block:: sh
52+
53+ $ sudo -H dnf install -y \\
54+${fedora_build}
55+
56+.. END build-packages
57+
58+.. START create user
59+
60+.. tabs::
61+
62+ .. group-tab:: bash
63+
64+ .. code-block:: sh
65+
66+ $ sudo -H useradd --shell /bin/bash --system \\
67+ --home-dir "$SERVICE_HOME" \\
68+ --comment "Privacy-respecting metasearch engine" $SERVICE_USER
69+
70+ $ sudo -H mkdir "$SERVICE_HOME"
71+ $ sudo -H chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
72+
73+.. END create user
74+
75+.. START clone searx
76+
77+.. tabs::
78+
79+ .. group-tab:: bash
80+
81+ .. code-block:: sh
82+
83+ $ sudo -H -u ${SERVICE_USER} -i
84+ (${SERVICE_USER})$ git clone "https://github.com/asciimoo/searx.git" "$SEARX_SRC"
85+
86+.. END clone searx
87+
88+.. START create virtualenv
89+
90+.. tabs::
91+
92+ .. group-tab:: bash
93+
94+ .. code-block:: sh
95+
96+ (${SERVICE_USER})$ python3 -m venv "${SEARX_PYENV}"
97+ (${SERVICE_USER})$ echo ". ${SEARX_PYENV}/bin/activate" >> "$SERVICE_HOME/.profile"
98+
99+.. END create virtualenv
100+
101+.. START manage.sh update_packages
102+
103+.. tabs::
104+
105+ .. group-tab:: bash
106+
107+ .. code-block:: sh
108+
109+ $ sudo -H -u ${SERVICE_USER} -i
110+
111+ (${SERVICE_USER})$ command -v python && python --version
112+ $SEARX_PYENV/bin/python
113+ Python 3.8.1
114+
115+ # update pip's boilerplate ..
116+ pip install -U pip
117+ pip install -U setuptools
118+ pip install -U wheel
119+
120+ # jump to searx's working tree and install searx into virtualenv
121+ (${SERVICE_USER})$ cd "$SEARX_SRC"
122+ (${SERVICE_USER})$ pip install -e .
123+
124+
125+.. END manage.sh update_packages
126+
127+.. START searx config
128+
129+.. tabs::
130+
131+ .. group-tab:: bash
132+
133+ .. code-block:: sh
134+
135+ $ sudo -H cp "$SEARX_SRC/searx/settings.yml" "${SEARX_SETTINGS_PATH}"
136+ $ sudo -H sed -i -e "s/ultrasecretkey/\\$(openssl rand -hex 16)/g" "$SEARX_SETTINGS_PATH"
137+ $ sudo -H sed -i -e "s/{instance_name}/searx@\\$(uname -n)/g" "$SEARX_SETTINGS_PATH"
138+
139+.. END searx config
140+
141+.. START check searx installation
142+
143+.. tabs::
144+
145+ .. group-tab:: bash
146+
147+ .. code-block:: sh
148+
149+ # enable debug ..
150+ $ sudo -H sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS_PATH"
151+
152+ # start webapp
153+ $ sudo -H -u ${SERVICE_USER} -i
154+ (${SERVICE_USER})$ cd ${SEARX_SRC}
155+ (${SERVICE_USER})$ export SEARX_SETTINGS_PATH="${SEARX_SETTINGS_PATH}"
156+ (${SERVICE_USER})$ python searx/webapp.py
157+
158+ # disable debug
159+ $ sudo -H sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS_PATH"
160+
161+Open WEB browser and visit http://$SEARX_INTERNAL_URL . If you are inside a
162+container or in a script, test with curl:
163+
164+.. tabs::
165+
166+ .. group-tab:: WEB browser
167+
168+ .. code-block:: sh
169+
170+ $ xgd-open http://$SEARX_INTERNAL_URL
171+
172+ .. group-tab:: curl
173+
174+ .. code-block:: none
175+
176+ $ curl --location --verbose --head --insecure $SEARX_INTERNAL_URL
177+
178+ * Trying 127.0.0.1:8888...
179+ * TCP_NODELAY set
180+ * Connected to 127.0.0.1 (127.0.0.1) port 8888 (#0)
181+ > HEAD / HTTP/1.1
182+ > Host: 127.0.0.1:8888
183+ > User-Agent: curl/7.68.0
184+ > Accept: */*
185+ >
186+ * Mark bundle as not supporting multiuse
187+ * HTTP 1.0, assume close after body
188+ < HTTP/1.0 200 OK
189+ HTTP/1.0 200 OK
190+ ...
191+
192+.. END check searx installation
docs/conf.py+13−1
@@ -1,10 +1,12 @@
11# -*- coding: utf-8 -*-
22
33import sys, os
4+from sphinx_build_tools import load_sphinx_config
45from searx.version import VERSION_STRING
56from pallets_sphinx_themes import ProjectLink
67
78from searx.brand import GIT_URL
9+GIT_BRANCH = os.environ.get("GIT_BRANCH", "master")
810from searx.brand import SEARX_URL
911from searx.brand import DOCS_URL
1012
@@ -22,6 +24,8 @@ master_doc = "index"
2224source_suffix = '.rst'
2325numfig = True
2426
27+exclude_patterns = ['build-templates/*.rst']
28+
2529from searx import webapp
2630jinja_contexts = {
2731 'webapp': dict(**webapp.__dict__)
@@ -35,7 +39,7 @@ extlinks['wiki'] = ('https://github.com/asciimoo/searx/wiki/%s', ' ')
3539extlinks['pull'] = ('https://github.com/asciimoo/searx/pull/%s', 'PR ')
3640
3741# links to custom brand
38−extlinks['origin'] = (GIT_URL + '/blob/master/%s', 'git://')
42+extlinks['origin'] = (GIT_URL + '/blob/' + GIT_BRANCH + '/%s', 'git://')
3943extlinks['patch'] = (GIT_URL + '/commit/%s', '#')
4044extlinks['search'] = (SEARX_URL + '/%s', '#')
4145extlinks['docs'] = (DOCS_URL + '/%s', 'docs: ')
@@ -61,6 +65,8 @@ extensions = [
6165 "pallets_sphinx_themes",
6266 "sphinx_issues", # https://github.com/sloria/sphinx-issues/blob/master/README.rst
6367 "sphinxcontrib.jinja", # https://github.com/tardyp/sphinx-jinja
68+ "sphinxcontrib.programoutput", # https://github.com/NextThought/sphinxcontrib-programoutput
69+ 'linuxdoc.kernel_include', # Implementation of the 'kernel-include' reST-directive.
6470 'linuxdoc.rstFlatTable', # Implementation of the 'flat-table' reST-directive.
6571 'linuxdoc.kfigure', # Sphinx extension which implements scalable image handling.
6672 "sphinx_tabs.tabs", # https://github.com/djungelorm/sphinx-tabs
@@ -112,3 +118,9 @@ html_show_sourcelink = False
112118latex_documents = [
113119 (master_doc, "searx-{}.tex".format(VERSION_STRING), html_title, author, "manual")
114120]
121+
122+# ------------------------------------------------------------------------------
123+# Since loadConfig overwrites settings from the global namespace, it has to be
124+# the last statement in the conf.py file
125+# ------------------------------------------------------------------------------
126+load_sphinx_config(globals())
docs/dev/contribution_guide.rst+5
@@ -4,6 +4,11 @@
44How to contribute
55=================
66
7+.. contents:: Contents
8+ :depth: 2
9+ :local:
10+ :backlinks: entry
11+
712Prime directives: Privacy, Hackability
813======================================
914
docs/dev/index.rst+2−1
@@ -3,7 +3,8 @@ Developer documentation
33=======================
44
55.. toctree::
6− :maxdepth: 1
6+ :maxdepth: 2
7+ :caption: Contents
78
89 quickstart
910 contribution_guide
docs/dev/makefile.rst+24−24
@@ -11,23 +11,17 @@ Makefile Targets
1111 Before looking deeper at the targets, first read about :ref:`makefile setup`
1212 and :ref:`make pyenv`.
1313
14+ To install system requirements follow :ref:`buildhosts`.
15+
1416With the aim to simplify development cycles, started with :pull:`1756` a
1517``Makefile`` based boilerplate was added. If you are not familiar with
1618Makefiles, we recommend to read gnu-make_ introduction.
1719
1820The usage is simple, just type ``make {target-name}`` to *build* a target.
19−Calling the ``help`` target gives a first overview::
20−
21− $ make help
22− test - run developer tests
23− docs - build documentation
24− docs-live - autobuild HTML documentation while editing
25− run - run developer instance
26− install - developer install (./local)
27− uninstall - uninstall (./local)
28− gh-pages - build docs & deploy on gh-pages branch
29− clean - drop builds and environments
30− ...
21+Calling the ``help`` target gives a first overview (``make help``):
22+
23+.. program-output:: bash -c "cd ..; make --no-print-directory help"
24+
3125
3226.. contents:: Contents
3327 :depth: 2
@@ -37,27 +31,33 @@ Calling the ``help`` target gives a first overview::
3731
3832.. _makefile setup:
3933
40−Setup
41−=====
34+Makefile setup
35+==============
4236
4337.. _git stash: https://git-scm.com/docs/git-stash
4438
45−The main setup is done in the :origin:`Makefile`::
46−
47− export GIT_URL=https://github.com/asciimoo/searx
48− export SEARX_URL=https://searx.me
49− export DOCS_URL=https://asciimoo.github.io/searx
50−
5139.. sidebar:: fork & upstream
5240
5341 Commit changes in your (local) branch, fork or whatever, but do not push them
5442 upstream / `git stash`_ is your friend.
5543
56−:GIT_URL: Changes this, to point to your searx fork.
44+The main setup is done in the :origin:`Makefile`.
45+
46+.. literalinclude:: ../../Makefile
47+ :start-after: START Makefile setup
48+ :end-before: END Makefile setup
49+
50+:GIT_URL: Changes this, to point to your searx fork.
51+:GIT_BRANCH: Changes this, to point to your searx branch.
52+:SEARX_URL: Changes this, to point to your searx instance.
53+:DOCS_URL: If you host your own (*brand*) documentation, change this URL.
5754
58−:SEARX_URL: Changes this, to point to your searx instance.
55+If you change any of this build environment variables, you have to run ``make
56+buildenv``::
5957
60−:DOCS_URL: If you host your own (branded) documentation, change this URL.
58+ $ make buildenv
59+ build searx/brand.py
60+ build utils/brand.env
6161
6262.. _make pyenv:
6363
@@ -170,7 +170,7 @@ e.g.:
170170
171171.. code:: sh
172172
173− $ make test.pep8 test.unit
173+ $ make test.pep8 test.unit test.sh
174174 . ./local/py3/bin/activate; ./manage.sh pep8_check
175175 [!] Running pep8 check
176176 . ./local/py3/bin/activate; ./manage.sh unit_tests
docs/dev/quickstart.rst+1−1
@@ -27,7 +27,7 @@ searx-ve virtualenv and install the required packages using ``manage.sh``.
2727 cd ~/myprojects
2828 git clone https://github.com/asciimoo/searx.git
2929 cd searx
30− virtualenv searx-ve
30+ python3 -m venv searx-ve
3131 . ./searx-ve/bin/activate
3232 ./manage.sh update_dev_packages
3333
docs/dev/reST.rst+5−5
@@ -325,8 +325,9 @@ Literal blocks
325325
326326The simplest form of :duref:`literal-blocks` is a indented block introduced by
327327two colons (``::``). For highlighting use :dudir:`highlight` or :ref:`reST
328−code` directive. To include literals from external files use directive
329−:dudir:`literalinclude`.
328+code` directive. To include literals from external files use
329+:rst:dir:`literalinclude` or :ref:`kernel-include <kernel-include-directive>`
330+directive (latter one expands environment variables in the path name).
330331
331332.. _reST literal:
332333
@@ -1312,9 +1313,8 @@ others are basic-tabs_ and code-tabs_. Below a *group-tab* example from
13121313
13131314.. literalinclude:: ../admin/buildhosts.rst
13141315 :language: reST
1315− :start-after: .. _system requirements:
1316− :end-before: .. _system requirements END:
1317−
1316+ :start-after: .. SNIP sh lint requirements
1317+ :end-before: .. SNAP sh lint requirements
13181318
13191319.. _math:
13201320
docs/index.rst+10−8
@@ -2,7 +2,14 @@
22Welcome to searx
33================
44
5−Search without being tracked.
5+ *Search without being tracked.*
6+
7+Searx is a free internet metasearch engine which aggregates results from more
8+than 70 search services. Users are neither tracked nor profiled. Additionally,
9+searx can be used over Tor for online anonymity.
10+
11+Get started with searx by using one of the Searx-instances_. If you don't trust
12+anyone, you can set up your own, see :ref:`installation`.
613
714.. sidebar:: Features
815
@@ -16,19 +23,14 @@ Search without being tracked.
1623 - Hosted by organizations, such as *La Quadrature du Net*, which promote
1724 digital rights
1825
19−Searx is a free internet metasearch engine which aggregates results from more
20−than 70 search services. Users are neither tracked nor profiled. Additionally,
21−searx can be used over Tor for online anonymity.
22−
23−Get started with searx by using one of the Searx-instances_. If you don't trust
24−anyone, you can set up your own, see :ref:`installation`.
25−
2626.. toctree::
2727 :maxdepth: 2
28+ :caption: Contents
2829
2930 user/index
3031 admin/index
3132 dev/index
33+ utils/index
3234 blog/index
3335
3436.. _Searx-instances: https://searx.space
docs/user/conf.pynew file+19
@@ -0,0 +1,19 @@
1+# -*- coding: utf-8; mode: python -*-
2+"""Configuration for the Searx user handbook
3+"""
4+project = 'Searx User-HB'
5+version = release = VERSION_STRING
6+
7+# Grouping the document tree into LaTeX files. List of tuples
8+# (source start file, target name, title,
9+# author, documentclass [howto, manual, or own class]).
10+latex_documents = [
11+ ('index' # startdocname
12+ , 'searx-user-hb.tex' # targetname
13+ , '' # take title from .rst
14+ , author # author
15+ , 'howto' # documentclass
16+ , False # toctree_only
17+ ),
18+]
19+
docs/user/index.rst+2−1
@@ -3,7 +3,8 @@ User documentation
33==================
44
55.. toctree::
6− :maxdepth: 1
6+ :maxdepth: 2
7+ :caption: Contents
78
89 search_syntax
910 own-instance
docs/user/own-instance.rst+4−2
@@ -2,8 +2,10 @@
22Why use a private instance?
33===========================
44
5−"Is it worth to run my own instance?" is a common question among searx users.
6−Before answering this question, see what options a searx user has.
5+ *"Is it worth to run my own instance?"*
6+
7+\.\. is a common question among searx users. Before answering this question,
8+see what options a searx user has.
79
810Public instances are open to everyone who has access to its URL. Usually, these
911are operated by unknown parties (from the users' point of view). Private
docs/user/public_instances.rstdeleted−3
@@ -1,3 +0,0 @@
1−:orphan:
2−
3−This page page has been moved to `searx.space <https://searx.space/>`__
docs/utils/filtron.sh.rstnew file+80
@@ -0,0 +1,80 @@
1+
2+.. _filtron.sh:
3+
4+====================
5+``utils/filtron.sh``
6+====================
7+
8+.. sidebar:: further reading
9+
10+ - :ref:`searx filtron`
11+ - :ref:`architecture`
12+ - :ref:`installation` (:ref:`nginx <installation nginx>` & :ref:`apache
13+ <installation apache>`)
14+
15+.. _Go: https://golang.org/
16+.. _filtron: https://github.com/asciimoo/filtron
17+.. _filtron README: https://github.com/asciimoo/filtron/blob/master/README.md
18+
19+To simplify installation and maintenance of a filtron instance you can use the
20+script :origin:`utils/filtron.sh`. In most cases you will install filtron_
21+simply by running the command:
22+
23+.. code:: bash
24+
25+ sudo -H ./utils/filtron.sh install all
26+
27+The script adds a ``${SERVICE_USER}`` (default:``filtron``) and installs filtron_
28+into this user account:
29+
30+#. Create a separated user account (``filtron``).
31+#. Download and install Go_ binary in user's $HOME (``~filtron``).
32+#. Install filtron with the package management from Go_ (``go get -v -u
33+ github.com/asciimoo/filtron``)
34+#. Setup a proper rule configuration :origin:`[ref]
35+ <utils/templates/etc/filtron/rules.json>` (``/etc/filtron/rules.json``).
36+#. Setup a systemd service unit :origin:`[ref]
37+ <utils/templates/lib/systemd/system/filtron.service>`
38+ (``/lib/systemd/system/filtron.service``).
39+
40+
41+Create user
42+===========
43+
44+.. kernel-include:: $DOCS_BUILD/includes/filtron.rst
45+ :start-after: START create user
46+ :end-before: END create user
47+
48+
49+Install go
50+==========
51+
52+.. kernel-include:: $DOCS_BUILD/includes/filtron.rst
53+ :start-after: START install go
54+ :end-before: END install go
55+
56+
57+Install filtron
58+===============
59+
60+Install :origin:`rules.json <utils/templates/etc/filtron/rules.json>` at
61+``/etc/filtron/rules.json`` (see :ref:`Sample configuration of filtron`) and
62+install filtron software and systemd unit:
63+
64+.. kernel-include:: $DOCS_BUILD/includes/filtron.rst
65+ :start-after: START install filtron
66+ :end-before: END install filtron
67+
68+.. kernel-include:: $DOCS_BUILD/includes/filtron.rst
69+ :start-after: START install systemd unit
70+ :end-before: END install systemd unit
71+
72+.. _filtron.sh overview:
73+
74+Overview
75+========
76+
77+The ``--help`` output of the script is largely self-explanatory
78+(:ref:`toolboxing common`):
79+
80+.. program-output:: ../utils/filtron.sh --help
docs/utils/index.rstnew file+53
@@ -0,0 +1,53 @@
1+.. _searx_utils:
2+.. _toolboxing:
3+
4+=======================
5+Tooling box ``utils/*``
6+=======================
7+
8+In the folder :origin:`utils/` we maintain some tools useful for admins and
9+developers.
10+
11+.. toctree::
12+ :maxdepth: 2
13+ :caption: Contents
14+
15+ searx.sh
16+ filtron.sh
17+ morty.sh
18+ lxc.sh
19+
20+.. _toolboxing common:
21+
22+Common commands & environment
23+=============================
24+
25+Scripts to maintain services often dispose of common commands and environments.
26+
27+``shell`` : command
28+ Opens a shell from the service user ``${SERVICE_USSR}``, very helpful for
29+ troubleshooting.
30+
31+``inspect service`` : command
32+ Shows status and log of the service, most often you have a option to enable
33+ more verbose debug logs. Very helpful for debugging, but be careful not to
34+ enable debugging in a production environment!
35+
36+``FORCE_TIMEOUT`` : environment
37+ Sets timeout for interactive prompts. If you want to run a script in batch
38+ job, with defaults choices, set ``FORCE_TIMEOUT=0``. By example; to install a
39+ reverse proxy for filtron on all containers of the :ref:`searx suite
40+ <lxc-searx.env>` use ::
41+
42+ sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh apache install
43+
44+.. _toolboxing setup:
45+
46+Tooling box setup
47+=================
48+
49+The main setup is done in the :origin:`.config.sh` (read also :ref:`makefile
50+setup`).
51+
52+.. literalinclude:: ../../.config.sh
53+ :language: bash
docs/utils/lxc.sh.rstnew file+148
@@ -0,0 +1,148 @@
1+
2+.. _snap: https://snapcraft.io
3+.. _snapcraft LXD: https://snapcraft.io/lxd
4+.. _LXC/LXD Image Server: https://uk.images.linuxcontainers.org/
5+.. _LXC: https://linuxcontainers.org/lxc/introduction/
6+.. _LXD: https://linuxcontainers.org/lxd/introduction/
7+.. _`LXD@github`: https://github.com/lxc/lxd
8+
9+.. _archlinux: https://www.archlinux.org/
10+
11+.. _lxc.sh:
12+
13+================
14+``utils/lxc.sh``
15+================
16+
17+.. sidebar:: further reading
18+
19+ - snap_, `snapcraft LXD`_
20+ - LXC_, LXD_
21+ - `LXC/LXD Image Server`_
22+ - `LXD@github`_
23+
24+With the use of *Linux Containers* (LXC_) we can scale our tasks over a stack of
25+containers, what we call the: *lxc suite*. The *searx suite*
26+(:origin:`lxc-searx.env <utils/lxc-searx.env>`) is loaded by default, every time
27+you start the ``lxc.sh`` script (*you do not need to care about*).
28+
29+Before you can start with containers, you need to install and initiate LXD_
30+once::
31+
32+ $ snap install lxd
33+ $ lxd init --auto
34+
35+To make use of the containers from the *searx suite*, you have to build the
36+:ref:`LXC suite containers <lxc.sh help>` initial. But be warned, **this might
37+take some time**::
38+
39+ $ sudo -H ./utils/lxc.sh build
40+
41+A cup of coffee later, your LXC suite is build up and you can run whatever task
42+you want / in a selected or even in all :ref:`LXC suite containers <lxc.sh
43+help>`. If you do not want to build all containers, **you can build just
44+one**::
45+
46+ $ sudo -H ./utils/lxc.sh build searx-ubu1804
47+
48+*Good to know ...*
49+
50+Eeach container shares the root folder of the repository and the
51+command ``utils/lxc.sh cmd`` **handles relative path names transparent**,
52+compare output of::
53+
54+ $ sudo -H ./utils/lxc.sh cmd -- ls -la Makefile
55+ ...
56+
57+In the containers, you can run what ever you want, e.g. to start a bash use::
58+
59+ $ sudo -H ./utils/lxc.sh cmd searx-ubu1804 bash
60+ INFO: [searx-ubu1804] bash
61+ root@searx-ubu1804:/share/searx#
62+
63+If there comes the time you want to **get rid off all** the containers and
64+**clean up local images** just type::
65+
66+ $ sudo -H ./utils/lxc.sh remove
67+ $ sudo -H ./utils/lxc.sh remove images
68+
69+
70+Install suite
71+=============
72+
73+To install the complete :ref:`searx suite (includes searx, morty & filtron)
74+<lxc-searx.env>` into all LXC_ use::
75+
76+ $ sudo -H ./utils/lxc.sh install suite
77+
78+The command above installs a searx suite (see :ref:`installation scripts`). To
79+get the IP (URL) of the filtron service in the containers use ``show suite``
80+command. To test instances from containers just open the URLs in your
81+WEB-Browser::
82+
83+ $ sudo ./utils/lxc.sh show suite | grep filtron
84+ [searx-ubu1604] INFO: (eth0) filtron: http://n.n.n.246:4004/ http://n.n.n.246/searx
85+ [searx-ubu1804] INFO: (eth0) filtron: http://n.n.n.147:4004/ http://n.n.n.147/searx
86+ [searx-ubu1910] INFO: (eth0) filtron: http://n.n.n.140:4004/ http://n.n.n.140/searx
87+ [searx-ubu2004] INFO: (eth0) filtron: http://n.n.n.18:4004/ http://n.n.n.18/searx
88+ [searx-fedora31] INFO: (eth0) filtron: http://n.n.n.46:4004/ http://n.n.n.46/searx
89+ [searx-archlinux] INFO: (eth0) filtron: http://n.n.n.32:4004/ http://n.n.n.32/searx
90+
91+To :ref:`install a nginx <installation nginx>` reverse proxy for filtron and
92+morty use (or alternatively use :ref:`apache <installation apache>`)::
93+
94+ sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh nginx install
95+ sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/morty.sh nginx install
96+
97+
98+Running commands
99+================
100+
101+**Inside containers, you can use make or run scripts** from the
102+:ref:`toolboxing`. By example: to setup a :ref:`buildhosts` and run the
103+Makefile target ``test`` in the archlinux_ container::
104+
105+ sudo -H ./utils/lxc.sh cmd searx-archlinux ./utils/searx.sh install buildhost
106+ sudo -H ./utils/lxc.sh cmd searx-archlinux make test
107+
108+
109+Setup searx buildhost
110+=====================
111+
112+You can **install the searx buildhost environment** into one or all containers.
113+The installation procedure to set up a :ref:`build host<buildhosts>` takes its
114+time. Installation in all containers will take more time (time for another cup
115+of coffee).::
116+
117+ sudo -H ./utils/lxc.sh cmd -- ./utils/searx.sh install buildhost
118+
119+To build (live) documentation inside a archlinux_ container::
120+
121+ sudo -H ./utils/lxc.sh cmd searx-archlinux make docs-clean docs-live
122+ ...
123+ [I 200331 15:00:42 server:296] Serving on http://0.0.0.0:8080
124+
125+To get IP of the container and the port number *live docs* is listening::
126+
127+ $ sudo ./utils/lxc.sh show suite | grep docs-live
128+ ...
129+ [searx-archlinux] INFO: (eth0) docs-live: http://n.n.n.12:8080/
130+
131+
132+.. _lxc.sh help:
133+
134+Overview
135+========
136+
137+The ``--help`` output of the script is largely self-explanatory:
138+
139+.. program-output:: ../utils/lxc.sh --help
140+
141+
142+.. _lxc-searx.env:
143+
144+searx suite
145+===========
146+
147+.. literalinclude:: ../../utils/lxc-searx.env
148+ :language: bash
docs/utils/morty.sh.rstnew file+80
@@ -0,0 +1,80 @@
1+
2+.. _morty: https://github.com/asciimoo/morty
3+.. _morty's README: https://github.com/asciimoo/morty
4+.. _Go: https://golang.org/
5+
6+.. _morty.sh:
7+
8+==================
9+``utils/morty.sh``
10+==================
11+
12+.. sidebar:: further reading
13+
14+ - :ref:`architecture`
15+ - :ref:`installation` (:ref:`nginx <installation nginx>` & :ref:`apache
16+ <installation apache>`)
17+ - :ref:`searx morty`
18+
19+To simplify installation and maintenance of a morty_ instance you can use the
20+script :origin:`utils/morty.sh`. In most cases you will install morty_ simply by
21+running the command:
22+
23+.. code:: bash
24+
25+ sudo -H ./utils/morty.sh install all
26+
27+The script adds a ``${SERVICE_USER}`` (default:``morty``) and installs morty_
28+into this user account:
29+
30+#. Create a separated user account (``morty``).
31+#. Download and install Go_ binary in user's $HOME (``~morty``).
32+#. Install morty_ with the package management from Go_ (``go get -v -u
33+ github.com/asciimoo/morty``)
34+#. Setup a systemd service unit :origin:`[ref]
35+ <utils/templates/lib/systemd/system/morty.service>`
36+ (``/lib/systemd/system/morty.service``).
37+
38+.. hint::
39+
40+ To add morty to your searx instance read chapter :ref:`searx morty`.
41+
42+Create user
43+===========
44+
45+.. kernel-include:: $DOCS_BUILD/includes/morty.rst
46+ :start-after: START create user
47+ :end-before: END create user
48+
49+
50+Install go
51+==========
52+
53+.. kernel-include:: $DOCS_BUILD/includes/morty.rst
54+ :start-after: START install go
55+ :end-before: END install go
56+
57+
58+Install morty
59+=============
60+
61+Install morty software and systemd unit:
62+
63+.. kernel-include:: $DOCS_BUILD/includes/morty.rst
64+ :start-after: START install morty
65+ :end-before: END install morty
66+
67+.. kernel-include:: $DOCS_BUILD/includes/morty.rst
68+ :start-after: START install systemd unit
69+ :end-before: END install systemd unit
70+
71+.. _morty.sh overview:
72+
73+Overview
74+========
75+
76+The ``--help`` output of the script is largely self-explanatory
77+(:ref:`toolboxing common`):
78+
79+.. program-output:: ../utils/morty.sh --help
80+
docs/utils/searx.sh.rstnew file+39
@@ -0,0 +1,39 @@
1+
2+.. _searx.sh:
3+
4+==================
5+``utils/searx.sh``
6+==================
7+
8+.. sidebar:: further reading
9+
10+ - :ref:`architecture`
11+ - :ref:`installation`
12+ - :ref:`installation nginx`
13+ - :ref:`installation apache`
14+
15+To simplify installation and maintenance of a searx instance you can use the
16+script :origin:`utils/searx.sh`.
17+
18+Install
19+=======
20+
21+In most cases you will install searx simply by running the command:
22+
23+.. code:: bash
24+
25+ sudo -H ./utils/searx.sh install all
26+
27+The script adds a ``${SERVICE_USER}`` (default:``searx``) and installs searx
28+into this user account. The installation is described in chapter
29+:ref:`installation basic`.
30+
31+.. _intranet reverse proxy:
32+
33+Overview
34+========
35+
36+The ``--help`` output of the script is largely self-explanatory
37+(:ref:`toolboxing common`):
38+
39+.. program-output:: ../utils/searx.sh --help
requirements-dev.txt+2
@@ -15,3 +15,5 @@ selenium==3.141.0
1515linuxdoc @ git+http://github.com/return42/linuxdoc.git
1616sphinx-jinja
1717sphinx-tabs
18+sphinxcontrib-programoutput
19+twine
searx/brand.py+1
@@ -1,4 +1,5 @@
11GIT_URL = 'https://github.com/asciimoo/searx'
2+GIT_BRANCH = 'master'
23ISSUE_URL = 'https://github.com/asciimoo/searx/issues'
34SEARX_URL = 'https://searx.me'
45DOCS_URL = 'https://asciimoo.github.io/searx'
searx/static/plugins/js/search_on_category_select.js+18
@@ -6,19 +6,37 @@ $(document).ready(function() {
66 });
77 $(document.getElementById($(this).attr("for"))).prop('checked', true);
88 if($('#q').val()) {
9+ if (getHttpRequest() == "GET") {
10+ $('#search_form').attr('action', $('#search_form').serialize());
11+ }
912 $('#search_form').submit();
1013 }
1114 return false;
1215 });
1316 $('#time-range').change(function(e) {
1417 if($('#q').val()) {
18+ if (getHttpRequest() == "GET") {
19+ $('#search_form').attr('action', $('#search_form').serialize());
20+ }
1521 $('#search_form').submit();
1622 }
1723 });
1824 $('#language').change(function(e) {
1925 if($('#q').val()) {
26+ if (getHttpRequest() == "GET") {
27+ $('#search_form').attr('action', $('#search_form').serialize());
28+ }
2029 $('#search_form').submit();
2130 }
2231 });
2332 }
2433});
34+
35+function getHttpRequest() {
36+ httpRequest = "POST";
37+ urlParams = new URLSearchParams(window.location.search);
38+ if (urlParams.has('method')) {
39+ httpRequest = urlParams.get('method');
40+ }
41+ return httpRequest;
42+}
searx/templates/__common__/opensearch.xml+5−15
@@ -6,23 +6,13 @@
66 <Image>{{ urljoin(host, url_for('static', filename='img/favicon.png')) }}</Image>
77 <LongName>searx metasearch</LongName>
88 {% if opensearch_method == 'get' %}
9− <Url type="text/html" method="get" template="{{ host }}search?q={searchTerms}"/>
10− {% if autocomplete %}
11− <Url type="application/x-suggestions+json" method="get" template="{{ host }}autocompleter">
12− <Param name="format" value="x-suggestions" />
13− <Param name="q" value="{searchTerms}" />
14− </Url>
15− {% endif %}
9+ <Url rel="results" type="text/html" method="get" template="{{ host }}search?q={searchTerms}"/>
1610 {% else %}
17− <Url type="text/html" method="post" template="{{ host }}">
11+ <Url rel="results" type="text/html" method="post" template="{{ host }}">
1812 <Param name="q" value="{searchTerms}" />
1913 </Url>
20− {% if autocomplete %}
21− <!-- TODO, POST REQUEST doesn't work -->
22− <Url type="application/x-suggestions+json" method="get" template="{{ host }}autocompleter">
23− <Param name="format" value="x-suggestions" />
24− <Param name="q" value="{searchTerms}" />
25− </Url>
26− {% endif %}
14+ {% endif %}
15+ {% if autocomplete %}
16+ <Url rel="suggestions" type="application/json" template="{{ host }}autocompleter"/>
2717 {% endif %}
2818</OpenSearchDescription>
searx/webapp.py+1−1
@@ -949,7 +949,7 @@ def opensearch():
949949
950950 resp = Response(response=ret,
951951 status=200,
952− mimetype="text/xml")
952+ mimetype="application/opensearchdescription+xml")
953953 return resp
954954
955955
utils/brand.env+1
@@ -1,4 +1,5 @@
11export GIT_URL='https://github.com/asciimoo/searx'
2+export GIT_BRANCH='master'
23export ISSUE_URL='https://github.com/asciimoo/searx/issues'
34export SEARX_URL='https://searx.me'
45export DOCS_URL='https://asciimoo.github.io/searx'
utils/filtron.shnew file+561
@@ -0,0 +1,561 @@
1+#!/usr/bin/env bash
2+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
3+# SPDX-License-Identifier: AGPL-3.0-or-later
4+# shellcheck disable=SC2119,SC2001
5+
6+# shellcheck source=utils/lib.sh
7+source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
8+# shellcheck source=utils/brand.env
9+source "${REPO_ROOT}/utils/brand.env"
10+source_dot_config
11+source "${REPO_ROOT}/utils/lxc-searx.env"
12+in_container && lxc_set_suite_env
13+
14+# ----------------------------------------------------------------------------
15+# config
16+# ----------------------------------------------------------------------------
17+
18+PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
19+PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}"
20+
21+FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \
22+| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}"
23+[[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/
24+
25+FILTRON_ETC="/etc/filtron"
26+FILTRON_RULES="$FILTRON_ETC/rules.json"
27+
28+FILTRON_API="${FILTRON_API:-127.0.0.1:4005}"
29+FILTRON_LISTEN="${FILTRON_LISTEN:-127.0.0.1:4004}"
30+FILTRON_TARGET="${FILTRON_TARGET:-127.0.0.1:8888}"
31+
32+SERVICE_NAME="filtron"
33+SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}"
34+SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}"
35+SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}"
36+SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service"
37+# shellcheck disable=SC2034
38+SERVICE_GROUP="${SERVICE_USER}"
39+
40+# shellcheck disable=SC2034
41+SERVICE_GROUP="${SERVICE_USER}"
42+
43+GO_ENV="${SERVICE_HOME}/.go_env"
44+GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz"
45+GO_TAR=$(basename "$GO_PKG_URL")
46+
47+APACHE_FILTRON_SITE="searx.conf"
48+NGINX_FILTRON_SITE="searx.conf"
49+
50+# shellcheck disable=SC2034
51+CONFIG_FILES=(
52+ "${FILTRON_RULES}"
53+ "${SERVICE_SYSTEMD_UNIT}"
54+)
55+
56+# ----------------------------------------------------------------------------
57+usage() {
58+# ----------------------------------------------------------------------------
59+
60+ # shellcheck disable=SC1117
61+ cat <<EOF
62+usage::
63+ $(basename "$0") shell
64+ $(basename "$0") install [all|user|rules]
65+ $(basename "$0") update [filtron]
66+ $(basename "$0") remove [all]
67+ $(basename "$0") activate [service]
68+ $(basename "$0") deactivate [service]
69+ $(basename "$0") inspect [service]
70+ $(basename "$0") option [debug-on|debug-off]
71+ $(basename "$0") apache [install|remove]
72+ $(basename "$0") nginx [install|remove]
73+
74+shell
75+ start interactive shell from user ${SERVICE_USER}
76+install / remove
77+ :all: complete setup of filtron service
78+ :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME)
79+ :rules: reinstall filtron rules $FILTRON_RULES
80+update filtron
81+ Update filtron installation ($SERVICE_HOME)
82+activate service
83+ activate and start service daemon (systemd unit)
84+deactivate service
85+ stop and deactivate service daemon (systemd unit)
86+inspect service
87+ show service status and log
88+option
89+ set one of the available options
90+apache (${PUBLIC_URL})
91+ :install: apache site with a reverse proxy (ProxyPass)
92+ :remove: apache site ${APACHE_FILTRON_SITE}
93+nginx (${PUBLIC_URL})
94+ :install: nginx site with a reverse proxy (ProxyPass)
95+ :remove: nginx site ${NGINX_FILTRON_SITE}
96+
97+filtron rules: ${FILTRON_RULES}
98+
99+If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file::
100+ PUBLIC_URL : ${PUBLIC_URL}
101+ PUBLIC_HOST : ${PUBLIC_HOST}
102+ SERVICE_USER : ${SERVICE_USER}
103+ FILTRON_TARGET : ${FILTRON_TARGET}
104+ FILTRON_API : ${FILTRON_API}
105+ FILTRON_LISTEN : ${FILTRON_LISTEN}
106+EOF
107+ if in_container; then
108+ # in containers the service is listening on 0.0.0.0 (see lxc-searx.env)
109+ for ip in $(global_IPs) ; do
110+ if [[ $ip =~ .*:.* ]]; then
111+ echo " container URL (IPv6): http://[${ip#*|}]:4005/"
112+ else
113+ # IPv4:
114+ echo " container URL (IPv4): http://${ip#*|}:4005/"
115+ fi
116+ done
117+ fi
118+ [[ -n ${1} ]] && err_msg "$1"
119+}
120+
121+main() {
122+ required_commands \
123+ sudo install git wget curl \
124+ || exit
125+
126+ local _usage="unknown or missing $1 command $2"
127+
128+ case $1 in
129+ --getenv) var="$2"; echo "${!var}"; exit 0;;
130+ -h|--help) usage; exit 0;;
131+
132+ shell)
133+ sudo_or_exit
134+ interactive_shell "${SERVICE_USER}"
135+ ;;
136+ inspect)
137+ case $2 in
138+ service)
139+ sudo_or_exit
140+ inspect_service
141+ ;;
142+ *) usage "$_usage"; exit 42;;
143+ esac ;;
144+ install)
145+ rst_title "$SERVICE_NAME" part
146+ sudo_or_exit
147+ case $2 in
148+ all) install_all ;;
149+ user) assert_user ;;
150+ rules)
151+ rst_title "Re-Install filtron rules"
152+ echo
153+ install_template --no-eval "$FILTRON_RULES" root root 644
154+ systemd_restart_service "${SERVICE_NAME}"
155+ ;;
156+ *) usage "$_usage"; exit 42;;
157+ esac ;;
158+ update)
159+ sudo_or_exit
160+ case $2 in
161+ filtron) update_filtron ;;
162+ *) usage "$_usage"; exit 42;;
163+ esac ;;
164+ remove)
165+ sudo_or_exit
166+ case $2 in
167+ all) remove_all;;
168+ user) drop_service_account "${SERVICE_USER}" ;;
169+ *) usage "$_usage"; exit 42;;
170+ esac ;;
171+ activate)
172+ sudo_or_exit
173+ case $2 in
174+ service) systemd_activate_service "${SERVICE_NAME}" ;;
175+ *) usage "$_usage"; exit 42;;
176+ esac ;;
177+ deactivate)
178+ sudo_or_exit
179+ case $2 in
180+ service) systemd_deactivate_service "${SERVICE_NAME}" ;;
181+ *) usage "$_usage"; exit 42;;
182+ esac ;;
183+ apache)
184+ sudo_or_exit
185+ case $2 in
186+ install) install_apache_site ;;
187+ remove) remove_apache_site ;;
188+ *) usage "$_usage"; exit 42;;
189+ esac ;;
190+ nginx)
191+ sudo_or_exit
192+ case $2 in
193+ install) install_nginx_site ;;
194+ remove) remove_nginx_site ;;
195+ *) usage "$_usage"; exit 42;;
196+ esac ;;
197+ option)
198+ sudo_or_exit
199+ case $2 in
200+ debug-on) echo; enable_debug ;;
201+ debug-off) echo; disable_debug ;;
202+ *) usage "$_usage"; exit 42;;
203+ esac ;;
204+ doc) rst-doc ;;
205+ *) usage "unknown or missing command $1"; exit 42;;
206+ esac
207+}
208+
209+install_all() {
210+ rst_title "Install $SERVICE_NAME (service)"
211+ assert_user
212+ wait_key
213+ install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}"
214+ wait_key
215+ install_filtron
216+ wait_key
217+ systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
218+ wait_key
219+ echo
220+ if ! service_is_available "http://${FILTRON_LISTEN}" ; then
221+ err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}"
222+ fi
223+ if apache_is_installed; then
224+ info_msg "Apache is installed on this host."
225+ if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
226+ install_apache_site
227+ fi
228+ elif nginx_is_installed; then
229+ info_msg "nginx is installed on this host."
230+ if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
231+ install_nginx_site
232+ fi
233+ fi
234+ if ask_yn "Do you want to inspect the installation?" Ny; then
235+ inspect_service
236+ fi
237+
238+}
239+
240+remove_all() {
241+ rst_title "De-Install $SERVICE_NAME (service)"
242+
243+ rst_para "\
244+It goes without saying that this script can only be used to remove
245+installations that were installed with this script."
246+
247+ if ! systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then
248+ return 42
249+ fi
250+ drop_service_account "${SERVICE_USER}"
251+ rm -r "$FILTRON_ETC" 2>&1 | prefix_stdout
252+ if service_is_available "${PUBLIC_URL}"; then
253+ MSG="** Don't forget to remove your public site! (${PUBLIC_URL}) **" wait_key 10
254+ fi
255+}
256+
257+assert_user() {
258+ rst_title "user $SERVICE_USER" section
259+ echo
260+ tee_stderr 1 <<EOF | bash | prefix_stdout
261+useradd --shell /bin/bash --system \
262+ --home-dir "$SERVICE_HOME" \
263+ --comment 'Reverse HTTP proxy to filter requests' $SERVICE_USER
264+mkdir "$SERVICE_HOME"
265+chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
266+groups $SERVICE_USER
267+EOF
268+ SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)"
269+ export SERVICE_HOME
270+ echo "export SERVICE_HOME=$SERVICE_HOME"
271+
272+ cat > "$GO_ENV" <<EOF
273+export GOPATH=\$HOME/go-apps
274+export PATH=\$PATH:\$HOME/local/go/bin:\$GOPATH/bin
275+EOF
276+ echo "Environment $GO_ENV has been setup."
277+
278+ tee_stderr <<EOF | sudo -i -u "$SERVICE_USER"
279+grep -qFs -- 'source $GO_ENV' ~/.profile || echo 'source $GO_ENV' >> ~/.profile
280+EOF
281+}
282+
283+filtron_is_installed() {
284+ [[ -f $SERVICE_HOME/go-apps/bin/filtron ]]
285+}
286+
287+_svcpr=" ${_Yellow}|${SERVICE_USER}|${_creset} "
288+
289+install_filtron() {
290+ rst_title "Install filtron in user's ~/go-apps" section
291+ echo
292+ tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
293+go get -v -u github.com/asciimoo/filtron
294+EOF
295+ install_template --no-eval "$FILTRON_RULES" root root 644
296+}
297+
298+update_filtron() {
299+ rst_title "Update filtron" section
300+ echo
301+ tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
302+go get -v -u github.com/asciimoo/filtron
303+EOF
304+}
305+
306+inspect_service() {
307+
308+ rst_title "service status & log"
309+
310+ cat <<EOF
311+
312+sourced ${DOT_CONFIG#"$REPO_ROOT/"} :
313+
314+ PUBLIC_URL : ${PUBLIC_URL}
315+ PUBLIC_HOST : ${PUBLIC_HOST}
316+ FILTRON_URL_PATH : ${FILTRON_URL_PATH}
317+ FILTRON_API : ${FILTRON_API}
318+ FILTRON_LISTEN : ${FILTRON_LISTEN}
319+ FILTRON_TARGET : ${FILTRON_TARGET}
320+
321+EOF
322+
323+ if service_account_is_available "$SERVICE_USER"; then
324+ info_msg "service account $SERVICE_USER available."
325+ else
326+ err_msg "service account $SERVICE_USER not available!"
327+ fi
328+ if go_is_available "$SERVICE_USER"; then
329+ info_msg "~$SERVICE_USER: go is installed"
330+ else
331+ err_msg "~$SERVICE_USER: go is not installed"
332+ fi
333+ if filtron_is_installed; then
334+ info_msg "~$SERVICE_USER: filtron app is installed"
335+ else
336+ err_msg "~$SERVICE_USER: filtron app is not installed!"
337+ fi
338+
339+ if ! service_is_available "http://${FILTRON_API}"; then
340+ err_msg "API not available at: http://${FILTRON_API}"
341+ fi
342+
343+ if ! service_is_available "http://${FILTRON_LISTEN}" ; then
344+ err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}"
345+ fi
346+
347+ if service_is_available "http://${FILTRON_TARGET}" ; then
348+ info_msg "Filtron's target is available at: http://${FILTRON_TARGET}"
349+ fi
350+
351+ if ! service_is_available "${PUBLIC_URL}"; then
352+ warn_msg "Public service at ${PUBLIC_URL} is not available!"
353+ if ! in_container; then
354+ warn_msg "Check if public name is correct and routed or use the public IP from above."
355+ fi
356+ fi
357+
358+ if in_container; then
359+ lxc_suite_info
360+ else
361+ info_msg "public URL --> ${PUBLIC_URL}"
362+ info_msg "internal URL --> http://${FILTRON_LISTEN}"
363+ fi
364+
365+
366+ local _debug_on
367+ if ask_yn "Enable filtron debug mode?"; then
368+ enable_debug
369+ _debug_on=1
370+ fi
371+ echo
372+ systemctl --no-pager -l status "${SERVICE_NAME}"
373+ echo
374+
375+ info_msg "public URL --> ${PUBLIC_URL}"
376+ # shellcheck disable=SC2059
377+ printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log"
378+ read -r -s -n1 -t 5
379+ echo
380+ while true; do
381+ trap break 2
382+ journalctl -f -u "${SERVICE_NAME}"
383+ done
384+
385+ if [[ $_debug_on == 1 ]]; then
386+ disable_debug
387+ fi
388+ return 0
389+}
390+
391+
392+enable_debug() {
393+ info_msg "try to enable debug mode ..."
394+ python <<EOF
395+import sys, json
396+
397+debug = {
398+ u'name': u'debug request'
399+ , u'filters': []
400+ , u'interval': 0
401+ , u'limit': 0
402+ , u'actions': [{u'name': u'log'}]
403+}
404+
405+with open('$FILTRON_RULES') as rules:
406+ j = json.load(rules)
407+
408+pos = None
409+for i in range(len(j)):
410+ if j[i].get('name') == 'debug request':
411+ pos = i
412+ break
413+if pos is not None:
414+ j[pos] = debug
415+else:
416+ j.append(debug)
417+with open('$FILTRON_RULES', 'w') as rules:
418+ json.dump(j, rules, indent=2, sort_keys=True)
419+
420+EOF
421+ systemctl restart "${SERVICE_NAME}.service"
422+}
423+
424+disable_debug() {
425+ info_msg "try to disable debug mode ..."
426+ python <<EOF
427+import sys, json
428+with open('$FILTRON_RULES') as rules:
429+ j = json.load(rules)
430+
431+pos = None
432+for i in range(len(j)):
433+ if j[i].get('name') == 'debug request':
434+ pos = i
435+ break
436+if pos is not None:
437+ del j[pos]
438+ with open('$FILTRON_RULES', 'w') as rules:
439+ json.dump(j, rules, indent=2, sort_keys=True)
440+EOF
441+ systemctl restart "${SERVICE_NAME}.service"
442+}
443+
444+install_apache_site() {
445+
446+ rst_title "Install Apache site $APACHE_FILTRON_SITE"
447+
448+ rst_para "\
449+This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SITE})"
450+
451+ ! apache_is_installed && info_msg "Apache is not installed."
452+
453+ if ! ask_yn "Do you really want to continue?" Yn; then
454+ return
455+ else
456+ install_apache
457+ fi
458+
459+ "${REPO_ROOT}/utils/searx.sh" install uwsgi
460+
461+ apache_install_site --variant=filtron "${APACHE_FILTRON_SITE}"
462+
463+ info_msg "testing public url .."
464+ if ! service_is_available "${PUBLIC_URL}"; then
465+ err_msg "Public service at ${PUBLIC_URL} is not available!"
466+ fi
467+}
468+
469+remove_apache_site() {
470+
471+ rst_title "Remove Apache site $APACHE_FILTRON_SITE"
472+
473+ rst_para "\
474+This removes apache site ${APACHE_FILTRON_SITE}."
475+
476+ ! apache_is_installed && err_msg "Apache is not installed."
477+
478+ if ! ask_yn "Do you really want to continue?" Yn; then
479+ return
480+ fi
481+
482+ apache_remove_site "$APACHE_FILTRON_SITE"
483+
484+}
485+
486+install_nginx_site() {
487+
488+ rst_title "Install nginx site $NGINX_FILTRON_SITE"
489+
490+ rst_para "\
491+This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_FILTRON_SITE})"
492+
493+ ! nginx_is_installed && info_msg "nginx is not installed."
494+
495+ if ! ask_yn "Do you really want to continue?" Yn; then
496+ return
497+ else
498+ install_nginx
499+ fi
500+
501+ "${REPO_ROOT}/utils/searx.sh" install uwsgi
502+
503+ # shellcheck disable=SC2034
504+ SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC)
505+ # shellcheck disable=SC2034
506+ SEARX_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_URL_PATH)
507+ nginx_install_app --variant=filtron "${NGINX_FILTRON_SITE}"
508+
509+ info_msg "testing public url .."
510+ if ! service_is_available "${PUBLIC_URL}"; then
511+ err_msg "Public service at ${PUBLIC_URL} is not available!"
512+ fi
513+}
514+
515+remove_nginx_site() {
516+
517+ rst_title "Remove nginx site $NGINX_FILTRON_SITE"
518+
519+ rst_para "\
520+This removes nginx site ${NGINX_FILTRON_SITE}."
521+
522+ ! nginx_is_installed && err_msg "nginx is not installed."
523+
524+ if ! ask_yn "Do you really want to continue?" Yn; then
525+ return
526+ fi
527+
528+ nginx_remove_site "$FILTRON_FILTRON_SITE"
529+
530+}
531+
532+
533+rst-doc() {
534+
535+ eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/filtron.rst")\""
536+
537+ echo -e "\n.. START install systemd unit"
538+ cat <<EOF
539+.. tabs::
540+
541+ .. group-tab:: systemd
542+
543+ .. code:: bash
544+
545+EOF
546+ eval "echo \"$(< "${TEMPLATES}/${SERVICE_SYSTEMD_UNIT}")\"" | prefix_stdout " "
547+ echo -e "\n.. END install systemd unit"
548+
549+ # for DIST_NAME in ubuntu-20.04 arch fedora; do
550+ # (
551+ # DIST_ID=${DIST_NAME%-*}
552+ # DIST_VERS=${DIST_NAME#*-}
553+ # [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS=
554+ # # ...
555+ # )
556+ # done
557+}
558+
559+# ----------------------------------------------------------------------------
560+main "$@"
561+# ----------------------------------------------------------------------------
utils/lib.shnew file+1519
@@ -0,0 +1,1519 @@
1+#!/usr/bin/env bash
2+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
3+# SPDX-License-Identifier: AGPL-3.0-or-later
4+# shellcheck disable=SC2059,SC1117
5+
6+# ubuntu, debian, arch, fedora ...
7+DIST_ID=$(source /etc/os-release; echo "$ID");
8+# shellcheck disable=SC2034
9+DIST_VERS=$(source /etc/os-release; echo "$VERSION_ID");
10+
11+ADMIN_NAME="${ADMIN_NAME:-$(git config user.name)}"
12+ADMIN_NAME="${ADMIN_NAME:-$USER}"
13+
14+ADMIN_EMAIL="${ADMIN_EMAIL:-$(git config user.email)}"
15+ADMIN_EMAIL="${ADMIN_EMAIL:-$USER@$(hostname)}"
16+
17+if [[ -z "${REPO_ROOT}" ]]; then
18+ REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")
19+ while [ -h "${REPO_ROOT}" ] ; do
20+ REPO_ROOT=$(readlink "${REPO_ROOT}")
21+ done
22+ REPO_ROOT=$(cd "${REPO_ROOT}/.." && pwd -P )
23+fi
24+
25+if [[ -z ${TEMPLATES} ]]; then
26+ TEMPLATES="${REPO_ROOT}/utils/templates"
27+fi
28+
29+if [[ -z "$CACHE" ]]; then
30+ CACHE="${REPO_ROOT}/cache"
31+fi
32+
33+if [[ -z ${DIFF_CMD} ]]; then
34+ DIFF_CMD="diff -u"
35+ if command -v colordiff >/dev/null; then
36+ DIFF_CMD="colordiff -u"
37+ fi
38+fi
39+
40+DOT_CONFIG="${DOT_CONFIG:-${REPO_ROOT}/.config.sh}"
41+
42+source_dot_config() {
43+ if [[ ! -e "${DOT_CONFIG}" ]]; then
44+ err_msg "configuration does not extsts at: ${DOT_CONFIG}"
45+ return 42
46+ fi
47+ # shellcheck disable=SC1090
48+ source "${DOT_CONFIG}"
49+}
50+
51+sudo_or_exit() {
52+ # usage: sudo_or_exit
53+
54+ if [ ! "$(id -u)" -eq 0 ]; then
55+ err_msg "this command requires root (sudo) privilege!" >&2
56+ exit 42
57+ fi
58+}
59+
60+required_commands() {
61+
62+ # usage: required_commands [cmd1 ...]
63+
64+ local exit_val=0
65+ while [ -n "$1" ]; do
66+
67+ if ! command -v "$1" &>/dev/null; then
68+ err_msg "missing command $1"
69+ exit_val=42
70+ fi
71+ shift
72+ done
73+ return $exit_val
74+}
75+
76+# colors
77+# ------
78+
79+# shellcheck disable=SC2034
80+set_terminal_colors() {
81+ _colors=8
82+ _creset='\e[0m' # reset all attributes
83+
84+ _Black='\e[0;30m'
85+ _White='\e[1;37m'
86+ _Red='\e[0;31m'
87+ _Green='\e[0;32m'
88+ _Yellow='\e[0;33m'
89+ _Blue='\e[0;34m'
90+ _Violet='\e[0;35m'
91+ _Cyan='\e[0;36m'
92+
93+ _BBlack='\e[1;30m'
94+ _BWhite='\e[1;37m'
95+ _BRed='\e[1;31m'
96+ _BGreen='\e[1;32m'
97+ _BYellow='\e[1;33m'
98+ _BBlue='\e[1;34m'
99+ _BPurple='\e[1;35m'
100+ _BCyan='\e[1;36m'
101+}
102+
103+if [ ! -p /dev/stdout ]; then
104+ set_terminal_colors
105+fi
106+
107+# reST
108+# ----
109+
110+if command -v fmt >/dev/null; then
111+ export FMT="fmt -u"
112+else
113+ export FMT="cat"
114+fi
115+
116+rst_title() {
117+ # usage: rst_title <header-text> [part|chapter|section]
118+
119+ case ${2-chapter} in
120+ part) printf "\n${_BGreen}${1//?/=}${_creset}\n${_BCyan}${1}${_creset}\n${_BGreen}${1//?/=}${_creset}\n";;
121+ chapter) printf "\n${_BCyan}${1}${_creset}\n${_BGreen}${1//?/=}${_creset}\n";;
122+ section) printf "\n${_BCyan}${1}${_creset}\n${_BGreen}${1//?/-}${_creset}\n";;
123+ *)
124+ err_msg "invalid argument '${2}' in line $(caller)"
125+ return 42
126+ ;;
127+ esac
128+}
129+
130+rst_para() {
131+ # usage: RST_INDENT=1 rst_para "lorem ipsum ..."
132+ local prefix=''
133+ if [[ -n $RST_INDENT ]] && [[ $RST_INDENT -gt 0 ]]; then
134+ prefix="$(for i in $(seq 1 "$RST_INDENT"); do printf " "; done)"
135+ echo -en "\n$*\n" | $FMT | prefix_stdout "$prefix"
136+ else
137+ echo -en "\n$*\n" | $FMT
138+ fi
139+}
140+
141+die() {
142+ echo -e "${_BRed}ERROR:${_creset} ${BASH_SOURCE[1]}: line ${BASH_LINENO[0]}: ${2-died ${1-1}}" >&2;
143+ exit "${1-1}"
144+}
145+
146+die_caller() {
147+ echo -e "${_BRed}ERROR:${_creset} ${BASH_SOURCE[2]}: line ${BASH_LINENO[1]}: ${FUNCNAME[1]}(): ${2-died ${1-1}}" >&2;
148+ exit "${1-1}"
149+}
150+
151+err_msg() { echo -e "${_BRed}ERROR:${_creset} $*" >&2; }
152+warn_msg() { echo -e "${_BBlue}WARN:${_creset} $*" >&2; }
153+info_msg() { echo -e "${_BYellow}INFO:${_creset} $*" >&2; }
154+
155+clean_stdin() {
156+ if [[ $(uname -s) != 'Darwin' ]]; then
157+ while read -r -n1 -t 0.1; do : ; done
158+ fi
159+}
160+
161+wait_key(){
162+ # usage: waitKEY [<timeout in sec>]
163+
164+ clean_stdin
165+ local _t=$1
166+ local msg="${MSG}"
167+ [[ -z "$msg" ]] && msg="${_Green}** press any [${_BCyan}KEY${_Green}] to continue **${_creset}"
168+
169+ [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT
170+ [[ -n $_t ]] && _t="-t $_t"
171+ printf "$msg"
172+ # shellcheck disable=SC2086
173+ read -r -s -n1 $_t
174+ echo
175+ clean_stdin
176+}
177+
178+ask_yn() {
179+ # usage: ask_yn <prompt-text> [Ny|Yn] [<timeout in sec>]
180+
181+ local EXIT_YES=0 # exit status 0 --> successful
182+ local EXIT_NO=1 # exit status 1 --> error code
183+
184+ local _t=$3
185+ [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT
186+ [[ -n $_t ]] && _t="-t $_t"
187+ case "${FORCE_SELECTION:-${2}}" in
188+ Y) return ${EXIT_YES} ;;
189+ N) return ${EXIT_NO} ;;
190+ Yn)
191+ local exit_val=${EXIT_YES}
192+ local choice="[${_BGreen}YES${_creset}/no]"
193+ local default="Yes"
194+ ;;
195+ *)
196+ local exit_val=${EXIT_NO}
197+ local choice="[${_BGreen}NO${_creset}/yes]"
198+ local default="No"
199+ ;;
200+ esac
201+ echo
202+ while true; do
203+ clean_stdin
204+ printf "$1 ${choice} "
205+ # shellcheck disable=SC2086
206+ read -r -n1 $_t
207+ if [[ -z $REPLY ]]; then
208+ printf "$default\n"; break
209+ elif [[ $REPLY =~ ^[Yy]$ ]]; then
210+ exit_val=${EXIT_YES}
211+ printf "\n"
212+ break
213+ elif [[ $REPLY =~ ^[Nn]$ ]]; then
214+ exit_val=${EXIT_NO}
215+ printf "\n"
216+ break
217+ fi
218+ _t=""
219+ err_msg "invalid choice"
220+ done
221+ clean_stdin
222+ return $exit_val
223+}
224+
225+tee_stderr () {
226+
227+ # usage::
228+ # tee_stderr 1 <<EOF | python -i
229+ # print("hello")
230+ # EOF
231+ # ...
232+ # >>> print("hello")
233+ # hello
234+
235+ local _t="0";
236+ if [[ -n $1 ]] ; then _t="$1"; fi
237+
238+ (while read -r line; do
239+ # shellcheck disable=SC2086
240+ sleep $_t
241+ echo -e "$line" >&2
242+ echo "$line"
243+ done)
244+}
245+
246+prefix_stdout () {
247+ # usage: <cmd> | prefix_stdout [prefix]
248+
249+ local prefix="${_BYellow}-->|${_creset}"
250+
251+ if [[ -n $1 ]] ; then prefix="$1"; fi
252+
253+ # shellcheck disable=SC2162
254+ (while IFS= read line; do
255+ echo -e "${prefix}$line"
256+ done)
257+}
258+
259+append_line() {
260+
261+ # usage: append_line <line> <file>
262+ #
263+ # Append line if not exists, create file if not exists. E.g::
264+ #
265+ # append_line 'source ~/.foo' ~/bashrc
266+
267+ local LINE=$1
268+ local FILE=$2
269+ grep -qFs -- "$LINE" "$FILE" || echo "$LINE" >> "$FILE"
270+}
271+
272+cache_download() {
273+
274+ # usage: cache_download <url> <local-filename>
275+
276+ local exit_value=0
277+
278+ if [[ -n ${SUDO_USER} ]]; then
279+ sudo -u "${SUDO_USER}" mkdir -p "${CACHE}"
280+ else
281+ mkdir -p "${CACHE}"
282+ fi
283+
284+ if [[ -f "${CACHE}/$2" ]] ; then
285+ info_msg "already cached: $1"
286+ info_msg " --> ${CACHE}/$2"
287+ fi
288+
289+ if [[ ! -f "${CACHE}/$2" ]]; then
290+ info_msg "caching: $1"
291+ info_msg " --> ${CACHE}/$2"
292+ if [[ -n ${SUDO_USER} ]]; then
293+ sudo -u "${SUDO_USER}" wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$?
294+ else
295+ wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$?
296+ fi
297+ if [[ ! $exit_value = 0 ]]; then
298+ err_msg "failed to download: $1"
299+ fi
300+ fi
301+}
302+
303+backup_file() {
304+
305+ # usage: backup_file /path/to/file.foo
306+
307+ local stamp
308+ stamp=$(date +"_%Y%m%d_%H%M%S")
309+ info_msg "create backup: ${1}${stamp}"
310+ cp -a "${1}" "${1}${stamp}"
311+}
312+
313+choose_one() {
314+
315+ # usage:
316+ #
317+ # DEFAULT_SELECT= 2 \
318+ # choose_one <name> "your selection?" "Coffee" "Coffee with milk"
319+
320+ local default=${DEFAULT_SELECT-1}
321+ local REPLY
322+ local env_name=$1 && shift
323+ local choice=$1;
324+ local max="${#@}"
325+ local _t
326+ [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT
327+ [[ -n $_t ]] && _t="-t $_t"
328+
329+ list=("$@")
330+ echo -e "${_BGreen}Menu::${_creset}"
331+ for ((i=1; i<= $((max -1)); i++)); do
332+ if [[ "$i" == "$default" ]]; then
333+ echo -e " ${_BGreen}$i.${_creset}) ${list[$i]} [default]"
334+ else
335+ echo -e " $i.) ${list[$i]}"
336+ fi
337+ done
338+ while true; do
339+ clean_stdin
340+ printf "$1 [${_BGreen}$default${_creset}] "
341+
342+ if (( 10 > max )); then
343+ # shellcheck disable=SC2086
344+ read -r -n1 $_t
345+ else
346+ # shellcheck disable=SC2086,SC2229
347+ read -r $_t
348+ fi
349+ # selection fits
350+ [[ $REPLY =~ ^-?[0-9]+$ ]] && (( REPLY > 0 )) && (( REPLY < max )) && break
351+
352+ # take default
353+ [[ -z $REPLY ]] && REPLY=$default && break
354+
355+ _t=""
356+ err_msg "invalid choice"
357+ done
358+ eval "$env_name"='${list[${REPLY}]}'
359+ echo
360+ clean_stdin
361+}
362+
363+install_template() {
364+
365+ # usage:
366+ #
367+ # install_template [--no-eval] [--variant=<name>] \
368+ # {file} [{owner} [{group} [{chmod}]]]
369+ #
370+ # E.g. the origin of variant 'raw' of /etc/updatedb.conf is::
371+ #
372+ # ${TEMPLATES}/etc/updatedb.conf:raw
373+ #
374+ # To install variant 'raw' of /etc/updatedb.conf without evaluated
375+ # replacements you can use::
376+ #
377+ # install_template --variant=raw --no-eval \
378+ # /etc/updatedb.conf root root 644
379+
380+ local _reply=""
381+ local do_eval=1
382+ local variant=""
383+ local pos_args=("$0")
384+
385+ for i in "$@"; do
386+ case $i in
387+ --no-eval) do_eval=0; shift ;;
388+ --variant=*) variant=":${i#*=}"; shift ;;
389+ *) pos_args+=("$i") ;;
390+ esac
391+ done
392+
393+ local dst="${pos_args[1]}"
394+ local template_origin="${TEMPLATES}${dst}${variant}"
395+ local template_file="${TEMPLATES}${dst}"
396+
397+ local owner="${pos_args[2]-$(id -un)}"
398+ local group="${pos_args[3]-$(id -gn)}"
399+ local chmod="${pos_args[4]-644}"
400+
401+ info_msg "install (eval=$do_eval): ${dst}"
402+ [[ -n $variant ]] && info_msg "variant --> ${variant}"
403+
404+ if [[ ! -f "${template_origin}" ]] ; then
405+ err_msg "${template_origin} does not exists"
406+ err_msg "... can't install $dst"
407+ wait_key 30
408+ return 42
409+ fi
410+
411+ if [[ "$do_eval" == "1" ]]; then
412+ template_file="${CACHE}${dst}${variant}"
413+ info_msg "BUILD template ${template_file}"
414+ if [[ -n ${SUDO_USER} ]]; then
415+ sudo -u "${SUDO_USER}" mkdir -p "$(dirname "${template_file}")"
416+ else
417+ mkdir -p "$(dirname "${template_file}")"
418+ fi
419+ # shellcheck disable=SC2086
420+ eval "echo \"$(cat ${template_origin})\"" > "${template_file}"
421+ if [[ -n ${SUDO_USER} ]]; then
422+ chown "${SUDO_USER}:${SUDO_USER}" "${template_file}"
423+ fi
424+ else
425+ template_file=$template_origin
426+ fi
427+
428+ mkdir -p "$(dirname "${dst}")"
429+
430+ if [[ ! -f "${dst}" ]]; then
431+ info_msg "install: ${template_file}"
432+ sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \
433+ "${template_file}" "${dst}" | prefix_stdout
434+ return $?
435+ fi
436+
437+ if [[ -f "${dst}" ]] && cmp --silent "${template_file}" "${dst}" ; then
438+ info_msg "file ${dst} allready installed"
439+ return 0
440+ fi
441+
442+ info_msg "diffrent file ${dst} allready exists on this host"
443+
444+ while true; do
445+ choose_one _reply "choose next step with file $dst" \
446+ "replace file" \
447+ "leave file unchanged" \
448+ "interactiv shell" \
449+ "diff files"
450+
451+ case $_reply in
452+ "replace file")
453+ info_msg "install: ${template_file}"
454+ sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \
455+ "${template_file}" "${dst}" | prefix_stdout
456+ break
457+ ;;
458+ "leave file unchanged")
459+ break
460+ ;;
461+ "interactiv shell")
462+ echo -e "// edit ${_Red}${dst}${_creset} to your needs"
463+ echo -e "// exit with [${_BCyan}CTRL-D${_creset}]"
464+ sudo -H -u "${owner}" -i
465+ $DIFF_CMD "${dst}" "${template_file}"
466+ echo
467+ echo -e "// ${_BBlack}did you edit file ...${_creset}"
468+ echo -en "// ${_Red}${dst}${_creset}"
469+ if ask_yn "//${_BBlack}... to your needs?${_creset}"; then
470+ break
471+ fi
472+ ;;
473+ "diff files")
474+ $DIFF_CMD "${dst}" "${template_file}" | prefix_stdout
475+ esac
476+ done
477+}
478+
479+
480+service_is_available() {
481+
482+ # usage: service_is_available <URL>
483+
484+ [[ -z $1 ]] && die_caller 42 "missing argument <URL>"
485+ local URL="$1"
486+ http_code=$(curl -H 'Cache-Control: no-cache' \
487+ --silent -o /dev/null --head --write-out '%{http_code}' --insecure \
488+ "${URL}")
489+ exit_val=$?
490+ if [[ $exit_val = 0 ]]; then
491+ info_msg "got $http_code from ${URL}"
492+ fi
493+ case "$http_code" in
494+ 404|410|423) exit_val=$http_code;;
495+ esac
496+ return "$exit_val"
497+}
498+
499+# golang
500+# ------
501+
502+go_is_available() {
503+
504+ # usage: go_is_available $SERVICE_USER && echo "go is installed!"
505+
506+ sudo -i -u "${1}" which go &>/dev/null
507+}
508+
509+install_go() {
510+
511+ # usage: install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}"
512+
513+ local _svcpr=" ${_Yellow}|${3}|${_creset} "
514+
515+ rst_title "Install Go in user's HOME" section
516+
517+ rst_para "download and install go binary .."
518+ cache_download "${1}" "${2}"
519+
520+ tee_stderr 0.1 <<EOF | sudo -i -u "${3}" | prefix_stdout "$_svcpr"
521+echo \$PATH
522+echo \$GOPATH
523+mkdir -p \$HOME/local
524+rm -rf \$HOME/local/go
525+tar -C \$HOME/local -xzf ${CACHE}/${2}
526+EOF
527+ sudo -i -u "${3}" <<EOF | prefix_stdout
528+! which go >/dev/null && echo "ERROR - Go Installation not found in PATH!?!"
529+which go >/dev/null && go version && echo "congratulations -- Go installation OK :)"
530+EOF
531+}
532+
533+# system accounts
534+# ---------------
535+
536+service_account_is_available() {
537+
538+ # usage: service_account_is_available "$SERVICE_USER" && echo "OK"
539+
540+ sudo -i -u "$1" echo \$HOME &>/dev/null
541+}
542+
543+drop_service_account() {
544+
545+ # usage: drop_service_account "${SERVICE_USER}"
546+
547+ rst_title "Drop ${1} HOME" section
548+ if ask_yn "Do you really want to drop ${1} home folder?"; then
549+ userdel -r -f "${1}" 2>&1 | prefix_stdout
550+ else
551+ rst_para "Leave HOME folder $(du -sh "${1}") unchanged."
552+ fi
553+}
554+
555+interactive_shell(){
556+
557+ # usage: interactive_shell "${SERVICE_USER}"
558+
559+ echo -e "// exit with [${_BCyan}CTRL-D${_creset}]"
560+ sudo -H -u "${1}" -i
561+}
562+
563+
564+# systemd
565+# -------
566+
567+SYSTEMD_UNITS="${SYSTEMD_UNITS:-/lib/systemd/system}"
568+
569+systemd_install_service() {
570+
571+ # usage: systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
572+
573+ rst_title "Install System-D Unit ${1}" section
574+ echo
575+ install_template "${2}" root root 644
576+ wait_key
577+ systemd_activate_service "${1}"
578+}
579+
580+systemd_remove_service() {
581+
582+ # usage: systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
583+
584+ if ! ask_yn "Do you really want to deinstall systemd unit ${1}?"; then
585+ return 42
586+ fi
587+ systemd_deactivate_service "${1}"
588+ rm "${2}" 2>&1 | prefix_stdout
589+}
590+
591+systemd_activate_service() {
592+
593+ # usage: systemd_activate_service "${SERVICE_NAME}"
594+
595+ rst_title "Activate ${1} (service)" section
596+ echo
597+ tee_stderr <<EOF | bash 2>&1
598+systemctl enable ${1}.service
599+systemctl restart ${1}.service
600+EOF
601+ tee_stderr <<EOF | bash 2>&1
602+systemctl status --no-pager ${1}.service
603+EOF
604+}
605+
606+systemd_deactivate_service() {
607+
608+ # usage: systemd_deactivate_service "${SERVICE_NAME}"
609+
610+ rst_title "De-Activate ${1} (service)" section
611+ echo
612+ tee_stderr <<EOF | bash 2>&1 | prefix_stdout
613+systemctl stop ${1}.service
614+systemctl disable ${1}.service
615+EOF
616+}
617+
618+systemd_restart_service() {
619+
620+ # usage: systemd_restart_service "${SERVICE_NAME}"
621+
622+ rst_title "Restart ${1} (service)" section
623+ echo
624+ tee_stderr <<EOF | bash 2>&1
625+systemctl restart ${1}.service
626+EOF
627+ tee_stderr <<EOF | bash 2>&1
628+systemctl status --no-pager ${1}.service
629+EOF
630+}
631+
632+
633+# nginx
634+# -----
635+
636+nginx_distro_setup() {
637+ # shellcheck disable=SC2034
638+
639+ NGINX_DEFAULT_SERVER=/etc/nginx/nginx.conf
640+
641+ # Including *location* directives from a dedicated config-folder into the
642+ # server directive is, what what fedora (already) does.
643+ NGINX_APPS_ENABLED="/etc/nginx/default.d"
644+
645+ # We add a apps-available folder and linking configurations into the
646+ # NGINX_APPS_ENABLED folder. See also nginx_include_apps_enabled().
647+ NGINX_APPS_AVAILABLE="/etc/nginx/default.apps-available"
648+
649+ case $DIST_ID-$DIST_VERS in
650+ ubuntu-*|debian-*)
651+ NGINX_PACKAGES="nginx"
652+ NGINX_DEFAULT_SERVER=/etc/nginx/sites-available/default
653+ ;;
654+ arch-*)
655+ NGINX_PACKAGES="nginx-mainline"
656+ ;;
657+ fedora-*)
658+ NGINX_PACKAGES="nginx"
659+ ;;
660+ *)
661+ err_msg "$DIST_ID-$DIST_VERS: nginx not yet implemented"
662+ ;;
663+ esac
664+}
665+nginx_distro_setup
666+
667+install_nginx(){
668+ info_msg "installing nginx ..."
669+ pkg_install "${NGINX_PACKAGES}"
670+ case $DIST_ID-$DIST_VERS in
671+ arch-*|fedora-*)
672+ systemctl enable nginx
673+ systemctl start nginx
674+ ;;
675+ esac
676+}
677+
678+nginx_is_installed() {
679+ command -v nginx &>/dev/null
680+}
681+
682+nginx_reload() {
683+
684+ info_msg "reload nginx .."
685+ echo
686+ if ! nginx -t; then
687+ err_msg "testing nginx configuration failed"
688+ return 42
689+ fi
690+ systemctl restart nginx
691+}
692+
693+nginx_install_app() {
694+
695+ # usage: nginx_install_app [<template option> ...] <myapp>
696+ #
697+ # <template option>: see install_template
698+
699+ local template_opts=()
700+ local pos_args=("$0")
701+
702+ for i in "$@"; do
703+ case $i in
704+ -*) template_opts+=("$i");;
705+ *) pos_args+=("$i");;
706+ esac
707+ done
708+
709+ nginx_include_apps_enabled "${NGINX_DEFAULT_SERVER}"
710+
711+ install_template "${template_opts[@]}" \
712+ "${NGINX_APPS_AVAILABLE}/${pos_args[1]}" \
713+ root root 644
714+ nginx_enable_app "${pos_args[1]}"
715+ info_msg "installed nginx app: ${pos_args[1]}"
716+}
717+
718+nginx_include_apps_enabled() {
719+
720+ # Add the *NGINX_APPS_ENABLED* infrastruture to a nginx server block. Such
721+ # infrastruture is already known from fedora, including apps (location
722+ # directives) from the /etc/nginx/default.d folder into the *default* nginx
723+ # server.
724+
725+ # usage: nginx_include_apps_enabled <config-file>
726+ #
727+ # config-file: Config file with server directive in.
728+
729+ [[ -z $1 ]] && die_caller 42 "missing argument <config-file>"
730+ local server_conf="$1"
731+
732+ # include /etc/nginx/default.d/*.conf;
733+ local include_directive="include ${NGINX_APPS_ENABLED}/*.conf;"
734+ local include_directive_re="^\s*include ${NGINX_APPS_ENABLED}/\*\.conf;"
735+
736+ info_msg "checking existence: '${include_directive}' in file ${server_conf}"
737+ if grep "${include_directive_re}" "${server_conf}"; then
738+ info_msg "OK, already exists."
739+ return
740+ fi
741+
742+ info_msg "add missing directive: '${include_directive}'"
743+ cp "${server_conf}" "${server_conf}.bak"
744+
745+ (
746+ local line
747+ local stage=0
748+ while IFS= read -r line
749+ do
750+ echo "$line"
751+ if [[ $stage = 0 ]]; then
752+ if [[ $line =~ ^[[:space:]]*server*[[:space:]]*\{ ]]; then
753+ stage=1
754+ fi
755+ fi
756+
757+ if [[ $stage = 1 ]]; then
758+ echo " # Load configuration files for the default server block."
759+ echo " $include_directive"
760+ echo ""
761+ stage=2
762+ fi
763+ done < "${server_conf}.bak"
764+ ) > "${server_conf}"
765+
766+}
767+
768+nginx_remove_app() {
769+
770+ # usage: nginx_remove_app <myapp.conf>
771+
772+ info_msg "remove nginx app: $1"
773+ nginx_dissable_app "$1"
774+ rm -f "${NGINX_APPS_AVAILABLE}/$1"
775+}
776+
777+nginx_enable_app() {
778+
779+ # usage: nginx_enable_app <myapp.conf>
780+
781+ local CONF="$1"
782+
783+ info_msg "enable nginx app: ${CONF}"
784+ mkdir -p "${NGINX_APPS_ENABLED}"
785+ rm -f "${NGINX_APPS_ENABLED}/${CONF}"
786+ ln -s "${NGINX_APPS_AVAILABLE}/${CONF}" "${NGINX_APPS_ENABLED}/${CONF}"
787+ nginx_reload
788+}
789+
790+nginx_dissable_app() {
791+
792+ # usage: nginx_disable_app <myapp.conf>
793+
794+ local CONF="$1"
795+
796+ info_msg "disable nginx app: ${CONF}"
797+ rm -f "${NGINX_APPS_ENABLED}/${CONF}"
798+ nginx_reload
799+}
800+
801+
802+# Apache
803+# ------
804+
805+apache_distro_setup() {
806+ # shellcheck disable=SC2034
807+ case $DIST_ID-$DIST_VERS in
808+ ubuntu-*|debian-*)
809+ # debian uses the /etc/apache2 path, while other distros use
810+ # the apache default at /etc/httpd
811+ APACHE_SITES_AVAILABLE="/etc/apache2/sites-available"
812+ APACHE_SITES_ENABLED="/etc/apache2/sites-enabled"
813+ APACHE_MODULES="/usr/lib/apache2/modules"
814+ APACHE_PACKAGES="apache2"
815+ ;;
816+ arch-*)
817+ APACHE_SITES_AVAILABLE="/etc/httpd/sites-available"
818+ APACHE_SITES_ENABLED="/etc/httpd/sites-enabled"
819+ APACHE_MODULES="modules"
820+ APACHE_PACKAGES="apache"
821+ ;;
822+ fedora-*)
823+ APACHE_SITES_AVAILABLE="/etc/httpd/sites-available"
824+ APACHE_SITES_ENABLED="/etc/httpd/sites-enabled"
825+ APACHE_MODULES="modules"
826+ APACHE_PACKAGES="httpd"
827+ ;;
828+ *)
829+ err_msg "$DIST_ID-$DIST_VERS: apache not yet implemented"
830+ ;;
831+ esac
832+}
833+
834+apache_distro_setup
835+
836+install_apache(){
837+ info_msg "installing apache ..."
838+ pkg_install "$APACHE_PACKAGES"
839+ case $DIST_ID-$DIST_VERS in
840+ arch-*|fedora-*)
841+ if ! grep "IncludeOptional sites-enabled" "/etc/httpd/conf/httpd.conf"; then
842+ echo "IncludeOptional sites-enabled/*.conf" >> "/etc/httpd/conf/httpd.conf"
843+ fi
844+ systemctl enable httpd
845+ systemctl start httpd
846+ ;;
847+ esac
848+}
849+
850+apache_is_installed() {
851+ case $DIST_ID-$DIST_VERS in
852+ ubuntu-*|debian-*) (command -v apachectl) &>/dev/null;;
853+ arch-*) (command -v httpd) &>/dev/null;;
854+ fedora-*) (command -v httpd) &>/dev/null;;
855+ esac
856+}
857+
858+apache_reload() {
859+
860+ info_msg "reload apache .."
861+ echo
862+ case $DIST_ID-$DIST_VERS in
863+ ubuntu-*|debian-*)
864+ sudo -H apachectl configtest
865+ sudo -H systemctl force-reload apache2
866+ ;;
867+ arch-*| fedora-*)
868+ sudo -H httpd -t
869+ sudo -H systemctl force-reload httpd
870+ ;;
871+ esac
872+}
873+
874+apache_install_site() {
875+
876+ # usage: apache_install_site [<template option> ...] <mysite.conf>
877+ #
878+ # <template option>: see install_template
879+
880+ local template_opts=()
881+ local pos_args=("$0")
882+
883+ for i in "$@"; do
884+ case $i in
885+ -*) template_opts+=("$i");;
886+ *) pos_args+=("$i");;
887+ esac
888+ done
889+
890+ install_template "${template_opts[@]}" \
891+ "${APACHE_SITES_AVAILABLE}/${pos_args[1]}" \
892+ root root 644
893+ apache_enable_site "${pos_args[1]}"
894+ info_msg "installed apache site: ${pos_args[1]}"
895+}
896+
897+apache_remove_site() {
898+
899+ # usage: apache_remove_site <mysite.conf>
900+
901+ info_msg "remove apache site: $1"
902+ apache_dissable_site "$1"
903+ rm -f "${APACHE_SITES_AVAILABLE}/$1"
904+}
905+
906+apache_enable_site() {
907+
908+ # usage: apache_enable_site <mysite.conf>
909+
910+ local CONF="$1"
911+
912+ info_msg "enable apache site: ${CONF}"
913+
914+ case $DIST_ID-$DIST_VERS in
915+ ubuntu-*|debian-*)
916+ sudo -H a2ensite -q "${CONF}"
917+ ;;
918+ arch-*)
919+ mkdir -p "${APACHE_SITES_ENABLED}"
920+ rm -f "${APACHE_SITES_ENABLED}/${CONF}"
921+ ln -s "${APACHE_SITES_AVAILABLE}/${CONF}" "${APACHE_SITES_ENABLED}/${CONF}"
922+ ;;
923+ fedora-*)
924+ mkdir -p "${APACHE_SITES_ENABLED}"
925+ rm -f "${APACHE_SITES_ENABLED}/${CONF}"
926+ ln -s "${APACHE_SITES_AVAILABLE}/${CONF}" "${APACHE_SITES_ENABLED}/${CONF}"
927+ ;;
928+ esac
929+ apache_reload
930+}
931+
932+apache_dissable_site() {
933+
934+ # usage: apache_disable_site <mysite.conf>
935+
936+ local CONF="$1"
937+
938+ info_msg "disable apache site: ${CONF}"
939+
940+ case $DIST_ID-$DIST_VERS in
941+ ubuntu-*|debian-*)
942+ sudo -H a2dissite -q "${CONF}"
943+ ;;
944+ arch-*)
945+ rm -f "${APACHE_SITES_ENABLED}/${CONF}"
946+ ;;
947+ fedora-*)
948+ rm -f "${APACHE_SITES_ENABLED}/${CONF}"
949+ ;;
950+ esac
951+ apache_reload
952+}
953+
954+# uWSGI
955+# -----
956+
957+uWSGI_SETUP="${uWSGI_SETUP:=/etc/uwsgi}"
958+uWSGI_USER=
959+uWSGI_GROUP=
960+
961+# How distros manage uWSGI apps is very different. From uWSGI POV read:
962+# - https://uwsgi-docs.readthedocs.io/en/latest/Management.html
963+
964+uWSGI_distro_setup() {
965+ case $DIST_ID-$DIST_VERS in
966+ ubuntu-*|debian-*)
967+ # init.d --> /usr/share/doc/uwsgi/README.Debian.gz
968+ # For uWSGI debian uses the LSB init process, this might be changed
969+ # one day, see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=833067
970+ uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-available"
971+ uWSGI_APPS_ENABLED="${uWSGI_SETUP}/apps-enabled"
972+ uWSGI_PACKAGES="uwsgi"
973+ ;;
974+ arch-*)
975+ # systemd --> /usr/lib/systemd/system/uwsgi@.service
976+ # For uWSGI archlinux uses systemd template units, see
977+ # - http://0pointer.de/blog/projects/instances.html
978+ # - https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd
979+ uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-archlinux"
980+ uWSGI_APPS_ENABLED="${uWSGI_SETUP}"
981+ uWSGI_PACKAGES="uwsgi"
982+ ;;
983+ fedora-*)
984+ # systemd --> /usr/lib/systemd/system/uwsgi.service
985+ # The unit file starts uWSGI in emperor mode (/etc/uwsgi.ini), see
986+ # - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html
987+ uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-available"
988+ uWSGI_APPS_ENABLED="${uWSGI_SETUP}.d"
989+ uWSGI_PACKAGES="uwsgi"
990+ uWSGI_USER="uwsgi"
991+ uWSGI_GROUP="uwsgi"
992+ ;;
993+ *)
994+ err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented"
995+ ;;
996+esac
997+}
998+
999+uWSGI_distro_setup
1000+
1001+install_uwsgi(){
1002+ info_msg "installing uwsgi ..."
1003+ pkg_install "$uWSGI_PACKAGES"
1004+ case $DIST_ID-$DIST_VERS in
1005+ fedora-*)
1006+ # enable & start should be called once at uWSGI installation time
1007+ systemctl enable uwsgi
1008+ systemctl restart uwsgi
1009+ ;;
1010+ esac
1011+}
1012+
1013+uWSGI_restart() {
1014+
1015+ # usage: uWSGI_restart() <myapp.ini>
1016+
1017+ local CONF="$1"
1018+
1019+ [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
1020+ info_msg "restart uWSGI service"
1021+ case $DIST_ID-$DIST_VERS in
1022+ ubuntu-*|debian-*)
1023+ # the 'service' method seems broken in that way, that it (re-)starts
1024+ # the whole uwsgi process.
1025+ service uwsgi restart "${CONF%.*}"
1026+ ;;
1027+ arch-*)
1028+ # restart systemd template instance
1029+ if uWSGI_app_available "${CONF}"; then
1030+ systemctl restart "uwsgi@${CONF%.*}"
1031+ else
1032+ info_msg "[uWSGI:systemd-template] ${CONF} not installed (no need to restart)"
1033+ fi
1034+ ;;
1035+ fedora-*)
1036+ # in emperor mode, just touch the file to restart
1037+ if uWSGI_app_enabled "${CONF}"; then
1038+ touch "${uWSGI_APPS_ENABLED}/${CONF}"
1039+ # it seems, there is a polling time in between touch and restart
1040+ # of the service.
1041+ sleep 3
1042+ else
1043+ info_msg "[uWSGI:emperor] ${CONF} not installed (no need to restart)"
1044+ fi
1045+ ;;
1046+ *)
1047+ err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented"
1048+ return 42
1049+ ;;
1050+ esac
1051+}
1052+
1053+uWSGI_prepare_app() {
1054+
1055+ # usage: uWSGI_prepare_app <myapp.ini>
1056+
1057+ [[ -z $1 ]] && die_caller 42 "missing argument <myapp.ini>"
1058+
1059+ local APP="${1%.*}"
1060+
1061+ case $DIST_ID-$DIST_VERS in
1062+ fedora-*)
1063+ # in emperor mode, the uwsgi user is the owner of the sockets
1064+ info_msg "prepare (uwsgi:uwsgi) /run/uwsgi/app/${APP}"
1065+ mkdir -p "/run/uwsgi/app/${APP}"
1066+ chown -R "uwsgi:uwsgi" "/run/uwsgi/app/${APP}"
1067+ ;;
1068+ *)
1069+ info_msg "prepare (${SERVICE_USER}:${SERVICE_GROUP}) /run/uwsgi/app/${APP}"
1070+ mkdir -p "/run/uwsgi/app/${APP}"
1071+ chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "/run/uwsgi/app/${APP}"
1072+ ;;
1073+ esac
1074+}
1075+
1076+
1077+uWSGI_app_available() {
1078+ # usage: uWSGI_app_available <myapp.ini>
1079+ local CONF="$1"
1080+
1081+ [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
1082+ [[ -f "${uWSGI_APPS_AVAILABLE}/${CONF}" ]]
1083+}
1084+
1085+uWSGI_install_app() {
1086+
1087+ # usage: uWSGI_install_app [<template option> ...] <myapp.ini>
1088+ #
1089+ # <template option>: see install_template
1090+
1091+ local pos_args=("$0")
1092+
1093+ for i in "$@"; do
1094+ case $i in
1095+ -*) template_opts+=("$i");;
1096+ *) pos_args+=("$i");;
1097+ esac
1098+ done
1099+ uWSGI_prepare_app "${pos_args[1]}"
1100+ mkdir -p "${uWSGI_APPS_AVAILABLE}"
1101+ install_template "${template_opts[@]}" \
1102+ "${uWSGI_APPS_AVAILABLE}/${pos_args[1]}" \
1103+ root root 644
1104+ uWSGI_enable_app "${pos_args[1]}"
1105+ uWSGI_restart "${pos_args[1]}"
1106+ info_msg "uWSGI app: ${pos_args[1]} is installed"
1107+}
1108+
1109+uWSGI_remove_app() {
1110+
1111+ # usage: uWSGI_remove_app <myapp.ini>
1112+
1113+ local CONF="$1"
1114+
1115+ [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
1116+ info_msg "remove uWSGI app: ${CONF}"
1117+ uWSGI_disable_app "${CONF}"
1118+ uWSGI_restart "${CONF}"
1119+ rm -f "${uWSGI_APPS_AVAILABLE}/${CONF}"
1120+}
1121+
1122+uWSGI_app_enabled() {
1123+ # usage: uWSGI_app_enabled <myapp.ini>
1124+
1125+ local exit_val=0
1126+ local CONF="$1"
1127+
1128+ [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
1129+ case $DIST_ID-$DIST_VERS in
1130+ ubuntu-*|debian-*)
1131+ [[ -f "${uWSGI_APPS_ENABLED}/${CONF}" ]]
1132+ exit_val=$?
1133+ ;;
1134+ arch-*)
1135+ systemctl -q is-enabled "uwsgi@${CONF%.*}"
1136+ exit_val=$?
1137+ ;;
1138+ fedora-*)
1139+ [[ -f "${uWSGI_APPS_ENABLED}/${CONF}" ]]
1140+ exit_val=$?
1141+ ;;
1142+ *)
1143+ # FIXME
1144+ err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented"
1145+ exit_val=1
1146+ ;;
1147+ esac
1148+ return $exit_val
1149+}
1150+
1151+# shellcheck disable=SC2164
1152+uWSGI_enable_app() {
1153+
1154+ # usage: uWSGI_enable_app <myapp.ini>
1155+
1156+ local CONF="$1"
1157+
1158+ [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
1159+ case $DIST_ID-$DIST_VERS in
1160+ ubuntu-*|debian-*)
1161+ mkdir -p "${uWSGI_APPS_ENABLED}"
1162+ rm -f "${uWSGI_APPS_ENABLED}/${CONF}"
1163+ ln -s "${uWSGI_APPS_AVAILABLE}/${CONF}" "${uWSGI_APPS_ENABLED}/${CONF}"
1164+ info_msg "enabled uWSGI app: ${CONF} (restart required)"
1165+ ;;
1166+ arch-*)
1167+ mkdir -p "${uWSGI_APPS_ENABLED}"
1168+ rm -f "${uWSGI_APPS_ENABLED}/${CONF}"
1169+ ln -s "${uWSGI_APPS_AVAILABLE}/${CONF}" "${uWSGI_APPS_ENABLED}/${CONF}"
1170+ systemctl enable "uwsgi@${CONF%.*}"
1171+ info_msg "enabled uWSGI app: ${CONF} (restart required)"
1172+ ;;
1173+ fedora-*)
1174+ mkdir -p "${uWSGI_APPS_ENABLED}"
1175+ rm -f "${uWSGI_APPS_ENABLED}/${CONF}"
1176+ ln -s "${uWSGI_APPS_AVAILABLE}/${CONF}" "${uWSGI_APPS_ENABLED}/${CONF}"
1177+ chown "${uWSGI_USER}:${uWSGI_GROUP}" "${uWSGI_APPS_ENABLED}/${CONF}"
1178+ info_msg "enabled uWSGI app: ${CONF}"
1179+ ;;
1180+ *)
1181+ # FIXME
1182+ err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented"
1183+ ;;
1184+ esac
1185+}
1186+
1187+uWSGI_disable_app() {
1188+
1189+ # usage: uWSGI_disable_app <myapp.ini>
1190+
1191+ local CONF="$1"
1192+
1193+ [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
1194+ case $DIST_ID-$DIST_VERS in
1195+ ubuntu-*|debian-*)
1196+ service uwsgi stop "${CONF%.*}"
1197+ rm -f "${uWSGI_APPS_ENABLED}/${CONF}"
1198+ info_msg "disabled uWSGI app: ${CONF} (restart uWSGI required)"
1199+ ;;
1200+ arch-*)
1201+ systemctl stop "uwsgi@${CONF%.*}"
1202+ systemctl disable "uwsgi@${CONF%.*}"
1203+ rm -f "${uWSGI_APPS_ENABLED}/${CONF}"
1204+ ;;
1205+ fedora-*)
1206+ # in emperor mode, just remove the app.ini file
1207+ rm -f "${uWSGI_APPS_ENABLED}/${CONF}"
1208+ ;;
1209+ *)
1210+ # FIXME
1211+ err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented"
1212+ ;;
1213+ esac
1214+}
1215+
1216+# distro's package manager
1217+# ------------------------
1218+
1219+_apt_pkg_info_is_updated=0
1220+
1221+pkg_install() {
1222+
1223+ # usage: TITEL='install foobar' pkg_install foopkg barpkg
1224+
1225+ rst_title "${TITLE:-installation of packages}" section
1226+ echo -e "\npackage(s)::\n"
1227+ # shellcheck disable=SC2068
1228+ echo " " $@ | $FMT
1229+
1230+ if ! ask_yn "Should packages be installed?" Yn 30; then
1231+ return 42
1232+ fi
1233+ case $DIST_ID in
1234+ ubuntu|debian)
1235+ if [[ $_apt_pkg_info_is_updated == 0 ]]; then
1236+ export _apt_pkg_info_is_updated=1
1237+ apt update
1238+ fi
1239+ # shellcheck disable=SC2068
1240+ apt-get install -m -y $@
1241+ ;;
1242+ arch)
1243+ # shellcheck disable=SC2068
1244+ pacman -Sy --noconfirm $@
1245+ ;;
1246+ fedora)
1247+ # shellcheck disable=SC2068
1248+ dnf install -y $@
1249+ ;;
1250+ esac
1251+}
1252+
1253+pkg_remove() {
1254+
1255+ # usage: TITEL='remove foobar' pkg_remove foopkg barpkg
1256+
1257+ rst_title "${TITLE:-remove packages}" section
1258+ echo -e "\npackage(s)::\n"
1259+ # shellcheck disable=SC2068
1260+ echo " " $@ | $FMT
1261+
1262+ if ! ask_yn "Should packages be removed (purge)?" Yn 30; then
1263+ return 42
1264+ fi
1265+ case $DIST_ID in
1266+ ubuntu|debian)
1267+ # shellcheck disable=SC2068
1268+ apt-get purge --autoremove --ignore-missing -y $@
1269+ ;;
1270+ arch)
1271+ # shellcheck disable=SC2068
1272+ pacman -R --noconfirm $@
1273+ ;;
1274+ fedora)
1275+ # shellcheck disable=SC2068
1276+ dnf remove -y $@
1277+ ;;
1278+ esac
1279+}
1280+
1281+pkg_is_installed() {
1282+
1283+ # usage: pkg_is_install foopkg || pkg_install foopkg
1284+
1285+ case $DIST_ID in
1286+ ubuntu|debian)
1287+ dpkg -l "$1" &> /dev/null
1288+ return $?
1289+ ;;
1290+ arch)
1291+ pacman -Qsq "$1" &> /dev/null
1292+ return $?
1293+ ;;
1294+ fedora)
1295+ dnf list -q --installed "$1" &> /dev/null
1296+ return $?
1297+ ;;
1298+ esac
1299+}
1300+
1301+# git tooling
1302+# -----------
1303+
1304+# shellcheck disable=SC2164
1305+git_clone() {
1306+
1307+ # usage:
1308+ #
1309+ # git_clone <url> <name> [<branch> [<user>]]
1310+ # git_clone <url> <path> [<branch> [<user>]]
1311+ #
1312+ # First form uses $CACHE/<name> as destination folder, second form clones
1313+ # into <path>. If repository is allready cloned, pull from <branch> and
1314+ # update working tree (if needed, the caller has to stash local changes).
1315+ #
1316+ # git clone https://github.com/asciimoo/searx searx-src origin/master searxlogin
1317+ #
1318+
1319+ local url="$1"
1320+ local dest="$2"
1321+ local branch="$3"
1322+ local user="$4"
1323+ local bash_cmd="bash"
1324+ local remote="origin"
1325+
1326+ if [[ ! "${dest:0:1}" = "/" ]]; then
1327+ dest="$CACHE/$dest"
1328+ fi
1329+
1330+ [[ -z $branch ]] && branch=master
1331+ [[ -z $user ]] && [[ -n "${SUDO_USER}" ]] && user="${SUDO_USER}"
1332+ [[ -n $user ]] && bash_cmd="sudo -H -u $user -i"
1333+
1334+ if [[ -d "${dest}" ]] ; then
1335+ info_msg "already cloned: $dest"
1336+ tee_stderr 0.1 <<EOF | $bash_cmd 2>&1 | prefix_stdout " ${_Yellow}|$user|${_creset} "
1337+cd "${dest}"
1338+git checkout -m -B "$branch" --track "$remote/$branch"
1339+git pull --all
1340+EOF
1341+ else
1342+ info_msg "clone into: $dest"
1343+ tee_stderr 0.1 <<EOF | $bash_cmd 2>&1 | prefix_stdout " ${_Yellow}|$user|${_creset} "
1344+mkdir -p "$(dirname "$dest")"
1345+cd "$(dirname "$dest")"
1346+git clone --branch "$branch" --origin "$remote" "$url" "$(basename "$dest")"
1347+EOF
1348+ fi
1349+}
1350+
1351+# containers
1352+# ----------
1353+
1354+in_container() {
1355+ # Test if shell runs in a container.
1356+ #
1357+ # usage: in_container && echo "process running inside a LXC container"
1358+ # in_container || echo "process is not running inside a LXC container"
1359+ #
1360+ # sudo_or_exit
1361+ # hint: Reads init process environment, therefore root access is required!
1362+ # to be safe, take a look at the environment of process 1 (/sbin/init)
1363+ # grep -qa 'container=lxc' /proc/1/environ
1364+
1365+ # see lxc_init_container_env
1366+ [[ -f /.lxcenv ]]
1367+}
1368+
1369+LXC_ENV_FOLDER=
1370+if in_container; then
1371+ # shellcheck disable=SC2034
1372+ LXC_ENV_FOLDER="lxc/$(hostname)/"
1373+fi
1374+
1375+lxc_init_container_env() {
1376+
1377+ # usage: lxc_init_container_env <name>
1378+
1379+ # Create a /.lxcenv file in the root folder. Call this once after the
1380+ # container is inital started and before installing any boilerplate stuff.
1381+
1382+ info_msg "create /.lxcenv in container $1"
1383+ cat <<EOF | lxc exec "${1}" -- bash | prefix_stdout "[${_BBlue}${1}${_creset}] "
1384+touch "/.lxcenv"
1385+ls -l "/.lxcenv"
1386+EOF
1387+}
1388+
1389+# apt packages
1390+LXC_BASE_PACKAGES_debian="bash git build-essential python3 virtualenv"
1391+
1392+# pacman packages
1393+LXC_BASE_PACKAGES_arch="bash git base-devel python python-virtualenv"
1394+
1395+# dnf packages
1396+LXC_BASE_PACKAGES_fedora="bash git @development-tools python virtualenv"
1397+
1398+case $DIST_ID in
1399+ ubuntu|debian) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_debian}" ;;
1400+ arch) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_arch}" ;;
1401+ fedora) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_fedora}" ;;
1402+ *) err_msg "$DIST_ID-$DIST_VERS: pkg_install LXC_BASE_PACKAGES not yet implemented" ;;
1403+esac
1404+
1405+lxc_install_base_packages() {
1406+ info_msg "install LXC_BASE_PACKAGES in container $1"
1407+ pkg_install "${LXC_BASE_PACKAGES}"
1408+}
1409+
1410+
1411+lxc_image_copy() {
1412+
1413+ # usage: lxc_copy_image <remote image> <local image>
1414+ #
1415+ # lxc_copy_image "images:ubuntu/19.10" "ubu1910"
1416+
1417+ if lxc_image_exists "local:${LXC_SUITE[i+1]}"; then
1418+ info_msg "image ${LXC_SUITE[i]} already copied --> ${LXC_SUITE[i+1]}"
1419+ else
1420+ info_msg "copy image locally ${LXC_SUITE[i]} --> ${LXC_SUITE[i+1]}"
1421+ lxc image copy "${LXC_SUITE[i]}" local: \
1422+ --alias "${LXC_SUITE[i+1]}" | prefix_stdout
1423+ fi
1424+}
1425+
1426+lxc_init_container() {
1427+
1428+ # usage: lxc_init_container <image name> <container name>
1429+
1430+ local image_name="$1"
1431+ local container_name="$2"
1432+
1433+ if lxc info "${container_name}" &>/dev/null; then
1434+ info_msg "container '${container_name}' already exists"
1435+ else
1436+ info_msg "create container instance: ${container_name}"
1437+ lxc init "local:${image_name}" "${container_name}"
1438+ fi
1439+}
1440+
1441+lxc_exists(){
1442+
1443+ # usage: lxc_exists <name> || echo "container <name> does not exists"
1444+
1445+ lxc info "$1" &>/dev/null
1446+}
1447+
1448+lxc_image_exists(){
1449+ # usage: lxc_image_exists <alias> || echo "image <alias> does locally not exists"
1450+
1451+ lxc image info "local:$1" &>/dev/null
1452+
1453+}
1454+
1455+lxc_delete_container() {
1456+
1457+ # usage: lxc_delete_container <container-name>
1458+
1459+ if lxc info "$1" &>/dev/null; then
1460+ info_msg "stop & delete instance ${_BBlue}${1}${_creset}"
1461+ lxc stop "$1" &>/dev/null
1462+ lxc delete "$1" | prefix_stdout
1463+ else
1464+ warn_msg "instance '$1' does not exist / can't delete :o"
1465+ fi
1466+}
1467+
1468+lxc_delete_local_image() {
1469+
1470+ # usage: lxc_delete_local_image <container-name>
1471+
1472+ info_msg "delete image 'local:$i'"
1473+ lxc image delete "local:$i"
1474+}
1475+
1476+
1477+# IP
1478+# --
1479+
1480+global_IPs(){
1481+ # usage: global_IPS
1482+ #
1483+ # print list of host's SCOPE global addresses and adapters e.g::
1484+ #
1485+ # $ global_IPs
1486+ # enp4s0|192.168.1.127
1487+ # lxdbr0|10.246.86.1
1488+ # lxdbr0|fd42:8c58:2cd:b73f::1
1489+
1490+ ip -o addr show | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\1|\2/p'
1491+}
1492+
1493+primary_ip() {
1494+
1495+ case $DIST_ID in
1496+ arch)
1497+ ip -o addr show \
1498+ | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\2/p' \
1499+ | head -n 1
1500+ ;;
1501+ *) hostname -I | cut -d' ' -f1 ;;
1502+ esac
1503+}
1504+
1505+# URL
1506+# ---
1507+
1508+url_replace_hostname(){
1509+
1510+ # usage: url_replace_hostname <url> <new hostname>
1511+
1512+ # to replace hostname by primary IP::
1513+ #
1514+ # url_replace_hostname http://searx-ubu1604/morty $(primary_ip)
1515+ # http://10.246.86.250/morty
1516+
1517+ # shellcheck disable=SC2001
1518+ echo "$1" | sed "s|\(http[s]*://\)[^/]*\(.*\)|\1$2\2|"
1519+}
utils/lxc-searx.envnew file+95
@@ -0,0 +1,95 @@
1+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
2+# SPDX-License-Identifier: AGPL-3.0-or-later
3+# shellcheck shell=bash
4+
5+# This file is a setup of a LXC suite. It is sourced from different context, do
6+# not manipulate the environment directly, implement functions and manipulate
7+# environment only is subshells!
8+
9+# ----------------------------------------------------------------------------
10+# config
11+# ----------------------------------------------------------------------------
12+
13+# shellcheck disable=SC2034
14+LXC_SUITE_NAME="searx"
15+lxc_set_suite_env() {
16+ # name of https://images.linuxcontainers.org
17+ export LINUXCONTAINERS_ORG_NAME="${LINUXCONTAINERS_ORG_NAME:-images}"
18+ export LXC_HOST_PREFIX="${LXC_SUITE_NAME:-searx}"
19+ export LXC_SUITE=(
20+
21+ # to disable containers, comment out lines ..
22+
23+ # end of standard support see https://wiki.ubuntu.com/Releases
24+ "$LINUXCONTAINERS_ORG_NAME:ubuntu/16.04" "ubu1604" # April 2021
25+ "$LINUXCONTAINERS_ORG_NAME:ubuntu/18.04" "ubu1804" # April 2023
26+ "$LINUXCONTAINERS_ORG_NAME:ubuntu/19.10" "ubu1910" # July 2020
27+ "$LINUXCONTAINERS_ORG_NAME:ubuntu/20.04" "ubu2004" # future (EOL 2030)
28+
29+ # EOL see https://fedoraproject.org/wiki/Releases
30+ "$LINUXCONTAINERS_ORG_NAME:fedora/31" "fedora31"
31+
32+ # rolling releases see https://www.archlinux.org/releng/releases/
33+ "$LINUXCONTAINERS_ORG_NAME:archlinux" "archlinux"
34+ )
35+
36+ PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
37+ if in_container; then
38+ # container hostnames do not have a DNS entry: use primary IP!
39+ PUBLIC_URL="http://$(primary_ip)/searx"
40+
41+ # make GUEST's services public to the HOST
42+ FILTRON_API="0.0.0.0:4005"
43+ FILTRON_LISTEN="0.0.0.0:4004"
44+ MORTY_LISTEN="0.0.0.0:3000"
45+
46+ # export LXC specific environment
47+ export PUBLIC_URL FILTRON_API FILTRON_LISTEN MORTY_LISTEN
48+ fi
49+}
50+
51+lxc_suite_install_info() {
52+ (
53+ lxc_set_suite_env
54+ cat <<EOF
55+LXC suite: ${LXC_SUITE_NAME} --> ${PUBLIC_URL}
56+ suite includes searx, morty & filtron
57+suite images:
58+$(echo " ${LOCAL_IMAGES[*]}" | $FMT)
59+suite containers:
60+$(echo " ${CONTAINERS[*]}" | $FMT)
61+EOF
62+ )
63+ }
64+
65+lxc_suite_install() {
66+ (
67+ lxc_set_suite_env
68+ FORCE_TIMEOUT=0
69+ export FORCE_TIMEOUT
70+ "${LXC_REPO_ROOT}/utils/searx.sh" install all
71+ "${LXC_REPO_ROOT}/utils/morty.sh" install all
72+ "${LXC_REPO_ROOT}/utils/filtron.sh" install all
73+
74+ rst_title "suite installation finished ($(hostname))" part
75+ lxc_suite_info
76+ echo
77+ )
78+}
79+
80+lxc_suite_info() {
81+ (
82+ lxc_set_suite_env
83+ for ip in $(global_IPs) ; do
84+ if [[ $ip =~ .*:.* ]]; then
85+ info_msg "(${ip%|*}) IPv6: http://[${ip#*|}]"
86+ else
87+ # IPv4:
88+ # shellcheck disable=SC2034,SC2031
89+ info_msg "(${ip%|*}) filtron: http://${ip#*|}:4004/ $PUBLIC_URL"
90+ info_msg "(${ip%|*}) morty: http://${ip#*|}:3000/ $PUBLIC_URL_MORTY"
91+ info_msg "(${ip%|*}) docs-live: http://${ip#*|}:8080/"
92+ fi
93+ done
94+ )
95+}
utils/lxc.shnew file+552
@@ -0,0 +1,552 @@
1+#!/usr/bin/env bash
2+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
3+# SPDX-License-Identifier: AGPL-3.0-or-later
4+
5+# shellcheck source=utils/lib.sh
6+source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
7+source_dot_config
8+
9+# load environment of the LXC suite
10+LXC_ENV="${LXC_ENV:-${REPO_ROOT}/utils/lxc-searx.env}"
11+source "$LXC_ENV"
12+lxc_set_suite_env
13+
14+# ----------------------------------------------------------------------------
15+# config
16+# ----------------------------------------------------------------------------
17+#
18+# read also:
19+# - https://lxd.readthedocs.io/en/latest/
20+
21+LXC_HOST_PREFIX="${LXC_HOST_PREFIX:-test}"
22+
23+# where all folders from HOST are mounted
24+LXC_SHARE_FOLDER="/share"
25+LXC_REPO_ROOT="${LXC_SHARE_FOLDER}/$(basename "${REPO_ROOT}")"
26+
27+ubu1604_boilerplate="
28+export DEBIAN_FRONTEND=noninteractive
29+apt-get update -y
30+apt-get upgrade -y
31+apt-get install -y git curl wget
32+"
33+ubu1804_boilerplate="$ubu1604_boilerplate"
34+ubu1904_boilerplate="$ubu1804_boilerplate"
35+ubu1910_boilerplate="$ubu1904_boilerplate"
36+
37+# shellcheck disable=SC2034
38+ubu2004_boilerplate="
39+$ubu1910_boilerplate
40+echo 'Set disable_coredump false' >> /etc/sudo.conf
41+"
42+
43+# shellcheck disable=SC2034
44+archlinux_boilerplate="
45+pacman -Syu --noconfirm
46+pacman -S --noconfirm inetutils git curl wget sudo
47+echo 'Set disable_coredump false' >> /etc/sudo.conf
48+"
49+
50+# shellcheck disable=SC2034
51+fedora31_boilerplate="
52+dnf update -y
53+dnf install -y git curl wget hostname
54+echo 'Set disable_coredump false' >> /etc/sudo.conf
55+"
56+
57+REMOTE_IMAGES=()
58+CONTAINERS=()
59+LOCAL_IMAGES=()
60+
61+for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do
62+ REMOTE_IMAGES=("${REMOTE_IMAGES[@]}" "${LXC_SUITE[i]}")
63+ CONTAINERS=("${CONTAINERS[@]}" "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}")
64+ LOCAL_IMAGES=("${LOCAL_IMAGES[@]}" "${LXC_SUITE[i+1]}")
65+done
66+
67+HOST_USER="${SUDO_USER:-$USER}"
68+HOST_USER_ID=$(id -u "${HOST_USER}")
69+HOST_GROUP_ID=$(id -g "${HOST_USER}")
70+
71+# ----------------------------------------------------------------------------
72+usage() {
73+# ----------------------------------------------------------------------------
74+ _cmd="$(basename "$0")"
75+ cat <<EOF
76+usage::
77+ $_cmd build [containers|<name>]
78+ $_cmd copy [images]
79+ $_cmd remove [containers|<name>|images]
80+ $_cmd [start|stop] [containers|<name>]
81+ $_cmd show [images|suite|info|config [<name>]]
82+ $_cmd cmd [--|<name>] '...'
83+ $_cmd install [suite|base [<name>]]
84+
85+build
86+ :containers: build, launch all containers and 'install base' packages
87+ :<name>: build, launch container <name> and 'install base' packages
88+copy:
89+ :images: copy remote images of the suite into local storage
90+remove
91+ :containers: delete all 'containers' or only <container-name>
92+ :images: delete local images of the suite
93+start/stop
94+ :containers: start/stop all 'containers' from the suite
95+ :<name>: start/stop container <name> from suite
96+show
97+ :info: show info of all (or <name>) containers from LXC suite
98+ :config: show config of all (or <name>) containers from the LXC suite
99+ :suite: show services of all (or <name>) containers from the LXC suite
100+ :images: show information of local images
101+cmd
102+ use single qoutes to evaluate in container's bash, e.g.: 'echo \$(hostname)'
103+ -- run command '...' in all containers of the LXC suite
104+ :<name>: run command '...' in container <name>
105+install
106+ :base: prepare LXC; install basic packages
107+ :suite: install LXC ${LXC_SUITE_NAME} suite into all (or <name>) containers
108+
109+EOF
110+ usage_containers
111+ [ -n "${1+x}" ] && err_msg "$1"
112+}
113+
114+usage_containers() {
115+ lxc_suite_install_info
116+ [ -n "${1+x}" ] && err_msg "$1"
117+}
118+
119+lxd_info() {
120+
121+ cat <<EOF
122+
123+LXD is needed, to install run::
124+
125+ snap install lxd
126+ lxd init --auto
127+
128+EOF
129+}
130+
131+main() {
132+
133+ local exit_val
134+ local _usage="unknown or missing $1 command $2"
135+
136+ # don't check prerequisite when in recursion
137+ if [[ ! $1 == __* ]]; then
138+ if ! in_container; then
139+ ! required_commands lxc && lxd_info && exit 42
140+ fi
141+ [[ -z $LXC_SUITE ]] && err_msg "missing LXC_SUITE" && exit 42
142+ fi
143+
144+ case $1 in
145+ --getenv) var="$2"; echo "${!var}"; exit 0;;
146+ -h|--help) usage; exit 0;;
147+
148+ build)
149+ sudo_or_exit
150+ case $2 in
151+ ${LXC_HOST_PREFIX}-*) build_container "$2" ;;
152+ ''|--|containers) build_all_containers ;;
153+ *) usage "$_usage"; exit 42;;
154+ esac
155+ ;;
156+ copy)
157+ case $2 in
158+ ''|images) lxc_copy_images_localy;;
159+ *) usage "$_usage"; exit 42;;
160+ esac
161+ ;;
162+ remove)
163+ sudo_or_exit
164+ case $2 in
165+ ''|--|containers) remove_containers ;;
166+ images) lxc_delete_images_localy ;;
167+ ${LXC_HOST_PREFIX}-*)
168+ ! lxc_exists "$2" && warn_msg "container not yet exists: $2" && exit 0
169+ if ask_yn "Do you really want to delete container $2"; then
170+ lxc_delete_container "$2"
171+ fi
172+ ;;
173+ *) usage "uknown or missing container <name> $2"; exit 42;;
174+ esac
175+ ;;
176+ start|stop)
177+ sudo_or_exit
178+ case $2 in
179+ ''|--|containers) lxc_cmd "$1" ;;
180+ ${LXC_HOST_PREFIX}-*)
181+ ! lxc_exists "$2" && usage_containers "unknown container: $2" && exit 42
182+ info_msg "lxc $1 $2"
183+ lxc "$1" "$2" | prefix_stdout "[${_BBlue}${i}${_creset}] "
184+ ;;
185+ *) usage "uknown or missing container <name> $2"; exit 42;;
186+ esac
187+ ;;
188+ show)
189+ sudo_or_exit
190+ case $2 in
191+ suite)
192+ case $3 in
193+ ${LXC_HOST_PREFIX}-*)
194+ lxc exec -t "$3" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __show suite \
195+ | prefix_stdout "[${_BBlue}$3${_creset}] "
196+ ;;
197+ *) show_suite;;
198+ esac
199+ ;;
200+ images) show_images ;;
201+ config)
202+ case $3 in
203+ ${LXC_HOST_PREFIX}-*)
204+ ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42
205+ lxc config show "$3" | prefix_stdout "[${_BBlue}${3}${_creset}] "
206+ ;;
207+ *)
208+ rst_title "container configurations"
209+ echo
210+ lxc list "$LXC_HOST_PREFIX-"
211+ echo
212+ lxc_cmd config show
213+ ;;
214+ esac
215+ ;;
216+ info)
217+ case $3 in
218+ ${LXC_HOST_PREFIX}-*)
219+ ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42
220+ lxc info "$3" | prefix_stdout "[${_BBlue}${3}${_creset}] "
221+ ;;
222+ *)
223+ rst_title "container info"
224+ echo
225+ lxc_cmd info
226+ ;;
227+ esac
228+ ;;
229+ *) usage "$_usage"; exit 42;;
230+ esac
231+ ;;
232+ __show)
233+ # wrapped show commands, called once in each container
234+ case $2 in
235+ suite) lxc_suite_info ;;
236+ esac
237+ ;;
238+ cmd)
239+ sudo_or_exit
240+ shift
241+ case $1 in
242+ --) shift; lxc_exec "$@" ;;
243+ ${LXC_HOST_PREFIX}-*)
244+ ! lxc_exists "$1" && usage_containers "unknown container: $1" && exit 42
245+ local name=$1
246+ shift
247+ lxc_exec_cmd "${name}" "$@"
248+ ;;
249+ *) usage_containers "unknown container: $1" && exit 42
250+ esac
251+ ;;
252+ install)
253+ sudo_or_exit
254+ case $2 in
255+ suite|base)
256+ case $3 in
257+ ${LXC_HOST_PREFIX}-*)
258+ ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42
259+ lxc_exec_cmd "$3" "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2"
260+ ;;
261+ ''|--) lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2" ;;
262+ *) usage_containers "unknown container: $3" && exit 42
263+ esac
264+ ;;
265+ *) usage "$_usage"; exit 42 ;;
266+ esac
267+ ;;
268+ __install)
269+ # wrapped install commands, called once in each container
270+ # shellcheck disable=SC2119
271+ case $2 in
272+ suite) lxc_suite_install ;;
273+ base) FORCE_TIMEOUT=0 lxc_install_base_packages ;;
274+ esac
275+ ;;
276+ doc)
277+ echo
278+ echo ".. generic utils/lxc.sh documentation"
279+ ;;
280+ -*) usage "unknown option $1"; exit 42;;
281+ *) usage "unknown or missing command $1"; exit 42;;
282+ esac
283+}
284+
285+
286+build_all_containers() {
287+ rst_title "Build all LXC containers of suite"
288+ echo
289+ usage_containers
290+ lxc_copy_images_localy
291+ lxc_init_all_containers
292+ lxc_config_all_containers
293+ lxc_boilerplate_all_containers
294+ rst_title "install LXC base packages" section
295+ echo
296+ lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install base
297+ echo
298+ lxc list "$LXC_HOST_PREFIX"
299+}
300+
301+build_container() {
302+ rst_title "Build container $1"
303+
304+ local remote_image
305+ local container
306+ local image
307+ local boilerplate_script
308+
309+ for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do
310+ if [ "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}" = "$1" ]; then
311+ remote_image="${LXC_SUITE[i]}"
312+ container="${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}"
313+ image="${LXC_SUITE[i+1]}"
314+ boilerplate_script="${image}_boilerplate"
315+ boilerplate_script="${!boilerplate_script}"
316+ break
317+ fi
318+ done
319+ echo
320+ if [ -z "$container" ]; then
321+ err_msg "container $1 unknown"
322+ usage_containers
323+ return 42
324+ fi
325+ lxc_image_copy "${remote_image}" "${image}"
326+ rst_title "init container" section
327+ lxc_init_container "${image}" "${container}"
328+ rst_title "configure container" section
329+ lxc_config_container "${container}"
330+ rst_title "run LXC boilerplate scripts" section
331+ lxc_install_boilerplate "${container}" "$boilerplate_script"
332+ echo
333+ rst_title "install LXC base packages" section
334+ lxc_exec_cmd "${container}" "${LXC_REPO_ROOT}/utils/lxc.sh" __install base \
335+ | prefix_stdout "[${_BBlue}${container}${_creset}] "
336+ echo
337+ lxc list "$container"
338+}
339+
340+remove_containers() {
341+ rst_title "Remove all LXC containers of suite"
342+ rst_para "existing containers matching ${_BGreen}$LXC_HOST_PREFIX-*${_creset}"
343+ echo
344+ lxc list "$LXC_HOST_PREFIX-"
345+ echo -en "\\n${_BRed}LXC containers to delete::${_creset}\\n\\n ${CONTAINERS[*]}\\n" | $FMT
346+ local default=Ny
347+ [[ $FORCE_TIMEOUT = 0 ]] && default=Yn
348+ if ask_yn "Do you really want to delete these containers" $default; then
349+ for i in "${CONTAINERS[@]}"; do
350+ lxc_delete_container "$i"
351+ done
352+ fi
353+ echo
354+ lxc list "$LXC_HOST_PREFIX-"
355+}
356+
357+# images
358+# ------
359+
360+lxc_copy_images_localy() {
361+ rst_title "copy images" section
362+ for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do
363+ lxc_image_copy "${LXC_SUITE[i]}" "${LXC_SUITE[i+1]}"
364+ done
365+ # lxc image list local: && wait_key
366+}
367+
368+lxc_delete_images_localy() {
369+ rst_title "Delete LXC images"
370+ rst_para "local existing images"
371+ echo
372+ lxc image list local:
373+ echo -en "\\n${_BRed}LXC images to delete::${_creset}\\n\\n ${LOCAL_IMAGES[*]}\\n"
374+ if ask_yn "Do you really want to delete these images"; then
375+ for i in "${LOCAL_IMAGES[@]}"; do
376+ lxc_delete_local_image "$i"
377+ done
378+ fi
379+
380+ for i in $(lxc image list --format csv | grep '^,' | sed 's/,\([^,]*\).*$/\1/'); do
381+ if ask_yn "Image $i has no alias, do you want to delete the image?" Yn; then
382+ lxc_delete_local_image "$i"
383+ fi
384+ done
385+
386+ echo
387+ lxc image list local:
388+}
389+
390+show_images(){
391+ rst_title "local images"
392+ echo
393+ lxc image list local:
394+ echo -en "\\n${_Green}LXC suite images::${_creset}\\n\\n ${LOCAL_IMAGES[*]}\\n"
395+ wait_key
396+ for i in "${LOCAL_IMAGES[@]}"; do
397+ if lxc_image_exists "$i"; then
398+ info_msg "lxc image info ${_BBlue}${i}${_creset}"
399+ lxc image info "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] "
400+ else
401+ warn_msg "image ${_BBlue}$i${_creset} does not yet exists"
402+ fi
403+ done
404+
405+}
406+
407+
408+# container
409+# ---------
410+
411+show_suite(){
412+ rst_title "LXC suite ($LXC_HOST_PREFIX-*)"
413+ echo
414+ lxc list "$LXC_HOST_PREFIX-"
415+ echo
416+ for i in "${CONTAINERS[@]}"; do
417+ if ! lxc_exists "$i"; then
418+ warn_msg "container ${_BBlue}$i${_creset} does not yet exists"
419+ else
420+ lxc exec -t "${i}" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __show suite \
421+ | prefix_stdout "[${_BBlue}${i}${_creset}] "
422+ echo
423+ fi
424+ done
425+}
426+
427+lxc_cmd() {
428+ for i in "${CONTAINERS[@]}"; do
429+ if ! lxc_exists "$i"; then
430+ warn_msg "container ${_BBlue}$i${_creset} does not yet exists"
431+ else
432+ info_msg "lxc $* $i"
433+ lxc "$@" "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] "
434+ fi
435+ done
436+}
437+
438+lxc_exec_cmd() {
439+ local name="$1"
440+ shift
441+ exit_val=
442+ info_msg "[${_BBlue}${name}${_creset}] ${_BGreen}${*}${_creset}"
443+ lxc exec -t --cwd "${LXC_REPO_ROOT}" "${name}" -- bash -c "$*"
444+ exit_val=$?
445+ if [[ $exit_val -ne 0 ]]; then
446+ warn_msg "[${_BBlue}${name}${_creset}] exit code (${_BRed}${exit_val}${_creset}) from ${_BGreen}${*}${_creset}"
447+ else
448+ info_msg "[${_BBlue}${name}${_creset}] exit code (${exit_val}) from ${_BGreen}${*}${_creset}"
449+ fi
450+}
451+
452+lxc_exec() {
453+ for i in "${CONTAINERS[@]}"; do
454+ if ! lxc_exists "$i"; then
455+ warn_msg "container ${_BBlue}$i${_creset} does not yet exists"
456+ else
457+ lxc_exec_cmd "${i}" "$@" | prefix_stdout "[${_BBlue}${i}${_creset}] "
458+ fi
459+ done
460+}
461+
462+lxc_init_all_containers() {
463+ rst_title "init all containers" section
464+
465+ local image_name
466+ local container_name
467+
468+ for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do
469+ lxc_init_container "${LXC_SUITE[i+1]}" "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}"
470+ done
471+}
472+
473+lxc_config_all_containers() {
474+ rst_title "configure all containers" section
475+
476+ for i in "${CONTAINERS[@]}"; do
477+ lxc_config_container "${i}"
478+ done
479+}
480+
481+lxc_config_container() {
482+ info_msg "[${_BBlue}$1${_creset}] configure container ..."
483+
484+ info_msg "[${_BBlue}$1${_creset}] map uid/gid from host to container"
485+ # https://lxd.readthedocs.io/en/latest/userns-idmap/#custom-idmaps
486+ echo -e -n "uid $HOST_USER_ID 0\\ngid $HOST_GROUP_ID 0"\
487+ | lxc config set "$1" raw.idmap -
488+
489+ info_msg "[${_BBlue}$1${_creset}] share ${REPO_ROOT} (repo_share) from HOST into container"
490+ # https://lxd.readthedocs.io/en/latest/instances/#type-disk
491+ lxc config device add "$1" repo_share disk \
492+ source="${REPO_ROOT}" \
493+ path="${LXC_REPO_ROOT}" &>/dev/null
494+ # lxc config show "$1" && wait_key
495+}
496+
497+lxc_boilerplate_all_containers() {
498+ rst_title "run LXC boilerplate scripts" section
499+
500+ local boilerplate_script
501+ local image_name
502+
503+ for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do
504+
505+ image_name="${LXC_SUITE[i+1]}"
506+ boilerplate_script="${image_name}_boilerplate"
507+ boilerplate_script="${!boilerplate_script}"
508+
509+ lxc_install_boilerplate "${LXC_HOST_PREFIX}-${image_name}" "$boilerplate_script"
510+
511+ if [[ -z "${boilerplate_script}" ]]; then
512+ err_msg "[${_BBlue}${container_name}${_creset}] no boilerplate for image '${image_name}'"
513+ fi
514+ done
515+}
516+
517+lxc_install_boilerplate() {
518+
519+ # usage: lxc_install_boilerplate <container-name> <string: shell commands ..>
520+ #
521+ # usage: lxc_install_boilerplate searx-archlinux "${archlinux_boilerplate}"
522+
523+ local container_name="$1"
524+ local boilerplate_script="$2"
525+
526+ info_msg "[${_BBlue}${container_name}${_creset}] init .."
527+ if lxc start -q "${container_name}" &>/dev/null; then
528+ sleep 5 # guest needs some time to come up and get an IP
529+ fi
530+ lxc_init_container_env "${container_name}"
531+ info_msg "[${_BBlue}${container_name}${_creset}] install /.lxcenv.mk .."
532+ cat <<EOF | lxc exec "${container_name}" -- bash | prefix_stdout "[${_BBlue}${container_name}${_creset}] "
533+rm -f "/.lxcenv.mk"
534+ln -s "${LXC_REPO_ROOT}/utils/makefile.lxc" "/.lxcenv.mk"
535+ls -l "/.lxcenv.mk"
536+EOF
537+
538+ info_msg "[${_BBlue}${container_name}${_creset}] run LXC boilerplate scripts .."
539+ if lxc start -q "${container_name}" &>/dev/null; then
540+ sleep 5 # guest needs some time to come up and get an IP
541+ fi
542+ if [[ -n "${boilerplate_script}" ]]; then
543+ echo "${boilerplate_script}" \
544+ | lxc exec "${container_name}" -- bash \
545+ | prefix_stdout "[${_BBlue}${container_name}${_creset}] "
546+ fi
547+}
548+
549+
550+# ----------------------------------------------------------------------------
551+main "$@"
552+# ----------------------------------------------------------------------------
utils/makefile.include+15−1
@@ -1,12 +1,25 @@
11# -*- coding: utf-8; mode: makefile-gmake -*-
22
3+ifeq (,$(wildcard /.lxcenv.mk))
4+PHONY += lxc-activate lxc-purge
5+lxc-activate:
6+ @$(MAKE) -s -f /share/searx/utils/makefile.lxc lxc-activate
7+lxc-purge:
8+ $(Q)rm -rf ./lxc
9+else
10+ include /.lxcenv.mk
11+endif
12+
13+ifeq (,$(wildcard /.lxcenv.mk))
314make-help:
15+else
16+make-help: lxc-help
17+endif
418 @echo ' make V=0|1 [targets] 0 => quiet build (default), 1 => verbose build'
519 @echo ' make V=2 [targets] 2 => give reason for rebuild of target'
620
721quiet_cmd_common_clean = CLEAN $@
822 cmd_common_clean = \
9− rm -rf tests/build ;\
1023 find . -name '*.orig' -exec rm -f {} + ;\
1124 find . -name '*.rej' -exec rm -f {} + ;\
1225 find . -name '*~' -exec rm -f {} + ;\
@@ -126,3 +139,4 @@ echo-cmd = $(if $($(quiet)cmd_$(1)),echo '$(call escsq,$($(quiet)cmd_$(1)))$(ech
126139# printing commands
127140cmd = @$(echo-cmd) $(cmd_$(1))
128141
142+.PHONY: $(PHONY)
utils/makefile.lxcnew file+29
@@ -0,0 +1,29 @@
1+# -*- coding: utf-8; mode: makefile-gmake -*-
2+#
3+# LXC environment
4+# ===============
5+#
6+# To activate/deactivate LXC makefile environment in a container, set/unset link
7+# from root '/.lxcenv.mk' to *this* file::
8+#
9+# sudo make ./utils/makefile.lxc lxc-activate
10+# sudo make ./utils/makefile.lxc lxc-deactivate
11+
12+LXC_ENV_FOLDER=lxc/$(shell hostname)/
13+
14+lxc-help::
15+ @echo 'LXC: running in container LXC_ENV_FOLDER=$(LXC_ENV_FOLDER)'
16+
17+# If not activated, serve target 'lxc-activate' ..
18+ifeq (,$(wildcard /.lxcenv.mk))
19+PHONY += lxc-activate
20+lxc-activate:
21+ ln -s "$(abspath $(lastword $(MAKEFILE_LIST)))" "/.lxcenv.mk"
22+else
23+# .. and if activated, serve target 'lxc-deactivate'.
24+PHONY += lxc-deactivate
25+lxc-deactivate:
26+ rm /.lxcenv.mk
27+endif
28+
29+.PHONY: $(PHONY)
utils/makefile.python+4−4
@@ -8,9 +8,9 @@ export PYTHONPATH := $(SITE_PYTHON):$$PYTHONPATH
88export PY_ENV PYDIST PYBUILD
99
1010# folder where the python distribution takes place
11−PYDIST ?= ./py_dist
11+PYDIST = ./$(LXC_ENV_FOLDER)dist
1212# folder where the python intermediate build files take place
13−PYBUILD ?= ./py_build
13+PYBUILD = ./$(LXC_ENV_FOLDER)build
1414# python version to use
1515PY ?=3
1616# $(PYTHON) points to the python interpreter from the OS! The python from the
@@ -30,8 +30,7 @@ PYLINT_RC ?= .pylintrc
3030TEST_FOLDER ?= ./tests
3131TEST ?= .
3232
33−VTENV_OPTS = "--no-site-packages"
34−PY_ENV = ./local/py$(PY)
33+PY_ENV = ./$(LXC_ENV_FOLDER)local/py$(PY)
3534PY_ENV_BIN = $(PY_ENV)/bin
3635PY_ENV_ACT = . $(PY_ENV_BIN)/activate
3736
@@ -41,6 +40,7 @@ ifeq ($(OS),Windows_NT)
4140 PY_ENV_ACT = $(PY_ENV_BIN)/activate
4241endif
4342
43+VTENV_OPTS ?=
4444ifeq ($(PYTHON),python)
4545 VIRTUALENV = virtualenv
4646else
utils/makefile.sphinx+15−11
@@ -1,17 +1,19 @@
11# -*- coding: utf-8; mode: makefile-gmake -*-
22
3+export DOCS_FOLDER DOCS_BUILD DOCS_DIST BOOKS_FOLDER BOOKS_DIST
4+
35# You can set these variables from the command line.
46SPHINXOPTS ?=
57SPHINXBUILD ?= $(PY_ENV_BIN)/sphinx-build
68SPHINX_CONF ?= conf.py
79
8−DOCS_FOLDER ?= docs
9−DOCS_BUILD ?= build/docs
10−DOCS_DIST ?= dist/docs
10+DOCS_FOLDER = ./docs
11+DOCS_BUILD = ./$(LXC_ENV_FOLDER)build/docs
12+DOCS_DIST = ./$(LXC_ENV_FOLDER)dist/docs
1113GH_PAGES ?= gh-pages
1214
13−BOOKS_FOLDER ?= docs
14−BOOKS_DIST ?= dist/books
15+BOOKS_FOLDER = ./docs
16+BOOKS_DIST = ./$(LXC_ENV_FOLDER)dist/books
1517
1618ifeq ($(KBUILD_VERBOSE),1)
1719 SPHINX_VERBOSE = "-v"
@@ -54,11 +56,13 @@ docs-help:
5456# requirements
5557# ------------------------------------------------------------------------------
5658
57−sphinx-doc: $(PY_ENV)
59+sphinx-doc-prebuilds:: $(PY_ENV)
60+
61+sphinx-doc: sphinx-doc-prebuilds
5862 @echo "PYENV installing Sphinx$(SPHINXVERS)"
5963 $(Q)$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) 'Sphinx$(SPHINXVERS)'
6064
61−sphinx-live: $(PY_ENV)
65+sphinx-live: sphinx-doc-prebuilds
6266 @echo "PYENV installing Sphinx$(SPHINXVERS)"
6367 $(Q)$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) 'Sphinx$(SPHINXVERS)' sphinx-autobuild
6468
@@ -113,7 +117,7 @@ quiet_cmd_sphinx_clean = CLEAN $@
113117# targets
114118# ------------------------------------------------------------------------------
115119
116−# build PDF of whole documentation in: $(DOCS_DIST)/pdf
120+# build PDF of whole documentation in: $(DOCS_DIST)/pdf
117121
118122PHONY += sphinx-pdf
119123sphinx-pdf: sphinx-latex
@@ -154,7 +158,7 @@ $(BOOKS_HTML): sphinx-doc | $(BOOKS_DIST)
154158 -b html \
155159 -c $(DOCS_FOLDER) \
156160 -d $(DOCS_BUILD)/books/$(patsubst books/%.html,%,$@)/.doctrees \
157− $(patsubst books/%.html,%,$@) \
161+ $(BOOKS_FOLDER)/$(patsubst books/%.html,%,$@) \
158162 $(BOOKS_DIST)/$(patsubst books/%.html,%,$@)
159163 @echo "SPHINX $@ --> file://$(abspath $(BOOKS_DIST)/$(patsubst books/%.html,%,$@))"
160164
@@ -166,7 +170,7 @@ $(BOOKS_LIVE): sphinx-live | $(BOOKS_DIST)
166170 -b html \
167171 -c $(DOCS_FOLDER) \
168172 -d $(DOCS_BUILD)/books/$(patsubst books/%.live,%,$@)/.doctrees \
169− $(patsubst books/%.live,%,$@) \
173+ $(BOOKS_FOLDER)/$(patsubst books/%.live,%,$@) \
170174 $(BOOKS_DIST)/$(patsubst books/%.live,%,$@)
171175
172176$(BOOKS_PDF): %.pdf : %.latex
@@ -182,7 +186,7 @@ $(BOOKS_LATEX): sphinx-doc | $(BOOKS_DIST)
182186 -b latex \
183187 -c $(DOCS_FOLDER) \
184188 -d $(DOCS_BUILD)/books/$(patsubst books/%.latex,%,$@)/.doctrees \
185− $(patsubst books/%.latex,%,$@) \
189+ $(BOOKS_FOLDER)/$(patsubst books/%.latex,%,$@) \
186190 $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@)
187191 @echo "SPHINX $@ --> file://$(abspath $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@))"
188192
utils/morty.shnew file+546
@@ -0,0 +1,546 @@
1+#!/usr/bin/env bash
2+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
3+# SPDX-License-Identifier: AGPL-3.0-or-later
4+
5+# shellcheck source=utils/lib.sh
6+source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
7+# shellcheck source=utils/brand.env
8+source "${REPO_ROOT}/utils/brand.env"
9+source_dot_config
10+SEARX_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
11+source "${REPO_ROOT}/utils/lxc-searx.env"
12+in_container && lxc_set_suite_env
13+
14+# ----------------------------------------------------------------------------
15+# config
16+# ----------------------------------------------------------------------------
17+
18+MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}"
19+PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty/}"
20+
21+PUBLIC_URL_MORTY="${PUBLIC_URL_MORTY:-$(echo "$SEARX_URL" | sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}}"
22+
23+# shellcheck disable=SC2034
24+MORTY_TIMEOUT=5
25+
26+SERVICE_NAME="morty"
27+SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}"
28+SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}"
29+SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}"
30+SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service"
31+# shellcheck disable=SC2034
32+SERVICE_GROUP="${SERVICE_USER}"
33+# shellcheck disable=SC2034
34+SERVICE_ENV_DEBUG=false
35+
36+GO_ENV="${SERVICE_HOME}/.go_env"
37+GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz"
38+GO_TAR=$(basename "$GO_PKG_URL")
39+
40+# shellcheck disable=SC2034
41+CONFIG_FILES=()
42+
43+# Apache Settings
44+
45+APACHE_MORTY_SITE="morty.conf"
46+NGINX_MORTY_SITE="morty.conf"
47+
48+# ----------------------------------------------------------------------------
49+usage() {
50+# ----------------------------------------------------------------------------
51+
52+ # shellcheck disable=SC1117
53+ cat <<EOF
54+usage::
55+ $(basename "$0") shell
56+ $(basename "$0") install [all|user]
57+ $(basename "$0") update [morty]
58+ $(basename "$0") remove [all]
59+ $(basename "$0") activate [service]
60+ $(basename "$0") deactivate [service]
61+ $(basename "$0") inspect [service]
62+ $(basename "$0") option [debug-on|debug-off|new-key]
63+ $(basename "$0") apache [install|remove]
64+ $(basename "$0") nginx [install|remove]
65+ $(basename "$0") info [searx]
66+
67+shell
68+ start interactive shell from user ${SERVICE_USER}
69+install / remove
70+ all: complete setup of morty service
71+ user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME)
72+update morty
73+ Update morty installation ($SERVICE_HOME)
74+activate service
75+ activate and start service daemon (systemd unit)
76+deactivate service
77+ stop and deactivate service daemon (systemd unit)
78+inspect service
79+ show service status and log
80+option
81+ set one of the available options
82+ :new-key: set new morty key
83+apache : ${PUBLIC_URL_MORTY}
84+ :install: apache site with a reverse proxy (ProxyPass)
85+ :remove: apache site ${APACHE_MORTY_SITE}
86+nginx (${PUBLIC_URL_MORTY})
87+ :install: nginx site with a reverse proxy (ProxyPass)
88+ :remove: nginx site ${NGINX_MORTY_SITE}
89+
90+If needed, set the environment variables in the '${DOT_CONFIG#"$REPO_ROOT/"}' file::
91+ PUBLIC_URL_MORTY: ${PUBLIC_URL_MORTY}
92+ MORTY_LISTEN: ${MORTY_LISTEN}
93+ SERVICE_USER: ${SERVICE_USER}
94+EOF
95+ if in_container; then
96+ # in containers the service is listening on 0.0.0.0 (see lxc-searx.env)
97+ for ip in $(global_IPs) ; do
98+ if [[ $ip =~ .*:.* ]]; then
99+ echo " container URL (IPv6): http://[${ip#*|}]:3000/"
100+ else
101+ # IPv4:
102+ echo " container URL (IPv4): http://${ip#*|}:3000/"
103+ fi
104+ done
105+ fi
106+ echo
107+ info_searx
108+
109+ [[ -n ${1} ]] && err_msg "$1"
110+}
111+
112+info_searx() {
113+ # shellcheck disable=SC1117
114+ cat <<EOF
115+To activate result and image proxy in searx, edit settings.yml (read:
116+${DOCS_URL}/admin/morty.html)::
117+ result_proxy:
118+ url : ${PUBLIC_URL_MORTY}
119+ server:
120+ image_proxy : True
121+EOF
122+}
123+
124+main() {
125+ required_commands \
126+ sudo install git wget curl \
127+ || exit
128+
129+ local _usage="ERROR: unknown or missing $1 command $2"
130+
131+ case $1 in
132+ --getenv) var="$2"; echo "${!var}"; exit 0;;
133+ -h|--help) usage; exit 0;;
134+
135+ shell)
136+ sudo_or_exit
137+ interactive_shell "${SERVICE_USER}"
138+ ;;
139+ inspect)
140+ case $2 in
141+ service)
142+ sudo_or_exit
143+ inspect_service
144+ ;;
145+ *) usage "$_usage"; exit 42;;
146+ esac ;;
147+ install)
148+ rst_title "$SERVICE_NAME" part
149+ sudo_or_exit
150+ case $2 in
151+ all) install_all ;;
152+ user) assert_user ;;
153+ *) usage "$_usage"; exit 42;;
154+ esac ;;
155+ update)
156+ sudo_or_exit
157+ case $2 in
158+ morty) update_morty ;;
159+ *) usage "$_usage"; exit 42;;
160+ esac ;;
161+ remove)
162+ sudo_or_exit
163+ case $2 in
164+ all) remove_all;;
165+ user) drop_service_account "${SERVICE_USER}" ;;
166+ *) usage "$_usage"; exit 42;;
167+ esac ;;
168+ activate)
169+ sudo_or_exit
170+ case $2 in
171+ service) systemd_activate_service "${SERVICE_NAME}" ;;
172+ *) usage "$_usage"; exit 42;;
173+ esac ;;
174+ deactivate)
175+ sudo_or_exit
176+ case $2 in
177+ service) systemd_deactivate_service "${SERVICE_NAME}" ;;
178+ *) usage "$_usage"; exit 42;;
179+ esac ;;
180+ apache)
181+ sudo_or_exit
182+ case $2 in
183+ install) install_apache_site ;;
184+ remove) remove_apache_site ;;
185+ *) usage "$_usage"; exit 42;;
186+ esac ;;
187+ nginx)
188+ sudo_or_exit
189+ case $2 in
190+ install) install_nginx_site ;;
191+ remove) remove_nginx_site ;;
192+ *) usage "$_usage"; exit 42;;
193+ esac ;;
194+ info)
195+ case $2 in
196+ searx) info_searx ;;
197+ *) usage "$_usage"; exit 42;;
198+ esac ;;
199+ option)
200+ sudo_or_exit
201+ case $2 in
202+ new-key) set_new_key ;;
203+ debug-on) enable_debug ;;
204+ debug-off) disable_debug ;;
205+ *) usage "$_usage"; exit 42;;
206+ esac ;;
207+ doc) rst-doc ;;
208+ *) usage "ERROR: unknown or missing command $1"; exit 42;;
209+ esac
210+}
211+
212+install_all() {
213+
214+ MORTY_KEY="$(head -c 32 /dev/urandom | base64)"
215+
216+ rst_title "Install $SERVICE_NAME (service)"
217+ assert_user
218+ wait_key
219+ install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}"
220+ wait_key
221+ install_morty
222+ wait_key
223+ systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
224+ wait_key
225+ if ! service_is_available "http://${MORTY_LISTEN}" ; then
226+ err_msg "Morty does not listening on: http://${MORTY_LISTEN}"
227+ fi
228+ if apache_is_installed; then
229+ info_msg "Apache is installed on this host."
230+ if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
231+ install_apache_site
232+ fi
233+ elif nginx_is_installed; then
234+ info_msg "nginx is installed on this host."
235+ if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
236+ install_nginx_site
237+ fi
238+ fi
239+ info_searx
240+ if ask_yn "Add image and result proxy to searx settings.yml?" Yn; then
241+ "${REPO_ROOT}/utils/searx.sh" option result-proxy "${PUBLIC_URL_MORTY}" "${MORTY_KEY}"
242+ "${REPO_ROOT}/utils/searx.sh" option image-proxy-on
243+ fi
244+
245+ if ask_yn "Do you want to inspect the installation?" Ny; then
246+ inspect_service
247+ fi
248+
249+}
250+
251+remove_all() {
252+ rst_title "De-Install $SERVICE_NAME (service)"
253+
254+ rst_para "\
255+It goes without saying that this script can only be used to remove
256+installations that were installed with this script."
257+
258+ if systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then
259+ drop_service_account "${SERVICE_USER}"
260+ fi
261+}
262+
263+assert_user() {
264+ rst_title "user $SERVICE_USER" section
265+ echo
266+ tee_stderr 1 <<EOF | bash | prefix_stdout
267+useradd --shell /bin/bash --system \
268+ --home-dir "$SERVICE_HOME" \
269+ --comment 'Web content sanitizer proxy' $SERVICE_USER
270+mkdir "$SERVICE_HOME"
271+chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
272+groups $SERVICE_USER
273+EOF
274+ SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)"
275+ export SERVICE_HOME
276+ echo "export SERVICE_HOME=$SERVICE_HOME"
277+
278+ cat > "$GO_ENV" <<EOF
279+export GOPATH=\$HOME/go-apps
280+export PATH=\$PATH:\$HOME/local/go/bin:\$GOPATH/bin
281+EOF
282+ echo "Environment $GO_ENV has been setup."
283+
284+ tee_stderr <<EOF | sudo -i -u "$SERVICE_USER"
285+grep -qFs -- 'source $GO_ENV' ~/.profile || echo 'source $GO_ENV' >> ~/.profile
286+EOF
287+}
288+
289+morty_is_installed() {
290+ [[ -f $SERVICE_HOME/go-apps/bin/morty ]]
291+}
292+
293+_svcpr=" ${_Yellow}|${SERVICE_USER}|${_creset} "
294+
295+install_morty() {
296+ rst_title "Install morty in user's ~/go-apps" section
297+ echo
298+ tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
299+go get -v -u github.com/asciimoo/morty
300+EOF
301+ tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
302+cd \$GOPATH/src/github.com/asciimoo/morty
303+go test
304+go test -benchmem -bench .
305+EOF
306+}
307+
308+update_morty() {
309+ rst_title "Update morty" section
310+ echo
311+ tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
312+go get -v -u github.com/asciimoo/morty
313+EOF
314+ tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
315+cd \$GOPATH/src/github.com/asciimoo/morty
316+go test
317+go test -benchmem -bench .
318+EOF
319+}
320+
321+set_service_env_debug() {
322+
323+ # usage: set_service_env_debug [false|true]
324+
325+ # shellcheck disable=SC2034
326+ local SERVICE_ENV_DEBUG="${1:-false}"
327+ if systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then
328+ systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
329+ fi
330+}
331+
332+inspect_service() {
333+
334+ rst_title "service status & log"
335+
336+ cat <<EOF
337+
338+sourced ${DOT_CONFIG#"$REPO_ROOT/"} :
339+
340+ MORTY_LISTEN : ${MORTY_LISTEN}
341+
342+EOF
343+
344+ if service_account_is_available "$SERVICE_USER"; then
345+ info_msg "service account $SERVICE_USER available."
346+ else
347+ err_msg "service account $SERVICE_USER not available!"
348+ fi
349+ if go_is_available "$SERVICE_USER"; then
350+ info_msg "~$SERVICE_USER: go is installed"
351+ else
352+ err_msg "~$SERVICE_USER: go is not installed"
353+ fi
354+ if morty_is_installed; then
355+ info_msg "~$SERVICE_USER: morty app is installed"
356+ else
357+ err_msg "~$SERVICE_USER: morty app is not installed!"
358+ fi
359+
360+ if ! service_is_available "http://${MORTY_LISTEN}" ; then
361+ err_msg "Morty does not listening on: http://${MORTY_LISTEN}"
362+ echo -e "${_Green}stop with [${_BCyan}CTRL-C${_Green}] or .."
363+ wait_key
364+ fi
365+
366+ if ! service_is_available "${PUBLIC_URL_MORTY}"; then
367+ warn_msg "Public service at ${PUBLIC_URL_MORTY} is not available!"
368+ if ! in_container; then
369+ warn_msg "Check if public name is correct and routed or use the public IP from above."
370+ fi
371+ fi
372+
373+ if in_container; then
374+ lxc_suite_info
375+ else
376+ info_msg "public URL --> ${PUBLIC_URL_MORTY}"
377+ info_msg "morty URL --> http://${MORTY_LISTEN}"
378+ fi
379+
380+ local _debug_on
381+ if ask_yn "Enable morty debug mode (needs reinstall of systemd service)?"; then
382+ enable_debug
383+ _debug_on=1
384+ else
385+ systemctl --no-pager -l status "${SERVICE_NAME}"
386+ fi
387+ echo
388+
389+ # shellcheck disable=SC2059
390+ printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log"
391+ read -r -s -n1 -t 5
392+ echo
393+ while true; do
394+ trap break 2
395+ journalctl -f -u "${SERVICE_NAME}"
396+ done
397+
398+ if [[ $_debug_on == 1 ]]; then
399+ FORCE_SELECTION=Y disable_debug
400+ fi
401+ return 0
402+}
403+
404+enable_debug() {
405+ warn_msg "Do not enable debug in production enviroments!!"
406+ info_msg "Enabling debug option needs to reinstall systemd service!"
407+ set_service_env_debug true
408+}
409+
410+disable_debug() {
411+ info_msg "Disabling debug option needs to reinstall systemd service!"
412+ set_service_env_debug false
413+}
414+
415+
416+set_new_key() {
417+ rst_title "Set morty key"
418+ echo
419+
420+ MORTY_KEY="$(head -c 32 /dev/urandom | base64)"
421+ info_msg "morty key: '${MORTY_KEY}'"
422+
423+ warn_msg "this will need to reinstall services .."
424+ MSG="${_Green}press any [${_BCyan}KEY${_Green}] to continue // stop with [${_BCyan}CTRL-C${_creset}]" wait_key
425+
426+ systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
427+ "${REPO_ROOT}/utils/searx.sh" option result-proxy "${PUBLIC_URL_MORTY}" "${MORTY_KEY}"
428+ "${REPO_ROOT}/utils/searx.sh" option image-proxy-on
429+}
430+
431+
432+install_apache_site() {
433+
434+ rst_title "Install Apache site $APACHE_MORTY_SITE"
435+
436+ rst_para "\
437+This installs a reverse proxy (ProxyPass) into apache site (${APACHE_MORTY_SITE})"
438+
439+ ! apache_is_installed && err_msg "Apache is not installed."
440+
441+ if ! ask_yn "Do you really want to continue?" Yn; then
442+ return
443+ else
444+ install_apache
445+ fi
446+
447+ apache_install_site "${APACHE_MORTY_SITE}"
448+
449+ info_msg "testing public url .."
450+ if ! service_is_available "${PUBLIC_URL_MORTY}"; then
451+ err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!"
452+ fi
453+}
454+
455+remove_apache_site() {
456+
457+ rst_title "Remove Apache site $APACHE_MORTY_SITE"
458+
459+ rst_para "\
460+This removes apache site ${APACHE_MORTY_SITE}."
461+
462+ ! apache_is_installed && err_msg "Apache is not installed."
463+
464+ if ! ask_yn "Do you really want to continue?" Yn; then
465+ return
466+ fi
467+
468+ apache_remove_site "$APACHE_MORTY_SITE"
469+}
470+
471+install_nginx_site() {
472+
473+ rst_title "Install nginx site $NGINX_MORTY_SITE"
474+
475+ rst_para "\
476+This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_MORTY_SITE})"
477+
478+ ! nginx_is_installed && err_msg "nginx is not installed."
479+
480+ if ! ask_yn "Do you really want to continue?" Yn; then
481+ return
482+ else
483+ install_nginx
484+ fi
485+
486+ "${REPO_ROOT}/utils/searx.sh" install uwsgi
487+
488+ # shellcheck disable=SC2034
489+ SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC)
490+ # shellcheck disable=SC2034
491+ SEARX_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_URL_PATH)
492+ nginx_install_app "${NGINX_MORTY_SITE}"
493+
494+ info_msg "testing public url .."
495+ if ! service_is_available "${PUBLIC_URL_MORTY}"; then
496+ err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!"
497+ fi
498+}
499+
500+remove_nginx_site() {
501+
502+ rst_title "Remove nginx site $NGINX_MORTY_SITE"
503+
504+ rst_para "\
505+This removes nginx site ${NGINX_MORTY_SITE}."
506+
507+ ! nginx_is_installed && err_msg "nginx is not installed."
508+
509+ if ! ask_yn "Do you really want to continue?" Yn; then
510+ return
511+ fi
512+
513+ nginx_remove_site "$NGINX_MORTY_SITE"
514+
515+}
516+
517+rst-doc() {
518+
519+ eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/morty.rst")\""
520+
521+ echo -e "\n.. START install systemd unit"
522+ cat <<EOF
523+.. tabs::
524+
525+ .. group-tab:: systemd
526+
527+ .. code:: bash
528+
529+EOF
530+ eval "echo \"$(< "${TEMPLATES}/${SERVICE_SYSTEMD_UNIT}")\"" | prefix_stdout " "
531+ echo -e "\n.. END install systemd unit"
532+
533+ # for DIST_NAME in ubuntu-20.04 arch fedora; do
534+ # (
535+ # DIST_ID=${DIST_NAME%-*}
536+ # DIST_VERS=${DIST_NAME#*-}
537+ # [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS=
538+ # # ...
539+ # )
540+ # done
541+}
542+
543+
544+# ----------------------------------------------------------------------------
545+main "$@"
546+# ----------------------------------------------------------------------------
utils/searx.shnew file+869
@@ -0,0 +1,869 @@
1+#!/usr/bin/env bash
2+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
3+# SPDX-License-Identifier: AGPL-3.0-or-later
4+# shellcheck disable=SC2001
5+
6+# shellcheck source=utils/lib.sh
7+source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
8+# shellcheck source=utils/brand.env
9+source "${REPO_ROOT}/utils/brand.env"
10+source_dot_config
11+source "${REPO_ROOT}/utils/lxc-searx.env"
12+in_container && lxc_set_suite_env
13+
14+# ----------------------------------------------------------------------------
15+# config
16+# ----------------------------------------------------------------------------
17+
18+PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
19+
20+SEARX_INTERNAL_HTTP="${SEARX_INTERNAL_HTTP:-127.0.0.1:8888}"
21+
22+SEARX_URL_PATH="${SEARX_URL_PATH:-$(echo "${PUBLIC_URL}" \
23+| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}"
24+[[ "${SEARX_URL_PATH}" == "${PUBLIC_URL}" ]] && SEARX_URL_PATH=/
25+SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(echo "$PUBLIC_URL" \
26+| sed -e 's,^.*://\([^\:/]*\).*,\1,g') }"
27+
28+SERVICE_NAME="searx"
29+SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}"
30+SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}"
31+SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}"
32+# shellcheck disable=SC2034
33+SERVICE_GROUP="${SERVICE_USER}"
34+
35+GIT_BRANCH="${GIT_BRANCH:-master}"
36+SEARX_PYENV="${SERVICE_HOME}/searx-pyenv"
37+SEARX_SRC="${SERVICE_HOME}/searx-src"
38+SEARX_SETTINGS_PATH="/etc/searx/settings.yml"
39+SEARX_UWSGI_APP="searx.ini"
40+# shellcheck disable=SC2034
41+SEARX_UWSGI_SOCKET="/run/uwsgi/app/searx/socket"
42+
43+# apt packages
44+SEARX_PACKAGES_debian="\
45+virtualenv python3-dev python3-babel python3-venv
46+uwsgi uwsgi-plugin-python3
47+git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev
48+shellcheck"
49+
50+BUILD_PACKAGES_debian="\
51+firefox graphviz imagemagick texlive-xetex librsvg2-bin
52+texlive-latex-recommended texlive-extra-utils ttf-dejavu
53+latexmk"
54+
55+# pacman packages
56+SEARX_PACKAGES_arch="\
57+python-virtualenv python python-pip python-lxml python-babel
58+uwsgi uwsgi-plugin-python
59+git base-devel libxml2
60+shellcheck"
61+
62+BUILD_PACKAGES_arch="\
63+firefox graphviz imagemagick texlive-bin extra/librsvg
64+texlive-core texlive-latexextra ttf-dejavu"
65+
66+# dnf packages
67+SEARX_PACKAGES_fedora="\
68+virtualenv python python-pip python-lxml python-babel
69+uwsgi uwsgi-plugin-python3
70+git @development-tools libxml2
71+ShellCheck"
72+
73+BUILD_PACKAGES_fedora="\
74+firefox graphviz graphviz-gd ImageMagick librsvg2-tools
75+texlive-xetex-bin texlive-collection-fontsrecommended
76+texlive-collection-latex dejavu-sans-fonts dejavu-serif-fonts
77+dejavu-sans-mono-fonts"
78+
79+case $DIST_ID-$DIST_VERS in
80+ ubuntu-16.04|ubuntu-18.04)
81+ SEARX_PACKAGES="${SEARX_PACKAGES_debian}"
82+ BUILD_PACKAGES="${BUILD_PACKAGES_debian}"
83+ APACHE_PACKAGES="$APACHE_PACKAGES libapache2-mod-proxy-uwsgi"
84+ ;;
85+ ubuntu-20.04)
86+ # https://askubuntu.com/a/1224710
87+ SEARX_PACKAGES="${SEARX_PACKAGES_debian} python-is-python3"
88+ BUILD_PACKAGES="${BUILD_PACKAGES_debian}"
89+ ;;
90+ ubuntu-*|debian-*)
91+ SEARX_PACKAGES="${SEARX_PACKAGES_debian}"
92+ BUILD_PACKAGES="${BUILD_PACKAGES_debian}"
93+ ;;
94+ arch-*)
95+ SEARX_PACKAGES="${SEARX_PACKAGES_arch}"
96+ BUILD_PACKAGES="${BUILD_PACKAGES_arch}"
97+ ;;
98+ fedora-*)
99+ SEARX_PACKAGES="${SEARX_PACKAGES_fedora}"
100+ BUILD_PACKAGES="${BUILD_PACKAGES_fedora}"
101+ ;;
102+esac
103+
104+# Apache Settings
105+APACHE_SEARX_SITE="searx.conf"
106+
107+# shellcheck disable=SC2034
108+CONFIG_FILES=(
109+ "${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP}"
110+)
111+
112+# shellcheck disable=SC2034
113+CONFIG_BACKUP_ENCRYPTED=(
114+ "${SEARX_SETTINGS_PATH}"
115+)
116+
117+# ----------------------------------------------------------------------------
118+usage() {
119+# ----------------------------------------------------------------------------
120+
121+ # shellcheck disable=SC1117
122+ cat <<EOF
123+usage::
124+ $(basename "$0") shell
125+ $(basename "$0") install [all|user|searx-src|pyenv|uwsgi|packages|buildhost]
126+ $(basename "$0") update [searx]
127+ $(basename "$0") remove [all|user|pyenv|searx-src]
128+ $(basename "$0") activate [service]
129+ $(basename "$0") deactivate [service]
130+ $(basename "$0") inspect [service]
131+ $(basename "$0") option [debug-[on|off]|image-proxy-[on|off]|result-proxy <url> <key>]
132+ $(basename "$0") apache [install|remove]
133+
134+shell
135+ start interactive shell from user ${SERVICE_USER}
136+install / remove
137+ :all: complete (de-) installation of searx service
138+ :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME)
139+ :searx-src: clone $GIT_URL
140+ :pyenv: create/remove virtualenv (python) in $SEARX_PYENV
141+ :uwsgi: install searx uWSGI application
142+ :settings: reinstall settings from ${REPO_ROOT}/searx/settings.yml
143+ :packages: install needed packages from OS package manager
144+ :buildhost: install packages from OS package manager needed by buildhosts
145+update searx
146+ Update searx installation ($SERVICE_HOME)
147+activate service
148+ activate and start service daemon (systemd unit)
149+deactivate service
150+ stop and deactivate service daemon (systemd unit)
151+inspect service
152+ run some small tests and inspect service's status and log
153+option
154+ set one of the available options
155+apache
156+ :install: apache site with the searx uwsgi app
157+ :remove: apache site ${APACHE_FILTRON_SITE}
158+
159+searx settings: ${SEARX_SETTINGS_PATH}
160+
161+If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file::
162+ PUBLIC_URL : ${PUBLIC_URL}
163+ SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME}
164+ SERVICE_USER : ${SERVICE_USER}
165+ SEARX_INTERNAL_HTTP : http://${SEARX_INTERNAL_HTTP}
166+EOF
167+ if in_container; then
168+ # searx is listening on 127.0.0.1 and not available from outside container
169+ # in containers the service is listening on 0.0.0.0 (see lxc-searx.env)
170+ echo -e "${_BBlack}HINT:${_creset} searx only listen on loopback device" \
171+ "${_BBlack}inside${_creset} the container."
172+ for ip in $(global_IPs) ; do
173+ if [[ $ip =~ .*:.* ]]; then
174+ echo " container (IPv6): [${ip#*|}]"
175+ else
176+ # IPv4:
177+ echo " container (IPv4): ${ip#*|}"
178+ fi
179+ done
180+ fi
181+ [[ -n ${1} ]] && err_msg "$1"
182+}
183+
184+main() {
185+ required_commands \
186+ sudo systemctl install git wget curl \
187+ || exit
188+
189+ local _usage="unknown or missing $1 command $2"
190+
191+ case $1 in
192+ --getenv) var="$2"; echo "${!var}"; exit 0;;
193+ -h|--help) usage; exit 0;;
194+ shell)
195+ sudo_or_exit
196+ interactive_shell "${SERVICE_USER}"
197+ ;;
198+ inspect)
199+ case $2 in
200+ service)
201+ sudo_or_exit
202+ inspect_service
203+ ;;
204+ *) usage "$_usage"; exit 42;;
205+ esac ;;
206+ install)
207+ rst_title "$SEARX_INSTANCE_NAME" part
208+ sudo_or_exit
209+ case $2 in
210+ all) install_all ;;
211+ user) assert_user ;;
212+ pyenv) create_pyenv ;;
213+ searx-src) clone_searx ;;
214+ settings) install_settings ;;
215+ uwsgi)
216+ install_searx_uwsgi
217+ if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then
218+ err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check searx & uwsgi setup!"
219+ fi
220+ ;;
221+ packages)
222+ pkg_install "$SEARX_PACKAGES"
223+ ;;
224+ buildhost)
225+ pkg_install "$SEARX_PACKAGES"
226+ pkg_install "$BUILD_PACKAGES"
227+ ;;
228+ *) usage "$_usage"; exit 42;;
229+ esac ;;
230+ update)
231+ sudo_or_exit
232+ case $2 in
233+ searx) update_searx;;
234+ *) usage "$_usage"; exit 42;;
235+ esac ;;
236+ remove)
237+ sudo_or_exit
238+ case $2 in
239+ all) remove_all;;
240+ user) drop_service_account "${SERVICE_USER}";;
241+ pyenv) remove_pyenv ;;
242+ searx-src) remove_searx ;;
243+ *) usage "$_usage"; exit 42;;
244+ esac ;;
245+ activate)
246+ sudo_or_exit
247+ case $2 in
248+ service)
249+ activate_service ;;
250+ *) usage "$_usage"; exit 42;;
251+ esac ;;
252+ deactivate)
253+ sudo_or_exit
254+ case $2 in
255+ service) deactivate_service ;;
256+ *) usage "$_usage"; exit 42;;
257+ esac ;;
258+ option)
259+ sudo_or_exit
260+ case $2 in
261+ debug-on) echo; enable_debug ;;
262+ debug-off) echo; disable_debug ;;
263+ result-proxy) set_result_proxy "$3" "$4" ;;
264+ image-proxy-on) enable_image_proxy ;;
265+ image-proxy-off) disable_image_proxy ;;
266+ *) usage "$_usage"; exit 42;;
267+ esac ;;
268+ apache)
269+ sudo_or_exit
270+ case $2 in
271+ install) install_apache_site ;;
272+ remove) remove_apache_site ;;
273+ *) usage "$_usage"; exit 42;;
274+ esac ;;
275+ doc) rst-doc;;
276+ *) usage "unknown or missing command $1"; exit 42;;
277+ esac
278+}
279+
280+_service_prefix=" ${_Yellow}|$SERVICE_USER|${_creset} "
281+
282+install_all() {
283+ rst_title "Install $SEARX_INSTANCE_NAME (service)"
284+ pkg_install "$SEARX_PACKAGES"
285+ wait_key
286+ assert_user
287+ wait_key
288+ clone_searx
289+ wait_key
290+ create_pyenv
291+ wait_key
292+ install_settings
293+ wait_key
294+ test_local_searx
295+ wait_key
296+ install_searx_uwsgi
297+ if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then
298+ err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check searx & uwsgi setup!"
299+ fi
300+ if ask_yn "Do you want to inspect the installation?" Ny; then
301+ inspect_service
302+ fi
303+}
304+
305+update_searx() {
306+ rst_title "Update searx instance"
307+
308+ echo
309+ tee_stderr 0.3 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
310+cd ${SEARX_SRC}
311+git checkout -B "$GIT_BRANCH"
312+git pull
313+pip install -U pip
314+pip install -U setuptools
315+pip install -U wheel
316+pip install -U -e .
317+EOF
318+ install_settings
319+ uWSGI_restart "$SEARX_UWSGI_APP"
320+}
321+
322+remove_all() {
323+ rst_title "De-Install $SEARX_INSTANCE_NAME (service)"
324+
325+ rst_para "\
326+It goes without saying that this script can only be used to remove
327+installations that were installed with this script."
328+
329+ if ! ask_yn "Do you really want to deinstall $SEARX_INSTANCE_NAME?"; then
330+ return
331+ fi
332+ remove_searx_uwsgi
333+ drop_service_account "${SERVICE_USER}"
334+ remove_settings
335+ wait_key
336+ if service_is_available "${PUBLIC_URL}"; then
337+ MSG="** Don't forgett to remove your public site! (${PUBLIC_URL}) **" wait_key 10
338+ fi
339+}
340+
341+assert_user() {
342+ rst_title "user $SERVICE_USER" section
343+ echo
344+ tee_stderr 1 <<EOF | bash | prefix_stdout
345+useradd --shell /bin/bash --system \
346+ --home-dir "$SERVICE_HOME" \
347+ --comment 'Privacy-respecting metasearch engine' $SERVICE_USER
348+mkdir "$SERVICE_HOME"
349+chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
350+groups $SERVICE_USER
351+EOF
352+ #SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)"
353+ #export SERVICE_HOME
354+ #echo "export SERVICE_HOME=$SERVICE_HOME"
355+}
356+
357+clone_is_available() {
358+ [[ -f "$SEARX_SRC/.git/config" ]]
359+}
360+
361+# shellcheck disable=SC2164
362+clone_searx() {
363+ rst_title "Clone searx sources" section
364+ echo
365+ if ! sudo -i -u "$SERVICE_USER" ls -d "$REPO_ROOT" > /dev/null; then
366+ die 42 "user '$SERVICE_USER' missed read permission: $REPO_ROOT"
367+ fi
368+ SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME 2>/dev/null)"
369+ if [[ ! "${SERVICE_HOME}" ]]; then
370+ err_msg "to clone searx sources, user $SERVICE_USER hast to be created first"
371+ return 42
372+ fi
373+ export SERVICE_HOME
374+ git_clone "$REPO_ROOT" "$SEARX_SRC" \
375+ "$GIT_BRANCH" "$SERVICE_USER"
376+
377+ pushd "${SEARX_SRC}" > /dev/null
378+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
379+cd "${SEARX_SRC}"
380+git remote set-url origin ${GIT_URL}
381+git config user.email "$ADMIN_EMAIL"
382+git config user.name "$ADMIN_NAME"
383+git config --list
384+EOF
385+ popd > /dev/null
386+}
387+
388+install_settings() {
389+ rst_title "${SEARX_SETTINGS_PATH}" section
390+ if ! clone_is_available; then
391+ err_msg "you have to install searx first"
392+ exit 42
393+ fi
394+ mkdir -p "$(dirname ${SEARX_SETTINGS_PATH})"
395+
396+ if [[ ! -f ${SEARX_SETTINGS_PATH} ]]; then
397+ info_msg "install settings ${REPO_ROOT}/searx/settings.yml"
398+ info_msg " --> ${SEARX_SETTINGS_PATH}"
399+ cp "${REPO_ROOT}/searx/settings.yml" "${SEARX_SETTINGS_PATH}"
400+ configure_searx
401+ return
402+ fi
403+
404+ rst_para "Diff between origin's setting file (+) and current (-):"
405+ echo
406+ $DIFF_CMD "${SEARX_SETTINGS_PATH}" "${SEARX_SRC}/searx/settings.yml"
407+
408+ local action
409+ choose_one action "What should happen to the settings file? " \
410+ "keep configuration unchanged" \
411+ "use origin settings" \
412+ "start interactiv shell"
413+ case $action in
414+ "keep configuration unchanged")
415+ info_msg "leave settings file unchanged"
416+ ;;
417+ "use origin settings")
418+ backup_file "${SEARX_SETTINGS_PATH}"
419+ info_msg "install origin settings"
420+ cp "${SEARX_SRC}/searx/settings.yml" "${SEARX_SETTINGS_PATH}"
421+ ;;
422+ "start interactiv shell")
423+ backup_file "${SEARX_SETTINGS_PATH}"
424+ echo -e "// exit with [${_BCyan}CTRL-D${_creset}]"
425+ sudo -H -i
426+ rst_para 'Diff between new setting file (-) and current (+):'
427+ echo
428+ $DIFF_CMD "${SEARX_SRC}/searx/settings.yml" "${SEARX_SETTINGS_PATH}"
429+ wait_key
430+ ;;
431+ esac
432+}
433+
434+remove_settings() {
435+ rst_title "remove searx settings" section
436+ echo
437+ info_msg "delete ${SEARX_SETTINGS_PATH}"
438+ rm -f "${SEARX_SETTINGS_PATH}"
439+}
440+
441+remove_searx() {
442+ rst_title "Drop searx sources" section
443+ if ask_yn "Do you really want to drop searx sources ($SEARX_SRC)?"; then
444+ rm -rf "$SEARX_SRC"
445+ else
446+ rst_para "Leave searx sources unchanged."
447+ fi
448+}
449+
450+pyenv_is_available() {
451+ [[ -f "${SEARX_PYENV}/bin/activate" ]]
452+}
453+
454+create_pyenv() {
455+ rst_title "Create virtualenv (python)" section
456+ echo
457+ if [[ ! -f "${SEARX_SRC}/manage.sh" ]]; then
458+ err_msg "to create pyenv for searx, searx has to be cloned first"
459+ return 42
460+ fi
461+ info_msg "create pyenv in ${SEARX_PYENV}"
462+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
463+rm -rf "${SEARX_PYENV}"
464+python3 -m venv "${SEARX_PYENV}"
465+grep -qFs -- 'source ${SEARX_PYENV}/bin/activate' ~/.profile \
466+ || echo 'source ${SEARX_PYENV}/bin/activate' >> ~/.profile
467+EOF
468+ info_msg "inspect python's virtual environment"
469+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
470+command -v python && python --version
471+EOF
472+ wait_key
473+ info_msg "install needed python packages"
474+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
475+pip install -U pip
476+pip install -U setuptools
477+pip install -U wheel
478+pip install -U -e .
479+cd ${SEARX_SRC}
480+pip install -e .
481+EOF
482+}
483+
484+remove_pyenv() {
485+ rst_title "Remove virtualenv (python)" section
486+ if ! ask_yn "Do you really want to drop ${SEARX_PYENV} ?"; then
487+ return
488+ fi
489+ info_msg "remove pyenv activation from ~/.profile"
490+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
491+grep -v 'source ${SEARX_PYENV}/bin/activate' ~/.profile > ~/.profile.##
492+mv ~/.profile.## ~/.profile
493+EOF
494+ rm -rf "${SEARX_PYENV}"
495+}
496+
497+configure_searx() {
498+ rst_title "Configure searx" section
499+ rst_para "Setup searx config located at $SEARX_SETTINGS_PATH"
500+ echo
501+ tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix"
502+cd ${SEARX_SRC}
503+sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "$SEARX_SETTINGS_PATH"
504+sed -i -e "s/{instance_name}/${SEARX_INSTANCE_NAME}/g" "$SEARX_SETTINGS_PATH"
505+EOF
506+}
507+
508+test_local_searx() {
509+ rst_title "Testing searx instance localy" section
510+ echo
511+
512+ if service_is_available "http://${SEARX_INTERNAL_HTTP}" &>/dev/null; then
513+ err_msg "URL/port http://${SEARX_INTERNAL_HTTP} is already in use, you"
514+ err_msg "should stop that service before starting local tests!"
515+ if ! ask_yn "Continue with local tests?"; then
516+ return
517+ fi
518+ fi
519+ sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS_PATH"
520+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
521+export SEARX_SETTINGS_PATH="${SEARX_SETTINGS_PATH}"
522+cd ${SEARX_SRC}
523+timeout 10 python searx/webapp.py &
524+sleep 3
525+curl --location --verbose --head --insecure $SEARX_INTERNAL_HTTP
526+EOF
527+ sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS_PATH"
528+}
529+
530+install_searx_uwsgi() {
531+ rst_title "Install searx's uWSGI app (searx.ini)" section
532+ echo
533+ install_uwsgi
534+ uWSGI_install_app "$SEARX_UWSGI_APP"
535+}
536+
537+remove_searx_uwsgi() {
538+ rst_title "Remove searx's uWSGI app (searx.ini)" section
539+ echo
540+ uWSGI_remove_app "$SEARX_UWSGI_APP"
541+}
542+
543+activate_service() {
544+ rst_title "Activate $SEARX_INSTANCE_NAME (service)" section
545+ echo
546+ uWSGI_enable_app "$SEARX_UWSGI_APP"
547+ uWSGI_restart "$SEARX_UWSGI_APP"
548+}
549+
550+deactivate_service() {
551+ rst_title "De-Activate $SEARX_INSTANCE_NAME (service)" section
552+ echo
553+ uWSGI_disable_app "$SEARX_UWSGI_APP"
554+ uWSGI_restart "$SEARX_UWSGI_APP"
555+}
556+
557+enable_image_proxy() {
558+ info_msg "try to enable image_proxy ..."
559+ tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix"
560+cd ${SEARX_SRC}
561+sed -i -e "s/image_proxy : False/image_proxy : True/g" "$SEARX_SETTINGS_PATH"
562+EOF
563+ uWSGI_restart "$SEARX_UWSGI_APP"
564+}
565+
566+disable_image_proxy() {
567+ info_msg "try to enable image_proxy ..."
568+ tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix"
569+cd ${SEARX_SRC}
570+sed -i -e "s/image_proxy : True/image_proxy : False/g" "$SEARX_SETTINGS_PATH"
571+EOF
572+ uWSGI_restart "$SEARX_UWSGI_APP"
573+}
574+
575+enable_debug() {
576+ warn_msg "Do not enable debug in production enviroments!!"
577+ info_msg "try to enable debug mode ..."
578+ tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix"
579+cd ${SEARX_SRC}
580+sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS_PATH"
581+EOF
582+ uWSGI_restart "$SEARX_UWSGI_APP"
583+}
584+
585+disable_debug() {
586+ info_msg "try to disable debug mode ..."
587+ tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix"
588+cd ${SEARX_SRC}
589+sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS_PATH"
590+EOF
591+ uWSGI_restart "$SEARX_UWSGI_APP"
592+}
593+
594+set_result_proxy() {
595+
596+ # usage: set_result_proxy <URL> [<key>]
597+
598+ info_msg "try to set result proxy: '$1' ($2)"
599+ cp "${SEARX_SETTINGS_PATH}" "${SEARX_SETTINGS_PATH}.bak"
600+ _set_result_proxy "$1" "$2" > "${SEARX_SETTINGS_PATH}"
601+}
602+
603+_set_result_proxy() {
604+ local line
605+ local stage=0
606+ local url=" url: $1"
607+ local key=" key: !!binary \"$2\""
608+ if [[ -z $2 ]]; then
609+ key=
610+ fi
611+
612+ while IFS= read -r line
613+ do
614+ if [[ $stage = 0 ]] || [[ $stage = 2 ]] ; then
615+ if [[ $line =~ ^[[:space:]]*#*[[:space:]]*result_proxy[[:space:]]*:[[:space:]]*$ ]]; then
616+ if [[ $stage = 0 ]]; then
617+ stage=1
618+ echo "result_proxy:"
619+ continue
620+ elif [[ $stage = 2 ]]; then
621+ continue
622+ fi
623+ fi
624+ fi
625+ if [[ $stage = 1 ]] || [[ $stage = 2 ]] ; then
626+ if [[ $line =~ ^[[:space:]]*#*[[:space:]]*url[[:space:]]*:[[:space:]] ]]; then
627+ [[ $stage = 1 ]] && echo "$url"
628+ continue
629+ elif [[ $line =~ ^[[:space:]]*#*[[:space:]]*key[[:space:]]*:[[:space:]] ]]; then
630+ [[ $stage = 1 ]] && [[ -n $key ]] && echo "$key"
631+ continue
632+ elif [[ $line =~ ^[[:space:]]*$ ]]; then
633+ stage=2
634+ fi
635+ fi
636+ echo "$line"
637+ done < "${SEARX_SETTINGS_PATH}.bak"
638+}
639+
640+function has_substring() {
641+ [[ "$1" != "${2/$1/}" ]]
642+}
643+inspect_service() {
644+ rst_title "service status & log"
645+ cat <<EOF
646+
647+sourced ${DOT_CONFIG#"$REPO_ROOT/"} :
648+
649+ PUBLIC_URL : ${PUBLIC_URL}
650+ SEARX_URL_PATH : ${SEARX_URL_PATH}
651+ SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME}
652+ SEARX_INTERNAL_HTTP : ${SEARX_INTERNAL_HTTP}
653+
654+EOF
655+
656+ if service_account_is_available "$SERVICE_USER"; then
657+ info_msg "Service account $SERVICE_USER exists."
658+ else
659+ err_msg "Service account $SERVICE_USER does not exists!"
660+ fi
661+
662+ if pyenv_is_available; then
663+ info_msg "~$SERVICE_USER: python environment is available."
664+ else
665+ err_msg "~$SERVICE_USER: python environment is not available!"
666+ fi
667+
668+ if clone_is_available; then
669+ info_msg "~$SERVICE_USER: Searx software is installed."
670+ else
671+ err_msg "~$SERVICE_USER: Missing searx software!"
672+ fi
673+
674+ if uWSGI_app_enabled "$SEARX_UWSGI_APP"; then
675+ info_msg "uWSGI app $SEARX_UWSGI_APP is enabled."
676+ else
677+ err_msg "uWSGI app $SEARX_UWSGI_APP not enabled!"
678+ fi
679+
680+ uWSGI_app_available "$SEARX_UWSGI_APP" \
681+ || err_msg "uWSGI app $SEARX_UWSGI_APP not available!"
682+
683+ if in_container; then
684+ lxc_suite_info
685+ else
686+ info_msg "public URL --> ${PUBLIC_URL}"
687+ info_msg "internal URL --> http://${SEARX_INTERNAL_HTTP}"
688+ fi
689+
690+ if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then
691+ err_msg "uWSGI app (service) at http://${SEARX_INTERNAL_HTTP} is not available!"
692+ MSG="${_Green}[${_BCyan}CTRL-C${_Green}] to stop or [${_BCyan}KEY${_Green}] to continue"\
693+ wait_key
694+ fi
695+
696+ if ! service_is_available "${PUBLIC_URL}"; then
697+ warn_msg "Public service at ${PUBLIC_URL} is not available!"
698+ if ! in_container; then
699+ warn_msg "Check if public name is correct and routed or use the public IP from above."
700+ fi
701+ fi
702+
703+ local _debug_on
704+ if ask_yn "Enable searx debug mode?"; then
705+ enable_debug
706+ _debug_on=1
707+ fi
708+ echo
709+
710+ case $DIST_ID-$DIST_VERS in
711+ ubuntu-*|debian-*)
712+ systemctl --no-pager -l status "${SERVICE_NAME}"
713+ ;;
714+ arch-*)
715+ systemctl --no-pager -l status "uwsgi@${SERVICE_NAME%.*}"
716+ ;;
717+ fedora-*)
718+ systemctl --no-pager -l status uwsgi
719+ ;;
720+ esac
721+
722+ # shellcheck disable=SC2059
723+ printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log"
724+ read -r -s -n1 -t 5
725+ echo
726+
727+ while true; do
728+ trap break 2
729+ case $DIST_ID-$DIST_VERS in
730+ ubuntu-*|debian-*) tail -f /var/log/uwsgi/app/searx.log ;;
731+ arch-*) journalctl -f -u "uwsgi@${SERVICE_NAME%.*}" ;;
732+ fedora-*) journalctl -f -u uwsgi ;;
733+ esac
734+ done
735+
736+ if [[ $_debug_on == 1 ]]; then
737+ disable_debug
738+ fi
739+ return 0
740+}
741+
742+install_apache_site() {
743+ rst_title "Install Apache site $APACHE_SEARX_SITE"
744+
745+ rst_para "\
746+This installs the searx uwsgi app as apache site. If your server is public to
747+the internet, you should instead use a reverse proxy (filtron) to block
748+excessively bot queries."
749+
750+ ! apache_is_installed && err_msg "Apache is not installed."
751+
752+ if ! ask_yn "Do you really want to continue?" Yn; then
753+ return
754+ else
755+ install_apache
756+ fi
757+
758+ apache_install_site --variant=uwsgi "${APACHE_SEARX_SITE}"
759+
760+ rst_title "Install searx's uWSGI app (searx.ini)" section
761+ echo
762+ uWSGI_install_app --variant=socket "$SEARX_UWSGI_APP"
763+
764+ if ! service_is_available "${PUBLIC_URL}"; then
765+ err_msg "Public service at ${PUBLIC_URL} is not available!"
766+ fi
767+}
768+
769+remove_apache_site() {
770+
771+ rst_title "Remove Apache site ${APACHE_SEARX_SITE}"
772+
773+ rst_para "\
774+This removes apache site ${APACHE_SEARX_SITE}."
775+
776+ ! apache_is_installed && err_msg "Apache is not installed."
777+
778+ if ! ask_yn "Do you really want to continue?" Yn; then
779+ return
780+ fi
781+
782+ apache_remove_site "${APACHE_SEARX_SITE}"
783+
784+ rst_title "Remove searx's uWSGI app (searx.ini)" section
785+ echo
786+ uWSGI_remove_app "$SEARX_UWSGI_APP"
787+}
788+
789+rst-doc() {
790+ local debian="${SEARX_PACKAGES_debian}"
791+ local arch="${SEARX_PACKAGES_arch}"
792+ local fedora="${SEARX_PACKAGES_fedora}"
793+ local debian_build="${BUILD_PACKAGES_debian}"
794+ local arch_build="${BUILD_PACKAGES_arch}"
795+ local fedora_build="${BUILD_PACKAGES_fedora}"
796+ debian="$(echo "${debian}" | sed 's/.*/ & \\/' | sed '$ s/.$//')"
797+ arch="$(echo "${arch}" | sed 's/.*/ & \\/' | sed '$ s/.$//')"
798+ fedora="$(echo "${fedora}" | sed 's/.*/ & \\/' | sed '$ s/.$//')"
799+ debian_build="$(echo "${debian_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')"
800+ arch_build="$(echo "${arch_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')"
801+ fedora_build="$(echo "${fedora_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')"
802+
803+ eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/searx.rst")\""
804+
805+ # I use ubuntu-20.04 here to demonstrate that versions are also suported,
806+ # normaly debian-* and ubuntu-* are most the same.
807+
808+ for DIST_NAME in ubuntu-20.04 arch fedora; do
809+ (
810+ DIST_ID=${DIST_NAME%-*}
811+ DIST_VERS=${DIST_NAME#*-}
812+ [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS=
813+ uWSGI_distro_setup
814+
815+ echo -e "\n.. START searx uwsgi-description $DIST_NAME"
816+
817+ case $DIST_ID-$DIST_VERS in
818+ ubuntu-*|debian-*) cat <<EOF
819+# init.d --> /usr/share/doc/uwsgi/README.Debian.gz
820+# For uWSGI debian uses the LSB init process, this might be changed
821+# one day, see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=833067
822+
823+create ${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP}
824+enable: sudo -H ln -s ${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP} ${uWSGI_APPS_ENABLED}/
825+start: sudo -H service uwsgi start ${SEARX_UWSGI_APP%.*}
826+restart: sudo -H service uwsgi restart ${SEARX_UWSGI_APP%.*}
827+stop: sudo -H service uwsgi stop ${SEARX_UWSGI_APP%.*}
828+disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP}
829+EOF
830+ ;;
831+ arch-*) cat <<EOF
832+# systemd --> /usr/lib/systemd/system/uwsgi@.service
833+# For uWSGI archlinux uses systemd template units, see
834+# - http://0pointer.de/blog/projects/instances.html
835+# - https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd
836+
837+create: ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP}
838+enable: sudo -H systemctl enable uwsgi@${SEARX_UWSGI_APP%.*}
839+start: sudo -H systemctl start uwsgi@${SEARX_UWSGI_APP%.*}
840+restart: sudo -H systemctl restart uwsgi@${SEARX_UWSGI_APP%.*}
841+stop: sudo -H systemctl stop uwsgi@${SEARX_UWSGI_APP%.*}
842+disable: sudo -H systemctl disable uwsgi@${SEARX_UWSGI_APP%.*}
843+EOF
844+ ;;
845+ fedora-*) cat <<EOF
846+# systemd --> /usr/lib/systemd/system/uwsgi.service
847+# The unit file starts uWSGI in emperor mode (/etc/uwsgi.ini), see
848+# - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html
849+
850+create: ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP}
851+restart: sudo -H touch ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP}
852+disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP}
853+EOF
854+ ;;
855+ esac
856+ echo -e ".. END searx uwsgi-description $DIST_NAME"
857+
858+ echo -e "\n.. START searx uwsgi-appini $DIST_NAME"
859+ eval "echo \"$(< "${TEMPLATES}/${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP}")\""
860+ echo -e "\n.. END searx uwsgi-appini $DIST_NAME"
861+
862+ )
863+ done
864+
865+}
866+
867+# ----------------------------------------------------------------------------
868+main "$@"
869+# ----------------------------------------------------------------------------
utils/site-python/sphinx_build_tools.pynew file+48
@@ -0,0 +1,48 @@
1+# -*- coding: utf-8; mode: python -*-
2+"""Implement some sphinx-build tools.
3+
4+"""
5+
6+import os
7+import sys
8+from sphinx.util.pycompat import execfile_
9+
10+# ------------------------------------------------------------------------------
11+def load_sphinx_config(namespace):
12+# ------------------------------------------------------------------------------
13+
14+ u"""Load an additional configuration file into *namespace*.
15+
16+ The name of the configuration file is taken from the environment
17+ ``SPHINX_CONF``. The external configuration file extends (or overwrites) the
18+ configuration values from the origin ``conf.py``. With this you are able to
19+ maintain *build themes*. To your docs/conf.py add::
20+
21+ from sphinx_build_tools import load_sphinx_config
22+ ...
23+
24+ # Since loadConfig overwrites settings from the global namespace, it has to be
25+ # the last statement in the conf.py file
26+
27+ load_sphinx_config(globals())
28+
29+ """
30+
31+ config_file = os.environ.get("SPHINX_CONF", None)
32+ if (config_file is not None
33+ and os.path.normpath(namespace["__file__"]) != os.path.normpath(config_file) ):
34+ config_file = os.path.abspath(config_file)
35+
36+ if os.path.isfile(config_file):
37+ sys.stdout.write(
38+ "load additional sphinx-config: %s\n"
39+ % config_file)
40+ config = namespace.copy()
41+ config['__file__'] = config_file
42+ execfile_(config_file, config)
43+ del config['__file__']
44+ namespace.update(config)
45+ else:
46+ sys.stderr.write(
47+ "WARNING: additional sphinx-config not found: %s\n"
48+ % config_file)
utils/templates/etc/apache2new file+1
@@ -0,0 +1 @@
1+httpd
utils/templates/etc/filtron/rules.jsonnew file+129
@@ -0,0 +1,129 @@
1+[
2+ {
3+ "name": "roboagent limit",
4+ "filters": [
5+ "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client|Ruby|UniversalFeedParser)"
6+ ],
7+ "limit": 0,
8+ "stop": true,
9+ "actions": [
10+ { "name": "log"},
11+ { "name": "block",
12+ "params": {
13+ "message": "Rate limit exceeded"
14+ }
15+ }
16+ ]
17+ },
18+ {
19+ "name": "botlimit",
20+ "filters": [
21+ "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
22+ ],
23+ "limit": 0,
24+ "stop": true,
25+ "actions": [
26+ { "name": "log"},
27+ { "name": "block",
28+ "params": {
29+ "message": "Rate limit exceeded"
30+ }
31+ }
32+ ]
33+ },
34+ {
35+ "name": "suspiciously frequent IP",
36+ "filters": [],
37+ "interval": 600,
38+ "limit": 30,
39+ "aggregations": [
40+ "Header:X-Forwarded-For"
41+ ],
42+ "actions":[
43+ {"name":"log"}
44+ ]
45+ },
46+ {
47+ "name": "search request",
48+ "filters": [
49+ "Param:q",
50+ "Path=^(/|/search)$"
51+ ],
52+ "interval": 61,
53+ "limit": 999,
54+ "subrules": [
55+ {
56+ "name": "missing Accept-Language",
57+ "filters": ["!Header:Accept-Language"],
58+ "limit": 0,
59+ "stop": true,
60+ "actions": [
61+ {"name":"log"},
62+ {"name": "block",
63+ "params": {"message": "Rate limit exceeded"}}
64+ ]
65+ },
66+ {
67+ "name": "suspiciously Connection=close header",
68+ "filters": ["Header:Connection=close"],
69+ "limit": 0,
70+ "stop": true,
71+ "actions": [
72+ {"name":"log"},
73+ {"name": "block",
74+ "params": {"message": "Rate limit exceeded"}}
75+ ]
76+ },
77+ {
78+ "name": "IP limit",
79+ "interval": 61,
80+ "limit": 9,
81+ "stop": true,
82+ "aggregations": [
83+ "Header:X-Forwarded-For"
84+ ],
85+ "actions": [
86+ { "name": "log"},
87+ { "name": "block",
88+ "params": {
89+ "message": "Rate limit exceeded"
90+ }
91+ }
92+ ]
93+ },
94+ {
95+ "name": "rss/json limit",
96+ "filters": [
97+ "Param:format=(csv|json|rss)"
98+ ],
99+ "interval": 121,
100+ "limit": 2,
101+ "stop": true,
102+ "actions": [
103+ { "name": "log"},
104+ { "name": "block",
105+ "params": {
106+ "message": "Rate limit exceeded"
107+ }
108+ }
109+ ]
110+ },
111+ {
112+ "name": "useragent limit",
113+ "interval": 61,
114+ "limit": 199,
115+ "aggregations": [
116+ "Header:User-Agent"
117+ ],
118+ "actions": [
119+ { "name": "log"},
120+ { "name": "block",
121+ "params": {
122+ "message": "Rate limit exceeded"
123+ }
124+ }
125+ ]
126+ }
127+ ]
128+ }
129+]
utils/templates/etc/httpd/sites-available/morty.confnew file+28
@@ -0,0 +1,28 @@
1+# -*- coding: utf-8; mode: apache -*-
2+
3+LoadModule headers_module ${APACHE_MODULES}/mod_headers.so
4+LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so
5+LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so
6+#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so
7+
8+# SetEnvIf Request_URI "${PUBLIC_URL_PATH_MORTY}" dontlog
9+# CustomLog /dev/null combined env=dontlog
10+
11+<Location ${PUBLIC_URL_PATH_MORTY} >
12+
13+ <IfModule mod_security2.c>
14+ SecRuleEngine Off
15+ </IfModule>
16+
17+ Require all granted
18+
19+ Order deny,allow
20+ Deny from all
21+ #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
22+ Allow from all
23+
24+ ProxyPreserveHost On
25+ ProxyPass http://${MORTY_LISTEN}
26+ RequestHeader set X-Script-Name ${PUBLIC_URL_PATH_MORTY}
27+
28+</Location>
utils/templates/etc/httpd/sites-available/searx.conf:filtronnew file+33
@@ -0,0 +1,33 @@
1+# -*- coding: utf-8; mode: apache -*-
2+
3+LoadModule headers_module ${APACHE_MODULES}/mod_headers.so
4+LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so
5+LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so
6+#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so
7+
8+# SetEnvIf Request_URI "${FILTRON_URL_PATH}" dontlog
9+# CustomLog /dev/null combined env=dontlog
10+
11+# SecRuleRemoveById 981054
12+# SecRuleRemoveById 981059
13+# SecRuleRemoveById 981060
14+# SecRuleRemoveById 950907
15+
16+<Location ${FILTRON_URL_PATH} >
17+
18+ <IfModule mod_security2.c>
19+ SecRuleEngine Off
20+ </IfModule>
21+
22+ Require all granted
23+
24+ Order deny,allow
25+ Deny from all
26+ #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
27+ Allow from all
28+
29+ ProxyPreserveHost On
30+ ProxyPass http://${FILTRON_LISTEN}
31+ RequestHeader set X-Script-Name ${FILTRON_URL_PATH}
32+
33+</Location>
utils/templates/etc/httpd/sites-available/searx.conf:uwsginew file+27
@@ -0,0 +1,27 @@
1+# -*- coding: utf-8; mode: apache -*-
2+
3+LoadModule headers_module ${APACHE_MODULES}/mod_headers.so
4+LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so
5+LoadModule proxy_uwsgi_module ${APACHE_MODULES}/mod_proxy_uwsgi.so
6+# LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so
7+
8+# SetEnvIf Request_URI "${SEARX_URL_PATH}" dontlog
9+# CustomLog /dev/null combined env=dontlog
10+
11+<Location ${SEARX_URL_PATH}>
12+
13+ <IfModule mod_security2.c>
14+ SecRuleEngine Off
15+ </IfModule>
16+
17+ Require all granted
18+
19+ Order deny,allow
20+ Deny from all
21+ # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
22+ Allow from all
23+
24+ ProxyPreserveHost On
25+ ProxyPass unix:${SEARX_UWSGI_SOCKET}|uwsgi://uwsgi-uds-searx/
26+
27+</Location>
utils/templates/etc/nginx/default.apps-available/morty.confnew file+11
@@ -0,0 +1,11 @@
1+# https://example.org/morty
2+
3+location /morty {
4+ proxy_pass http://127.0.0.1:3000/;
5+
6+ proxy_set_header Host \$http_host;
7+ proxy_set_header Connection \$http_connection;
8+ proxy_set_header X-Real-IP \$remote_addr;
9+ proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
10+ proxy_set_header X-Scheme \$scheme;
11+}
utils/templates/etc/nginx/default.apps-available/searx.conf:filtronnew file+16
@@ -0,0 +1,16 @@
1+# https://example.org/searx
2+
3+location ${SEARX_URL_PATH} {
4+ proxy_pass http://127.0.0.1:4004/;
5+
6+ proxy_set_header Host \$http_host;
7+ proxy_set_header Connection \$http_connection;
8+ proxy_set_header X-Real-IP \$remote_addr;
9+ proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
10+ proxy_set_header X-Scheme \$scheme;
11+ proxy_set_header X-Script-Name ${SEARX_URL_PATH};
12+}
13+
14+location ${SEARX_URL_PATH}/static {
15+ alias ${SEARX_SRC}/searx/static;
16+}
utils/templates/etc/uwsgi/apps-archlinux/searx.ininew file+80
@@ -0,0 +1,80 @@
1+[uwsgi]
2+
3+# uWSGI core
4+# ----------
5+#
6+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
7+
8+# Who will run the code
9+uid = ${SERVICE_USER}
10+gid = ${SERVICE_GROUP}
11+
12+# chdir to specified directory before apps loading
13+chdir = ${SEARX_SRC}/searx
14+
15+# searx configuration (settings.yml)
16+env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH}
17+
18+# disable logging for privacy
19+logger = systemd
20+disable-logging = true
21+
22+# The right granted on the created socket
23+chmod-socket = 666
24+
25+# Plugin to use and interpretor config
26+single-interpreter = true
27+
28+# enable master process
29+master = true
30+
31+# load apps in each worker instead of the master
32+lazy-apps = true
33+
34+# load uWSGI plugins
35+plugin = python
36+
37+# By default the Python plugin does not initialize the GIL. This means your
38+# app-generated threads will not run. If you need threads, remember to enable
39+# them with enable-threads. Running uWSGI in multithreading mode (with the
40+# threads options) will automatically enable threading support. This *strange*
41+# default behaviour is for performance reasons.
42+enable-threads = true
43+
44+
45+# plugin: python
46+# --------------
47+#
48+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python
49+
50+# load a WSGI module
51+module = searx.webapp
52+
53+# set PYTHONHOME/virtualenv
54+virtualenv = ${SEARX_PYENV}
55+
56+# add directory (or glob) to pythonpath
57+pythonpath = ${SEARX_SRC}
58+
59+
60+# speak to upstream
61+# -----------------
62+#
63+# Activate the 'http' configuration for filtron or activate the 'socket'
64+# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
65+
66+# using IP:
67+#
68+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
69+# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
70+
71+http = ${SEARX_INTERNAL_HTTP}
72+
73+# using unix-sockets:
74+#
75+# On some distributions you need to create the app folder for the sockets::
76+#
77+# mkdir -p /run/uwsgi/app/searx
78+# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
79+#
80+# socket = /run/uwsgi/app/searx/socket
utils/templates/etc/uwsgi/apps-archlinux/searx.ini:socketnew file+80
@@ -0,0 +1,80 @@
1+[uwsgi]
2+
3+# uWSGI core
4+# ----------
5+#
6+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
7+
8+# Who will run the code
9+uid = ${SERVICE_USER}
10+gid = ${SERVICE_GROUP}
11+
12+# chdir to specified directory before apps loading
13+chdir = ${SEARX_SRC}/searx
14+
15+# searx configuration (settings.yml)
16+env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH}
17+
18+# disable logging for privacy
19+logger = systemd
20+disable-logging = true
21+
22+# The right granted on the created socket
23+chmod-socket = 666
24+
25+# Plugin to use and interpretor config
26+single-interpreter = true
27+
28+# enable master process
29+master = true
30+
31+# load apps in each worker instead of the master
32+lazy-apps = true
33+
34+# load uWSGI plugins
35+plugin = python
36+
37+# By default the Python plugin does not initialize the GIL. This means your
38+# app-generated threads will not run. If you need threads, remember to enable
39+# them with enable-threads. Running uWSGI in multithreading mode (with the
40+# threads options) will automatically enable threading support. This *strange*
41+# default behaviour is for performance reasons.
42+enable-threads = true
43+
44+
45+# plugin: python
46+# --------------
47+#
48+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python
49+
50+# load a WSGI module
51+module = searx.webapp
52+
53+# set PYTHONHOME/virtualenv
54+virtualenv = ${SEARX_PYENV}
55+
56+# add directory (or glob) to pythonpath
57+pythonpath = ${SEARX_SRC}
58+
59+
60+# speak to upstream
61+# -----------------
62+#
63+# Activate the 'http' configuration for filtron or activate the 'socket'
64+# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
65+
66+# using IP:
67+#
68+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
69+# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
70+
71+# http = ${SEARX_INTERNAL_HTTP}
72+
73+# using unix-sockets:
74+#
75+# On some distributions you need to create the app folder for the sockets::
76+#
77+# mkdir -p /run/uwsgi/app/searx
78+# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
79+#
80+socket = /run/uwsgi/app/searx/socket
utils/templates/etc/uwsgi/apps-available/searx.ininew file+79
@@ -0,0 +1,79 @@
1+[uwsgi]
2+
3+# uWSGI core
4+# ----------
5+#
6+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
7+
8+# Who will run the code
9+uid = ${SERVICE_USER}
10+gid = ${SERVICE_GROUP}
11+
12+# chdir to specified directory before apps loading
13+chdir = ${SEARX_SRC}/searx
14+
15+# searx configuration (settings.yml)
16+env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH}
17+
18+# disable logging for privacy
19+disable-logging = true
20+
21+# The right granted on the created socket
22+chmod-socket = 666
23+
24+# Plugin to use and interpretor config
25+single-interpreter = true
26+
27+# enable master process
28+master = true
29+
30+# load apps in each worker instead of the master
31+lazy-apps = true
32+
33+# load uWSGI plugins
34+plugin = python3,http
35+
36+# By default the Python plugin does not initialize the GIL. This means your
37+# app-generated threads will not run. If you need threads, remember to enable
38+# them with enable-threads. Running uWSGI in multithreading mode (with the
39+# threads options) will automatically enable threading support. This *strange*
40+# default behaviour is for performance reasons.
41+enable-threads = true
42+
43+
44+# plugin: python
45+# --------------
46+#
47+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python
48+
49+# load a WSGI module
50+module = searx.webapp
51+
52+# set PYTHONHOME/virtualenv
53+virtualenv = ${SEARX_PYENV}
54+
55+# add directory (or glob) to pythonpath
56+pythonpath = ${SEARX_SRC}
57+
58+
59+# speak to upstream
60+# -----------------
61+#
62+# Activate the 'http' configuration for filtron or activate the 'socket'
63+# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
64+
65+# using IP:
66+#
67+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
68+# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
69+
70+http = ${SEARX_INTERNAL_HTTP}
71+
72+# using unix-sockets:
73+#
74+# On some distributions you need to create the app folder for the sockets::
75+#
76+# mkdir -p /run/uwsgi/app/searx
77+# chmod -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
78+#
79+# socket = /run/uwsgi/app/searx/socket
utils/templates/etc/uwsgi/apps-available/searx.ini:socketnew file+79
@@ -0,0 +1,79 @@
1+[uwsgi]
2+
3+# uWSGI core
4+# ----------
5+#
6+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
7+
8+# Who will run the code
9+uid = ${SERVICE_USER}
10+gid = ${SERVICE_GROUP}
11+
12+# chdir to specified directory before apps loading
13+chdir = ${SEARX_SRC}/searx
14+
15+# searx configuration (settings.yml)
16+env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH}
17+
18+# disable logging for privacy
19+disable-logging = true
20+
21+# The right granted on the created socket
22+chmod-socket = 666
23+
24+# Plugin to use and interpretor config
25+single-interpreter = true
26+
27+# enable master process
28+master = true
29+
30+# load apps in each worker instead of the master
31+lazy-apps = true
32+
33+# load uWSGI plugins
34+plugin = python3,http
35+
36+# By default the Python plugin does not initialize the GIL. This means your
37+# app-generated threads will not run. If you need threads, remember to enable
38+# them with enable-threads. Running uWSGI in multithreading mode (with the
39+# threads options) will automatically enable threading support. This *strange*
40+# default behaviour is for performance reasons.
41+enable-threads = true
42+
43+
44+# plugin: python
45+# --------------
46+#
47+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python
48+
49+# load a WSGI module
50+module = searx.webapp
51+
52+# set PYTHONHOME/virtualenv
53+virtualenv = ${SEARX_PYENV}
54+
55+# add directory (or glob) to pythonpath
56+pythonpath = ${SEARX_SRC}
57+
58+
59+# speak to upstream
60+# -----------------
61+#
62+# Activate the 'http' configuration for filtron or activate the 'socket'
63+# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
64+
65+# using IP:
66+#
67+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
68+# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
69+
70+# http = ${SEARX_INTERNAL_HTTP}
71+
72+# using unix-sockets:
73+#
74+# On some distributions you need to create the app folder for the sockets::
75+#
76+# mkdir -p /run/uwsgi/app/searx
77+# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
78+#
79+socket = /run/uwsgi/app/searx/socket
utils/templates/lib/systemd/system/filtron.servicenew file+29
@@ -0,0 +1,29 @@
1+[Unit]
2+
3+Description=${SERVICE_NAME}
4+After=syslog.target
5+After=network.target
6+
7+[Service]
8+
9+Type=simple
10+User=${SERVICE_USER}
11+Group=${SERVICE_GROUP}
12+WorkingDirectory=${SERVICE_HOME}
13+ExecStart=${SERVICE_HOME}/go-apps/bin/filtron -api '${FILTRON_API}' -listen '${FILTRON_LISTEN}' -rules '${FILTRON_RULES}' -target '${FILTRON_TARGET}'
14+
15+Restart=always
16+Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME}
17+
18+# Some distributions may not support these hardening directives. If you cannot
19+# start the service due to an unknown option, comment out the ones not supported
20+# by your version of systemd.
21+
22+ProtectSystem=full
23+PrivateDevices=yes
24+PrivateTmp=yes
25+NoNewPrivileges=true
26+
27+[Install]
28+
29+WantedBy=multi-user.target
utils/templates/lib/systemd/system/morty.servicenew file+29
@@ -0,0 +1,29 @@
1+[Unit]
2+
3+Description=${SERVICE_NAME}
4+After=syslog.target
5+After=network.target
6+
7+[Service]
8+
9+Type=simple
10+User=${SERVICE_USER}
11+Group=${SERVICE_GROUP}
12+WorkingDirectory=${SERVICE_HOME}
13+ExecStart=${SERVICE_HOME}/go-apps/bin/morty -key '${MORTY_KEY}' -listen '${MORTY_LISTEN}' -timeout ${MORTY_TIMEOUT}
14+
15+Restart=always
16+Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME} DEBUG=${SERVICE_ENV_DEBUG}
17+
18+# Some distributions may not support these hardening directives. If you cannot
19+# start the service due to an unknown option, comment out the ones not supported
20+# by your version of systemd.
21+
22+ProtectSystem=full
23+PrivateDevices=yes
24+PrivateTmp=yes
25+NoNewPrivileges=true
26+
27+[Install]
28+
29+WantedBy=multi-user.target