on master help sign in
Changes from the installation tests on (all) LXC containers.

Tested and fixed HTTP & uWSGI installation on: ubu1604 ubu1804 ubu1910 ubu2004 fedora31 archlinux Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

commitf693149cded4f783380f8f02154bd9288b72cdd5 authorMarkus Heiser <markus.heiser@darmarit.de> dateWed 8 Apr 2020, 18:38 (7 years ago) parentee39a09
13 files changed+262−24
docs/utils/index.rst+12−4 docs/utils/lxc.sh.rst+5−1 utils/filtron.sh+4−3 utils/lib.sh+58−3 utils/lxc.sh−1 utils/morty.sh+9−5 utils/searx.sh+9−1 utils/templates/etc/httpd/sites-available/morty.conf+1−1 utils/templates/etc/httpd/sites-available/searx.conf:filtron+1−1 utils/templates/etc/uwsgi/apps-archlinux/searx.ini+2−2 utils/templates/etc/uwsgi/apps-archlinux/searx.ini:socket+80 utils/templates/etc/uwsgi/apps-available/searx.ini+2−2 utils/templates/etc/uwsgi/apps-available/searx.ini:socket+79
docs/utils/index.rst+12−4
@@ -19,20 +19,28 @@ developers.
1919
2020.. _toolboxing common:
2121
22−Common commands
23−===============
22+Common commands & environment
23+=============================
2424
2525Scripts to maintain services often dispose of common commands and environments.
2626
27−``shell``:
27+``shell`` : command
2828 Opens a shell from the service user ``${SERVICE_USSR}``, very helpful for
2929 troubleshooting.
3030
31−``inspect service``:
31+``inspect service`` : command
3232 Shows status and log of the service, most often you have a option to enable
3333 more verbose debug logs. Very helpful for debugging, but be careful not to
3434 enable debugging in a production environment!
3535
36+``FORCE_TIMEOUT`` : environment
37+ Sets timeout for interactive prompts. If you want to run a script in batch
38+ job, with defaults choices, set ``FORCE_TIMEOUT=0``. By example; to install a
39+ reverse proxy for filtron on all containers of the :ref:`searx suite
40+ <lxc-searx.env>` use ::
41+
42+ sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh apache install
43+
3644.. _toolboxing setup:
3745
3846Tooling box setup
docs/utils/lxc.sh.rst+5−1
@@ -88,7 +88,11 @@ WEB-Browser::
8888 [searx-fedora31] INFO: (eth0) filtron: http://n.n.n.18:4004/
8989 [searx-archlinux] INFO: (eth0) filtron: http://n.n.n.12:4004/
9090
91−
91+To install a reverse proxy for filtron and morty use::
92+
93+ sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh apache install
94+ sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/morty.sh apache install
95+
9296Running commands
9397================
9498
utils/filtron.sh+4−3
@@ -23,7 +23,6 @@ FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \
2323[[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/
2424
2525FILTRON_ETC="/etc/filtron"
26−
2726FILTRON_RULES="$FILTRON_ETC/rules.json"
2827
2928FILTRON_API="${FILTRON_API:-127.0.0.1:4005}"
@@ -447,7 +446,8 @@ This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SIT
447446 install_apache
448447 fi
449448
450− echo
449+ "${REPO_ROOT}/utils/searx.sh" install uwsgi
450+
451451 apache_install_site --variant=filtron "${APACHE_FILTRON_SITE}"
452452
453453 info_msg "testing public url .."
@@ -465,11 +465,12 @@ This removes apache site ${APACHE_FILTRON_SITE}."
465465
466466 ! apache_is_installed && err_msg "Apache is not installed."
467467
468− if ! ask_yn "Do you really want to continue?"; then
468+ if ! ask_yn "Do you really want to continue?" Yn; then
469469 return
470470 fi
471471
472472 apache_remove_site "$APACHE_FILTRON_SITE"
473+
473474}
474475
475476rst-doc() {
utils/lib.sh+58−3
@@ -389,7 +389,7 @@ install_template() {
389389 local chmod="${pos_args[4]-644}"
390390
391391 info_msg "install (eval=$do_eval): ${dst}"
392− [[ -n $variant ]] && info_msg "variant: ${variant}"
392+ [[ -n $variant ]] && info_msg "variant --> ${variant}"
393393
394394 if [[ ! -f "${template_origin}" ]] ; then
395395 err_msg "${template_origin} does not exists"
@@ -777,6 +777,7 @@ apache_dissable_site() {
777777 ln -s "${APACHE_SITES_AVAILABLE}/${CONF}" "${APACHE_SITES_ENABLED}/${CONF}"
778778 ;;
779779 esac
780+ apache_reload
780781}
781782
782783# uWSGI
@@ -846,7 +847,7 @@ uWSGI_restart() {
846847 if uWSGI_app_available "${CONF}"; then
847848 systemctl restart "uwsgi@${CONF%.*}"
848849 else
849− info_msg "in systemd template mode: ${CONF} not installed (nothing to restart)"
850+ info_msg "[uWSGI:systemd-template] ${CONF} not installed (no need to restart)"
850851 fi
851852 ;;
852853 fedora-*)
@@ -854,7 +855,7 @@ uWSGI_restart() {
854855 if uWSGI_app_enabled "${CONF}"; then
855856 touch "${uWSGI_APPS_ENABLED}/${CONF}"
856857 else
857− info_msg "in uWSGI emperor mode: ${CONF} not installed (nothing to restart)"
858+ info_msg "[uWSGI:emperor] ${CONF} not installed (no need to restart)"
858859 fi
859860 ;;
860861 *)
@@ -864,6 +865,32 @@ uWSGI_restart() {
864865 esac
865866}
866867
868+uWSGI_prepare_app() {
869+
870+ # usage: uWSGI_prepare_app <myapp.ini>
871+
872+ local APP="${1%.*}"
873+ if [[ -z $APP ]]; then
874+ err_msg "uWSGI_prepare_app: missing arguments"
875+ return 42
876+ fi
877+
878+ case $DIST_ID-$DIST_VERS in
879+ fedora-*)
880+ # in emperor mode, the uwsgi user is the owner of the sockets
881+ info_msg "prepare (uwsgi:uwsgi) /run/uwsgi/app/${APP}"
882+ mkdir -p "/run/uwsgi/app/${APP}"
883+ chown -R "uwsgi:uwsgi" "/run/uwsgi/app/${APP}"
884+ ;;
885+ *)
886+ info_msg "prepare (${SERVICE_USER}:${SERVICE_GROUP}) /run/uwsgi/app/${APP}"
887+ mkdir -p "/run/uwsgi/app/${APP}"
888+ chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "/run/uwsgi/app/${APP}"
889+ ;;
890+ esac
891+}
892+
893+
867894uWSGI_app_available() {
868895 # usage: uWSGI_app_available <myapp.ini>
869896 local CONF="$1"
@@ -888,6 +915,7 @@ uWSGI_install_app() {
888915 *) pos_args+=("$i");;
889916 esac
890917 done
918+ uWSGI_prepare_app "${pos_args[1]}"
891919 mkdir -p "${uWSGI_APPS_AVAILABLE}"
892920 install_template "${template_opts[@]}" \
893921 "${uWSGI_APPS_AVAILABLE}/${pos_args[1]}" \
@@ -1281,3 +1309,30 @@ global_IPs(){
12811309
12821310 ip -o addr show | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\1|\2/p'
12831311}
1312+
1313+primary_ip() {
1314+
1315+ case $DIST_ID in
1316+ arch)
1317+ echo "$(ip -o addr show \
1318+ | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\2/p' \
1319+ | head -n 1)"
1320+ ;;
1321+ *) echo "$(hostname -I | cut -d' ' -f1)" ;;
1322+ esac
1323+}
1324+
1325+# URL
1326+# ---
1327+
1328+url_replace_hostname(){
1329+
1330+ # usage: url_replace_hostname <url> <new hostname>
1331+
1332+ # to replace hostname by primary IP::
1333+ #
1334+ # url_replace_hostname http://searx-ubu1604/morty $(primary_ip)
1335+ # http://10.246.86.250/morty
1336+
1337+ echo "$1" | sed "s|\(http[s]*://\)[^/]*\(.*\)|\1$2\2|"
1338+}
utils/lxc.sh−1
@@ -438,7 +438,6 @@ lxc_cmd() {
438438 else
439439 info_msg "lxc $* $i"
440440 lxc "$@" "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] "
441− echo
442441 fi
443442 done
444443}
utils/morty.sh+9−5
@@ -14,12 +14,16 @@ in_container && lxc_set_suite_env
1414# config
1515# ----------------------------------------------------------------------------
1616
17−PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
18−PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}"
17+MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}"
1918PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty}"
20−PUBLIC_URL_MORTY="$(echo "$PUBLIC_URL" | sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}"
2119
22−MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}"
20+SEARX_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
21+PUBLIC_URL_MORTY="$(echo "$SEARX_URL" | sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}"
22+if in_container; then
23+ # container hostnames do not have a DNS entry, use primary IP
24+ PUBLIC_URL_MORTY="$(url_replace_hostname "$PUBLIC_URL_MORTY" "$(primary_ip)")"
25+fi
26+
2327# shellcheck disable=SC2034
2428MORTY_TIMEOUT=5
2529
@@ -425,7 +429,7 @@ This removes apache site ${APACHE_MORTY_SITE}."
425429
426430 ! apache_is_installed && err_msg "Apache is not installed."
427431
428− if ! ask_yn "Do you really want to continue?"; then
432+ if ! ask_yn "Do you really want to continue?" Yn; then
429433 return
430434 fi
431435
utils/searx.sh+9−1
@@ -748,6 +748,10 @@ excessively bot queries."
748748
749749 apache_install_site --variant=uwsgi "${APACHE_SEARX_SITE}"
750750
751+ rst_title "Install searx's uWSGI app (searx.ini)" section
752+ echo
753+ uWSGI_install_app --variant=socket "$SEARX_UWSGI_APP"
754+
751755 if ! service_is_available "${PUBLIC_URL}"; then
752756 err_msg "Public service at ${PUBLIC_URL} is not available!"
753757 fi
@@ -762,11 +766,15 @@ This removes apache site ${APACHE_SEARX_SITE}."
762766
763767 ! apache_is_installed && err_msg "Apache is not installed."
764768
765− if ! ask_yn "Do you really want to continue?"; then
769+ if ! ask_yn "Do you really want to continue?" Yn; then
766770 return
767771 fi
768772
769773 apache_remove_site "${APACHE_SEARX_SITE}"
774+
775+ rst_title "Remove searx's uWSGI app (searx.ini)" section
776+ echo
777+ uWSGI_remove_app "$SEARX_UWSGI_APP"
770778}
771779
772780rst-doc() {
utils/templates/etc/httpd/sites-available/morty.conf+1−1
@@ -2,7 +2,7 @@
22
33LoadModule headers_module ${APACHE_MODULES}/mod_headers.so
44LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so
5−LoadModule proxy_module ${APACHE_MODULES}/mod_proxy_http.so
5+LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so
66#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so
77
88# SetEnvIf Request_URI "${PUBLIC_URL_PATH_MORTY}" dontlog
utils/templates/etc/httpd/sites-available/searx.conf:filtron+1−1
@@ -2,7 +2,7 @@
22
33LoadModule headers_module ${APACHE_MODULES}/mod_headers.so
44LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so
5−LoadModule proxy_module ${APACHE_MODULES}/mod_proxy_http.so
5+LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so
66#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so
77
88# SetEnvIf Request_URI "${FILTRON_URL_PATH}" dontlog
utils/templates/etc/uwsgi/apps-archlinux/searx.ini+2−2
@@ -74,7 +74,7 @@ http = ${SEARX_INTERNAL_HTTP}
7474#
7575# On some distributions you need to create the app folder for the sockets::
7676#
77−# mkdir -p /run/uwsgi/app/searx/socket
78−# chmod -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx/socket
77+# mkdir -p /run/uwsgi/app/searx
78+# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
7979#
8080# socket = /run/uwsgi/app/searx/socket
utils/templates/etc/uwsgi/apps-archlinux/searx.ini:socketnew file+80
@@ -0,0 +1,80 @@
1+[uwsgi]
2+
3+# uWSGI core
4+# ----------
5+#
6+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
7+
8+# Who will run the code
9+uid = ${SERVICE_USER}
10+gid = ${SERVICE_GROUP}
11+
12+# chdir to specified directory before apps loading
13+chdir = ${SEARX_SRC}/searx
14+
15+# searx configuration (settings.yml)
16+env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH}
17+
18+# disable logging for privacy
19+logger = systemd
20+disable-logging = true
21+
22+# The right granted on the created socket
23+chmod-socket = 666
24+
25+# Plugin to use and interpretor config
26+single-interpreter = true
27+
28+# enable master process
29+master = true
30+
31+# load apps in each worker instead of the master
32+lazy-apps = true
33+
34+# load uWSGI plugins
35+plugin = python
36+
37+# By default the Python plugin does not initialize the GIL. This means your
38+# app-generated threads will not run. If you need threads, remember to enable
39+# them with enable-threads. Running uWSGI in multithreading mode (with the
40+# threads options) will automatically enable threading support. This *strange*
41+# default behaviour is for performance reasons.
42+enable-threads = true
43+
44+
45+# plugin: python
46+# --------------
47+#
48+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python
49+
50+# load a WSGI module
51+module = searx.webapp
52+
53+# set PYTHONHOME/virtualenv
54+virtualenv = ${SEARX_PYENV}
55+
56+# add directory (or glob) to pythonpath
57+pythonpath = ${SEARX_SRC}
58+
59+
60+# speak to upstream
61+# -----------------
62+#
63+# Activate the 'http' configuration for filtron or activate the 'socket'
64+# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
65+
66+# using IP:
67+#
68+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
69+# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
70+
71+# http = ${SEARX_INTERNAL_HTTP}
72+
73+# using unix-sockets:
74+#
75+# On some distributions you need to create the app folder for the sockets::
76+#
77+# mkdir -p /run/uwsgi/app/searx
78+# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
79+#
80+socket = /run/uwsgi/app/searx/socket
utils/templates/etc/uwsgi/apps-available/searx.ini+2−2
@@ -73,7 +73,7 @@ http = ${SEARX_INTERNAL_HTTP}
7373#
7474# On some distributions you need to create the app folder for the sockets::
7575#
76−# mkdir -p /run/uwsgi/app/searx/socket
77−# chmod -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx/socket
76+# mkdir -p /run/uwsgi/app/searx
77+# chmod -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
7878#
7979# socket = /run/uwsgi/app/searx/socket
utils/templates/etc/uwsgi/apps-available/searx.ini:socketnew file+79
@@ -0,0 +1,79 @@
1+[uwsgi]
2+
3+# uWSGI core
4+# ----------
5+#
6+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
7+
8+# Who will run the code
9+uid = ${SERVICE_USER}
10+gid = ${SERVICE_GROUP}
11+
12+# chdir to specified directory before apps loading
13+chdir = ${SEARX_SRC}/searx
14+
15+# searx configuration (settings.yml)
16+env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH}
17+
18+# disable logging for privacy
19+disable-logging = true
20+
21+# The right granted on the created socket
22+chmod-socket = 666
23+
24+# Plugin to use and interpretor config
25+single-interpreter = true
26+
27+# enable master process
28+master = true
29+
30+# load apps in each worker instead of the master
31+lazy-apps = true
32+
33+# load uWSGI plugins
34+plugin = python3,http
35+
36+# By default the Python plugin does not initialize the GIL. This means your
37+# app-generated threads will not run. If you need threads, remember to enable
38+# them with enable-threads. Running uWSGI in multithreading mode (with the
39+# threads options) will automatically enable threading support. This *strange*
40+# default behaviour is for performance reasons.
41+enable-threads = true
42+
43+
44+# plugin: python
45+# --------------
46+#
47+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python
48+
49+# load a WSGI module
50+module = searx.webapp
51+
52+# set PYTHONHOME/virtualenv
53+virtualenv = ${SEARX_PYENV}
54+
55+# add directory (or glob) to pythonpath
56+pythonpath = ${SEARX_SRC}
57+
58+
59+# speak to upstream
60+# -----------------
61+#
62+# Activate the 'http' configuration for filtron or activate the 'socket'
63+# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
64+
65+# using IP:
66+#
67+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
68+# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
69+
70+# http = ${SEARX_INTERNAL_HTTP}
71+
72+# using unix-sockets:
73+#
74+# On some distributions you need to create the app folder for the sockets::
75+#
76+# mkdir -p /run/uwsgi/app/searx
77+# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
78+#
79+socket = /run/uwsgi/app/searx/socket