f693149 · searx · git.iko.soy

docs/utils/index.rst+12−4

compare with master

unifiedsplit

@@ -19,20 +19,28 @@ developers.

19

20.. _toolboxing common:

21

22−Common commands

22+Common commands & environment

23−===============

23+=============================

24

25Scripts to maintain services often dispose of common commands and environments.

26

27−``shell``:

27+``shell`` : command

28 Opens a shell from the service user ``${SERVICE_USSR}``, very helpful for

29 troubleshooting.

30

31−``inspect service``:

31+``inspect service`` : command

32 Shows status and log of the service, most often you have a option to enable

33 more verbose debug logs. Very helpful for debugging, but be careful not to

34 enable debugging in a production environment!

35

36+``FORCE_TIMEOUT`` : environment

37+ Sets timeout for interactive prompts. If you want to run a script in batch

38+ job, with defaults choices, set ``FORCE_TIMEOUT=0``. By example; to install a

39+ reverse proxy for filtron on all containers of the :ref:`searx suite

40+ <lxc-searx.env>` use ::

41+

42+ sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh apache install

43+

36.. _toolboxing setup:

44.. _toolboxing setup:

37

45

38Tooling box setup

46Tooling box setup

docs/utils/lxc.sh.rst+5−1

@@ -88,7 +88,11 @@ WEB-Browser::

88 [searx-fedora31] INFO: (eth0) filtron: http://n.n.n.18:4004/

89 [searx-archlinux] INFO: (eth0) filtron: http://n.n.n.12:4004/

90

91−

91+To install a reverse proxy for filtron and morty use::

92+

93+ sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh apache install

94+ sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/morty.sh apache install

95+

92Running commands

96Running commands

93================

97================

94

98

utils/filtron.sh+4−3

@@ -23,7 +23,6 @@ FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \

23[[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/

24

25FILTRON_ETC="/etc/filtron"

26−

27FILTRON_RULES="$FILTRON_ETC/rules.json"

26FILTRON_RULES="$FILTRON_ETC/rules.json"

28

27

29FILTRON_API="${FILTRON_API:-127.0.0.1:4005}"

28FILTRON_API="${FILTRON_API:-127.0.0.1:4005}"

@@ -447,7 +446,8 @@ This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SIT

447 install_apache

446 install_apache

448 fi

447 fi

449

448

450− echo

449+ "${REPO_ROOT}/utils/searx.sh" install uwsgi

450+

451 apache_install_site --variant=filtron "${APACHE_FILTRON_SITE}"

452

453 info_msg "testing public url .."

@@ -465,11 +465,12 @@ This removes apache site ${APACHE_FILTRON_SITE}."

465

466 ! apache_is_installed && err_msg "Apache is not installed."

467

468− if ! ask_yn "Do you really want to continue?"; then

468+ if ! ask_yn "Do you really want to continue?" Yn; then

469 return

470 fi

471

472 apache_remove_site "$APACHE_FILTRON_SITE"

473+

473}

474}

474

475

475rst-doc() {

476rst-doc() {

utils/lib.sh+58−3

@@ -389,7 +389,7 @@ install_template() {

389 local chmod="${pos_args[4]-644}"

390

391 info_msg "install (eval=$do_eval): ${dst}"

392− [[ -n $variant ]] && info_msg "variant: ${variant}"

392+ [[ -n $variant ]] && info_msg "variant --> ${variant}"

393

394 if [[ ! -f "${template_origin}" ]] ; then

395 err_msg "${template_origin} does not exists"

@@ -777,6 +777,7 @@ apache_dissable_site() {

777 ln -s "${APACHE_SITES_AVAILABLE}/${CONF}" "${APACHE_SITES_ENABLED}/${CONF}"

778 ;;

779 esac

780+ apache_reload

780}

781}

781

782

782# uWSGI

783# uWSGI

@@ -846,7 +847,7 @@ uWSGI_restart() {

846 if uWSGI_app_available "${CONF}"; then

847 if uWSGI_app_available "${CONF}"; then

847 systemctl restart "uwsgi@${CONF%.*}"

848 systemctl restart "uwsgi@${CONF%.*}"

848 else

849 else

849− info_msg "in systemd template mode: ${CONF} not installed (nothing to restart)"

850+ info_msg "[uWSGI:systemd-template] ${CONF} not installed (no need to restart)"

850 fi

851 fi

851 ;;

852 ;;

852 fedora-*)

853 fedora-*)

@@ -854,7 +855,7 @@ uWSGI_restart() {

854 if uWSGI_app_enabled "${CONF}"; then

855 if uWSGI_app_enabled "${CONF}"; then

855 touch "${uWSGI_APPS_ENABLED}/${CONF}"

856 touch "${uWSGI_APPS_ENABLED}/${CONF}"

856 else

857 else

857− info_msg "in uWSGI emperor mode: ${CONF} not installed (nothing to restart)"

858+ info_msg "[uWSGI:emperor] ${CONF} not installed (no need to restart)"

858 fi

859 fi

859 ;;

860 ;;

860 *)

861 *)

@@ -864,6 +865,32 @@ uWSGI_restart() {

864 esac

865 esac

865}

866}

866

867

868+uWSGI_prepare_app() {

869+

870+ # usage: uWSGI_prepare_app <myapp.ini>

871+

872+ local APP="${1%.*}"

873+ if [[ -z $APP ]]; then

874+ err_msg "uWSGI_prepare_app: missing arguments"

875+ return 42

876+ fi

877+

878+ case $DIST_ID-$DIST_VERS in

879+ fedora-*)

880+ # in emperor mode, the uwsgi user is the owner of the sockets

881+ info_msg "prepare (uwsgi:uwsgi) /run/uwsgi/app/${APP}"

882+ mkdir -p "/run/uwsgi/app/${APP}"

883+ chown -R "uwsgi:uwsgi" "/run/uwsgi/app/${APP}"

884+ ;;

885+ *)

886+ info_msg "prepare (${SERVICE_USER}:${SERVICE_GROUP}) /run/uwsgi/app/${APP}"

887+ mkdir -p "/run/uwsgi/app/${APP}"

888+ chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "/run/uwsgi/app/${APP}"

889+ ;;

890+ esac

891+}

892+

893+

867uWSGI_app_available() {

894uWSGI_app_available() {

868 # usage: uWSGI_app_available <myapp.ini>

895 # usage: uWSGI_app_available <myapp.ini>

869 local CONF="$1"

896 local CONF="$1"

@@ -888,6 +915,7 @@ uWSGI_install_app() {

888 *) pos_args+=("$i");;

915 *) pos_args+=("$i");;

889 esac

916 esac

890 done

917 done

918+ uWSGI_prepare_app "${pos_args[1]}"

891 mkdir -p "${uWSGI_APPS_AVAILABLE}"

919 mkdir -p "${uWSGI_APPS_AVAILABLE}"

892 install_template "${template_opts[@]}" \

920 install_template "${template_opts[@]}" \

893 "${uWSGI_APPS_AVAILABLE}/${pos_args[1]}" \

921 "${uWSGI_APPS_AVAILABLE}/${pos_args[1]}" \

@@ -1281,3 +1309,30 @@ global_IPs(){

1281

1309

1282 ip -o addr show | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\1|\2/p'

1310 ip -o addr show | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\1|\2/p'

1283}

1311}

1312+

1313+primary_ip() {

1314+

1315+ case $DIST_ID in

1316+ arch)

1317+ echo "$(ip -o addr show \

1318+ | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\2/p' \

1319+ | head -n 1)"

1320+ ;;

1321+ *) echo "$(hostname -I | cut -d' ' -f1)" ;;

1322+ esac

1323+}

1324+

1325+# URL

1326+# ---

1327+

1328+url_replace_hostname(){

1329+

1330+ # usage: url_replace_hostname <url> <new hostname>

1331+

1332+ # to replace hostname by primary IP::

1333+ #

1334+ # url_replace_hostname http://searx-ubu1604/morty $(primary_ip)

1335+ # http://10.246.86.250/morty

1336+

1337+ echo "$1" | sed "s|$http[s]*://$[^/]*$.*$|\1$2\2|"

1338+}

utils/lxc.sh−1

@@ -438,7 +438,6 @@ lxc_cmd() {

438 else

439 info_msg "lxc $* $i"

440 lxc "$@" "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] "

441− echo

442 fi

441 fi

443 done

442 done

444}

443}

utils/morty.sh+9−5

@@ -14,12 +14,16 @@ in_container && lxc_set_suite_env

14# config

15# ----------------------------------------------------------------------------

16

17−PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"

17+MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}"

18−PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/$[^@]*@$\?$[^:/]*$.*/\2/')}"

19PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty}"

18PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty}"

20−PUBLIC_URL_MORTY="$(echo "$PUBLIC_URL" | sed -e's,^$.*://[^/]*$.*,\1,g')${PUBLIC_URL_PATH_MORTY}"

21

19

22−MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}"

20+SEARX_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"

21+PUBLIC_URL_MORTY="$(echo "$SEARX_URL" | sed -e's,^$.*://[^/]*$.*,\1,g')${PUBLIC_URL_PATH_MORTY}"

22+if in_container; then

23+ # container hostnames do not have a DNS entry, use primary IP

24+ PUBLIC_URL_MORTY="$(url_replace_hostname "$PUBLIC_URL_MORTY" "$(primary_ip)")"

25+fi

26+

23# shellcheck disable=SC2034

27# shellcheck disable=SC2034

24MORTY_TIMEOUT=5

28MORTY_TIMEOUT=5

25

29

@@ -425,7 +429,7 @@ This removes apache site ${APACHE_MORTY_SITE}."

425

429

426 ! apache_is_installed && err_msg "Apache is not installed."

430 ! apache_is_installed && err_msg "Apache is not installed."

427

431

428− if ! ask_yn "Do you really want to continue?"; then

432+ if ! ask_yn "Do you really want to continue?" Yn; then

429 return

433 return

430 fi

434 fi

431

435

utils/searx.sh+9−1

@@ -748,6 +748,10 @@ excessively bot queries."

748

749 apache_install_site --variant=uwsgi "${APACHE_SEARX_SITE}"

750

751+ rst_title "Install searx's uWSGI app (searx.ini)" section

752+ echo

753+ uWSGI_install_app --variant=socket "$SEARX_UWSGI_APP"

754+

751 if ! service_is_available "${PUBLIC_URL}"; then

755 if ! service_is_available "${PUBLIC_URL}"; then

752 err_msg "Public service at ${PUBLIC_URL} is not available!"

756 err_msg "Public service at ${PUBLIC_URL} is not available!"

753 fi

757 fi

@@ -762,11 +766,15 @@ This removes apache site ${APACHE_SEARX_SITE}."

762

766

763 ! apache_is_installed && err_msg "Apache is not installed."

767 ! apache_is_installed && err_msg "Apache is not installed."

764

768

765− if ! ask_yn "Do you really want to continue?"; then

769+ if ! ask_yn "Do you really want to continue?" Yn; then

766 return

770 return

767 fi

771 fi

768

772

769 apache_remove_site "${APACHE_SEARX_SITE}"

773 apache_remove_site "${APACHE_SEARX_SITE}"

774+

775+ rst_title "Remove searx's uWSGI app (searx.ini)" section

776+ echo

777+ uWSGI_remove_app "$SEARX_UWSGI_APP"

770}

778}

771

779

772rst-doc() {

780rst-doc() {

utils/templates/etc/httpd/sites-available/morty.conf+1−1

@@ -2,7 +2,7 @@

2

3LoadModule headers_module ${APACHE_MODULES}/mod_headers.so

4LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so

5−LoadModule proxy_module ${APACHE_MODULES}/mod_proxy_http.so

5+LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so

6#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so

7

8# SetEnvIf Request_URI "${PUBLIC_URL_PATH_MORTY}" dontlog

utils/templates/etc/httpd/sites-available/searx.conf:filtron+1−1

@@ -2,7 +2,7 @@

2

3LoadModule headers_module ${APACHE_MODULES}/mod_headers.so

4LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so

5−LoadModule proxy_module ${APACHE_MODULES}/mod_proxy_http.so

5+LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so

6#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so

7

8# SetEnvIf Request_URI "${FILTRON_URL_PATH}" dontlog

utils/templates/etc/uwsgi/apps-archlinux/searx.ini+2−2

@@ -74,7 +74,7 @@ http = ${SEARX_INTERNAL_HTTP}

74#

75# On some distributions you need to create the app folder for the sockets::

76#

77−# mkdir -p /run/uwsgi/app/searx/socket

77+# mkdir -p /run/uwsgi/app/searx

78−# chmod -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx/socket

78+# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx

79#

80# socket = /run/uwsgi/app/searx/socket

utils/templates/etc/uwsgi/apps-archlinux/searx.ini:socketnew file+80

@@ -0,0 +1,80 @@

1+[uwsgi]

2+

3+# uWSGI core

4+# ----------

5+#

6+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core

7+

8+# Who will run the code

9+uid = ${SERVICE_USER}

10+gid = ${SERVICE_GROUP}

11+

12+# chdir to specified directory before apps loading

13+chdir = ${SEARX_SRC}/searx

14+

15+# searx configuration (settings.yml)

16+env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH}

17+

18+# disable logging for privacy

19+logger = systemd

20+disable-logging = true

21+

22+# The right granted on the created socket

23+chmod-socket = 666

24+

25+# Plugin to use and interpretor config

26+single-interpreter = true

27+

28+# enable master process

29+master = true

30+

31+# load apps in each worker instead of the master

32+lazy-apps = true

33+

34+# load uWSGI plugins

35+plugin = python

36+

37+# By default the Python plugin does not initialize the GIL. This means your

38+# app-generated threads will not run. If you need threads, remember to enable

39+# them with enable-threads. Running uWSGI in multithreading mode (with the

40+# threads options) will automatically enable threading support. This *strange*

41+# default behaviour is for performance reasons.

42+enable-threads = true

43+

44+

45+# plugin: python

46+# --------------

47+#

48+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python

49+

50+# load a WSGI module

51+module = searx.webapp

52+

53+# set PYTHONHOME/virtualenv

54+virtualenv = ${SEARX_PYENV}

55+

56+# add directory (or glob) to pythonpath

57+pythonpath = ${SEARX_SRC}

58+

59+

60+# speak to upstream

61+# -----------------

62+#

63+# Activate the 'http' configuration for filtron or activate the 'socket'

64+# configuration if you setup your HTTP server to use uWSGI protocol via sockets.

65+

66+# using IP:

67+#

68+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http

69+# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html

70+

71+# http = ${SEARX_INTERNAL_HTTP}

72+

73+# using unix-sockets:

74+#

75+# On some distributions you need to create the app folder for the sockets::

76+#

77+# mkdir -p /run/uwsgi/app/searx

78+# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx

79+#

80+socket = /run/uwsgi/app/searx/socket

utils/templates/etc/uwsgi/apps-available/searx.ini+2−2

@@ -73,7 +73,7 @@ http = ${SEARX_INTERNAL_HTTP}

73#

74# On some distributions you need to create the app folder for the sockets::

75#

76−# mkdir -p /run/uwsgi/app/searx/socket

76+# mkdir -p /run/uwsgi/app/searx

77−# chmod -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx/socket

77+# chmod -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx

78#

79# socket = /run/uwsgi/app/searx/socket

utils/templates/etc/uwsgi/apps-available/searx.ini:socketnew file+79

@@ -0,0 +1,79 @@

1+[uwsgi]

2+

3+# uWSGI core

4+# ----------

5+#

6+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core

7+

8+# Who will run the code

9+uid = ${SERVICE_USER}

10+gid = ${SERVICE_GROUP}

11+

12+# chdir to specified directory before apps loading

13+chdir = ${SEARX_SRC}/searx

14+

15+# searx configuration (settings.yml)

16+env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH}

17+

18+# disable logging for privacy

19+disable-logging = true

20+

21+# The right granted on the created socket

22+chmod-socket = 666

23+

24+# Plugin to use and interpretor config

25+single-interpreter = true

26+

27+# enable master process

28+master = true

29+

30+# load apps in each worker instead of the master

31+lazy-apps = true

32+

33+# load uWSGI plugins

34+plugin = python3,http

35+

36+# By default the Python plugin does not initialize the GIL. This means your

37+# app-generated threads will not run. If you need threads, remember to enable

38+# them with enable-threads. Running uWSGI in multithreading mode (with the

39+# threads options) will automatically enable threading support. This *strange*

40+# default behaviour is for performance reasons.

41+enable-threads = true

42+

43+

44+# plugin: python

45+# --------------

46+#

47+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python

48+

49+# load a WSGI module

50+module = searx.webapp

51+

52+# set PYTHONHOME/virtualenv

53+virtualenv = ${SEARX_PYENV}

54+

55+# add directory (or glob) to pythonpath

56+pythonpath = ${SEARX_SRC}

57+

58+

59+# speak to upstream

60+# -----------------

61+#

62+# Activate the 'http' configuration for filtron or activate the 'socket'

63+# configuration if you setup your HTTP server to use uWSGI protocol via sockets.

64+

65+# using IP:

66+#

67+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http

68+# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html

69+

70+# http = ${SEARX_INTERNAL_HTTP}

71+

72+# using unix-sockets:

73+#

74+# On some distributions you need to create the app folder for the sockets::

75+#

76+# mkdir -p /run/uwsgi/app/searx

77+# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx

78+#

79+socket = /run/uwsgi/app/searx/socket