99ff16c · searx · git.iko.soy

git.iko.soy/searx

tooling box: added nginx + polished bash scripts and environment

- add installation method for nginx sites, morty and filtron - clean up PUBLIC_URL environment in and outside of containers - clean up comand lines - handle uWSGI quirks on fedora (emperor mode) - handle Python quirks on debian (there is no 'python' command anymore) - lib.sh: add die and die_caller functions - lxc_suite_install_info is now a function - lint: shellcheck Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

commit99ff16c465ed4d3b98041bf308dfeb0918b535ab authorMarkus Heiser <markus.heiser@darmarit.de> dateMon 13 Apr 2020, 11:34 (7 years ago) parent58d5da8

7 files changed+106−66

compare with master

unifiedsplit

.config.sh+2−1 utils/filtron.sh+9−4 utils/lib.sh+31−28 utils/lxc-searx.env+31−7 utils/lxc.sh+8−13 utils/morty.sh+9−6 utils/searx.sh+16−7

.config.sh+2−1

@@ -18,7 +18,8 @@

1818PUBLIC_URL="${SEARX_URL}"

1919

2020if [[ ${PUBLIC_URL} == "https://searx.me" ]]; then

21− PUBLIC_URL=

21+ # hint: Linux containers do not have DNS entries, lets use IPs

22+ PUBLIC_URL="http://$(primary_ip)/searx"

2223fi

2324

2425# searx.sh

utils/filtron.sh+9−4

@@ -225,6 +225,11 @@ install_all() {

225225 if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then

226226 install_apache_site

227227 fi

228+ elif nginx_is_installed; then

229+ info_msg "nginx is installed on this host."

230+ if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then

231+ install_nginx_site

232+ fi

228233 fi

229234 if ask_yn "Do you want to inspect the installation?" Ny; then

230235 inspect_service

@@ -315,8 +320,6 @@ sourced ${DOT_CONFIG#"$REPO_ROOT/"} :

315320

316321EOF

317322

318− apache_is_installed && info_msg "Apache is installed."

319−

320323 if service_account_is_available "$SERVICE_USER"; then

321324 info_msg "service account $SERVICE_USER available."

322325 else

@@ -445,7 +448,7 @@ install_apache_site() {

445448 rst_para "\

446449This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SITE})"

447450

448− ! apache_is_installed && err_msg "Apache is not installed."

451+ ! apache_is_installed && info_msg "Apache is not installed."

449452

450453 if ! ask_yn "Do you really want to continue?" Yn; then

451454 return

@@ -487,7 +490,7 @@ install_nginx_site() {

487490 rst_para "\

488491This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_FILTRON_SITE})"

489492

490− ! nginx_is_installed && err_msg "nginx is not installed."

493+ ! nginx_is_installed && info_msg "nginx is not installed."

491494

492495 if ! ask_yn "Do you really want to continue?" Yn; then

493496 return

@@ -497,7 +500,9 @@ This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_FILTRON_SITE}

497500

498501 "${REPO_ROOT}/utils/searx.sh" install uwsgi

499502

503+ # shellcheck disable=SC2034

500504 SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC)

505+ # shellcheck disable=SC2034

501506 SEARX_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_URL_PATH)

502507 nginx_install_app --variant=filtron "${NGINX_FILTRON_SITE}"

503508

utils/lib.sh+31−28

@@ -482,7 +482,7 @@ service_is_available() {

482482 # usage: service_is_available <URL>

483483

484484 [[ -z $1 ]] && die_caller 42 "missing argument <URL>"

485−

485+ local URL="$1"

486486 http_code=$(curl -H 'Cache-Control: no-cache' \

487487 --silent -o /dev/null --head --write-out '%{http_code}' --insecure \

488488 "${URL}")

@@ -969,6 +969,7 @@ uWSGI_distro_setup() {

969969 # one day, see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=833067

970970 uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-available"

971971 uWSGI_APPS_ENABLED="${uWSGI_SETUP}/apps-enabled"

972+ uWSGI_PACKAGES="uwsgi"

972973 ;;

973974 arch-*)

974975 # systemd --> /usr/lib/systemd/system/uwsgi@.service

@@ -977,6 +978,7 @@ uWSGI_distro_setup() {

977978 # - https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd

978979 uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-archlinux"

979980 uWSGI_APPS_ENABLED="${uWSGI_SETUP}"

981+ uWSGI_PACKAGES="uwsgi"

980982 ;;

981983 fedora-*)

982984 # systemd --> /usr/lib/systemd/system/uwsgi.service

@@ -984,6 +986,7 @@ uWSGI_distro_setup() {

984986 # - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html

985987 uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-available"

986988 uWSGI_APPS_ENABLED="${uWSGI_SETUP}.d"

989+ uWSGI_PACKAGES="uwsgi"

987990 uWSGI_USER="uwsgi"

988991 uWSGI_GROUP="uwsgi"

989992 ;;

@@ -995,18 +998,26 @@ esac

995998

996999uWSGI_distro_setup

9971000

1001+install_uwsgi(){

1002+ info_msg "installing uwsgi ..."

1003+ pkg_install "$uWSGI_PACKAGES"

1004+ case $DIST_ID-$DIST_VERS in

1005+ fedora-*)

1006+ # enable & start should be called once at uWSGI installation time

1007+ systemctl enable uwsgi

1008+ systemctl restart uwsgi

1009+ ;;

1010+ esac

1011+}

1012+

9981013uWSGI_restart() {

9991014

10001015 # usage: uWSGI_restart() <myapp.ini>

10011016

10021017 local CONF="$1"

1003− if [[ -z $CONF ]]; then

1004− err_msg "uWSGI_restart: missing arguments"

1005− return 42

1006− fi

10071018

1019+ [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"

10081020 info_msg "restart uWSGI service"

1009−

10101021 case $DIST_ID-$DIST_VERS in

10111022 ubuntu-*|debian-*)

10121023 # the 'service' method seems broken in that way, that it (re-)starts

@@ -1025,6 +1036,9 @@ uWSGI_restart() {

10251036 # in emperor mode, just touch the file to restart

10261037 if uWSGI_app_enabled "${CONF}"; then

10271038 touch "${uWSGI_APPS_ENABLED}/${CONF}"

1039+ # it seems, there is a polling time in between touch and restart

1040+ # of the service.

1041+ sleep 3

10281042 else

10291043 info_msg "[uWSGI:emperor] ${CONF} not installed (no need to restart)"

10301044 fi

@@ -1040,11 +1054,9 @@ uWSGI_prepare_app() {

10401054

10411055 # usage: uWSGI_prepare_app <myapp.ini>

10421056

1057+ [[ -z $1 ]] && die_caller 42 "missing argument <myapp.ini>"

1058+

10431059 local APP="${1%.*}"

1044− if [[ -z $APP ]]; then

1045− err_msg "uWSGI_prepare_app: missing arguments"

1046− return 42

1047− fi

10481060

10491061 case $DIST_ID-$DIST_VERS in

10501062 fedora-*)

@@ -1065,10 +1077,8 @@ uWSGI_prepare_app() {

10651077uWSGI_app_available() {

10661078 # usage: uWSGI_app_available <myapp.ini>

10671079 local CONF="$1"

1068− if [[ -z $CONF ]]; then

1069− err_msg "uWSGI_app_available: missing arguments"

1070− return 42

1071− fi

1080+

1081+ [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"

10721082 [[ -f "${uWSGI_APPS_AVAILABLE}/${CONF}" ]]

10731083}

10741084

@@ -1101,6 +1111,8 @@ uWSGI_remove_app() {

11011111 # usage: uWSGI_remove_app <myapp.ini>

11021112

11031113 local CONF="$1"

1114+

1115+ [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"

11041116 info_msg "remove uWSGI app: ${CONF}"

11051117 uWSGI_disable_app "${CONF}"

11061118 uWSGI_restart "${CONF}"

@@ -1110,12 +1122,10 @@ uWSGI_remove_app() {

11101122uWSGI_app_enabled() {

11111123 # usage: uWSGI_app_enabled <myapp.ini>

11121124

1113− local CONF="$1"

11141125 local exit_val=0

1115− if [[ -z $CONF ]]; then

1116− err_msg "uWSGI_app_enabled: missing arguments"

1117− return 42

1118− fi

1126+ local CONF="$1"

1127+

1128+ [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"

11191129 case $DIST_ID-$DIST_VERS in

11201130 ubuntu-*|debian-*)

11211131 [[ -f "${uWSGI_APPS_ENABLED}/${CONF}" ]]

@@ -1145,11 +1155,7 @@ uWSGI_enable_app() {

11451155

11461156 local CONF="$1"

11471157

1148− if [[ -z $CONF ]]; then

1149− err_msg "uWSGI_enable_app: missing arguments"

1150− return 42

1151− fi

1152−

1158+ [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"

11531159 case $DIST_ID-$DIST_VERS in

11541160 ubuntu-*|debian-*)

11551161 mkdir -p "${uWSGI_APPS_ENABLED}"

@@ -1183,11 +1189,8 @@ uWSGI_disable_app() {

11831189 # usage: uWSGI_disable_app <myapp.ini>

11841190

11851191 local CONF="$1"

1186− if [[ -z $CONF ]]; then

1187− err_msg "uWSGI_disable_app: missing arguments"

1188− return 42

1189− fi

11901192

1193+ [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"

11911194 case $DIST_ID-$DIST_VERS in

11921195 ubuntu-*|debian-*)

11931196 service uwsgi stop "${CONF%.*}"

utils/lxc-searx.env+31−7

@@ -32,13 +32,36 @@ lxc_set_suite_env() {

3232 # rolling releases see https://www.archlinux.org/releng/releases/

3333 "$LINUXCONTAINERS_ORG_NAME:archlinux" "archlinux"

3434 )

35− export FILTRON_API="0.0.0.0:4005"

36− export FILTRON_LISTEN="0.0.0.0:4004"

37− export MORTY_LISTEN="0.0.0.0:3000"

35+

36+ PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"

37+ if in_container; then

38+ # container hostnames do not have a DNS entry: use primary IP!

39+ PUBLIC_URL="http://$(primary_ip)/searx"

40+

41+ # make GUEST's services public to the HOST

42+ FILTRON_API="0.0.0.0:4005"

43+ FILTRON_LISTEN="0.0.0.0:4004"

44+ MORTY_LISTEN="0.0.0.0:3000"

45+

46+ # export LXC specific environment

47+ export PUBLIC_URL FILTRON_API FILTRON_LISTEN MORTY_LISTEN

48+ fi

3849}

3950

40−# shellcheck disable=SC2034

41−LXC_SUITE_INSTALL_INFO="suite includes searx, morty & filtron"

51+lxc_suite_install_info() {

52+ (

53+ lxc_set_suite_env

54+ cat <<EOF

55+LXC suite: ${LXC_SUITE_NAME} --> ${PUBLIC_URL}

56+ suite includes searx, morty & filtron

57+suite images:

58+$(echo " ${LOCAL_IMAGES[*]}" | $FMT)

59+suite containers:

60+$(echo " ${CONTAINERS[*]}" | $FMT)

61+EOF

62+ )

63+ }

64+

4265lxc_suite_install() {

4366 (

4467 lxc_set_suite_env

@@ -62,8 +85,9 @@ lxc_suite_info() {

6285 info_msg "(${ip%|*}) IPv6: http://[${ip#*|}]"

6386 else

6487 # IPv4:

65− info_msg "(${ip%|*}) filtron: http://${ip#*|}:4004/"

66− info_msg "(${ip%|*}) morty: http://${ip#*|}:3000/"

88+ # shellcheck disable=SC2034,SC2031

89+ info_msg "(${ip%|*}) filtron: http://${ip#*|}:4004/ $PUBLIC_URL"

90+ info_msg "(${ip%|*}) morty: http://${ip#*|}:3000/ $PUBLIC_URL_MORTY"

6791 info_msg "(${ip%|*}) docs-live: http://${ip#*|}:8080/"

6892 fi

6993 done

utils/lxc.sh+8−13

@@ -112,14 +112,7 @@ EOF

112112}

113113

114114usage_containers() {

115− cat <<EOF

116−LXC suite: ${LXC_SUITE_NAME}

117−$(echo " ${LXC_SUITE_INSTALL_INFO}" | $FMT)

118−suite images:

119−$(echo " ${LOCAL_IMAGES[*]}" | $FMT)

120−suite containers:

121−$(echo " ${CONTAINERS[*]}" | $FMT)

122−EOF

115+ lxc_suite_install_info

123116 [ -n "${1+x}" ] && err_msg "$1"

124117}

125118

@@ -172,7 +165,7 @@ main() {

172165 ''|--|containers) remove_containers ;;

173166 images) lxc_delete_images_localy ;;

174167 ${LXC_HOST_PREFIX}-*)

175− ! lxc_exists "$2" && usage_containers "unknown container: $2" && exit 42

168+ ! lxc_exists "$2" && warn_msg "container not yet exists: $2" && exit 0

176169 if ask_yn "Do you really want to delete container $2"; then

177170 lxc_delete_container "$2"

178171 fi

@@ -201,7 +194,7 @@ main() {

201194 lxc exec -t "$3" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __show suite \

202195 | prefix_stdout "[${_BBlue}$3${_creset}] "

203196 ;;

204− *|--) show_suite;;

197+ *) show_suite;;

205198 esac

206199 ;;

207200 images) show_images ;;

@@ -211,7 +204,7 @@ main() {

211204 ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42

212205 lxc config show "$3" | prefix_stdout "[${_BBlue}${3}${_creset}] "

213206 ;;

214− *|--)

207+ *)

215208 rst_title "container configurations"

216209 echo

217210 lxc list "$LXC_HOST_PREFIX-"

@@ -226,7 +219,7 @@ main() {

226219 ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42

227220 lxc info "$3" | prefix_stdout "[${_BBlue}${3}${_creset}] "

228221 ;;

229− *|--)

222+ *)

230223 rst_title "container info"

231224 echo

232225 lxc_cmd info

@@ -350,7 +343,9 @@ remove_containers() {

350343 echo

351344 lxc list "$LXC_HOST_PREFIX-"

352345 echo -en "\\n${_BRed}LXC containers to delete::${_creset}\\n\\n ${CONTAINERS[*]}\\n" | $FMT

353− if ask_yn "Do you really want to delete these containers"; then

346+ local default=Ny

347+ [[ $FORCE_TIMEOUT = 0 ]] && default=Yn

348+ if ask_yn "Do you really want to delete these containers" $default; then

354349 for i in "${CONTAINERS[@]}"; do

355350 lxc_delete_container "$i"

356351 done

utils/morty.sh+9−6

@@ -7,6 +7,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"

77# shellcheck source=utils/brand.env

88source "${REPO_ROOT}/utils/brand.env"

99source_dot_config

10+SEARX_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"

1011source "${REPO_ROOT}/utils/lxc-searx.env"

1112in_container && lxc_set_suite_env

1213

@@ -17,12 +18,7 @@ in_container && lxc_set_suite_env

1718MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}"

1819PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty/}"

1920

20−SEARX_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"

21−PUBLIC_URL_MORTY="$(echo "$SEARX_URL" | sed -e's,^$.*://[^/]*$.*,\1,g')${PUBLIC_URL_PATH_MORTY}"

22−if in_container; then

23− # container hostnames do not have a DNS entry, use primary IP

24− PUBLIC_URL_MORTY="$(url_replace_hostname "$PUBLIC_URL_MORTY" "$(primary_ip)")"

25−fi

21+PUBLIC_URL_MORTY="${PUBLIC_URL_MORTY:-$(echo "$SEARX_URL" | sed -e's,^$.*://[^/]*$.*,\1,g')${PUBLIC_URL_PATH_MORTY}}"

2622

2723# shellcheck disable=SC2034

2824MORTY_TIMEOUT=5

@@ -229,6 +225,11 @@ install_all() {

229225 if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then

230226 install_apache_site

231227 fi

228+ elif nginx_is_installed; then

229+ info_msg "nginx is installed on this host."

230+ if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then

231+ install_nginx_site

232+ fi

232233 fi

233234 info_searx

234235 if ask_yn "Add image and result proxy to searx settings.yml?" Yn; then

@@ -462,7 +463,9 @@ This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_MORTY_SITE})"

462463

463464 "${REPO_ROOT}/utils/searx.sh" install uwsgi

464465

466+ # shellcheck disable=SC2034

465467 SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC)

468+ # shellcheck disable=SC2034

466469 SEARX_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_URL_PATH)

467470 nginx_install_app "${NGINX_MORTY_SITE}"

468471

utils/searx.sh+16−7

@@ -81,6 +81,11 @@ case $DIST_ID-$DIST_VERS in

8181 BUILD_PACKAGES="${BUILD_PACKAGES_debian}"

8282 APACHE_PACKAGES="$APACHE_PACKAGES libapache2-mod-proxy-uwsgi"

8383 ;;

84+ ubuntu-20.04)

85+ # https://askubuntu.com/a/1224710

86+ SEARX_PACKAGES="${SEARX_PACKAGES_debian} python-is-python3"

87+ BUILD_PACKAGES="${BUILD_PACKAGES_debian}"

88+ ;;

8489 ubuntu-*|debian-*)

8590 SEARX_PACKAGES="${SEARX_PACKAGES_debian}"

8691 BUILD_PACKAGES="${BUILD_PACKAGES_debian}"

@@ -206,7 +211,12 @@ main() {

206211 pyenv) create_pyenv ;;

207212 searx-src) clone_searx ;;

208213 settings) install_settings ;;

209− uwsgi) install_searx_uwsgi;;

214+ uwsgi)

215+ install_searx_uwsgi

216+ if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then

217+ err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check searx & uwsgi setup!"

218+ fi

219+ ;;

210220 packages)

211221 pkg_install "$SEARX_PACKAGES"

212222 ;;

@@ -272,11 +282,6 @@ install_all() {

272282 rst_title "Install $SEARX_INSTANCE_NAME (service)"

273283 pkg_install "$SEARX_PACKAGES"

274284 wait_key

275− case $DIST_ID-$DIST_VERS in

276− fedora-*)

277− systemctl enable uwsgi

278− ;;

279− esac

280285 assert_user

281286 wait_key

282287 clone_searx

@@ -514,6 +519,7 @@ EOF

514519install_searx_uwsgi() {

515520 rst_title "Install searx's uWSGI app (searx.ini)" section

516521 echo

522+ install_uwsgi

517523 uWSGI_install_app "$SEARX_UWSGI_APP"

518524}

519525

@@ -575,7 +581,10 @@ EOF

575581}

576582

577583set_result_proxy() {

578− info_msg "try to set result proxy ..."

584+

585+ # usage: set_result_proxy <URL> [<key>]

586+

587+ info_msg "try to set result proxy: $1"

579588 cp "${SEARX_SETTINGS_PATH}" "${SEARX_SETTINGS_PATH}.bak"

580589 _set_result_proxy "$1" "$2" > "${SEARX_SETTINGS_PATH}"

581590}