4990b07 · searx · git.iko.soy

git.iko.soy/searx

utils/filtron.sh: various fix from first installation test (WIP)

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

commit4990b07b4bc42b0caf0d890f8c81c3545bbb807b authorMarkus Heiser <markus.heiser@darmarit.de> dateThu 9 Jan 2020, 16:25 (7 years ago) parentf201931

4 files changed+207−86

compare with master

unifiedsplit

utils/filtron.sh+60−17 utils/lib.sh+54−18 utils/templates/etc/filtron/rules.json+92−50 utils/templates/lib/systemd/system/filtron.service+1−1

utils/filtron.sh+60−17

@@ -11,6 +11,11 @@ source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"

1111

1212FILTRON_ETC="/etc/filtron"

1313

14+FILTRON_RULES="$FILTRON_ETC/rules.json"

15+FILTRON_API="127.0.0.1:4005"

16+FILTRON_LISTEN="127.0.0.1:4004"

17+FILTRON_TARGET="127.0.0.1:8888"

18+

1419SERVICE_NAME="filtron"

1520SERVICE_USER="${SERVICE_NAME}"

1621SERVICE_HOME="/home/${SERVICE_USER}"

@@ -23,6 +28,11 @@ GO_ENV="${SERVICE_HOME}/.go_env"

2328GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz"

2429GO_TAR=$(basename "$GO_PKG_URL")

2530

31+CONFIG_FILES=(

32+ "${FILTRON_RULES}"

33+ "${SERVICE_SYSTEMD_UNIT}"

34+)

35+

2636# ----------------------------------------------------------------------------

2737usage(){

2838# ----------------------------------------------------------------------------

@@ -37,10 +47,16 @@ usage:

3747 $(basename "$0") remove [all]

3848 $(basename "$0") activate [server]

3949 $(basename "$0") deactivate [server]

40−

41−shell - start interactive shell with user ${SERVICE_USER}

42−install user - add service user '$SERVICE_USER' at $SERVICE_HOME

43−

50+ $(basename "$0") show [server]

51+

52+shell

53+ start interactive shell from user ${SERVICE_USER}

54+show server

55+ show server status and log

56+install / remove

57+ all - complete setup of filtron server

58+install user

59+ add service user '$SERVICE_USER' at $SERVICE_HOME

4460EOF

4561 [ ! -z ${1+x} ] && echo -e "$1"

4662}

@@ -58,6 +74,14 @@ main(){

5874 sudo_or_exit

5975 interactive_shell

6076 ;;

77+ show)

78+ case $2 in

79+ server)

80+ sudo_or_exit

81+ show_server

82+ ;;

83+ *) usage "$_usage"; exit 42;;

84+ esac ;;

6185 install)

6286 sudo_or_exit

6387 case $2 in

@@ -91,21 +115,27 @@ main(){

91115install_all() {

92116 rst_title "Install $SERVICE_NAME (service)"

93117 assert_user

118+ wait_key

94119 install_go

120+ wait_key

95121 install_filtron

122+ wait_key

96123 install_server

124+ wait_key

97125}

98126

99127remove_all() {

100128 rst_title "De-Install $SERVICE_NAME (service)"

101129 remove_server

130+ wait_key

102131 remove_user

103− rm -rf "$FILTRON_ETC"

132+ rm -r "$FILTRON_ETC" 2>&1 | prefix_stdout

104133 wait_key

105134}

106135

107136install_server() {

108− rst_title "Install System-D Unit ${SERVICE_NAME}.service ..." section

137+ rst_title "Install System-D Unit ${SERVICE_NAME}.service" section

138+ echo

109139 install_template ${SERVICE_SYSTEMD_UNIT} root root 644

110140 wait_key

111141 activate_server

@@ -116,12 +146,12 @@ remove_server() {

116146 return

117147 fi

118148 deactivate_server

119− rm "${SERVICE_SYSTEMD_UNIT}"

149+ rm "${SERVICE_SYSTEMD_UNIT}" 2>&1 | prefix_stdout

120150}

121151

122−

123152activate_server () {

124153 rst_title "Activate $SERVICE_NAME (service)" section

154+ echo

125155 tee_stderr <<EOF | bash 2>&1 | prefix_stdout

126156systemctl enable $SERVICE_NAME.service

127157systemctl restart $SERVICE_NAME.service

@@ -129,7 +159,6 @@ EOF

129159 tee_stderr <<EOF | bash 2>&1 | prefix_stdout

130160systemctl status $SERVICE_NAME.service

131161EOF

132− wait_key

133162}

134163

135164deactivate_server () {

@@ -139,7 +168,6 @@ deactivate_server () {

139168systemctl stop $SERVICE_NAME.service

140169systemctl disable $SERVICE_NAME.service

141170EOF

142− wait_key

143171}

144172

145173assert_user() {

@@ -168,18 +196,18 @@ EOF

168196remove_user() {

169197 rst_title "Drop $SERVICE_USER HOME" section

170198 if ask_yn "Do you really want to drop $SERVICE_USER home folder?"; then

171− userdel -r -f "$SERVICE_USER"

199+ userdel -r -f "$SERVICE_USER" 2>&1 | prefix_stdout

172200 else

173201 rst_para "Leave HOME folder $(du -sh "$SERVICE_HOME") unchanged."

174202 fi

175203}

176204

177205interactive_shell(){

178− echo "// exit with STRG-D"

206+ echo "// exit with CTRL-D"

179207 sudo -H -u ${SERVICE_USER} -i

180208}

181209

182−_service_prefix="$SERVICE_USER@$(hostname) -->| "

210+_service_prefix=" |$SERVICE_USER| "

183211

184212install_go(){

185213 rst_title "Install Go in user's HOME" section

@@ -199,14 +227,29 @@ EOF

199227! which go >/dev/null && echo "Go Installation not found in PATH!?!"

200228which go >/dev/null && go version && echo "congratulations -- Go installation OK :)"

201229EOF

202− wait_key

203230}

204231

205232install_filtron() {

206− tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" | prefix_stdout "$_service_prefix"

207−go get -v -u github.com/asciimoo/filtron 2>&1

233+ rst_title "Install filtron in user's ~/go-apps" section

234+ echo

235+ tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_service_prefix"

236+go get -v -u github.com/asciimoo/filtron

208237EOF

209− install_template "$FILTRON_ETC/rules.json" root root 644

238+ install_template --no-eval "$FILTRON_RULES" root root 644

239+}

240+

241+show_server () {

242+ rst_title "server status & log"

243+ echo

244+ systemctl status filtron.service

245+ echo

246+ read -s -n1 -t 5 -p "// use CTRL-C to stop monitoring the log"

247+ echo

248+ while true; do

249+ trap break 2

250+ journalctl -f -u filtron

251+ done

252+ return 0

210253}

211254

212255# ----------------------------------------------------------------------------

utils/lib.sh+54−18

@@ -22,6 +22,13 @@ if [[ -z "$SYSTEMD_UNITS" ]]; then

2222 SYSTEMD_UNITS="/lib/systemd/system"

2323fi

2424

25+if [[ -z ${DIFF_CMD} ]]; then

26+ DIFF_CMD="diff -u"

27+ if command -v colordiff >/dev/null; then

28+ DIFF_CMD="colordiff -u"

29+ fi

30+fi

31+

2532sudo_or_exit() {

2633 # usage: sudo_or_exit

2734

@@ -55,10 +62,10 @@ rst_para() {

5562 # usage: RST_INDENT=1 rst_para "lorem ipsum ..."

5663 local prefix=''

5764 if ! [[ -z $RST_INDENT ]] && [[ $RST_INDENT -gt 0 ]]; then

58− prefix="$(for i in $(seq 1 "$RST_INDENT"); do printf " "; done)"

59− echo -en "\n$*\n" | $FMT | prefix_stdout "$prefix"

65+ prefix="$(for i in $(seq 1 "$RST_INDENT"); do printf " "; done)"

66+ echo -en "\n$*\n" | $FMT | prefix_stdout "$prefix"

6067 else

61− echo -en "\n$*\n" | $FMT

68+ echo -en "\n$*\n" | $FMT

6269 fi

6370}

6471

@@ -66,15 +73,23 @@ err_msg() { echo -e "ERROR: $*" >&2; }

6673warn_msg() { echo -e "WARN: $*" >&2; }

6774info_msg() { echo -e "INFO: $*"; }

6875

76+clean_stdin() {

77+ if [[ $(uname -s) != 'Darwin' ]]; then

78+ while $(read -n1 -t 0.1); do : ; done

79+ fi

80+}

81+

6982wait_key(){

7083 # usage: waitKEY [<timeout in sec>]

7184

85+ clean_stdin

7286 local _t=$1

7387 [[ ! -z $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT

7488 [[ ! -z $_t ]] && _t="-t $_t"

7589 # shellcheck disable=SC2086

76− read -n1 $_t -p "** press any [KEY] to continue **"

90+ read -s -n1 $_t -p "** press any [KEY] to continue **"

7791 echo

92+ clean_stdin

7893}

7994

8095ask_yn() {

@@ -100,6 +115,7 @@ ask_yn() {

100115 esac

101116 echo

102117 while true; do

118+ clean_stdin

103119 printf "$1 ${choice} "

104120 # shellcheck disable=SC2086

105121 read -n1 $_t

@@ -117,6 +133,7 @@ ask_yn() {

117133 _t=""

118134 err_msg "invalid choice"

119135 done

136+ clean_stdin

120137 return $exit_val

121138}

122139

@@ -144,7 +161,7 @@ tee_stderr () {

144161prefix_stdout () {

145162 # usage: <cmd> | prefix_stdout [prefix]

146163

147− local prefix="-->| "

164+ local prefix=" | "

148165

149166 if [[ ! -z $1 ]] ; then prefix="$1"; fi

150167

@@ -223,6 +240,7 @@ choose_one() {

223240 fi

224241 done

225242 while true; do

243+ clean_stdin

226244 printf "$1 [$default] "

227245

228246 if (( 10 > $max )); then

@@ -242,6 +260,7 @@ choose_one() {

242260 err_msg "invalid choice"

243261 done

244262 echo

263+ clean_stdin

245264 eval "$env_name"='${list[${REPLY}]}'

246265}

247266

@@ -288,31 +307,48 @@ install_template() {

288307

289308 mkdir -p "$(dirname "${dst}")"

290309

291− if [[ -f "${dst}" ]] ; then

292− info_msg "file ${dst} allready exists on this host"

310+ if [[ ! -f "${dst}" ]]; then

311+ info_msg "install: ${template_file}"

312+ sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \

313+ "${template_file}" "${dst}" | prefix_stdout

314+ return $?

315+ fi

316+

317+ if [[ -f "${dst}" ]] && cmp --silent "${template_file}" "${dst}" ; then

318+ info_msg "file ${dst} allready installed"

319+ return 0

320+ fi

321+

322+ info_msg "file ${dst} allready exists on this host"

323+

324+ while true; do

293325 choose_one _reply "choose next step with file $dst" \

294326 "replace file" \

295− "leave file unchanged"

327+ "leave file unchanged" \

328+ "interactiv shell" \

329+ "diff files"

296330

297331 case $_reply in

298332 "replace file")

299333 info_msg "install: ${template_file}"

300334 sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \

301335 "${template_file}" "${dst}" | prefix_stdout

336+ break

302337 ;;

303338 "leave file unchanged")

339+ break

304340 ;;

305341 "interactiv shell")

306− echo "// exit with STRG-D"

342+ echo "// edit ${dst} to your needs"

343+ echo "// exit with CTRL-D"

307344 sudo -H -u "${owner}" -i

308− ;;

345+ $DIFF_CMD "${dst}" "${template_file}"

346+ if ask_yn "did you edit ${template_file} to your needs?"; then

347+ break

348+ fi

349+ ;;

350+ "diff files")

351+ $DIFF_CMD "${dst}" "${template_file}" | prefix_stdout

309352 esac

310−

311− else

312− info_msg "install: ${template_file}"

313− sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \

314− "${template_file}" "${dst}" | prefix_stdout

315− fi

316−

353+ done

317354}

318−

utils/templates/etc/filtron/rules.json+92−50

@@ -1,56 +1,98 @@

1−[

1+[{

2+ "name":"search request",

3+ "filters":[

4+ "Param:q",

5+ "Path=^(/|/search)$"

6+ ],

7+ "interval":60,

8+ "limit":15,

9+ "subrules":[

210 {

3− "name": "api limit",

4− "interval": 60,

5− "limit": 1000,

6− "filters": ["Path=^/api"],

7− "aggregations": ["Path"],

8− "actions": [

9− {"name": "block"}

10− ],

11− "subrules": [

12− {

13− "name": "drop put",

14− "interval": 60,

15− "limit": 100,

16− "filters": ["Method=PUT"],

17− "aggregations": ["Header:X-Forwarded-For"],

18− "actions": [

19− {"name": "shell",

20− "params": {"cmd": "iptables -A INPUT -s %v -j DROP", "args": ["Header:X-Forwarded-For"]}}

21− ]

22− }

23− ]

11+ "name":"roboagent limit",

12+ "interval":60,

13+ "limit":15,

14+ "filters":[

16+ ],

17+ "actions":[

18+ {"name": "log"},

19+ {

20+ "name":"block",

21+ "params":{

22+ "message":"Rate limit exceeded"

23+ }

24+ }

25+ ]

2426 },

2527 {

26− "name": "log'n'block rss",

27− "interval": 300,

28− "limit": 2500,

29− "filters": ["Path=^/$", "GET:format=rss"],

30− "actions": [

31− {"name": "log"},

32− {"name": "block"}

33− ]

28+ "name":"botlimit",

29+ "limit":0,

30+ "stop":true,

31+ "filters":[

33+ ],

34+ "actions":[

35+ {"name": "log"},

36+ {

37+ "name":"block",

38+ "params":{

39+ "message":"Rate limit exceeded"

40+ }

41+ }

42+ ]

3443 },

3544 {

36− "name": "log rule",

37− "filters": ["Path=/"],

38− "actions": [ {"name": "log"} ],

39− "subrules": [

40− {

41− "name": "block missing accept-language",

42− "filters": ["!Header:Accept-Language"],

43− "actions": [

44− {"name": "block"}

45− ]

46− },

47− {

48− "name": "block curl",

49− "filters": ["Header:User-Agent=[Cc]url"],

50− "actions": [

51− {"name": "block"}

52− ]

53− }

54− ]

45+ "name":"IP limit",

46+ "interval":60,

47+ "limit":15,

48+ "stop":true,

49+ "aggregations":[

50+ "Header:X-Forwarded-For"

51+ ],

52+ "actions":[

53+ {"name": "log"},

54+ {

55+ "name":"block",

56+ "params":{

57+ "message":"Rate limit exceeded"

58+ }

59+ }

60+ ]

61+ },

62+ {

63+ "name":"rss/json limit",

64+ "interval":60,

65+ "limit":15,

66+ "stop":true,

67+ "filters":[

68+ "Param:format=(csv|json|rss)"

69+ ],

70+ "actions":[

71+ {"name": "log"},

72+ {

73+ "name":"block",

74+ "params":{

75+ "message":"Rate limit exceeded"

76+ }

77+ }

78+ ]

79+ },

80+ {

81+ "name":"useragent limit",

82+ "interval":60,

83+ "limit":15,

84+ "aggregations":[

85+ "Header:User-Agent"

86+ ],

87+ "actions":[

88+ {"name": "log"},

89+ {

90+ "name":"block",

91+ "params":{

92+ "message":"Rate limit exceeded"

93+ }

94+ }

95+ ]

5596 }

56−]

97+ ]

98+}]

utils/templates/lib/systemd/system/filtron.service+1−1

@@ -10,7 +10,7 @@ Type=simple

1010User=${SERVICE_USER}

1111Group=${SERVICE_GROUP}

1212WorkingDirectory=${SERVICE_HOME}

13−ExecStart=${SERVICE_HOME}/go-apps/bin/filtron -rules ${FILTRON_RULES}

13+ExecStart=${SERVICE_HOME}/go-apps/bin/filtron -api '${FILTRON_API}' -listen '${FILTRON_LISTEN}' -rules '${FILTRON_RULES}' -target '${FILTRON_TARGET}'

1414

1515Restart=always

1616Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME}