git.iko.soy/searx
on master help sign in
/etc/filtron/rules.json: normalize rules from docs & tooling box

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

commit2f40f61f83afb34f0c4b95019a53050a504ce43a authorMarkus Heiser <markus.heiser@darmarit.de> dateTue 4 Feb 2020, 17:59 (7 years ago) parentd07119c
5 files changed+206−203
compare with master
unifiedsplit
docs/admin/filtron.rst+98−94 utils/filtron.sh+4−3 utils/morty.sh+1−1 utils/searx.sh+3−3 utils/templates/etc/filtron/rules.json+100−102
docs/admin/filtron.rst+98−94
@@ -39,6 +39,10 @@ an installation and the maintenance of, use our script :ref:`filtron.sh`.
3939Sample configuration of filtron
4040===============================
4141
42+.. sidebar:: Tooling box
43+
44+ - :origin:`/etc/filtron/rules.json <utils/templates/etc/filtron/rules.json>`
45+
4246An example configuration can be find below. This configuration limits the access
4347of:
4448
@@ -50,100 +54,100 @@ of:
5054
5155.. code:: json
5256
53− [{
54− "name":"search request",
55− "filters":[
56− "Param:q",
57− "Path=^(/|/search)$"
58− ],
59− "interval":"<time-interval-in-sec (int)>",
60− "limit":"<max-request-number-in-interval (int)>",
61− "subrules":[
62− {
63− "name":"roboagent limit",
64− "interval":"<time-interval-in-sec (int)>",
65− "limit":"<max-request-number-in-interval (int)>",
66− "filters":[
67− "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"
68− ],
69− "actions":[
70− {
71− "name":"block",
72− "params":{
73− "message":"Rate limit exceeded"
74− }
75− }
76− ]
77− },
78− {
79− "name":"botlimit",
80− "limit":0,
81− "stop":true,
82− "filters":[
83− "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
84− ],
85− "actions":[
86− {
87− "name":"block",
88− "params":{
89− "message":"Rate limit exceeded"
90− }
91− }
92− ]
93− },
94− {
95− "name":"IP limit",
96− "interval":"<time-interval-in-sec (int)>",
97− "limit":"<max-request-number-in-interval (int)>",
98− "stop":true,
99− "aggregations":[
100− "Header:X-Forwarded-For"
101− ],
102− "actions":[
103− {
104− "name":"block",
105− "params":{
106− "message":"Rate limit exceeded"
107− }
108− }
109− ]
110− },
111− {
112− "name":"rss/json limit",
113− "interval":"<time-interval-in-sec (int)>",
114− "limit":"<max-request-number-in-interval (int)>",
115− "stop":true,
116− "filters":[
117− "Param:format=(csv|json|rss)"
118− ],
119− "actions":[
120− {
121− "name":"block",
122− "params":{
123− "message":"Rate limit exceeded"
124− }
125− }
126− ]
127− },
128− {
129− "name":"useragent limit",
130− "interval":"<time-interval-in-sec (int)>",
131− "limit":"<max-request-number-in-interval (int)>",
132− "aggregations":[
133− "Header:User-Agent"
134− ],
135− "actions":[
136− {
137− "name":"block",
138− "params":{
139− "message":"Rate limit exceeded"
140− }
141− }
142− ]
143− }
144− ]
145− }]
146−
57+ [
58+ { "name": "search request",
59+ "filters": [
60+ "Param:q",
61+ "Path=^(/|/search)$"
62+ ],
63+ "interval": "<time-interval-in-sec (int)>",
64+ "limit": "<max-request-number-in-interval (int)>",
65+ "subrules": [
66+ {
67+ "name": "roboagent limit",
68+ "interval": "<time-interval-in-sec (int)>",
69+ "limit": "<max-request-number-in-interval (int)>",
70+ "filters": [
71+ "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"
72+ ],
73+ "actions": [
74+ { "name": "log"},
75+ { "name": "block",
76+ "params": {
77+ "message": "Rate limit exceeded"
78+ }
79+ }
80+ ]
81+ },
82+ {
83+ "name": "botlimit",
84+ "limit": 0,
85+ "stop": true,
86+ "filters": [
87+ "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
88+ ],
89+ "actions": [
90+ { "name": "log"},
91+ { "name": "block",
92+ "params": {
93+ "message": "Rate limit exceeded"
94+ }
95+ }
96+ ]
97+ },
98+ {
99+ "name": "IP limit",
100+ "interval": "<time-interval-in-sec (int)>",
101+ "limit": "<max-request-number-in-interval (int)>",
102+ "stop": true,
103+ "aggregations": [
104+ "Header:X-Forwarded-For"
105+ ],
106+ "actions": [
107+ { "name": "log"},
108+ { "name": "block",
109+ "params": {
110+ "message": "Rate limit exceeded"
111+ }
112+ }
113+ ]
114+ },
115+ {
116+ "name": "rss/json limit",
117+ "interval": "<time-interval-in-sec (int)>",
118+ "limit": "<max-request-number-in-interval (int)>",
119+ "stop": true,
120+ "filters": [
121+ "Param:format=(csv|json|rss)"
122+ ],
123+ "actions": [
124+ { "name": "log"},
125+ { "name": "block",
126+ "params": {
127+ "message": "Rate limit exceeded"
128+ }
129+ }
130+ ]
131+ },
132+ {
133+ "name": "useragent limit",
134+ "interval": "<time-interval-in-sec (int)>",
135+ "limit": "<max-request-number-in-interval (int)>",
136+ "aggregations": [
137+ "Header:User-Agent"
138+ ],
139+ "actions": [
140+ { "name": "log"},
141+ { "name": "block",
142+ "params": {
143+ "message": "Rate limit exceeded"
144+ }
145+ }
146+ ]
147+ }
148+ ]
149+ }
150+ ]
147151
148152
149153Route request through filtron
utils/filtron.sh+4−3
@@ -94,8 +94,9 @@ If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"
9494 FILTRON_API : ${FILTRON_API}
9595 FILTRON_LISTEN : ${FILTRON_LISTEN}
9696 FILTRON_TARGET : ${FILTRON_TARGET}
97+
9798EOF
98− [ ! -z ${1+x} ] && echo -e "$1"
99+ [ ! -z ${1+x} ] && err_msg "$1"
99100}
100101
101102main() {
@@ -105,7 +106,7 @@ main() {
105106 dpkg apt-get install git wget curl \
106107 || exit
107108
108− local _usage="ERROR: unknown or missing $1 command $2"
109+ local _usage="unknown or missing $1 command $2"
109110
110111 case $1 in
111112 --source-only) ;;
@@ -170,7 +171,7 @@ main() {
170171 *) usage "$_usage"; exit 42;;
171172 esac ;;
172173
173− *) usage "ERROR: unknown or missing command $1"; exit 42;;
174+ *) usage "unknown or missing command $1"; exit 42;;
174175 esac
175176}
176177
utils/morty.sh+1−1
@@ -84,7 +84,7 @@ ${DOT_CONFIG#"$REPO_ROOT/"} file::
8484 SERVICE_USER : ${SERVICE_USER}
8585EOF
8686 info_searx
87− [ ! -z ${1+x} ] && echo -e "$1"
87+ [ ! -z ${1+x} ] && err_msg "$1"
8888}
8989
9090info_searx() {
utils/searx.sh+3−3
@@ -110,7 +110,7 @@ If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"
110110 SERVICE_USER : ${SERVICE_USER}
111111
112112EOF
113− [ ! -z ${1+x} ] && echo -e "$1"
113+ [ ! -z ${1+x} ] && err_msg "$1"
114114}
115115
116116main() {
@@ -120,7 +120,7 @@ main() {
120120 dpkg systemctl apt-get install git wget curl \
121121 || exit
122122
123− local _usage="ERROR: unknown or missing $1 command $2"
123+ local _usage="unknown or missing $1 command $2"
124124
125125 case $1 in
126126 --source-only) ;;
@@ -190,7 +190,7 @@ main() {
190190 *) usage "$_usage"; exit 42;;
191191 esac ;;
192192
193− *) usage "ERROR: unknown or missing command $1"; exit 42;;
193+ *) usage "unknown or missing command $1"; exit 42;;
194194 esac
195195}
196196
utils/templates/etc/filtron/rules.json+100−102
@@ -1,107 +1,105 @@
1−[{
2− "name":"search request",
3− "filters":[
4− "Param:q",
5− "Path=^(/|/search)$"
6− ],
7− "interval":120,
8− "limit":19,
9− "actions":[
10− {
11− "name":"block",
12− "params":{
13− "message":"common rate limit exceeded"
14− }
15− }
16− ],
17− "subrules":[
18− {
19− "name":"roboagent limit",
20− "interval":60,
21− "limit":3,
22− "filters":[
23− "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client|Ruby)"
24− ],
25− "actions":[
26− {"name":"log"},
27− {
28− "name":"block",
29− "params":{
30− "message":"rate limit exceeded"
1+[
2+ { "name": "suspiciously frequent IP",
3+ "filters": [],
4+ "interval": 600,
5+ "limit": 30,
6+ "aggregations": [
7+ "Header:X-Forwarded-For"
8+ ],
9+ "actions":[
10+ {"name":"log"}
11+ ]
12+ },
13+ { "name": "search request",
14+ "filters": [
15+ "Param:q",
16+ "Path=^(/|/search)$"
17+ ],
18+ "interval": 61,
19+ "limit": 999,
20+ "subrules": [
21+ {
22+ "name": "roboagent limit",
23+ "interval": 61,
24+ "limit": 1,
25+ "filters": [
26+ "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"
27+ ],
28+ "actions": [
29+ { "name": "log"},
30+ { "name": "block",
31+ "params": {
32+ "message": "Rate limit exceeded"
33+ }
3134 }
32− }
33− ]
34− },
35− {
36− "name":"botlimit",
37− "interval":60,
38− "limit":0,
39− "stop":true,
40− "filters":[
41− "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
42− ],
43− "actions":[
44− {"name":"log"},
45− {
46− "name":"block",
47− "params":{
48− "message":"rate limit exceeded"
35+ ]
36+ },
37+ {
38+ "name": "botlimit",
39+ "limit": 0,
40+ "stop": true,
41+ "filters": [
42+ "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
43+ ],
44+ "actions": [
45+ { "name": "log"},
46+ { "name": "block",
47+ "params": {
48+ "message": "Rate limit exceeded"
49+ }
4950 }
50− }
51− ]
52− },
53− {
54− "name":"IP limit",
55− "interval":60,
56− "limit":13,
57− "stop":true,
58− "aggregations":[
59− "Header:X-Forwarded-For"
60− ],
61− "actions":[
62− {"name":"log"},
63− {
64− "name":"block",
65− "params":{
66− "message":"rate limit exceeded"
51+ ]
52+ },
53+ {
54+ "name": "IP limit",
55+ "interval": 61,
56+ "limit": 9,
57+ "stop": true,
58+ "aggregations": [
59+ "Header:X-Forwarded-For"
60+ ],
61+ "actions": [
62+ { "name": "log"},
63+ { "name": "block",
64+ "params": {
65+ "message": "Rate limit exceeded"
66+ }
6767 }
68− }
69− ]
70− },
71− {
72− "name":"rss/json limit",
73− "interval":60,
74− "limit":13,
75− "stop":true,
76− "filters":[
77− "Param:format=(csv|json|rss)"
78− ],
79− "actions":[
80− {"name":"log"},
81− {
82− "name":"block",
83− "params":{
84− "message":"rate limit exceeded"
68+ ]
69+ },
70+ {
71+ "name": "rss/json limit",
72+ "interval": 121,
73+ "limit": 2,
74+ "stop": true,
75+ "filters": [
76+ "Param:format=(csv|json|rss)"
77+ ],
78+ "actions": [
79+ { "name": "log"},
80+ { "name": "block",
81+ "params": {
82+ "message": "Rate limit exceeded"
83+ }
8584 }
86− }
87− ]
88− },
89− {
90− "name":"useragent limit",
91− "interval":60,
92− "limit":13,
93− "aggregations":[
94− "Header:User-Agent"
95− ],
96− "actions":[
97− {"name":"log"},
98− {
99− "name":"block",
100− "params":{
101− "message":"rate limit exceeded"
85+ ]
86+ },
87+ {
88+ "name": "useragent limit",
89+ "interval": 61,
90+ "limit": 199,
91+ "aggregations": [
92+ "Header:User-Agent"
93+ ],
94+ "actions": [
95+ { "name": "log"},
96+ { "name": "block",
97+ "params": {
98+ "message": "Rate limit exceeded"
99+ }
102100 }
103− }
104− ]
105− }
106− ]
107−}]
101+ ]
102+ }
103+ ]
104+ }
105+]