git.iko.soy/searx
on master help sign in
[enh] Add command line engines: git grep, find, etc. (#2128)
f0ca1c3Noémi Ványi
searx/engines/command.pynew file+184
compare with master
unifiedsplit
@@ -0,0 +1,184 @@
1+'''
2+searx is free software: you can redistribute it and/or modify
3+it under the terms of the GNU Affero General Public License as published by
4+the Free Software Foundation, either version 3 of the License, or
5+(at your option) any later version.
6+
7+searx is distributed in the hope that it will be useful,
8+but WITHOUT ANY WARRANTY; without even the implied warranty of
9+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10+GNU Affero General Public License for more details.
11+
12+You should have received a copy of the GNU Affero General Public License
13+along with searx. If not, see < http://www.gnu.org/licenses/ >.
14+'''
15+
16+
17+from os.path import expanduser, isabs, realpath, commonprefix
18+from re import MULTILINE, search as re_search
19+from shlex import split as shlex_split
20+from subprocess import Popen, PIPE
21+from time import time
22+from threading import Thread
23+
24+from searx import logger
25+
26+
27+offline = True
28+paging = True
29+command = []
30+delimiter = {}
31+parse_regex = {}
32+query_type = ''
33+query_enum = []
34+environment_variables = {}
35+working_dir = realpath('.')
36+result_separator = '\n'
37+result_template = 'key-value.html'
38+timeout = 4.0
39+
40+_command_logger = logger.getChild('command')
41+_compiled_parse_regex = {}
42+
43+
44+def init(engine_settings):
45+ check_parsing_options(engine_settings)
46+
47+ if 'command' not in engine_settings:
48+ raise ValueError('engine command : missing configuration key: command')
49+
50+ global command, working_dir, result_template, delimiter, parse_regex, timeout, environment_variables
51+
52+ command = engine_settings['command']
53+
54+ if 'working_dir' in engine_settings:
55+ working_dir = engine_settings['working_dir']
56+ if not isabs(engine_settings['working_dir']):
57+ working_dir = realpath(working_dir)
58+
59+ if 'parse_regex' in engine_settings:
60+ parse_regex = engine_settings['parse_regex']
61+ for result_key, regex in parse_regex.items():
62+ _compiled_parse_regex[result_key] = re.compile(regex, flags=MULTILINE)
63+ if 'delimiter' in engine_settings:
64+ delimiter = engine_settings['delimiter']
65+
66+ if 'environment_variables' in engine_settings:
67+ environment_variables = engine_settings['environment_variables']
68+
69+
70+def search(query, params):
71+ cmd = _get_command_to_run(query)
72+ if not cmd:
73+ return []
74+
75+ results = []
76+ reader_thread = Thread(target=_get_results_from_process, args=(results, cmd, params['pageno']))
77+ reader_thread.start()
78+ reader_thread.join(timeout=timeout)
79+
80+ return results
81+
82+
83+def _get_command_to_run(query):
84+ params = shlex_split(query.decode('utf-8'))
85+ __check_query_params(params)
86+
87+ cmd = []
88+ for c in command:
89+ if c == '{{QUERY}}':
90+ cmd.extend(params)
91+ else:
92+ cmd.append(c)
93+
94+ return cmd
95+
96+
97+def _get_results_from_process(results, cmd, pageno):
98+ leftover = ''
99+ count = 0
100+ start, end = __get_results_limits(pageno)
101+ with Popen(cmd, stdout=PIPE, stderr=PIPE, env=environment_variables) as process:
102+ line = process.stdout.readline()
103+ while line:
104+ buf = leftover + line.decode('utf-8')
105+ raw_results = buf.split(result_separator)
106+ if raw_results[-1]:
107+ leftover = raw_results[-1]
108+ raw_results = raw_results[:-1]
109+
110+ for raw_result in raw_results:
111+ result = __parse_single_result(raw_result)
112+ if result is None:
113+ _command_logger.debug('skipped result:', raw_result)
114+ continue
115+
116+ if start <= count and count <= end:
117+ result['template'] = result_template
118+ results.append(result)
119+
120+ count += 1
121+ if end < count:
122+ return results
123+
124+ line = process.stdout.readline()
125+
126+ return_code = process.wait(timeout=timeout)
127+ if return_code != 0:
128+ raise RuntimeError('non-zero return code when running command', cmd, return_code)
129+
130+
131+def __get_results_limits(pageno):
132+ start = (pageno - 1) * 10
133+ end = start + 9
134+ return start, end
135+
136+
137+def __check_query_params(params):
138+ if not query_type:
139+ return
140+
141+ if query_type == 'path':
142+ query_path = params[-1]
143+ query_path = expanduser(query_path)
144+ if commonprefix([realpath(query_path), working_dir]) != working_dir:
145+ raise ValueError('requested path is outside of configured working directory')
146+ elif query_type == 'enum' and len(query_enum) > 0:
147+ for param in params:
148+ if param not in query_enum:
149+ raise ValueError('submitted query params is not allowed', param, 'allowed params:', query_enum)
150+
151+
152+def check_parsing_options(engine_settings):
153+ """ Checks if delimiter based parsing or regex parsing is configured correctly """
154+
155+ if 'delimiter' not in engine_settings and 'parse_regex' not in engine_settings:
156+ raise ValueError('failed to init settings for parsing lines: missing delimiter or parse_regex')
157+ if 'delimiter' in engine_settings and 'parse_regex' in engine_settings:
158+ raise ValueError('failed to init settings for parsing lines: too many settings')
159+
160+ if 'delimiter' in engine_settings:
161+ if 'chars' not in engine_settings['delimiter'] or 'keys' not in engine_settings['delimiter']:
162+ raise ValueError
163+
164+
165+def __parse_single_result(raw_result):
166+ """ Parses command line output based on configuration """
167+
168+ result = {}
169+
170+ if delimiter:
171+ elements = raw_result.split(delimiter['chars'], maxsplit=len(delimiter['keys']) - 1)
172+ if len(elements) != len(delimiter['keys']):
173+ return {}
174+ for i in range(len(elements)):
175+ result[delimiter['keys'][i]] = elements[i]
176+
177+ if parse_regex:
178+ for result_key, regex in _compiled_parse_regex.items():
179+ found = regex.search(raw_result)
180+ if not found:
181+ return {}
182+ result[result_key] = raw_result[found.start():found.end()]
183+
184+ return result
searx/settings.yml+71
@@ -920,6 +920,77 @@ engines:
920# shortcut : uw
920# shortcut : uw
921# base_url : 'http://doc.ubuntu-fr.org'
921# base_url : 'http://doc.ubuntu-fr.org'
922
922
923+# Be careful when enabling this engine if you are
924+# running a public instance. Do not expose any sensitive
925+# information. You can restrict access by configuring a list
926+# of access tokens under tokens.
927+# - name: git grep
928+# engine: command
929+# command: ['git', 'grep', '{{QUERY}}']
930+# shortcut: gg
931+# tokens: []
932+# disabled: True
933+# delimiter:
934+# chars: ':'
935+# keys: ['filepath', 'code']
936+
937+# Be careful when enabling this engine if you are
938+# running a public instance. Do not expose any sensitive
939+# information. You can restrict access by configuring a list
940+# of access tokens under tokens.
941+# - name: locate
942+# engine: command
943+# command: ['locate', '{{QUERY}}']
944+# shortcut: loc
945+# tokens: []
946+# disabled: True
947+# delimiter:
948+# chars: ' '
949+# keys: ['line']
950+
951+# Be careful when enabling this engine if you are
952+# running a public instance. Do not expose any sensitive
953+# information. You can restrict access by configuring a list
954+# of access tokens under tokens.
955+# - name: find
956+# engine: command
957+# command: ['find', '.', '-name', '{{QUERY}}']
958+# query_type: path
959+# shortcut: fnd
960+# tokens: []
961+# disabled: True
962+# delimiter:
963+# chars: ' '
964+# keys: ['line']
965+
966+# Be careful when enabling this engine if you are
967+# running a public instance. Do not expose any sensitive
968+# information. You can restrict access by configuring a list
969+# of access tokens under tokens.
970+# - name: pattern search in files
971+# engine: command
972+# command: ['fgrep', '{{QUERY}}']
973+# shortcut: fgr
974+# tokens: []
975+# disabled: True
976+# delimiter:
977+# chars: ' '
978+# keys: ['line']
979+
980+# Be careful when enabling this engine if you are
981+# running a public instance. Do not expose any sensitive
982+# information. You can restrict access by configuring a list
983+# of access tokens under tokens.
984+# - name: regex search in files
985+# engine: command
986+# command: ['grep', '{{QUERY}}']
987+# shortcut: gr
988+# tokens: []
989+# disabled: True
990+# delimiter:
991+# chars: ' '
992+# keys: ['line']
993+
923locales:
994locales:
924 en : English
995 en : English
925 ar : العَرَبِيَّة (Arabic)
996 ar : العَرَبِيَّة (Arabic)
searx/templates/oscar/result_templates/key-value.html+1−1
@@ -6,7 +6,7 @@
6 {% continue %}
6 {% continue %}
7 {% endif %}
7 {% endif %}
8 <tr>
8 <tr>
9− <td><b>{{ key|upper }}</b>: {{ value }}</td>
9+ <td><b>{{ key|upper }}</b>: {{ value|truncate }}</td>
10 </tr>
10 </tr>
11 {% endfor %}
11 {% endfor %}
12</table>
12</table>
tests/unit/engines/test_command.pynew file+241
@@ -0,0 +1,241 @@
1+'''
2+searx is free software: you can redistribute it and/or modify
3+it under the terms of the GNU Affero General Public License as published by
4+the Free Software Foundation, either version 3 of the License, or
5+(at your option) any later version.
6+
7+searx is distributed in the hope that it will be useful,
8+but WITHOUT ANY WARRANTY; without even the implied warranty of
9+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10+GNU Affero General Public License for more details.
11+
12+You should have received a copy of the GNU Affero General Public License
13+along with searx. If not, see < http://www.gnu.org/licenses/ >.
14+
15+'''
16+
17+from sys import version_info
18+
19+from searx.engines import command as command_engine
20+from searx.testing import SearxTestCase
21+
22+
23+class TestCommandEngine(SearxTestCase):
24+ def test_basic_seq_command_engine(self):
25+ ls_engine = command_engine
26+ ls_engine.command = ['seq', '{{QUERY}}']
27+ ls_engine.delimiter = {'chars': ' ', 'keys': ['number']}
28+ expected_results = [
29+ {'number': '1', 'template': 'key-value.html'},
30+ {'number': '2', 'template': 'key-value.html'},
31+ {'number': '3', 'template': 'key-value.html'},
32+ {'number': '4', 'template': 'key-value.html'},
33+ {'number': '5', 'template': 'key-value.html'},
34+ ]
35+ results = ls_engine.search('5'.encode('utf-8'), {'pageno': 1})
36+ self.assertEqual(results, expected_results)
37+
38+ def test_delimiter_parsing_command_engine(self):
39+ searx_logs = '''DEBUG:searx.webapp:static directory is /home/n/p/searx/searx/static
40+DEBUG:searx.webapp:templates directory is /home/n/p/searx/searx/templates
41+DEBUG:searx.engines:soundcloud engine: Starting background initialization
42+DEBUG:searx.engines:wolframalpha engine: Starting background initialization
43+DEBUG:searx.engines:locate engine: Starting background initialization
44+DEBUG:searx.engines:regex search in files engine: Starting background initialization
45+DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): www.wolframalpha.com
46+DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): soundcloud.com
47+DEBUG:searx.engines:find engine: Starting background initialization
48+DEBUG:searx.engines:pattern search in files engine: Starting background initialization
49+DEBUG:searx.webapp:starting webserver on 127.0.0.1:8888
50+WARNING:werkzeug: * Debugger is active!
51+INFO:werkzeug: * Debugger PIN: 299-578-362'''
52+ echo_engine = command_engine
53+ echo_engine.command = ['echo', searx_logs]
54+ echo_engine.delimiter = {'chars': ':', 'keys': ['level', 'component', 'message']}
55+
56+ expected_results_by_page = [
57+ [
58+ {
59+ 'component': 'searx.webapp',
60+ 'message': 'static directory is /home/n/p/searx/searx/static',
61+ 'template': 'key-value.html',
62+ 'level': 'DEBUG',
63+ },
64+ {
65+ 'component': 'searx.webapp',
66+ 'message': 'templates directory is /home/n/p/searx/searx/templates',
67+ 'template': 'key-value.html',
68+ 'level': 'DEBUG',
69+ },
70+ {
71+ 'component': 'searx.engines',
72+ 'message': 'soundcloud engine: Starting background initialization',
73+ 'template': 'key-value.html',
74+ 'level': 'DEBUG',
75+ },
76+ {
77+ 'component': 'searx.engines',
78+ 'message': 'wolframalpha engine: Starting background initialization',
79+ 'template': 'key-value.html',
80+ 'level': 'DEBUG',
81+ },
82+ {
83+ 'component': 'searx.engines',
84+ 'message': 'locate engine: Starting background initialization',
85+ 'template': 'key-value.html',
86+ 'level': 'DEBUG',
87+ },
88+ {
89+ 'component': 'searx.engines',
90+ 'message': 'regex search in files engine: Starting background initialization',
91+ 'template': 'key-value.html',
92+ 'level': 'DEBUG',
93+ },
94+ {
95+ 'component': 'urllib3.connectionpool',
96+ 'message': 'Starting new HTTPS connection (1): www.wolframalpha.com',
97+ 'template': 'key-value.html',
98+ 'level': 'DEBUG',
99+ },
100+ {
101+ 'component': 'urllib3.connectionpool',
102+ 'message': 'Starting new HTTPS connection (1): soundcloud.com',
103+ 'template': 'key-value.html',
104+ 'level': 'DEBUG',
105+ },
106+ {
107+ 'component': 'searx.engines',
108+ 'message': 'find engine: Starting background initialization',
109+ 'template': 'key-value.html',
110+ 'level': 'DEBUG',
111+ },
112+ {
113+ 'component': 'searx.engines',
114+ 'message': 'pattern search in files engine: Starting background initialization',
115+ 'template': 'key-value.html',
116+ 'level': 'DEBUG',
117+ },
118+
119+ ],
120+ [
121+ {
122+ 'component': 'searx.webapp',
123+ 'message': 'starting webserver on 127.0.0.1:8888',
124+ 'template': 'key-value.html',
125+ 'level': 'DEBUG',
126+ },
127+ {
128+ 'component': 'werkzeug',
129+ 'message': ' * Debugger is active!',
130+ 'template': 'key-value.html',
131+ 'level': 'WARNING',
132+ },
133+ {
134+ 'component': 'werkzeug',
135+ 'message': ' * Debugger PIN: 299-578-362',
136+ 'template': 'key-value.html',
137+ 'level': 'INFO',
138+ },
139+ ],
140+
141+ ]
142+
143+ for i in [0, 1]:
144+ results = echo_engine.search(''.encode('utf-8'), {'pageno': i + 1})
145+ self.assertEqual(results, expected_results_by_page[i])
146+
147+ def test_regex_parsing_command_engine(self):
148+ txt = '''commit 35f9a8c81d162a361b826bbcd4a1081a4fbe76a7
149+Author: Noémi Ványi <sitbackandwait@gmail.com>
150+Date: Tue Oct 15 11:31:33 2019 +0200
151+
152+first interesting message
153+
154+commit 6c3c206316153ccc422755512bceaa9ab0b14faa
155+Author: Noémi Ványi <sitbackandwait@gmail.com>
156+Date: Mon Oct 14 17:10:08 2019 +0200
157+
158+second interesting message
159+
160+commit d8594d2689b4d5e0d2f80250223886c3a1805ef5
161+Author: Noémi Ványi <sitbackandwait@gmail.com>
162+Date: Mon Oct 14 14:45:05 2019 +0200
163+
164+third interesting message
165+
166+commit '''
167+ git_log_engine = command_engine
168+ git_log_engine.command = ['echo', txt]
169+ git_log_engine.result_separator = '\n\ncommit '
170+ git_log_engine.delimiter = {}
171+ git_log_engine.parse_regex = {
172+ 'commit': '\w{40}',
173+ 'author': '[\w* ]* <\w*@?\w*\.?\w*>',
174+ 'date': 'Date: .*',
175+ 'message': '\n\n.*$'
176+ }
177+ expected_results = [
178+ {
179+ 'commit': '35f9a8c81d162a361b826bbcd4a1081a4fbe76a7',
180+ 'author': ' Noémi Ványi <sitbackandwait@gmail.com>',
181+ 'date': 'Date: Tue Oct 15 11:31:33 2019 +0200',
182+ 'message': '\n\nfirst interesting message',
183+ 'template': 'key-value.html',
184+ },
185+ {
186+ 'commit': '6c3c206316153ccc422755512bceaa9ab0b14faa',
187+ 'author': ' Noémi Ványi <sitbackandwait@gmail.com>',
188+ 'date': 'Date: Mon Oct 14 17:10:08 2019 +0200',
189+ 'message': '\n\nsecond interesting message',
190+ 'template': 'key-value.html',
191+ },
192+ {
193+ 'commit': 'd8594d2689b4d5e0d2f80250223886c3a1805ef5',
194+ 'author': ' Noémi Ványi <sitbackandwait@gmail.com>',
195+ 'date': 'Date: Mon Oct 14 14:45:05 2019 +0200',
196+ 'message': '\n\nthird interesting message',
197+ 'template': 'key-value.html',
198+ },
199+
200+ ]
201+
202+ results = git_log_engine.search(''.encode('utf-8'), {'pageno': 1})
203+ self.assertEqual(results, expected_results)
204+
205+ def test_working_dir_path_query(self):
206+ ls_engine = command_engine
207+ ls_engine.command = ['ls', '{{QUERY}}']
208+ ls_engine.result_separator = '\n'
209+ ls_engine.delimiter = {'chars': ' ', 'keys': ['file']}
210+ ls_engine.query_type = 'path'
211+
212+ results = ls_engine.search('.'.encode(), {'pageno': 1})
213+ self.assertTrue(len(results) != 0)
214+
215+ forbidden_paths = [
216+ '..',
217+ '../..',
218+ './..',
219+ '~',
220+ '/var',
221+ ]
222+ for forbidden_path in forbidden_paths:
223+ self.assertRaises(ValueError, ls_engine.search, '..'.encode(), {'pageno': 1})
224+
225+ def test_enum_queries(self):
226+ echo_engine = command_engine
227+ echo_engine.command = ['echo', '{{QUERY}}']
228+ echo_engine.query_type = 'enum'
229+ echo_engine.query_enum = ['i-am-allowed-to-say-this', 'and-that']
230+
231+ for allowed in echo_engine.query_enum:
232+ results = echo_engine.search(allowed.encode(), {'pageno': 1})
233+ self.assertTrue(len(results) != 0)
234+
235+ forbidden_queries = [
236+ 'forbidden',
237+ 'banned',
238+ 'prohibited',
239+ ]
240+ for forbidden in forbidden_queries:
241+ self.assertRaises(ValueError, echo_engine.search, forbidden.encode(), {'pageno': 1})