git.iko.soy/searx
on master help sign in
[enh] Add command line engines: git grep, find, etc. (#2128)

A new "base" engine called command is introduced. It is the foundation for all command line engines for now. You can use this engine to create your own command line engine. Add some engines (commented out to make sure no one enables anything accidentally): * git grep: This engine lets you grep in the searx repo. * locate: If locate is installed and initialized, you can search on the FS. * find: You can find files with a specific name from where you started searx. * pattern search in files: This engine utilizes the command fgrep. * regex search in files: This engine runs `grep` to find a file based on its contents.

commitf0ca1c34833e2c0c79af68e699e646d77167a269 authorNoémi Ványi <kvch@users.noreply.github.com> dateTue 8 Sep 2020, 09:51 (6 years ago) parent3397382
4 files changed+497−1
compare with master
unifiedsplit
searx/engines/command.py+184 searx/settings.yml+71 searx/templates/oscar/result_templates/key-value.html+1−1 tests/unit/engines/test_command.py+241
searx/engines/command.pynew file+184
@@ -0,0 +1,184 @@
1+'''
2+searx is free software: you can redistribute it and/or modify
3+it under the terms of the GNU Affero General Public License as published by
4+the Free Software Foundation, either version 3 of the License, or
5+(at your option) any later version.
6+
7+searx is distributed in the hope that it will be useful,
8+but WITHOUT ANY WARRANTY; without even the implied warranty of
9+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10+GNU Affero General Public License for more details.
11+
12+You should have received a copy of the GNU Affero General Public License
13+along with searx. If not, see < http://www.gnu.org/licenses/ >.
14+'''
15+
16+
17+from os.path import expanduser, isabs, realpath, commonprefix
18+from re import MULTILINE, search as re_search
19+from shlex import split as shlex_split
20+from subprocess import Popen, PIPE
21+from time import time
22+from threading import Thread
23+
24+from searx import logger
25+
26+
27+offline = True
28+paging = True
29+command = []
30+delimiter = {}
31+parse_regex = {}
32+query_type = ''
33+query_enum = []
34+environment_variables = {}
35+working_dir = realpath('.')
36+result_separator = '\n'
37+result_template = 'key-value.html'
38+timeout = 4.0
39+
40+_command_logger = logger.getChild('command')
41+_compiled_parse_regex = {}
42+
43+
44+def init(engine_settings):
45+ check_parsing_options(engine_settings)
46+
47+ if 'command' not in engine_settings:
48+ raise ValueError('engine command : missing configuration key: command')
49+
50+ global command, working_dir, result_template, delimiter, parse_regex, timeout, environment_variables
51+
52+ command = engine_settings['command']
53+
54+ if 'working_dir' in engine_settings:
55+ working_dir = engine_settings['working_dir']
56+ if not isabs(engine_settings['working_dir']):
57+ working_dir = realpath(working_dir)
58+
59+ if 'parse_regex' in engine_settings:
60+ parse_regex = engine_settings['parse_regex']
61+ for result_key, regex in parse_regex.items():
62+ _compiled_parse_regex[result_key] = re.compile(regex, flags=MULTILINE)
63+ if 'delimiter' in engine_settings:
64+ delimiter = engine_settings['delimiter']
65+
66+ if 'environment_variables' in engine_settings:
67+ environment_variables = engine_settings['environment_variables']
68+
69+
70+def search(query, params):
71+ cmd = _get_command_to_run(query)
72+ if not cmd:
73+ return []
74+
75+ results = []
76+ reader_thread = Thread(target=_get_results_from_process, args=(results, cmd, params['pageno']))
77+ reader_thread.start()
78+ reader_thread.join(timeout=timeout)
79+
80+ return results
81+
82+
83+def _get_command_to_run(query):
84+ params = shlex_split(query.decode('utf-8'))
85+ __check_query_params(params)
86+
87+ cmd = []
88+ for c in command:
89+ if c == '{{QUERY}}':
90+ cmd.extend(params)
91+ else:
92+ cmd.append(c)
93+
94+ return cmd
95+
96+
97+def _get_results_from_process(results, cmd, pageno):
98+ leftover = ''
99+ count = 0
100+ start, end = __get_results_limits(pageno)
101+ with Popen(cmd, stdout=PIPE, stderr=PIPE, env=environment_variables) as process:
102+ line = process.stdout.readline()
103+ while line:
104+ buf = leftover + line.decode('utf-8')
105+ raw_results = buf.split(result_separator)
106+ if raw_results[-1]:
107+ leftover = raw_results[-1]
108+ raw_results = raw_results[:-1]
109+
110+ for raw_result in raw_results:
111+ result = __parse_single_result(raw_result)
112+ if result is None:
113+ _command_logger.debug('skipped result:', raw_result)
114+ continue
115+
116+ if start <= count and count <= end:
117+ result['template'] = result_template
118+ results.append(result)
119+
120+ count += 1
121+ if end < count:
122+ return results
123+
124+ line = process.stdout.readline()
125+
126+ return_code = process.wait(timeout=timeout)
127+ if return_code != 0:
128+ raise RuntimeError('non-zero return code when running command', cmd, return_code)
129+
130+
131+def __get_results_limits(pageno):
132+ start = (pageno - 1) * 10
133+ end = start + 9
134+ return start, end
135+
136+
137+def __check_query_params(params):
138+ if not query_type:
139+ return
140+
141+ if query_type == 'path':
142+ query_path = params[-1]
143+ query_path = expanduser(query_path)
144+ if commonprefix([realpath(query_path), working_dir]) != working_dir:
145+ raise ValueError('requested path is outside of configured working directory')
146+ elif query_type == 'enum' and len(query_enum) > 0:
147+ for param in params:
148+ if param not in query_enum:
149+ raise ValueError('submitted query params is not allowed', param, 'allowed params:', query_enum)
150+
151+
152+def check_parsing_options(engine_settings):
153+ """ Checks if delimiter based parsing or regex parsing is configured correctly """
154+
155+ if 'delimiter' not in engine_settings and 'parse_regex' not in engine_settings:
156+ raise ValueError('failed to init settings for parsing lines: missing delimiter or parse_regex')
157+ if 'delimiter' in engine_settings and 'parse_regex' in engine_settings:
158+ raise ValueError('failed to init settings for parsing lines: too many settings')
159+
160+ if 'delimiter' in engine_settings:
161+ if 'chars' not in engine_settings['delimiter'] or 'keys' not in engine_settings['delimiter']:
162+ raise ValueError
163+
164+
165+def __parse_single_result(raw_result):
166+ """ Parses command line output based on configuration """
167+
168+ result = {}
169+
170+ if delimiter:
171+ elements = raw_result.split(delimiter['chars'], maxsplit=len(delimiter['keys']) - 1)
172+ if len(elements) != len(delimiter['keys']):
173+ return {}
174+ for i in range(len(elements)):
175+ result[delimiter['keys'][i]] = elements[i]
176+
177+ if parse_regex:
178+ for result_key, regex in _compiled_parse_regex.items():
179+ found = regex.search(raw_result)
180+ if not found:
181+ return {}
182+ result[result_key] = raw_result[found.start():found.end()]
183+
184+ return result
searx/settings.yml+71
@@ -920,6 +920,77 @@ engines:
920920# shortcut : uw
921921# base_url : 'http://doc.ubuntu-fr.org'
922922
923+# Be careful when enabling this engine if you are
924+# running a public instance. Do not expose any sensitive
925+# information. You can restrict access by configuring a list
926+# of access tokens under tokens.
927+# - name: git grep
928+# engine: command
929+# command: ['git', 'grep', '{{QUERY}}']
930+# shortcut: gg
931+# tokens: []
932+# disabled: True
933+# delimiter:
934+# chars: ':'
935+# keys: ['filepath', 'code']
936+
937+# Be careful when enabling this engine if you are
938+# running a public instance. Do not expose any sensitive
939+# information. You can restrict access by configuring a list
940+# of access tokens under tokens.
941+# - name: locate
942+# engine: command
943+# command: ['locate', '{{QUERY}}']
944+# shortcut: loc
945+# tokens: []
946+# disabled: True
947+# delimiter:
948+# chars: ' '
949+# keys: ['line']
950+
951+# Be careful when enabling this engine if you are
952+# running a public instance. Do not expose any sensitive
953+# information. You can restrict access by configuring a list
954+# of access tokens under tokens.
955+# - name: find
956+# engine: command
957+# command: ['find', '.', '-name', '{{QUERY}}']
958+# query_type: path
959+# shortcut: fnd
960+# tokens: []
961+# disabled: True
962+# delimiter:
963+# chars: ' '
964+# keys: ['line']
965+
966+# Be careful when enabling this engine if you are
967+# running a public instance. Do not expose any sensitive
968+# information. You can restrict access by configuring a list
969+# of access tokens under tokens.
970+# - name: pattern search in files
971+# engine: command
972+# command: ['fgrep', '{{QUERY}}']
973+# shortcut: fgr
974+# tokens: []
975+# disabled: True
976+# delimiter:
977+# chars: ' '
978+# keys: ['line']
979+
980+# Be careful when enabling this engine if you are
981+# running a public instance. Do not expose any sensitive
982+# information. You can restrict access by configuring a list
983+# of access tokens under tokens.
984+# - name: regex search in files
985+# engine: command
986+# command: ['grep', '{{QUERY}}']
987+# shortcut: gr
988+# tokens: []
989+# disabled: True
990+# delimiter:
991+# chars: ' '
992+# keys: ['line']
993+
923994locales:
924995 en : English
925996 ar : العَرَبِيَّة (Arabic)
searx/templates/oscar/result_templates/key-value.html+1−1
@@ -6,7 +6,7 @@
66 {% continue %}
77 {% endif %}
88 <tr>
9− <td><b>{{ key|upper }}</b>: {{ value }}</td>
9+ <td><b>{{ key|upper }}</b>: {{ value|truncate }}</td>
1010 </tr>
1111 {% endfor %}
1212</table>
tests/unit/engines/test_command.pynew file+241
@@ -0,0 +1,241 @@
1+'''
2+searx is free software: you can redistribute it and/or modify
3+it under the terms of the GNU Affero General Public License as published by
4+the Free Software Foundation, either version 3 of the License, or
5+(at your option) any later version.
6+
7+searx is distributed in the hope that it will be useful,
8+but WITHOUT ANY WARRANTY; without even the implied warranty of
9+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10+GNU Affero General Public License for more details.
11+
12+You should have received a copy of the GNU Affero General Public License
13+along with searx. If not, see < http://www.gnu.org/licenses/ >.
14+
15+'''
16+
17+from sys import version_info
18+
19+from searx.engines import command as command_engine
20+from searx.testing import SearxTestCase
21+
22+
23+class TestCommandEngine(SearxTestCase):
24+ def test_basic_seq_command_engine(self):
25+ ls_engine = command_engine
26+ ls_engine.command = ['seq', '{{QUERY}}']
27+ ls_engine.delimiter = {'chars': ' ', 'keys': ['number']}
28+ expected_results = [
29+ {'number': '1', 'template': 'key-value.html'},
30+ {'number': '2', 'template': 'key-value.html'},
31+ {'number': '3', 'template': 'key-value.html'},
32+ {'number': '4', 'template': 'key-value.html'},
33+ {'number': '5', 'template': 'key-value.html'},
34+ ]
35+ results = ls_engine.search('5'.encode('utf-8'), {'pageno': 1})
36+ self.assertEqual(results, expected_results)
37+
38+ def test_delimiter_parsing_command_engine(self):
39+ searx_logs = '''DEBUG:searx.webapp:static directory is /home/n/p/searx/searx/static
40+DEBUG:searx.webapp:templates directory is /home/n/p/searx/searx/templates
41+DEBUG:searx.engines:soundcloud engine: Starting background initialization
42+DEBUG:searx.engines:wolframalpha engine: Starting background initialization
43+DEBUG:searx.engines:locate engine: Starting background initialization
44+DEBUG:searx.engines:regex search in files engine: Starting background initialization
45+DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): www.wolframalpha.com
46+DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): soundcloud.com
47+DEBUG:searx.engines:find engine: Starting background initialization
48+DEBUG:searx.engines:pattern search in files engine: Starting background initialization
49+DEBUG:searx.webapp:starting webserver on 127.0.0.1:8888
50+WARNING:werkzeug: * Debugger is active!
51+INFO:werkzeug: * Debugger PIN: 299-578-362'''
52+ echo_engine = command_engine
53+ echo_engine.command = ['echo', searx_logs]
54+ echo_engine.delimiter = {'chars': ':', 'keys': ['level', 'component', 'message']}
55+
56+ expected_results_by_page = [
57+ [
58+ {
59+ 'component': 'searx.webapp',
60+ 'message': 'static directory is /home/n/p/searx/searx/static',
61+ 'template': 'key-value.html',
62+ 'level': 'DEBUG',
63+ },
64+ {
65+ 'component': 'searx.webapp',
66+ 'message': 'templates directory is /home/n/p/searx/searx/templates',
67+ 'template': 'key-value.html',
68+ 'level': 'DEBUG',
69+ },
70+ {
71+ 'component': 'searx.engines',
72+ 'message': 'soundcloud engine: Starting background initialization',
73+ 'template': 'key-value.html',
74+ 'level': 'DEBUG',
75+ },
76+ {
77+ 'component': 'searx.engines',
78+ 'message': 'wolframalpha engine: Starting background initialization',
79+ 'template': 'key-value.html',
80+ 'level': 'DEBUG',
81+ },
82+ {
83+ 'component': 'searx.engines',
84+ 'message': 'locate engine: Starting background initialization',
85+ 'template': 'key-value.html',
86+ 'level': 'DEBUG',
87+ },
88+ {
89+ 'component': 'searx.engines',
90+ 'message': 'regex search in files engine: Starting background initialization',
91+ 'template': 'key-value.html',
92+ 'level': 'DEBUG',
93+ },
94+ {
95+ 'component': 'urllib3.connectionpool',
96+ 'message': 'Starting new HTTPS connection (1): www.wolframalpha.com',
97+ 'template': 'key-value.html',
98+ 'level': 'DEBUG',
99+ },
100+ {
101+ 'component': 'urllib3.connectionpool',
102+ 'message': 'Starting new HTTPS connection (1): soundcloud.com',
103+ 'template': 'key-value.html',
104+ 'level': 'DEBUG',
105+ },
106+ {
107+ 'component': 'searx.engines',
108+ 'message': 'find engine: Starting background initialization',
109+ 'template': 'key-value.html',
110+ 'level': 'DEBUG',
111+ },
112+ {
113+ 'component': 'searx.engines',
114+ 'message': 'pattern search in files engine: Starting background initialization',
115+ 'template': 'key-value.html',
116+ 'level': 'DEBUG',
117+ },
118+
119+ ],
120+ [
121+ {
122+ 'component': 'searx.webapp',
123+ 'message': 'starting webserver on 127.0.0.1:8888',
124+ 'template': 'key-value.html',
125+ 'level': 'DEBUG',
126+ },
127+ {
128+ 'component': 'werkzeug',
129+ 'message': ' * Debugger is active!',
130+ 'template': 'key-value.html',
131+ 'level': 'WARNING',
132+ },
133+ {
134+ 'component': 'werkzeug',
135+ 'message': ' * Debugger PIN: 299-578-362',
136+ 'template': 'key-value.html',
137+ 'level': 'INFO',
138+ },
139+ ],
140+
141+ ]
142+
143+ for i in [0, 1]:
144+ results = echo_engine.search(''.encode('utf-8'), {'pageno': i + 1})
145+ self.assertEqual(results, expected_results_by_page[i])
146+
147+ def test_regex_parsing_command_engine(self):
148+ txt = '''commit 35f9a8c81d162a361b826bbcd4a1081a4fbe76a7
149+Author: Noémi Ványi <sitbackandwait@gmail.com>
150+Date: Tue Oct 15 11:31:33 2019 +0200
151+
152+first interesting message
153+
154+commit 6c3c206316153ccc422755512bceaa9ab0b14faa
155+Author: Noémi Ványi <sitbackandwait@gmail.com>
156+Date: Mon Oct 14 17:10:08 2019 +0200
157+
158+second interesting message
159+
160+commit d8594d2689b4d5e0d2f80250223886c3a1805ef5
161+Author: Noémi Ványi <sitbackandwait@gmail.com>
162+Date: Mon Oct 14 14:45:05 2019 +0200
163+
164+third interesting message
165+
166+commit '''
167+ git_log_engine = command_engine
168+ git_log_engine.command = ['echo', txt]
169+ git_log_engine.result_separator = '\n\ncommit '
170+ git_log_engine.delimiter = {}
171+ git_log_engine.parse_regex = {
172+ 'commit': '\w{40}',
173+ 'author': '[\w* ]* <\w*@?\w*\.?\w*>',
174+ 'date': 'Date: .*',
175+ 'message': '\n\n.*$'
176+ }
177+ expected_results = [
178+ {
179+ 'commit': '35f9a8c81d162a361b826bbcd4a1081a4fbe76a7',
180+ 'author': ' Noémi Ványi <sitbackandwait@gmail.com>',
181+ 'date': 'Date: Tue Oct 15 11:31:33 2019 +0200',
182+ 'message': '\n\nfirst interesting message',
183+ 'template': 'key-value.html',
184+ },
185+ {
186+ 'commit': '6c3c206316153ccc422755512bceaa9ab0b14faa',
187+ 'author': ' Noémi Ványi <sitbackandwait@gmail.com>',
188+ 'date': 'Date: Mon Oct 14 17:10:08 2019 +0200',
189+ 'message': '\n\nsecond interesting message',
190+ 'template': 'key-value.html',
191+ },
192+ {
193+ 'commit': 'd8594d2689b4d5e0d2f80250223886c3a1805ef5',
194+ 'author': ' Noémi Ványi <sitbackandwait@gmail.com>',
195+ 'date': 'Date: Mon Oct 14 14:45:05 2019 +0200',
196+ 'message': '\n\nthird interesting message',
197+ 'template': 'key-value.html',
198+ },
199+
200+ ]
201+
202+ results = git_log_engine.search(''.encode('utf-8'), {'pageno': 1})
203+ self.assertEqual(results, expected_results)
204+
205+ def test_working_dir_path_query(self):
206+ ls_engine = command_engine
207+ ls_engine.command = ['ls', '{{QUERY}}']
208+ ls_engine.result_separator = '\n'
209+ ls_engine.delimiter = {'chars': ' ', 'keys': ['file']}
210+ ls_engine.query_type = 'path'
211+
212+ results = ls_engine.search('.'.encode(), {'pageno': 1})
213+ self.assertTrue(len(results) != 0)
214+
215+ forbidden_paths = [
216+ '..',
217+ '../..',
218+ './..',
219+ '~',
220+ '/var',
221+ ]
222+ for forbidden_path in forbidden_paths:
223+ self.assertRaises(ValueError, ls_engine.search, '..'.encode(), {'pageno': 1})
224+
225+ def test_enum_queries(self):
226+ echo_engine = command_engine
227+ echo_engine.command = ['echo', '{{QUERY}}']
228+ echo_engine.query_type = 'enum'
229+ echo_engine.query_enum = ['i-am-allowed-to-say-this', 'and-that']
230+
231+ for allowed in echo_engine.query_enum:
232+ results = echo_engine.search(allowed.encode(), {'pageno': 1})
233+ self.assertTrue(len(results) != 0)
234+
235+ forbidden_queries = [
236+ 'forbidden',
237+ 'banned',
238+ 'prohibited',
239+ ]
240+ for forbidden in forbidden_queries:
241+ self.assertRaises(ValueError, echo_engine.search, forbidden.encode(), {'pageno': 1})