@@ -1,66 +1,397 @@
1−// pub struct Syntax {
2−// parser: Parser,
3−// }
1+use crate::{Change, Rope, RopeSlice, Transaction};
2+pub use helix_syntax::LANG;
3+pub use helix_syntax::{get_language, get_language_name};
4+
5+pub struct Syntax {
6+ grammar: Language,
7+ parser: Parser,
8+ cursors: Vec<QueryCursor>,
9+
10+ config: HighlightConfiguration,
411
5−//impl Syntax {
6−// // buffer, grammar, config, grammars, sync_timeout?
7−// pub fn new() -> Self {
8−// unimplemented!()
9−// // make a new root layer
10−// // track markers of injections
11−// //
12−// // track scope_descriptor: a Vec of scopes for item in tree
13−// //
14−// // fetch grammar for parser based on language string
15−// // update root layer
16−// }
17−
18−// // fn buffer_changed -> call layer.update(range, new_text) on root layer and then all marker layers
19−
20−// // call this on transaction.apply() -> buffer_changed(changes)
21−// //
22−// // fn parse(language, old_tree, ranges)
23−// //
24−// // fn tree() -> Tree
25−// //
26−// // <!--update_for_injection(grammar)-->
27−
28−// // Highlighting
29−// // fn highlight_iter() -> iterates over all the scopes
30−// // on_tokenize
31−// // on_change_highlighting
32−
33−// // Commenting
34−// // comment_strings_for_pos
35−// // is_commented
36−
37−// // Indentation
38−// // suggested_indent_for_line_at_buffer_row
39−// // suggested_indent_for_buffer_row
40−// // indent_level_for_line
41−
42−// // TODO: Folding
43−
44−// // Syntax APIs
45−// // get_syntax_node_containing_range ->
46−// // ...
47−// // get_syntax_node_at_pos
48−// // buffer_range_for_scope_at_pos
49−//}
12+ root_layer: LanguageLayer,
13+}
14+
15+impl Syntax {
16+ // buffer, grammar, config, grammars, sync_timeout?
17+ pub fn new(language: LANG, source: &Rope, config: HighlightConfiguration) -> Self {
18+ // fetch grammar for parser based on language string
19+ let grammar = get_language(&language);
20+ let parser = Parser::new();
21+
22+ let root_layer = LanguageLayer::new();
23+
24+ // track markers of injections
25+ // track scope_descriptor: a Vec of scopes for item in tree
26+
27+ let mut syntax = Self {
28+ grammar,
29+ parser,
30+ cursors: Vec::new(),
31+ config,
32+ root_layer,
33+ };
34+
35+ // update root layer
36+ syntax.root_layer.parse(
37+ &mut syntax.parser,
38+ &syntax.config,
39+ source,
40+ 0,
41+ vec![Range {
42+ start_byte: 0,
43+ end_byte: usize::MAX,
44+ start_point: Point::new(0, 0),
45+ end_point: Point::new(usize::MAX, usize::MAX),
46+ }],
47+ );
48+ syntax
49+ }
50+
51+ pub fn configure(&mut self, scopes: &[String]) {
52+ self.config.configure(scopes)
53+ }
54+
55+ pub fn update(&mut self, source: &Rope, changeset: &ChangeSet) -> Result<(), Error> {
56+ self.root_layer
57+ .update(&mut self.parser, &self.config, source, changeset)
58+
59+ // TODO: deal with injections and update them too
60+ }
61+
62+ // fn buffer_changed -> call layer.update(range, new_text) on root layer and then all marker layers
63+
64+ // call this on transaction.apply() -> buffer_changed(changes)
65+ //
66+ // fn parse(language, old_tree, ranges)
67+ //
68+ fn tree(&self) -> &Tree {
69+ self.root_layer.tree()
70+ }
71+ //
72+ // <!--update_for_injection(grammar)-->
73+
74+ // Highlighting
75+
76+ /// Iterate over the highlighted regions for a given slice of source code.
77+ pub fn highlight_iter<'a>(
78+ &'a mut self,
79+ source: &'a [u8],
80+ cancellation_flag: Option<&'a AtomicUsize>,
81+ mut injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
82+ ) -> Result<impl Iterator<Item = Result<HighlightEvent, Error>> + 'a, Error> {
83+ // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which
84+ // prevents them from being moved. But both of these values are really just
85+ // pointers, so it's actually ok to move them.
86+
87+ let mut cursor = QueryCursor::new(); // reuse a pool
88+ let tree_ref = unsafe { mem::transmute::<_, &'static Tree>(self.tree()) };
89+ let cursor_ref = unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) };
90+ let query_ref = unsafe { mem::transmute::<_, &'static mut Query>(&mut self.config.query) };
91+ let config_ref =
92+ unsafe { mem::transmute::<_, &'static HighlightConfiguration>(&self.config) };
93+ let captures = cursor_ref
94+ .captures(query_ref, tree_ref.root_node(), move |n: Node| {
95+ &source[n.byte_range()]
96+ })
97+ .peekable();
98+
99+ // manually craft the root layer based on the existing tree
100+ let layer = HighlightIterLayer {
101+ highlight_end_stack: Vec::new(),
102+ scope_stack: vec![LocalScope {
103+ inherits: false,
104+ range: 0..usize::MAX,
105+ local_defs: Vec::new(),
106+ }],
107+ cursor,
108+ depth: 0,
109+ _tree: None,
110+ captures,
111+ config: config_ref,
112+ ranges: vec![Range {
113+ start_byte: 0,
114+ end_byte: usize::MAX,
115+ start_point: Point::new(0, 0),
116+ end_point: Point::new(usize::MAX, usize::MAX),
117+ }],
118+ };
119+
120+ let mut result = HighlightIter {
121+ source,
122+ byte_offset: 0,
123+ injection_callback,
124+ cancellation_flag,
125+ highlighter: self,
126+ iter_count: 0,
127+ layers: vec![layer],
128+ next_event: None,
129+ last_highlight_range: None,
130+ };
131+ result.sort_layers();
132+ Ok(result)
133+ }
134+ // on_tokenize
135+ // on_change_highlighting
136+
137+ // Commenting
138+ // comment_strings_for_pos
139+ // is_commented
140+
141+ // Indentation
142+ // suggested_indent_for_line_at_buffer_row
143+ // suggested_indent_for_buffer_row
144+ // indent_level_for_line
145+
146+ // TODO: Folding
147+
148+ // Syntax APIs
149+ // get_syntax_node_containing_range ->
150+ // ...
151+ // get_syntax_node_at_pos
152+ // buffer_range_for_scope_at_pos
153+}
50154
51155pub struct LanguageLayer {
52156 // mode
53−// grammar
54−// depth
55−// tree: Tree,
157+ // grammar
158+ // depth
159+ tree: Option<Tree>,
56160}
57161
58−// impl LanguageLayer {
59−// // fn highlight_iter() -> same as Mode but for this layer. Mode composits these
60−// // fn buffer_changed
61−// // fn update(range)
62−// // fn update_injections()
63−// }
162+use crate::state::{coords_at_pos, Coords};
163+use crate::transaction::{ChangeSet, Operation};
164+use crate::Tendril;
165+
166+impl LanguageLayer {
167+ pub fn new() -> Self {
168+ Self { tree: None }
169+ }
170+
171+ fn tree(&self) -> &Tree {
172+ // TODO: no unwrap
173+ self.tree.as_ref().unwrap()
174+ }
175+
176+ fn parse<'a>(
177+ &mut self,
178+ parser: &mut Parser,
179+ config: &HighlightConfiguration,
180+ source: &Rope,
181+ mut depth: usize,
182+ mut ranges: Vec<Range>,
183+ ) -> Result<(), Error> {
184+ if parser.set_included_ranges(&ranges).is_ok() {
185+ parser
186+ .set_language(config.language)
187+ .map_err(|_| Error::InvalidLanguage)?;
188+
189+ // unsafe { syntax.parser.set_cancellation_flag(cancellation_flag) };
190+ let tree = parser
191+ .parse_with(
192+ &mut |byte, _| {
193+ if byte <= source.len_bytes() {
194+ let (chunk, start_byte, _, _) = source.chunk_at_byte(byte);
195+ chunk[byte - start_byte..].as_bytes()
196+ } else {
197+ // out of range
198+ &[]
199+ }
200+ },
201+ self.tree.as_ref(),
202+ )
203+ .ok_or(Error::Cancelled)?;
204+ // unsafe { syntax.parser.set_cancellation_flag(None) };
205+ // let mut cursor = syntax.cursors.pop().unwrap_or_else(QueryCursor::new);
206+
207+ // Process combined injections. (ERB, EJS, etc https://github.com/tree-sitter/tree-sitter/pull/526)
208+ // if let Some(combined_injections_query) = &config.combined_injections_query {
209+ // let mut injections_by_pattern_index =
210+ // vec![(None, Vec::new(), false); combined_injections_query.pattern_count()];
211+ // let matches =
212+ // cursor.matches(combined_injections_query, tree.root_node(), |n: Node| {
213+ // &source[n.byte_range()]
214+ // });
215+ // for mat in matches {
216+ // let entry = &mut injections_by_pattern_index[mat.pattern_index];
217+ // let (language_name, content_node, include_children) =
218+ // injection_for_match(config, combined_injections_query, &mat, source);
219+ // if language_name.is_some() {
220+ // entry.0 = language_name;
221+ // }
222+ // if let Some(content_node) = content_node {
223+ // entry.1.push(content_node);
224+ // }
225+ // entry.2 = include_children;
226+ // }
227+ // for (lang_name, content_nodes, includes_children) in injections_by_pattern_index {
228+ // if let (Some(lang_name), false) = (lang_name, content_nodes.is_empty()) {
229+ // if let Some(next_config) = (injection_callback)(lang_name) {
230+ // let ranges =
231+ // Self::intersect_ranges(&ranges, &content_nodes, includes_children);
232+ // if !ranges.is_empty() {
233+ // queue.push((next_config, depth + 1, ranges));
234+ // }
235+ // }
236+ // }
237+ // }
238+ // }
239+ self.tree = Some(tree)
240+ }
241+ Ok(())
242+ }
243+
244+ pub(crate) fn generate_edits(
245+ text: &RopeSlice,
246+ changeset: &ChangeSet,
247+ ) -> Vec<tree_sitter::InputEdit> {
248+ use Operation::*;
249+ let mut old_pos = 0;
250+ let mut new_pos = 0;
251+
252+ let mut edits = Vec::new();
253+
254+ let mut iter = changeset.changes.iter().peekable();
255+
256+ // TODO; this is a lot easier with Change instead of Operation.
257+
258+ fn point_at_pos(text: &RopeSlice, pos: usize) -> (usize, Point) {
259+ let byte = text.char_to_byte(pos);
260+ let line = text.char_to_line(pos);
261+ let line_start_byte = text.line_to_byte(line);
262+ let col = byte - line_start_byte;
263+
264+ (byte, Point::new(line, col))
265+ }
266+
267+ fn traverse(point: Point, text: &Tendril) -> Point {
268+ let Point {
269+ mut row,
270+ mut column,
271+ } = point;
272+
273+ // TODO: there should be a better way here
274+ for ch in text.bytes() {
275+ if ch == b'\n' {
276+ row += 1;
277+ column = 0;
278+ } else {
279+ column += 1;
280+ }
281+ }
282+ Point { row, column }
283+ }
284+
285+ while let Some(change) = iter.next() {
286+ let len = match change {
287+ Delete(i) | Retain(i) => *i,
288+ Insert(_) => 0,
289+ };
290+ let old_end = old_pos + len;
291+
292+ match change {
293+ Retain(_) => {
294+ new_pos += len;
295+ }
296+ Delete(_) => {
297+ let (start_byte, start_position) = point_at_pos(&text, old_pos);
298+ let (old_end_byte, old_end_position) = point_at_pos(&text, old_end);
299+
300+ // TODO: Position also needs to be byte based...
301+ // let byte = char_to_byte(old_pos)
302+ // let line = char_to_line(old_pos)
303+ // let line_start_byte = line_to_byte()
304+ // Position::new(line, line_start_byte - byte)
305+
306+ // a subsequent ins means a replace, consume it
307+ if let Some(Insert(s)) = iter.peek() {
308+ iter.next();
309+ let ins = s.chars().count();
310+
311+ // replacement
312+ edits.push(tree_sitter::InputEdit {
313+ start_byte, // old_pos to byte
314+ old_end_byte, // old_end to byte
315+ new_end_byte: start_byte + s.len(), // old_pos to byte + s.len()
316+ start_position, // old pos to coords
317+ old_end_position, // old_end to coords
318+ new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)
319+ });
320+
321+ new_pos += ins;
322+ } else {
323+ // deletion
324+ edits.push(tree_sitter::InputEdit {
325+ start_byte, // old_pos to byte
326+ old_end_byte, // old_end to byte
327+ new_end_byte: start_byte, // old_pos to byte
328+ start_position, // old pos to coords
329+ old_end_position, // old_end to coords
330+ new_end_position: start_position, // old pos to coords
331+ });
332+ };
333+ }
334+ Insert(s) => {
335+ let (start_byte, start_position) = point_at_pos(&text, old_pos);
336+
337+ let ins = s.chars().count();
338+
339+ // insert
340+ edits.push(tree_sitter::InputEdit {
341+ start_byte, // old_pos to byte
342+ old_end_byte: start_byte, // same
343+ new_end_byte: start_byte + s.len(), // old_pos + s.len()
344+ start_position, // old pos to coords
345+ old_end_position: start_position, // same
346+ new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)
347+ });
348+
349+ new_pos += ins;
350+ }
351+ }
352+ old_pos = old_end;
353+ }
354+ edits
355+ }
356+
357+ fn update(
358+ &mut self,
359+ parser: &mut Parser,
360+ config: &HighlightConfiguration,
361+ source: &Rope,
362+ changeset: &ChangeSet,
363+ ) -> Result<(), Error> {
364+ if changeset.is_empty() {
365+ return Ok(());
366+ }
367+
368+ let edits = Self::generate_edits(&source.slice(..), changeset);
369+
370+ // Notify the tree about all the changes
371+ for edit in edits {
372+ self.tree.as_mut().unwrap().edit(&edit);
373+ }
374+
375+ self.parse(
376+ parser,
377+ config,
378+ source,
379+ 0,
380+ // TODO: what to do about this range on update
381+ vec![Range {
382+ start_byte: 0,
383+ end_byte: usize::MAX,
384+ start_point: Point::new(0, 0),
385+ end_point: Point::new(usize::MAX, usize::MAX),
386+ }],
387+ )
388+ }
389+
390+ // fn highlight_iter() -> same as Mode but for this layer. Mode composits these
391+ // fn buffer_changed
392+ // fn update(range)
393+ // fn update_injections()
394+}
64395
65396// -- refactored from tree-sitter-highlight to be able to retain state
66397// TODO: add seek() to iter
@@ -169,7 +500,7 @@ where
169500{
170501 source: &'a [u8],
171502 byte_offset: usize,
172− highlighter: &'a mut Highlighter,
503+ highlighter: &'a mut Syntax,
173504 injection_callback: F,
174505 cancellation_flag: Option<&'a AtomicUsize>,
175506 layers: Vec<HighlightIterLayer<'a>>,
@@ -179,7 +510,7 @@ where
179510}
180511
181512struct HighlightIterLayer<'a> {
182− _tree: Tree,
513+ _tree: Option<Tree>,
183514 cursor: QueryCursor,
184515 captures: iter::Peekable<QueryCaptures<'a, &'a [u8]>>,
185516 config: &'a HighlightConfiguration,
@@ -207,43 +538,43 @@ impl Highlighter {
207538 &mut self.parser
208539 }
209540
210− /// Iterate over the highlighted regions for a given slice of source code.
211− pub fn highlight<'a>(
212− &'a mut self,
213− config: &'a HighlightConfiguration,
214− source: &'a [u8],
215− cancellation_flag: Option<&'a AtomicUsize>,
216− mut injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
217− ) -> Result<impl Iterator<Item = Result<HighlightEvent, Error>> + 'a, Error> {
218− let layers = HighlightIterLayer::new(
219− source,
220− self,
221− cancellation_flag,
222− &mut injection_callback,
223− config,
224− 0,
225− vec![Range {
226− start_byte: 0,
227− end_byte: usize::MAX,
228− start_point: Point::new(0, 0),
229− end_point: Point::new(usize::MAX, usize::MAX),
230− }],
231− )?;
232− assert_ne!(layers.len(), 0);
233− let mut result = HighlightIter {
234− source,
235− byte_offset: 0,
236− injection_callback,
237− cancellation_flag,
238− highlighter: self,
239− iter_count: 0,
240− layers,
241− next_event: None,
242− last_highlight_range: None,
243− };
244− result.sort_layers();
245− Ok(result)
246− }
541+ // /// Iterate over the highlighted regions for a given slice of source code.
542+ // pub fn highlight<'a>(
543+ // &'a mut self,
544+ // config: &'a HighlightConfiguration,
545+ // source: &'a [u8],
546+ // cancellation_flag: Option<&'a AtomicUsize>,
547+ // mut injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
548+ // ) -> Result<impl Iterator<Item = Result<HighlightEvent, Error>> + 'a, Error> {
549+ // let layers = HighlightIterLayer::new(
550+ // source,
551+ // self,
552+ // cancellation_flag,
553+ // &mut injection_callback,
554+ // config,
555+ // 0,
556+ // vec![Range {
557+ // start_byte: 0,
558+ // end_byte: usize::MAX,
559+ // start_point: Point::new(0, 0),
560+ // end_point: Point::new(usize::MAX, usize::MAX),
561+ // }],
562+ // )?;
563+ // assert_ne!(layers.len(), 0);
564+ // let mut result = HighlightIter {
565+ // source,
566+ // byte_offset: 0,
567+ // injection_callback,
568+ // cancellation_flag,
569+ // highlighter: self,
570+ // iter_count: 0,
571+ // layers,
572+ // next_event: None,
573+ // last_highlight_range: None,
574+ // };
575+ // result.sort_layers();
576+ // Ok(result)
577+ // }
247578}
248579
249580impl HighlightConfiguration {
@@ -413,7 +744,7 @@ impl<'a> HighlightIterLayer<'a> {
413744 /// added to the returned vector.
414745 fn new<F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a>(
415746 source: &'a [u8],
416− highlighter: &mut Highlighter,
747+ highlighter: &mut Syntax,
417748 cancellation_flag: Option<&'a AtomicUsize>,
418749 injection_callback: &mut F,
419750 mut config: &'a HighlightConfiguration,
@@ -423,6 +754,8 @@ impl<'a> HighlightIterLayer<'a> {
423754 let mut result = Vec::with_capacity(1);
424755 let mut queue = Vec::new();
425756 loop {
757+ // --> Tree parsing part
758+
426759 if highlighter.parser.set_included_ranges(&ranges).is_ok() {
427760 highlighter
428761 .parser
@@ -474,6 +807,8 @@ impl<'a> HighlightIterLayer<'a> {
474807 }
475808 }
476809
810+ // --> Highlighting query part
811+
477812 // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which
478813 // prevents them from being moved. But both of these values are really just
479814 // pointers, so it's actually ok to move them.
@@ -495,7 +830,7 @@ impl<'a> HighlightIterLayer<'a> {
495830 }],
496831 cursor,
497832 depth,
498− _tree: tree,
833+ _tree: Some(tree),
499834 captures,
500835 config,
501836 ranges,
@@ -1016,3 +1351,106 @@ fn shrink_and_clear<T>(vec: &mut Vec<T>, capacity: usize) {
10161351 }
10171352 vec.clear();
10181353}
1354+
1355+#[test]
1356+fn test_parser() {
1357+ let highlight_names: Vec<String> = [
1358+ "attribute",
1359+ "constant",
1360+ "function.builtin",
1361+ "function",
1362+ "keyword",
1363+ "operator",
1364+ "property",
1365+ "punctuation",
1366+ "punctuation.bracket",
1367+ "punctuation.delimiter",
1368+ "string",
1369+ "string.special",
1370+ "tag",
1371+ "type",
1372+ "type.builtin",
1373+ "variable",
1374+ "variable.builtin",
1375+ "variable.parameter",
1376+ ]
1377+ .iter()
1378+ .cloned()
1379+ .map(String::from)
1380+ .collect();
1381+
1382+ let language = get_language(&LANG::Rust);
1383+ let mut config = HighlightConfiguration::new(
1384+ language,
1385+ &std::fs::read_to_string(
1386+ "../helix-syntax/languages/tree-sitter-rust/queries/highlights.scm",
1387+ )
1388+ .unwrap(),
1389+ &std::fs::read_to_string(
1390+ "../helix-syntax/languages/tree-sitter-rust/queries/injections.scm",
1391+ )
1392+ .unwrap(),
1393+ "", // locals.scm
1394+ )
1395+ .unwrap();
1396+ config.configure(&highlight_names);
1397+
1398+ let source = Rope::from_str(
1399+ "
1400+ struct Stuff {}
1401+ fn main() {}
1402+ ",
1403+ );
1404+ let syntax = Syntax::new(LANG::Rust, &source, config);
1405+ let tree = syntax.root_layer.tree.unwrap();
1406+ let root = tree.root_node();
1407+ assert_eq!(root.kind(), "source_file");
1408+
1409+ assert_eq!(
1410+ root.to_sexp(),
1411+ concat!(
1412+ "(source_file ",
1413+ "(struct_item name: (type_identifier) body: (field_declaration_list)) ",
1414+ "(function_item name: (identifier) parameters: (parameters) body: (block)))"
1415+ )
1416+ );
1417+
1418+ let struct_node = root.child(0).unwrap();
1419+ assert_eq!(struct_node.kind(), "struct_item");
1420+}
1421+
1422+#[test]
1423+fn test_input_edits() {
1424+ use crate::State;
1425+ use tree_sitter::InputEdit;
1426+
1427+ let mut state = State::new("hello world!\ntest 123".into());
1428+ let transaction = Transaction::change(
1429+ &state,
1430+ vec![(6, 11, Some("test".into())), (12, 17, None)].into_iter(),
1431+ );
1432+ let edits = LanguageLayer::generate_edits(&state.doc.slice(..), &transaction.changes);
1433+ // transaction.apply(&mut state);
1434+
1435+ assert_eq!(
1436+ edits,
1437+ &[
1438+ InputEdit {
1439+ start_byte: 6,
1440+ old_end_byte: 11,
1441+ new_end_byte: 10,
1442+ start_position: Point { row: 0, column: 6 },
1443+ old_end_position: Point { row: 0, column: 11 },
1444+ new_end_position: Point { row: 0, column: 10 }
1445+ },
1446+ InputEdit {
1447+ start_byte: 12,
1448+ old_end_byte: 17,
1449+ new_end_byte: 12,
1450+ start_position: Point { row: 0, column: 12 },
1451+ old_end_position: Point { row: 1, column: 4 },
1452+ new_end_position: Point { row: 0, column: 12 }
1453+ }
1454+ ]
1455+ );
1456+}