on master help sign in
Update tree-sitter to 0.20

0.20 includes querying improvements, we no longer have to convert fragments to strings but can return an iterator of chunks instead.

commit3cb95be452491a72d18c98ebc619b0d2abb1b746 authorBlaž Hrastnik <blaz@mxxn.io> dateMon 6 Sep 2021, 13:18 (5 years ago) parent57ed518
4 files changed+45−42
Cargo.lock+2−2 helix-core/Cargo.toml+1−1 helix-core/src/syntax.rs+41−38 helix-syntax/Cargo.toml+1−1
Cargo.lock+2−2
@@ -1034,9 +1034,9 @@ dependencies = [
10341034
10351035[[package]]
10361036name = "tree-sitter"
1037−version = "0.19.5"
1037+version = "0.20.0"
10381038source = "registry+https://github.com/rust-lang/crates.io-index"
1039−checksum = "ad726ec26496bf4c083fff0f43d4eb3a2ad1bba305323af5ff91383c0b6ecac0"
1039+checksum = "63ec02a07a782abef91279b72fe8fd2bee4c168a22112cedec5d3b0d49b9e4f9"
10401040dependencies = [
10411041 "cc",
10421042 "regex",
helix-core/Cargo.toml+1−1
@@ -22,7 +22,7 @@ unicode-segmentation = "1.8"
2222unicode-width = "0.1"
2323unicode-general-category = "0.4"
2424# slab = "0.4.2"
25−tree-sitter = "0.19"
25+tree-sitter = "0.20"
2626once_cell = "1.8"
2727arc-swap = "1"
2828regex = "1"
helix-core/src/syntax.rs+41−38
@@ -276,16 +276,6 @@ fn byte_range_to_str(range: std::ops::Range<usize>, source: RopeSlice) -> Cow<st
276276 Cow::from(source.slice(start_char..end_char))
277277}
278278
279−fn node_to_bytes<'a>(node: Node, source: RopeSlice<'a>) -> Cow<'a, [u8]> {
280− let start_char = source.byte_to_char(node.start_byte());
281− let end_char = source.byte_to_char(node.end_byte());
282− let fragment = source.slice(start_char..end_char);
283− match fragment.as_str() {
284− Some(fragment) => Cow::Borrowed(fragment.as_bytes()),
285− None => Cow::Owned(String::from(fragment).into_bytes()),
286− }
287−}
288−
289279impl Syntax {
290280 // buffer, grammar, config, grammars, sync_timeout?
291281 pub fn new(
@@ -380,14 +370,11 @@ impl Syntax {
380370
381371 // TODO: if reusing cursors this might need resetting
382372 if let Some(range) = &range {
383− cursor_ref.set_byte_range(range.start, range.end);
373+ cursor_ref.set_byte_range(range.clone());
384374 }
385375
386376 let captures = cursor_ref
387− .captures(query_ref, tree_ref.root_node(), move |n: Node| {
388− // &source[n.byte_range()]
389− node_to_bytes(n, source)
390− })
377+ .captures(query_ref, tree_ref.root_node(), RopeProvider(source))
391378 .peekable();
392379
393380 // manually craft the root layer based on the existing tree
@@ -501,10 +488,7 @@ impl LanguageLayer {
501488 // let mut injections_by_pattern_index =
502489 // vec![(None, Vec::new(), false); combined_injections_query.pattern_count()];
503490 // let matches =
504− // cursor.matches(combined_injections_query, tree.root_node(), |n: Node| {
505− // // &source[n.byte_range()]
506− // node_to_bytes(n, source)
507− // });
491+ // cursor.matches(combined_injections_query, tree.root_node(), RopeProvider(source));
508492 // for mat in matches {
509493 // let entry = &mut injections_by_pattern_index[mat.pattern_index];
510494 // let (language_name, content_node, include_children) =
@@ -716,7 +700,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};
716700use std::{iter, mem, ops, str, usize};
717701use tree_sitter::{
718702 Language as Grammar, Node, Parser, Point, Query, QueryCaptures, QueryCursor, QueryError,
719− QueryMatch, Range, Tree,
703+ QueryMatch, Range, TextProvider, Tree,
720704};
721705
722706const CANCELLATION_CHECK_INTERVAL: usize = 100;
@@ -776,7 +760,7 @@ struct LocalScope<'a> {
776760}
777761
778762#[derive(Debug)]
779−struct HighlightIter<'a, 'tree: 'a, F>
763+struct HighlightIter<'a, F>
780764where
781765 F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
782766{
@@ -784,16 +768,41 @@ where
784768 byte_offset: usize,
785769 injection_callback: F,
786770 cancellation_flag: Option<&'a AtomicUsize>,
787− layers: Vec<HighlightIterLayer<'a, 'tree>>,
771+ layers: Vec<HighlightIterLayer<'a>>,
788772 iter_count: usize,
789773 next_event: Option<HighlightEvent>,
790774 last_highlight_range: Option<(usize, usize, usize)>,
791775}
792776
793−struct HighlightIterLayer<'a, 'tree: 'a> {
777+// Adapter to convert rope chunks to bytes
778+struct ChunksBytes<'a> {
779+ chunks: ropey::iter::Chunks<'a>,
780+}
781+impl<'a> Iterator for ChunksBytes<'a> {
782+ type Item = &'a [u8];
783+ fn next(&mut self) -> Option<Self::Item> {
784+ self.chunks.next().map(str::as_bytes)
785+ }
786+}
787+
788+struct RopeProvider<'a>(RopeSlice<'a>);
789+impl<'a> TextProvider<'a> for RopeProvider<'a> {
790+ type I = ChunksBytes<'a>;
791+
792+ fn text(&mut self, node: Node) -> Self::I {
793+ let start_char = self.0.byte_to_char(node.start_byte());
794+ let end_char = self.0.byte_to_char(node.end_byte());
795+ let fragment = self.0.slice(start_char..end_char);
796+ ChunksBytes {
797+ chunks: fragment.chunks(),
798+ }
799+ }
800+}
801+
802+struct HighlightIterLayer<'a> {
794803 _tree: Option<Tree>,
795804 cursor: QueryCursor,
796− captures: iter::Peekable<QueryCaptures<'a, 'tree, Cow<'a, [u8]>>>,
805+ captures: iter::Peekable<QueryCaptures<'a, 'a, RopeProvider<'a>>>,
797806 config: &'a HighlightConfiguration,
798807 highlight_end_stack: Vec<usize>,
799808 scope_stack: Vec<LocalScope<'a>>,
@@ -801,7 +810,7 @@ struct HighlightIterLayer<'a, 'tree: 'a> {
801810 depth: usize,
802811}
803812
804−impl<'a, 'tree: 'a> fmt::Debug for HighlightIterLayer<'a, 'tree> {
813+impl<'a> fmt::Debug for HighlightIterLayer<'a> {
805814 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
806815 f.debug_struct("HighlightIterLayer").finish()
807816 }
@@ -972,7 +981,7 @@ impl HighlightConfiguration {
972981 }
973982}
974983
975−impl<'a, 'tree: 'a> HighlightIterLayer<'a, 'tree> {
984+impl<'a> HighlightIterLayer<'a> {
976985 /// Create a new 'layer' of highlighting for this document.
977986 ///
978987 /// In the even that the new layer contains "combined injections" (injections where multiple
@@ -1029,10 +1038,7 @@ impl<'a, 'tree: 'a> HighlightIterLayer<'a, 'tree> {
10291038 let matches = cursor.matches(
10301039 combined_injections_query,
10311040 tree.root_node(),
1032− |n: Node| {
1033− // &source[n.byte_range()]
1034− node_to_bytes(n, source)
1035− },
1041+ RopeProvider(source),
10361042 );
10371043 for mat in matches {
10381044 let entry = &mut injections_by_pattern_index[mat.pattern_index];
@@ -1079,10 +1085,7 @@ impl<'a, 'tree: 'a> HighlightIterLayer<'a, 'tree> {
10791085 let cursor_ref =
10801086 unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) };
10811087 let captures = cursor_ref
1082− .captures(&config.query, tree_ref.root_node(), move |n: Node| {
1083− // &source[n.byte_range()]
1084− node_to_bytes(n, source)
1085− })
1088+ .captures(&config.query, tree_ref.root_node(), RopeProvider(source))
10861089 .peekable();
10871090
10881091 result.push(HighlightIterLayer {
@@ -1236,7 +1239,7 @@ impl<'a, 'tree: 'a> HighlightIterLayer<'a, 'tree> {
12361239 }
12371240}
12381241
1239−impl<'a, 'tree: 'a, F> HighlightIter<'a, 'tree, F>
1242+impl<'a, F> HighlightIter<'a, F>
12401243where
12411244 F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
12421245{
@@ -1287,7 +1290,7 @@ where
12871290 }
12881291 }
12891292
1290− fn insert_layer(&mut self, mut layer: HighlightIterLayer<'a, 'tree>) {
1293+ fn insert_layer(&mut self, mut layer: HighlightIterLayer<'a>) {
12911294 if let Some(sort_key) = layer.sort_key() {
12921295 let mut i = 1;
12931296 while i < self.layers.len() {
@@ -1306,7 +1309,7 @@ where
13061309 }
13071310}
13081311
1309−impl<'a, 'tree: 'a, F> Iterator for HighlightIter<'a, 'tree, F>
1312+impl<'a, F> Iterator for HighlightIter<'a, F>
13101313where
13111314 F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
13121315{
@@ -1570,7 +1573,7 @@ where
15701573fn injection_for_match<'a>(
15711574 config: &HighlightConfiguration,
15721575 query: &'a Query,
1573− query_match: &QueryMatch<'a>,
1576+ query_match: &QueryMatch<'a, 'a>,
15741577 source: RopeSlice<'a>,
15751578) -> (Option<Cow<'a, str>>, Option<Node<'a>>, bool) {
15761579 let content_capture_index = config.injection_content_capture_index;
helix-syntax/Cargo.toml+1−1
@@ -11,7 +11,7 @@ homepage = "https://helix-editor.com"
1111include = ["src/**/*", "languages/**/*", "build.rs", "!**/docs/**/*", "!**/test/**/*", "!**/examples/**/*", "!**/build/**/*"]
1212
1313[dependencies]
14−tree-sitter = "0.19"
14+tree-sitter = "0.20"
1515libloading = "0.7"
1616anyhow = "1"
1717