diff --git a/Cargo.lock b/Cargo.lock index 68221f37bd..9538893c33 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5850,7 +5850,7 @@ dependencies = [ [[package]] name = "tree-sitter" version = "0.20.8" -source = "git+https://github.com/tree-sitter/tree-sitter?rev=1f1b1eb4501ed0a2d195d37f7de15f72aa10acd0#1f1b1eb4501ed0a2d195d37f7de15f72aa10acd0" +source = "git+https://github.com/tree-sitter/tree-sitter?rev=366210ae925d7ea0891bc7a0c738f60c77c04d7b#366210ae925d7ea0891bc7a0c738f60c77c04d7b" dependencies = [ "cc", "regex", diff --git a/Cargo.toml b/Cargo.toml index 74c36d7006..31a9118a1a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ default-members = ["crates/zed"] resolver = "2" [patch.crates-io] -tree-sitter = { git = "https://github.com/tree-sitter/tree-sitter", rev = "1f1b1eb4501ed0a2d195d37f7de15f72aa10acd0" } +tree-sitter = { git = "https://github.com/tree-sitter/tree-sitter", rev = "366210ae925d7ea0891bc7a0c738f60c77c04d7b" } async-task = { git = "https://github.com/zed-industries/async-task", rev = "341b57d6de98cdfd7b418567b8de2022ca993a6e" } # TODO - Remove when a version is released with this PR: https://github.com/servo/core-foundation-rs/pull/457 diff --git a/crates/language/src/buffer.rs b/crates/language/src/buffer.rs index 1deb7f15c9..2a7a7e45b2 100644 --- a/crates/language/src/buffer.rs +++ b/crates/language/src/buffer.rs @@ -6,13 +6,15 @@ pub use crate::{ use crate::{ diagnostic_set::{DiagnosticEntry, DiagnosticGroup}, outline::OutlineItem, + syntax_map::{ + SyntaxMap, SyntaxMapCapture, SyntaxMapCaptures, SyntaxSnapshot, ToTreeSitterPoint, + }, CodeLabel, Outline, }; use anyhow::{anyhow, Result}; use clock::ReplicaId; use futures::FutureExt as _; use gpui::{fonts::HighlightStyle, AppContext, Entity, ModelContext, MutableAppContext, Task}; -use lazy_static::lazy_static; use parking_lot::Mutex; use settings::Settings; use similar::{ChangeTag, TextDiff}; @@ -25,7 +27,7 @@ use std::{ future::Future, iter::{self, Iterator, Peekable}, mem, - ops::{Deref, DerefMut, Range}, + ops::{Deref, Range}, path::{Path, PathBuf}, str, sync::Arc, @@ -36,7 +38,6 @@ use sum_tree::TreeMap; use text::operation_queue::OperationQueue; pub use text::{Buffer as TextBuffer, BufferSnapshot as TextBufferSnapshot, Operation as _, *}; use theme::SyntaxTheme; -use tree_sitter::{InputEdit, QueryCursor, Tree}; use util::TryFutureExt as _; #[cfg(any(test, feature = "test-support"))] @@ -44,10 +45,6 @@ pub use {tree_sitter_rust, tree_sitter_typescript}; pub use lsp::DiagnosticSeverity; -lazy_static! { - static ref QUERY_CURSORS: Mutex> = Default::default(); -} - pub struct Buffer { text: TextBuffer, file: Option>, @@ -60,7 +57,7 @@ pub struct Buffer { autoindent_requests: Vec>, pending_autoindent: Option>, sync_parse_timeout: Duration, - syntax_tree: Mutex>, + syntax_map: Mutex, parsing_in_background: bool, parse_count: usize, diagnostics: DiagnosticSet, @@ -76,7 +73,7 @@ pub struct Buffer { pub struct BufferSnapshot { text: text::BufferSnapshot, - tree: Option, + pub(crate) syntax: SyntaxSnapshot, file: Option>, diagnostics: DiagnosticSet, diagnostics_update_count: usize, @@ -222,14 +219,6 @@ pub trait LocalFile: File { ); } -pub(crate) struct QueryCursorHandle(Option); - -#[derive(Clone)] -struct SyntaxTree { - tree: Tree, - version: clock::Global, -} - #[derive(Clone, Debug)] pub enum AutoindentMode { /// Indent each line of inserted text. @@ -269,14 +258,11 @@ struct IndentSuggestion { delta: Ordering, } -pub(crate) struct TextProvider<'a>(pub(crate) &'a Rope); - struct BufferChunkHighlights<'a> { - captures: tree_sitter::QueryCaptures<'a, 'a, TextProvider<'a>>, - next_capture: Option<(tree_sitter::QueryMatch<'a, 'a>, usize)>, + captures: SyntaxMapCaptures<'a>, + next_capture: Option>, stack: Vec<(usize, HighlightId)>, - highlight_map: HighlightMap, - _query_cursor: QueryCursorHandle, + highlight_maps: Vec, } pub struct BufferChunks<'a> { @@ -433,7 +419,7 @@ impl Buffer { was_dirty_before_starting_transaction: None, text: buffer, file, - syntax_tree: Mutex::new(None), + syntax_map: Mutex::new(SyntaxMap::new()), parsing_in_background: false, parse_count: 0, sync_parse_timeout: Duration::from_millis(1), @@ -453,9 +439,14 @@ impl Buffer { } pub fn snapshot(&self) -> BufferSnapshot { + let text = self.text.snapshot(); + let mut syntax_map = self.syntax_map.lock(); + syntax_map.interpolate(&text); + let syntax = syntax_map.snapshot(); + BufferSnapshot { - text: self.text.snapshot(), - tree: self.syntax_tree(), + text, + syntax, file: self.file.clone(), remote_selections: self.remote_selections.clone(), diagnostics: self.diagnostics.clone(), @@ -511,11 +502,17 @@ impl Buffer { } pub fn set_language(&mut self, language: Option>, cx: &mut ModelContext) { - *self.syntax_tree.lock() = None; + self.syntax_map.lock().clear(); self.language = language; self.reparse(cx); } + pub fn set_language_registry(&mut self, language_registry: Arc) { + self.syntax_map + .lock() + .set_language_registry(language_registry); + } + pub fn did_save( &mut self, version: clock::Global, @@ -660,15 +657,6 @@ impl Buffer { self.file_update_count } - pub(crate) fn syntax_tree(&self) -> Option { - if let Some(syntax_tree) = self.syntax_tree.lock().as_mut() { - self.interpolate_tree(syntax_tree); - Some(syntax_tree.tree.clone()) - } else { - None - } - } - #[cfg(any(test, feature = "test-support"))] pub fn is_parsing(&self) -> bool { self.parsing_in_background @@ -679,75 +667,73 @@ impl Buffer { self.sync_parse_timeout = timeout; } - fn reparse(&mut self, cx: &mut ModelContext) -> bool { + fn reparse(&mut self, cx: &mut ModelContext) { if self.parsing_in_background { - return false; + return; } + let language = if let Some(language) = self.language.clone() { + language + } else { + return; + }; - if let Some(grammar) = self.grammar().cloned() { - let old_tree = self.syntax_tree(); - let text = self.as_rope().clone(); - let parsed_version = self.version(); - let parse_task = cx.background().spawn({ - let grammar = grammar.clone(); - async move { grammar.parse_text(&text, old_tree) } - }); + let text = self.text_snapshot(); + let parsed_version = self.version(); - match cx - .background() - .block_with_timeout(self.sync_parse_timeout, parse_task) - { - Ok(new_tree) => { - self.did_finish_parsing(new_tree, parsed_version, cx); - return true; - } - Err(parse_task) => { - self.parsing_in_background = true; - cx.spawn(move |this, mut cx| async move { - let new_tree = parse_task.await; - this.update(&mut cx, move |this, cx| { - let grammar_changed = this - .grammar() - .map_or(true, |curr_grammar| !Arc::ptr_eq(&grammar, curr_grammar)); - let parse_again = - this.version.changed_since(&parsed_version) || grammar_changed; - this.parsing_in_background = false; - this.did_finish_parsing(new_tree, parsed_version, cx); + let mut syntax_map = self.syntax_map.lock(); + syntax_map.interpolate(&text); + let language_registry = syntax_map.language_registry(); + let mut syntax_snapshot = syntax_map.snapshot(); + let syntax_map_version = syntax_map.parsed_version(); + drop(syntax_map); - if parse_again && this.reparse(cx) {} - }); - }) - .detach(); - } + let parse_task = cx.background().spawn({ + let language = language.clone(); + async move { + syntax_snapshot.reparse(&syntax_map_version, &text, language_registry, language); + syntax_snapshot + } + }); + + match cx + .background() + .block_with_timeout(self.sync_parse_timeout, parse_task) + { + Ok(new_syntax_snapshot) => { + self.did_finish_parsing(new_syntax_snapshot, parsed_version, cx); + return; + } + Err(parse_task) => { + self.parsing_in_background = true; + cx.spawn(move |this, mut cx| async move { + let new_syntax_map = parse_task.await; + this.update(&mut cx, move |this, cx| { + let grammar_changed = + this.language.as_ref().map_or(true, |current_language| { + !Arc::ptr_eq(&language, current_language) + }); + let parse_again = + this.version.changed_since(&parsed_version) || grammar_changed; + this.did_finish_parsing(new_syntax_map, parsed_version, cx); + this.parsing_in_background = false; + if parse_again { + this.reparse(cx); + } + }); + }) + .detach(); } } - false - } - - fn interpolate_tree(&self, tree: &mut SyntaxTree) { - for edit in self.edits_since::<(usize, Point)>(&tree.version) { - let (bytes, lines) = edit.flatten(); - tree.tree.edit(&InputEdit { - start_byte: bytes.new.start, - old_end_byte: bytes.new.start + bytes.old.len(), - new_end_byte: bytes.new.end, - start_position: lines.new.start.to_ts_point(), - old_end_position: (lines.new.start + (lines.old.end - lines.old.start)) - .to_ts_point(), - new_end_position: lines.new.end.to_ts_point(), - }); - } - tree.version = self.version(); } fn did_finish_parsing( &mut self, - tree: Tree, + syntax_snapshot: SyntaxSnapshot, version: clock::Global, cx: &mut ModelContext, ) { self.parse_count += 1; - *self.syntax_tree.lock() = Some(SyntaxTree { tree, version }); + self.syntax_map.lock().did_parse(syntax_snapshot, version); self.request_autoindent(cx); cx.emit(Event::Reparsed); cx.notify(); @@ -786,10 +772,7 @@ impl Buffer { fn compute_autoindents(&self) -> Option>> { let max_rows_between_yields = 100; let snapshot = self.snapshot(); - if snapshot.language.is_none() - || snapshot.tree.is_none() - || self.autoindent_requests.is_empty() - { + if snapshot.syntax.is_empty() || self.autoindent_requests.is_empty() { return None; } @@ -1288,10 +1271,6 @@ impl Buffer { cx.notify(); } - fn grammar(&self) -> Option<&Arc> { - self.language.as_ref().and_then(|l| l.grammar.as_ref()) - } - pub fn apply_ops>( &mut self, ops: I, @@ -1626,38 +1605,38 @@ impl BufferSnapshot { &self, row_range: Range, ) -> Option> + '_> { - let language = self.language.as_ref()?; - let grammar = language.grammar.as_ref()?; - let config = &language.config; + let config = &self.language.as_ref()?.config; let prev_non_blank_row = self.prev_non_blank_row(row_range.start); // Find the suggested indentation ranges based on the syntax tree. - let indents_query = grammar.indents_query.as_ref()?; - let mut query_cursor = QueryCursorHandle::new(); - let indent_capture_ix = indents_query.capture_index_for_name("indent"); - let end_capture_ix = indents_query.capture_index_for_name("end"); - query_cursor.set_point_range( - Point::new(prev_non_blank_row.unwrap_or(row_range.start), 0).to_ts_point() - ..Point::new(row_range.end, 0).to_ts_point(), - ); + let start = Point::new(prev_non_blank_row.unwrap_or(row_range.start), 0); + let end = Point::new(row_range.end, 0); + let range = (start..end).to_offset(&self.text); + let mut matches = self.syntax.matches(range, &self.text, |grammar| { + Some(&grammar.indents_config.as_ref()?.query) + }); + let indent_configs = matches + .grammars() + .iter() + .map(|grammar| grammar.indents_config.as_ref().unwrap()) + .collect::>(); let mut indent_ranges = Vec::>::new(); - for mat in query_cursor.matches( - indents_query, - self.tree.as_ref()?.root_node(), - TextProvider(self.as_rope()), - ) { + while let Some(mat) = matches.peek() { let mut start: Option = None; let mut end: Option = None; + + let config = &indent_configs[mat.grammar_index]; for capture in mat.captures { - if Some(capture.index) == indent_capture_ix { + if capture.index == config.indent_capture_ix { start.get_or_insert(Point::from_ts_point(capture.node.start_position())); end.get_or_insert(Point::from_ts_point(capture.node.end_position())); - } else if Some(capture.index) == end_capture_ix { + } else if Some(capture.index) == config.end_capture_ix { end = Some(Point::from_ts_point(capture.node.start_position())); } } + matches.advance(); if let Some((start, end)) = start.zip(end) { if start.row == end.row { continue; @@ -1789,10 +1768,18 @@ impl BufferSnapshot { pub fn chunks(&self, range: Range, language_aware: bool) -> BufferChunks { let range = range.start.to_offset(self)..range.end.to_offset(self); - let mut tree = None; + let mut syntax = None; let mut diagnostic_endpoints = Vec::new(); if language_aware { - tree = self.tree.as_ref(); + let captures = self.syntax.captures(range.clone(), &self.text, |grammar| { + grammar.highlights_query.as_ref() + }); + let highlight_maps = captures + .grammars() + .into_iter() + .map(|grammar| grammar.highlight_map()) + .collect(); + syntax = Some((captures, highlight_maps)); for entry in self.diagnostics_in_range::<_, usize>(range.clone(), false) { diagnostic_endpoints.push(DiagnosticEndpoint { offset: entry.range.start, @@ -1811,13 +1798,7 @@ impl BufferSnapshot { .sort_unstable_by_key(|endpoint| (endpoint.offset, !endpoint.is_start)); } - BufferChunks::new( - self.text.as_rope(), - range, - tree, - self.grammar(), - diagnostic_endpoints, - ) + BufferChunks::new(self.text.as_rope(), range, syntax, diagnostic_endpoints) } pub fn for_each_line(&self, range: Range, mut callback: impl FnMut(u32, &str)) { @@ -1843,12 +1824,6 @@ impl BufferSnapshot { self.language.as_ref() } - fn grammar(&self) -> Option<&Arc> { - self.language - .as_ref() - .and_then(|language| language.grammar.as_ref()) - } - pub fn surrounding_word(&self, start: T) -> (Range, Option) { let mut start = start.to_offset(self); let mut end = start; @@ -1879,61 +1854,71 @@ impl BufferSnapshot { } pub fn range_for_syntax_ancestor(&self, range: Range) -> Option> { - let tree = self.tree.as_ref()?; let range = range.start.to_offset(self)..range.end.to_offset(self); - let mut cursor = tree.root_node().walk(); + let mut result: Option> = None; + 'outer: for (_, _, node) in self.syntax.layers_for_range(range.clone(), &self.text) { + let mut cursor = node.walk(); - // Descend to the first leaf that touches the start of the range, - // and if the range is non-empty, extends beyond the start. - while cursor.goto_first_child_for_byte(range.start).is_some() { - if !range.is_empty() && cursor.node().end_byte() == range.start { - cursor.goto_next_sibling(); + // Descend to the first leaf that touches the start of the range, + // and if the range is non-empty, extends beyond the start. + while cursor.goto_first_child_for_byte(range.start).is_some() { + if !range.is_empty() && cursor.node().end_byte() == range.start { + cursor.goto_next_sibling(); + } } - } - // Ascend to the smallest ancestor that strictly contains the range. - loop { - let node_range = cursor.node().byte_range(); - if node_range.start <= range.start - && node_range.end >= range.end - && node_range.len() > range.len() - { - break; - } - if !cursor.goto_parent() { - break; - } - } - - let left_node = cursor.node(); - - // For an empty range, try to find another node immediately to the right of the range. - if left_node.end_byte() == range.start { - let mut right_node = None; - while !cursor.goto_next_sibling() { + // Ascend to the smallest ancestor that strictly contains the range. + loop { + let node_range = cursor.node().byte_range(); + if node_range.start <= range.start + && node_range.end >= range.end + && node_range.len() > range.len() + { + break; + } if !cursor.goto_parent() { - break; + continue 'outer; } } - while cursor.node().start_byte() == range.start { - right_node = Some(cursor.node()); - if !cursor.goto_first_child() { - break; + let left_node = cursor.node(); + let mut layer_result = left_node.byte_range(); + + // For an empty range, try to find another node immediately to the right of the range. + if left_node.end_byte() == range.start { + let mut right_node = None; + while !cursor.goto_next_sibling() { + if !cursor.goto_parent() { + break; + } + } + + while cursor.node().start_byte() == range.start { + right_node = Some(cursor.node()); + if !cursor.goto_first_child() { + break; + } + } + + // If there is a candidate node on both sides of the (empty) range, then + // decide between the two by favoring a named node over an anonymous token. + // If both nodes are the same in that regard, favor the right one. + if let Some(right_node) = right_node { + if right_node.is_named() || !left_node.is_named() { + layer_result = right_node.byte_range(); + } } } - // If there is a candidate node on both sides of the (empty) range, then - // decide between the two by favoring a named node over an anonymous token. - // If both nodes are the same in that regard, favor the right one. - if let Some(right_node) = right_node { - if right_node.is_named() || !left_node.is_named() { - return Some(right_node.byte_range()); + if let Some(previous_result) = &result { + if previous_result.len() < layer_result.len() { + continue; } } + result = Some(layer_result); } - Some(left_node.byte_range()) + result } pub fn outline(&self, theme: Option<&SyntaxTheme>) -> Option> { @@ -1947,8 +1932,10 @@ impl BufferSnapshot { theme: Option<&SyntaxTheme>, ) -> Option>> { let position = position.to_offset(self); - let mut items = - self.outline_items_containing(position.saturating_sub(1)..position + 1, theme)?; + let mut items = self.outline_items_containing( + position.saturating_sub(1)..self.len().min(position + 1), + theme, + )?; let mut prev_depth = None; items.retain(|item| { let result = prev_depth.map_or(true, |prev_depth| item.depth > prev_depth); @@ -1963,109 +1950,107 @@ impl BufferSnapshot { range: Range, theme: Option<&SyntaxTheme>, ) -> Option>> { - let tree = self.tree.as_ref()?; - let grammar = self - .language - .as_ref() - .and_then(|language| language.grammar.as_ref())?; - - let outline_query = grammar.outline_query.as_ref()?; - let mut cursor = QueryCursorHandle::new(); - cursor.set_byte_range(range.clone()); - let matches = cursor.matches( - outline_query, - tree.root_node(), - TextProvider(self.as_rope()), - ); + let mut matches = self.syntax.matches(range.clone(), &self.text, |grammar| { + grammar.outline_config.as_ref().map(|c| &c.query) + }); + let configs = matches + .grammars() + .iter() + .map(|g| g.outline_config.as_ref().unwrap()) + .collect::>(); let mut chunks = self.chunks(0..self.len(), true); - - let item_capture_ix = outline_query.capture_index_for_name("item")?; - let name_capture_ix = outline_query.capture_index_for_name("name")?; - let context_capture_ix = outline_query - .capture_index_for_name("context") - .unwrap_or(u32::MAX); - let mut stack = Vec::>::new(); - let items = matches - .filter_map(|mat| { - let item_node = mat.nodes_for_capture_index(item_capture_ix).next()?; - let item_range = item_node.start_byte()..item_node.end_byte(); - if item_range.end < range.start || item_range.start > range.end { - return None; + let mut items = Vec::new(); + while let Some(mat) = matches.peek() { + let config = &configs[mat.grammar_index]; + let item_node = mat.captures.iter().find_map(|cap| { + if cap.index == config.item_capture_ix { + Some(cap.node) + } else { + None } - let mut text = String::new(); - let mut name_ranges = Vec::new(); - let mut highlight_ranges = Vec::new(); + })?; - for capture in mat.captures { - let node_is_name; - if capture.index == name_capture_ix { - node_is_name = true; - } else if capture.index == context_capture_ix { - node_is_name = false; + let item_range = item_node.byte_range(); + if item_range.end < range.start || item_range.start > range.end { + matches.advance(); + continue; + } + + // TODO - move later, after processing captures + + let mut text = String::new(); + let mut name_ranges = Vec::new(); + let mut highlight_ranges = Vec::new(); + for capture in mat.captures { + let node_is_name; + if capture.index == config.name_capture_ix { + node_is_name = true; + } else if Some(capture.index) == config.context_capture_ix { + node_is_name = false; + } else { + continue; + } + + let range = capture.node.start_byte()..capture.node.end_byte(); + if !text.is_empty() { + text.push(' '); + } + if node_is_name { + let mut start = text.len(); + let end = start + range.len(); + + // When multiple names are captured, then the matcheable text + // includes the whitespace in between the names. + if !name_ranges.is_empty() { + start -= 1; + } + + name_ranges.push(start..end); + } + + let mut offset = range.start; + chunks.seek(offset); + for mut chunk in chunks.by_ref() { + if chunk.text.len() > range.end - offset { + chunk.text = &chunk.text[0..(range.end - offset)]; + offset = range.end; } else { - continue; + offset += chunk.text.len(); } - - let range = capture.node.start_byte()..capture.node.end_byte(); - if !text.is_empty() { - text.push(' '); + let style = chunk + .syntax_highlight_id + .zip(theme) + .and_then(|(highlight, theme)| highlight.style(theme)); + if let Some(style) = style { + let start = text.len(); + let end = start + chunk.text.len(); + highlight_ranges.push((start..end, style)); } - if node_is_name { - let mut start = text.len(); - let end = start + range.len(); - - // When multiple names are captured, then the matcheable text - // includes the whitespace in between the names. - if !name_ranges.is_empty() { - start -= 1; - } - - name_ranges.push(start..end); - } - - let mut offset = range.start; - chunks.seek(offset); - for mut chunk in chunks.by_ref() { - if chunk.text.len() > range.end - offset { - chunk.text = &chunk.text[0..(range.end - offset)]; - offset = range.end; - } else { - offset += chunk.text.len(); - } - let style = chunk - .syntax_highlight_id - .zip(theme) - .and_then(|(highlight, theme)| highlight.style(theme)); - if let Some(style) = style { - let start = text.len(); - let end = start + chunk.text.len(); - highlight_ranges.push((start..end, style)); - } - text.push_str(chunk.text); - if offset >= range.end { - break; - } + text.push_str(chunk.text); + if offset >= range.end { + break; } } + } - while stack.last().map_or(false, |prev_range| { - prev_range.start > item_range.start || prev_range.end < item_range.end - }) { - stack.pop(); - } - stack.push(item_range.clone()); + matches.advance(); + while stack.last().map_or(false, |prev_range| { + prev_range.start > item_range.start || prev_range.end < item_range.end + }) { + stack.pop(); + } + stack.push(item_range.clone()); - Some(OutlineItem { - depth: stack.len() - 1, - range: self.anchor_after(item_range.start)..self.anchor_before(item_range.end), - text, - highlight_ranges, - name_ranges, - }) + items.push(OutlineItem { + depth: stack.len() - 1, + range: self.anchor_after(item_range.start)..self.anchor_before(item_range.end), + text, + highlight_ranges, + name_ranges, }) - .collect::>(); + } Some(items) } @@ -2073,28 +2058,48 @@ impl BufferSnapshot { &self, range: Range, ) -> Option<(Range, Range)> { - let (grammar, tree) = self.grammar().zip(self.tree.as_ref())?; - let brackets_query = grammar.brackets_query.as_ref()?; - let open_capture_ix = brackets_query.capture_index_for_name("open")?; - let close_capture_ix = brackets_query.capture_index_for_name("close")?; - // Find bracket pairs that *inclusively* contain the given range. let range = range.start.to_offset(self).saturating_sub(1)..range.end.to_offset(self) + 1; - let mut cursor = QueryCursorHandle::new(); - let matches = cursor.set_byte_range(range).matches( - brackets_query, - tree.root_node(), - TextProvider(self.as_rope()), - ); + let mut matches = self.syntax.matches(range, &self.text, |grammar| { + grammar.brackets_config.as_ref().map(|c| &c.query) + }); + let configs = matches + .grammars() + .iter() + .map(|grammar| grammar.brackets_config.as_ref().unwrap()) + .collect::>(); // Get the ranges of the innermost pair of brackets. - matches - .filter_map(|mat| { - let open = mat.nodes_for_capture_index(open_capture_ix).next()?; - let close = mat.nodes_for_capture_index(close_capture_ix).next()?; - Some((open.byte_range(), close.byte_range())) - }) - .min_by_key(|(open_range, close_range)| close_range.end - open_range.start) + let mut result: Option<(Range, Range)> = None; + while let Some(mat) = matches.peek() { + let mut open = None; + let mut close = None; + let config = &configs[mat.grammar_index]; + for capture in mat.captures { + if capture.index == config.open_capture_ix { + open = Some(capture.node.byte_range()); + } else if capture.index == config.close_capture_ix { + close = Some(capture.node.byte_range()); + } + } + + matches.advance(); + + if let Some((open, close)) = open.zip(close) { + let len = close.end - open.start; + + if let Some((existing_open, existing_close)) = &result { + let existing_len = existing_close.end - existing_open.start; + if len > existing_len { + continue; + } + } + + result = Some((open, close)); + } + } + + result } #[allow(clippy::type_complexity)] @@ -2206,7 +2211,7 @@ impl Clone for BufferSnapshot { fn clone(&self) -> Self { Self { text: self.text.clone(), - tree: self.tree.clone(), + syntax: self.syntax.clone(), file: self.file.clone(), remote_selections: self.remote_selections.clone(), diagnostics: self.diagnostics.clone(), @@ -2227,56 +2232,23 @@ impl Deref for BufferSnapshot { } } -impl<'a> tree_sitter::TextProvider<'a> for TextProvider<'a> { - type I = ByteChunks<'a>; - - fn text(&mut self, node: tree_sitter::Node) -> Self::I { - ByteChunks(self.0.chunks_in_range(node.byte_range())) - } -} - -pub(crate) struct ByteChunks<'a>(rope::Chunks<'a>); - -impl<'a> Iterator for ByteChunks<'a> { - type Item = &'a [u8]; - - fn next(&mut self) -> Option { - self.0.next().map(str::as_bytes) - } -} - unsafe impl<'a> Send for BufferChunks<'a> {} impl<'a> BufferChunks<'a> { pub(crate) fn new( text: &'a Rope, range: Range, - tree: Option<&'a Tree>, - grammar: Option<&'a Arc>, + syntax: Option<(SyntaxMapCaptures<'a>, Vec)>, diagnostic_endpoints: Vec, ) -> Self { let mut highlights = None; - if let Some((grammar, tree)) = grammar.zip(tree) { - if let Some(highlights_query) = grammar.highlights_query.as_ref() { - let mut query_cursor = QueryCursorHandle::new(); - - // TODO - add a Tree-sitter API to remove the need for this. - let cursor = unsafe { - std::mem::transmute::<_, &'static mut QueryCursor>(query_cursor.deref_mut()) - }; - let captures = cursor.set_byte_range(range.clone()).captures( - highlights_query, - tree.root_node(), - TextProvider(text), - ); - highlights = Some(BufferChunkHighlights { - captures, - next_capture: None, - stack: Default::default(), - highlight_map: grammar.highlight_map(), - _query_cursor: query_cursor, - }) - } + if let Some((captures, highlight_maps)) = syntax { + highlights = Some(BufferChunkHighlights { + captures, + next_capture: None, + stack: Default::default(), + highlight_maps, + }) } let diagnostic_endpoints = diagnostic_endpoints.into_iter().peekable(); @@ -2302,14 +2274,13 @@ impl<'a> BufferChunks<'a> { highlights .stack .retain(|(end_offset, _)| *end_offset > offset); - if let Some((mat, capture_ix)) = &highlights.next_capture { - let capture = mat.captures[*capture_ix as usize]; + if let Some(capture) = &highlights.next_capture { if offset >= capture.node.start_byte() { let next_capture_end = capture.node.end_byte(); if offset < next_capture_end { highlights.stack.push(( next_capture_end, - highlights.highlight_map.get(capture.index), + highlights.highlight_maps[capture.grammar_index].get(capture.index), )); } highlights.next_capture.take(); @@ -2385,13 +2356,13 @@ impl<'a> Iterator for BufferChunks<'a> { highlights.next_capture = highlights.captures.next(); } - while let Some((mat, capture_ix)) = highlights.next_capture.as_ref() { - let capture = mat.captures[*capture_ix as usize]; + while let Some(capture) = highlights.next_capture.as_ref() { if self.range.start < capture.node.start_byte() { next_capture_start = capture.node.start_byte(); break; } else { - let highlight_id = highlights.highlight_map.get(capture.index); + let highlight_id = + highlights.highlight_maps[capture.grammar_index].get(capture.index); highlights .stack .push((capture.node.end_byte(), highlight_id)); @@ -2443,52 +2414,6 @@ impl<'a> Iterator for BufferChunks<'a> { } } -impl QueryCursorHandle { - pub(crate) fn new() -> Self { - let mut cursor = QUERY_CURSORS.lock().pop().unwrap_or_else(QueryCursor::new); - cursor.set_match_limit(64); - QueryCursorHandle(Some(cursor)) - } -} - -impl Deref for QueryCursorHandle { - type Target = QueryCursor; - - fn deref(&self) -> &Self::Target { - self.0.as_ref().unwrap() - } -} - -impl DerefMut for QueryCursorHandle { - fn deref_mut(&mut self) -> &mut Self::Target { - self.0.as_mut().unwrap() - } -} - -impl Drop for QueryCursorHandle { - fn drop(&mut self) { - let mut cursor = self.0.take().unwrap(); - cursor.set_byte_range(0..usize::MAX); - cursor.set_point_range(Point::zero().to_ts_point()..Point::MAX.to_ts_point()); - QUERY_CURSORS.lock().push(cursor) - } -} - -trait ToTreeSitterPoint { - fn to_ts_point(self) -> tree_sitter::Point; - fn from_ts_point(point: tree_sitter::Point) -> Self; -} - -impl ToTreeSitterPoint for Point { - fn to_ts_point(self) -> tree_sitter::Point { - tree_sitter::Point::new(self.row as usize, self.column as usize) - } - - fn from_ts_point(point: tree_sitter::Point) -> Self { - Point::new(point.row as u32, point.column as u32) - } -} - impl operation_queue::Operation for Operation { fn lamport_timestamp(&self) -> clock::Lamport { match self { diff --git a/crates/language/src/language.rs b/crates/language/src/language.rs index fbcc983df9..780f6e75b5 100644 --- a/crates/language/src/language.rs +++ b/crates/language/src/language.rs @@ -3,6 +3,7 @@ mod diagnostic_set; mod highlight_map; mod outline; pub mod proto; +mod syntax_map; #[cfg(test)] mod tests; @@ -29,8 +30,12 @@ use std::{ ops::Range, path::{Path, PathBuf}, str, - sync::Arc, + sync::{ + atomic::{AtomicUsize, Ordering::SeqCst}, + Arc, + }, }; +use syntax_map::SyntaxSnapshot; use theme::{SyntaxTheme, Theme}; use tree_sitter::{self, Query}; use util::ResultExt; @@ -49,6 +54,7 @@ thread_local! { } lazy_static! { + pub static ref NEXT_GRAMMAR_ID: AtomicUsize = Default::default(); pub static ref PLAIN_TEXT: Arc = Arc::new(Language::new( LanguageConfig { name: "Plain Text".into(), @@ -285,14 +291,42 @@ pub struct Language { } pub struct Grammar { + id: usize, pub(crate) ts_language: tree_sitter::Language, pub(crate) highlights_query: Option, - pub(crate) brackets_query: Option, - pub(crate) indents_query: Option, - pub(crate) outline_query: Option, + pub(crate) brackets_config: Option, + pub(crate) indents_config: Option, + pub(crate) outline_config: Option, + pub(crate) injection_config: Option, pub(crate) highlight_map: Mutex, } +struct IndentConfig { + query: Query, + indent_capture_ix: u32, + end_capture_ix: Option, +} + +struct OutlineConfig { + query: Query, + item_capture_ix: u32, + name_capture_ix: u32, + context_capture_ix: Option, +} + +struct InjectionConfig { + query: Query, + content_capture_ix: u32, + language_capture_ix: Option, + languages_by_pattern_ix: Vec>>, +} + +struct BracketConfig { + query: Query, + open_capture_ix: u32, + close_capture_ix: u32, +} + #[derive(Clone)] pub enum LanguageServerBinaryStatus { CheckingForUpdate, @@ -490,6 +524,13 @@ impl LanguageRegistry { } } +#[cfg(any(test, feature = "test-support"))] +impl Default for LanguageRegistry { + fn default() -> Self { + Self::test() + } +} + async fn get_server_binary_path( adapter: Arc, language: Arc, @@ -567,10 +608,12 @@ impl Language { config, grammar: ts_language.map(|ts_language| { Arc::new(Grammar { + id: NEXT_GRAMMAR_ID.fetch_add(1, SeqCst), highlights_query: None, - brackets_query: None, - indents_query: None, - outline_query: None, + brackets_config: None, + outline_config: None, + indents_config: None, + injection_config: None, ts_language, highlight_map: Default::default(), }) @@ -594,19 +637,104 @@ impl Language { pub fn with_brackets_query(mut self, source: &str) -> Result { let grammar = self.grammar_mut(); - grammar.brackets_query = Some(Query::new(grammar.ts_language, source)?); + let query = Query::new(grammar.ts_language, source)?; + let mut open_capture_ix = None; + let mut close_capture_ix = None; + get_capture_indices( + &query, + &mut [ + ("open", &mut open_capture_ix), + ("close", &mut close_capture_ix), + ], + ); + if let Some((open_capture_ix, close_capture_ix)) = open_capture_ix.zip(close_capture_ix) { + grammar.brackets_config = Some(BracketConfig { + query, + open_capture_ix, + close_capture_ix, + }); + } Ok(self) } pub fn with_indents_query(mut self, source: &str) -> Result { let grammar = self.grammar_mut(); - grammar.indents_query = Some(Query::new(grammar.ts_language, source)?); + let query = Query::new(grammar.ts_language, source)?; + let mut indent_capture_ix = None; + let mut end_capture_ix = None; + get_capture_indices( + &query, + &mut [ + ("indent", &mut indent_capture_ix), + ("end", &mut end_capture_ix), + ], + ); + if let Some(indent_capture_ix) = indent_capture_ix { + grammar.indents_config = Some(IndentConfig { + query, + indent_capture_ix, + end_capture_ix, + }); + } Ok(self) } pub fn with_outline_query(mut self, source: &str) -> Result { let grammar = self.grammar_mut(); - grammar.outline_query = Some(Query::new(grammar.ts_language, source)?); + let query = Query::new(grammar.ts_language, source)?; + let mut item_capture_ix = None; + let mut name_capture_ix = None; + let mut context_capture_ix = None; + get_capture_indices( + &query, + &mut [ + ("item", &mut item_capture_ix), + ("name", &mut name_capture_ix), + ("context", &mut context_capture_ix), + ], + ); + if let Some((item_capture_ix, name_capture_ix)) = item_capture_ix.zip(name_capture_ix) { + grammar.outline_config = Some(OutlineConfig { + query, + item_capture_ix, + name_capture_ix, + context_capture_ix, + }); + } + Ok(self) + } + + pub fn with_injection_query(mut self, source: &str) -> Result { + let grammar = self.grammar_mut(); + let query = Query::new(grammar.ts_language, source)?; + let mut language_capture_ix = None; + let mut content_capture_ix = None; + get_capture_indices( + &query, + &mut [ + ("language", &mut language_capture_ix), + ("content", &mut content_capture_ix), + ], + ); + let languages_by_pattern_ix = (0..query.pattern_count()) + .map(|ix| { + query.property_settings(ix).iter().find_map(|setting| { + if setting.key.as_ref() == "language" { + return setting.value.clone(); + } else { + None + } + }) + }) + .collect(); + if let Some(content_capture_ix) = content_capture_ix { + grammar.injection_config = Some(InjectionConfig { + query, + language_capture_ix, + content_capture_ix, + languages_by_pattern_ix, + }); + } Ok(self) } @@ -685,9 +813,16 @@ impl Language { let mut result = Vec::new(); if let Some(grammar) = &self.grammar { let tree = grammar.parse_text(text, None); + let captures = SyntaxSnapshot::single_tree_captures( + range.clone(), + text, + &tree, + grammar, + |grammar| grammar.highlights_query.as_ref(), + ); + let highlight_maps = vec![grammar.highlight_map()]; let mut offset = 0; - for chunk in BufferChunks::new(text, range, Some(&tree), self.grammar.as_ref(), vec![]) - { + for chunk in BufferChunks::new(text, range, Some((captures, highlight_maps)), vec![]) { let end_offset = offset + chunk.text.len(); if let Some(highlight_id) = chunk.syntax_highlight_id { if !highlight_id.is_default() { @@ -727,6 +862,10 @@ impl Language { } impl Grammar { + pub fn id(&self) -> usize { + self.id + } + fn parse_text(&self, text: &Rope, old_tree: Option) -> Tree { PARSER.with(|parser| { let mut parser = parser.borrow_mut(); @@ -826,6 +965,17 @@ impl LspAdapter for Arc { } } +fn get_capture_indices(query: &Query, captures: &mut [(&str, &mut Option)]) { + for (ix, name) in query.capture_names().iter().enumerate() { + for (capture_name, index) in captures.iter_mut() { + if capture_name == name { + **index = Some(ix as u32); + break; + } + } + } +} + pub fn point_to_lsp(point: PointUtf16) -> lsp::Position { lsp::Position::new(point.row, point.column) } diff --git a/crates/language/src/syntax_map.rs b/crates/language/src/syntax_map.rs new file mode 100644 index 0000000000..d1bf698e52 --- /dev/null +++ b/crates/language/src/syntax_map.rs @@ -0,0 +1,1930 @@ +use crate::{Grammar, InjectionConfig, Language, LanguageRegistry}; +use lazy_static::lazy_static; +use parking_lot::Mutex; +use std::{ + borrow::Cow, + cell::RefCell, + cmp::{Ordering, Reverse}, + collections::BinaryHeap, + ops::{Deref, DerefMut, Range}, + sync::Arc, +}; +use sum_tree::{Bias, SeekTarget, SumTree}; +use text::{rope, Anchor, BufferSnapshot, OffsetRangeExt, Point, Rope, ToOffset, ToPoint}; +use tree_sitter::{ + Node, Parser, Query, QueryCapture, QueryCaptures, QueryCursor, QueryMatches, Tree, +}; + +thread_local! { + static PARSER: RefCell = RefCell::new(Parser::new()); +} + +lazy_static! { + static ref QUERY_CURSORS: Mutex> = Default::default(); +} + +#[derive(Default)] +pub struct SyntaxMap { + parsed_version: clock::Global, + interpolated_version: clock::Global, + snapshot: SyntaxSnapshot, + language_registry: Option>, +} + +#[derive(Clone, Default)] +pub struct SyntaxSnapshot { + layers: SumTree, +} + +#[derive(Default)] +pub struct SyntaxMapCaptures<'a> { + layers: Vec>, + active_layer_count: usize, + grammars: Vec<&'a Grammar>, +} + +#[derive(Default)] +pub struct SyntaxMapMatches<'a> { + layers: Vec>, + active_layer_count: usize, + grammars: Vec<&'a Grammar>, +} + +#[derive(Debug)] +pub struct SyntaxMapCapture<'a> { + pub depth: usize, + pub node: Node<'a>, + pub index: u32, + pub grammar_index: usize, +} + +#[derive(Debug)] +pub struct SyntaxMapMatch<'a> { + pub depth: usize, + pub pattern_index: usize, + pub captures: &'a [QueryCapture<'a>], + pub grammar_index: usize, +} + +struct SyntaxMapCapturesLayer<'a> { + depth: usize, + captures: QueryCaptures<'a, 'a, TextProvider<'a>>, + next_capture: Option>, + grammar_index: usize, + _query_cursor: QueryCursorHandle, +} + +struct SyntaxMapMatchesLayer<'a> { + depth: usize, + next_pattern_index: usize, + next_captures: Vec>, + has_next: bool, + matches: QueryMatches<'a, 'a, TextProvider<'a>>, + grammar_index: usize, + _query_cursor: QueryCursorHandle, +} + +#[derive(Clone)] +struct SyntaxLayer { + depth: usize, + range: Range, + tree: tree_sitter::Tree, + language: Arc, +} + +#[derive(Debug, Clone)] +struct SyntaxLayerSummary { + min_depth: usize, + max_depth: usize, + range: Range, + last_layer_range: Range, +} + +#[derive(Clone, Debug)] +struct DepthAndRange(usize, Range); + +#[derive(Clone, Debug)] +struct DepthAndMaxPosition(usize, Anchor); + +#[derive(Clone, Debug)] +struct DepthAndRangeOrMaxPosition(DepthAndRange, DepthAndMaxPosition); + +struct ReparseStep { + depth: usize, + language: Arc, + ranges: Vec, + range: Range, +} + +#[derive(Debug, PartialEq, Eq)] +struct ChangedRegion { + depth: usize, + range: Range, +} + +#[derive(Default)] +struct ChangeRegionSet(Vec); + +struct TextProvider<'a>(&'a Rope); + +struct ByteChunks<'a>(rope::Chunks<'a>); + +struct QueryCursorHandle(Option); + +impl SyntaxMap { + pub fn new() -> Self { + Self::default() + } + + pub fn set_language_registry(&mut self, registry: Arc) { + self.language_registry = Some(registry); + } + + pub fn snapshot(&self) -> SyntaxSnapshot { + self.snapshot.clone() + } + + pub fn language_registry(&self) -> Option> { + self.language_registry.clone() + } + + pub fn parsed_version(&self) -> clock::Global { + self.parsed_version.clone() + } + + pub fn interpolate(&mut self, text: &BufferSnapshot) { + self.snapshot.interpolate(&self.interpolated_version, text); + self.interpolated_version = text.version.clone(); + } + + #[cfg(test)] + pub fn reparse(&mut self, language: Arc, text: &BufferSnapshot) { + self.snapshot.reparse( + &self.parsed_version, + text, + self.language_registry.clone(), + language, + ); + self.parsed_version = text.version.clone(); + self.interpolated_version = text.version.clone(); + } + + pub fn did_parse(&mut self, snapshot: SyntaxSnapshot, version: clock::Global) { + self.interpolated_version = version.clone(); + self.parsed_version = version; + self.snapshot = snapshot; + } + + pub fn clear(&mut self) { + self.snapshot = SyntaxSnapshot::default(); + } +} + +impl SyntaxSnapshot { + pub fn is_empty(&self) -> bool { + self.layers.is_empty() + } + + pub fn interpolate(&mut self, from_version: &clock::Global, text: &BufferSnapshot) { + let edits = text + .anchored_edits_since::<(usize, Point)>(&from_version) + .collect::>(); + if edits.is_empty() { + return; + } + + let mut layers = SumTree::new(); + let mut first_edit_ix_for_depth = 0; + let mut prev_depth = 0; + let mut cursor = self.layers.cursor::(); + cursor.next(text); + + 'outer: loop { + let depth = cursor.end(text).max_depth; + if depth > prev_depth { + first_edit_ix_for_depth = 0; + prev_depth = depth; + } + + // Preserve any layers at this depth that precede the first edit. + if let Some((_, edit_range)) = edits.get(first_edit_ix_for_depth) { + let target = DepthAndMaxPosition(depth, edit_range.start); + if target.cmp(&cursor.start(), text).is_gt() { + let slice = cursor.slice(&target, Bias::Left, text); + layers.push_tree(slice, text); + } + } + // If this layer follows all of the edits, then preserve it and any + // subsequent layers at this same depth. + else if cursor.item().is_some() { + let slice = cursor.slice( + &DepthAndRange(depth + 1, Anchor::MIN..Anchor::MAX), + Bias::Left, + text, + ); + layers.push_tree(slice, text); + continue; + }; + + let layer = if let Some(layer) = cursor.item() { + layer + } else { + break; + }; + let (start_byte, start_point) = layer.range.start.summary::<(usize, Point)>(text); + + + // Ignore edits that end before the start of this layer, and don't consider them + // for any subsequent layers at this same depth. + loop { + if let Some((_, edit_range)) = edits.get(first_edit_ix_for_depth) { + if edit_range.end.cmp(&layer.range.start, text).is_le() { + first_edit_ix_for_depth += 1; + } else { + break; + } + } else { + continue 'outer; + } + } + + let mut layer = layer.clone(); + for (edit, edit_range) in &edits[first_edit_ix_for_depth..] { + // Ignore any edits that follow this layer. + if edit_range.start.cmp(&layer.range.end, text).is_ge() { + break; + } + + // Apply any edits that intersect this layer to the layer's syntax tree. + let tree_edit = if edit_range.start.cmp(&layer.range.start, text).is_ge() { + tree_sitter::InputEdit { + start_byte: edit.new.start.0 - start_byte, + old_end_byte: edit.new.start.0 - start_byte + + (edit.old.end.0 - edit.old.start.0), + new_end_byte: edit.new.end.0 - start_byte, + start_position: (edit.new.start.1 - start_point).to_ts_point(), + old_end_position: (edit.new.start.1 - start_point + + (edit.old.end.1 - edit.old.start.1)) + .to_ts_point(), + new_end_position: (edit.new.end.1 - start_point).to_ts_point(), + } + } else { + let node = layer.tree.root_node(); + tree_sitter::InputEdit { + start_byte: 0, + old_end_byte: node.end_byte(), + new_end_byte: 0, + start_position: Default::default(), + old_end_position: node.end_position(), + new_end_position: Default::default(), + } + }; + + layer.tree.edit(&tree_edit); + } + + debug_assert!( + layer.tree.root_node().end_byte() <= text.len(), + "tree's size {}, is larger than text size {}", + layer.tree.root_node().end_byte(), + text.len(), + ); + + layers.push(layer, text); + cursor.next(text); + } + + layers.push_tree(cursor.suffix(&text), &text); + drop(cursor); + self.layers = layers; + } + + pub fn reparse( + &mut self, + from_version: &clock::Global, + text: &BufferSnapshot, + registry: Option>, + language: Arc, + ) { + let edits = text.edits_since::(from_version).collect::>(); + let max_depth = self.layers.summary().max_depth; + let mut cursor = self.layers.cursor::(); + cursor.next(&text); + let mut layers = SumTree::new(); + + let mut changed_regions = ChangeRegionSet::default(); + let mut queue = BinaryHeap::new(); + queue.push(ReparseStep { + depth: 0, + language: language.clone(), + ranges: Vec::new(), + range: Anchor::MIN..Anchor::MAX, + }); + + loop { + let step = queue.pop(); + let (depth, range) = if let Some(step) = &step { + (step.depth, step.range.clone()) + } else { + (max_depth + 1, Anchor::MAX..Anchor::MAX) + }; + + let target = DepthAndRange(depth, range.clone()); + let mut done = cursor.item().is_none(); + while !done && target.cmp(&cursor.end(text), &text).is_gt() { + done = true; + + let bounded_target = + DepthAndRangeOrMaxPosition(target.clone(), changed_regions.start_position()); + if bounded_target.cmp(&cursor.start(), &text).is_gt() { + let slice = cursor.slice(&bounded_target, Bias::Left, text); + if !slice.is_empty() { + layers.push_tree(slice, &text); + if changed_regions.prune(cursor.end(text), text) { + done = false; + } + } + } + + while target.cmp(&cursor.end(text), text).is_gt() { + let layer = if let Some(layer) = cursor.item() { + layer + } else { + break; + }; + + if changed_regions.intersects(&layer, text) { + changed_regions.insert( + ChangedRegion { + depth: layer.depth + 1, + range: layer.range.clone(), + }, + text, + ); + } else { + layers.push(layer.clone(), text); + } + + cursor.next(text); + if changed_regions.prune(cursor.end(text), text) { + done = false; + } + } + } + + let (ranges, language) = if let Some(step) = step { + (step.ranges, step.language) + } else { + break; + }; + + let start_point; + let start_byte; + let end_byte; + if let Some((first, last)) = ranges.first().zip(ranges.last()) { + start_point = first.start_point; + start_byte = first.start_byte; + end_byte = last.end_byte; + } else { + start_point = Point::zero().to_ts_point(); + start_byte = 0; + end_byte = text.len(); + }; + + let mut old_layer = cursor.item(); + if let Some(layer) = old_layer { + if layer.range.to_offset(text) == (start_byte..end_byte) { + cursor.next(&text); + } else { + old_layer = None; + } + } + + let grammar = if let Some(grammar) = language.grammar.as_deref() { + grammar + } else { + continue; + }; + + let tree; + let changed_ranges; + if let Some(old_layer) = old_layer { + tree = parse_text( + grammar, + text.as_rope(), + Some(old_layer.tree.clone()), + ranges, + ); + changed_ranges = join_ranges( + edits + .iter() + .map(|e| e.new.clone()) + .filter(|range| range.start < end_byte && range.end > start_byte), + old_layer + .tree + .changed_ranges(&tree) + .map(|r| start_byte + r.start_byte..start_byte + r.end_byte), + ); + } else { + tree = parse_text(grammar, text.as_rope(), None, ranges); + changed_ranges = vec![start_byte..end_byte]; + } + + layers.push( + SyntaxLayer { + depth, + range, + tree: tree.clone(), + language: language.clone(), + }, + &text, + ); + + if let (Some((config, registry)), false) = ( + grammar.injection_config.as_ref().zip(registry.as_ref()), + changed_ranges.is_empty(), + ) { + let depth = depth + 1; + for range in &changed_ranges { + changed_regions.insert( + ChangedRegion { + depth, + range: text.anchor_before(range.start)..text.anchor_after(range.end), + }, + text, + ); + } + get_injections( + config, + text, + tree.root_node_with_offset(start_byte, start_point), + registry, + depth, + &changed_ranges, + &mut queue, + ); + } + } + + drop(cursor); + self.layers = layers; + } + + pub fn single_tree_captures<'a>( + range: Range, + text: &'a Rope, + tree: &'a Tree, + grammar: &'a Grammar, + query: fn(&Grammar) -> Option<&Query>, + ) -> SyntaxMapCaptures<'a> { + SyntaxMapCaptures::new( + range.clone(), + text, + [(grammar, 0, tree.root_node())].into_iter(), + query, + ) + } + + pub fn captures<'a>( + &'a self, + range: Range, + buffer: &'a BufferSnapshot, + query: fn(&Grammar) -> Option<&Query>, + ) -> SyntaxMapCaptures { + SyntaxMapCaptures::new( + range.clone(), + buffer.as_rope(), + self.layers_for_range(range, buffer).into_iter(), + query, + ) + } + + pub fn matches<'a>( + &'a self, + range: Range, + buffer: &'a BufferSnapshot, + query: fn(&Grammar) -> Option<&Query>, + ) -> SyntaxMapMatches { + SyntaxMapMatches::new( + range.clone(), + buffer.as_rope(), + self.layers_for_range(range, buffer).into_iter(), + query, + ) + } + + #[cfg(test)] + pub fn layers(&self, buffer: &BufferSnapshot) -> Vec<(&Grammar, usize, Node)> { + self.layers_for_range(0..buffer.len(), buffer) + } + + pub fn layers_for_range<'a, T: ToOffset>( + &self, + range: Range, + buffer: &BufferSnapshot, + ) -> Vec<(&Grammar, usize, Node)> { + let start = buffer.anchor_before(range.start.to_offset(buffer)); + let end = buffer.anchor_after(range.end.to_offset(buffer)); + + let mut cursor = self.layers.filter::<_, ()>(|summary| { + if summary.max_depth > summary.min_depth { + true + } else { + let is_before_start = summary.range.end.cmp(&start, buffer).is_lt(); + let is_after_end = summary.range.start.cmp(&end, buffer).is_gt(); + !is_before_start && !is_after_end + } + }); + + let mut result = Vec::new(); + cursor.next(buffer); + while let Some(layer) = cursor.item() { + if let Some(grammar) = &layer.language.grammar { + result.push(( + grammar.as_ref(), + layer.depth, + layer.tree.root_node_with_offset( + layer.range.start.to_offset(buffer), + layer.range.start.to_point(buffer).to_ts_point(), + ), + )); + } + cursor.next(buffer) + } + + result + } +} + +impl<'a> SyntaxMapCaptures<'a> { + fn new( + range: Range, + text: &'a Rope, + layers: impl Iterator)>, + query: fn(&Grammar) -> Option<&Query>, + ) -> Self { + let mut result = Self { + layers: Vec::new(), + grammars: Vec::new(), + active_layer_count: 0, + }; + for (grammar, depth, node) in layers { + let query = if let Some(query) = query(grammar) { + query + } else { + continue; + }; + + let mut query_cursor = QueryCursorHandle::new(); + + // TODO - add a Tree-sitter API to remove the need for this. + let cursor = unsafe { + std::mem::transmute::<_, &'static mut QueryCursor>(query_cursor.deref_mut()) + }; + + cursor.set_byte_range(range.clone()); + let captures = cursor.captures(query, node, TextProvider(text)); + let grammar_index = result + .grammars + .iter() + .position(|g| g.id == grammar.id()) + .unwrap_or_else(|| { + result.grammars.push(grammar); + result.grammars.len() - 1 + }); + let mut layer = SyntaxMapCapturesLayer { + depth, + grammar_index, + next_capture: None, + captures, + _query_cursor: query_cursor, + }; + + layer.advance(); + if layer.next_capture.is_some() { + let key = layer.sort_key(); + let ix = match result.layers[..result.active_layer_count] + .binary_search_by_key(&key, |layer| layer.sort_key()) + { + Ok(ix) | Err(ix) => ix, + }; + result.layers.insert(ix, layer); + result.active_layer_count += 1; + } else { + result.layers.push(layer); + } + } + + result + } + + pub fn grammars(&self) -> &[&'a Grammar] { + &self.grammars + } + + pub fn peek(&self) -> Option> { + let layer = self.layers[..self.active_layer_count].first()?; + let capture = layer.next_capture?; + Some(SyntaxMapCapture { + depth: layer.depth, + grammar_index: layer.grammar_index, + index: capture.index, + node: capture.node, + }) + } + + pub fn advance(&mut self) -> bool { + let layer = if let Some(layer) = self.layers[..self.active_layer_count].first_mut() { + layer + } else { + return false; + }; + + layer.advance(); + if layer.next_capture.is_some() { + let key = layer.sort_key(); + let i = 1 + self.layers[1..self.active_layer_count] + .iter() + .position(|later_layer| key < later_layer.sort_key()) + .unwrap_or(self.active_layer_count - 1); + self.layers[0..i].rotate_left(1); + } else { + self.layers[0..self.active_layer_count].rotate_left(1); + self.active_layer_count -= 1; + } + + true + } + + pub fn set_byte_range(&mut self, range: Range) { + for layer in &mut self.layers { + layer.captures.set_byte_range(range.clone()); + if let Some(capture) = &layer.next_capture { + if capture.node.end_byte() > range.start { + continue; + } + } + layer.advance(); + } + self.layers.sort_unstable_by_key(|layer| layer.sort_key()); + self.active_layer_count = self + .layers + .iter() + .position(|layer| layer.next_capture.is_none()) + .unwrap_or(self.layers.len()); + } +} + +impl<'a> SyntaxMapMatches<'a> { + fn new( + range: Range, + text: &'a Rope, + layers: impl Iterator)>, + query: fn(&Grammar) -> Option<&Query>, + ) -> Self { + let mut result = Self::default(); + for (grammar, depth, node) in layers { + let query = if let Some(query) = query(grammar) { + query + } else { + continue; + }; + + let mut query_cursor = QueryCursorHandle::new(); + + // TODO - add a Tree-sitter API to remove the need for this. + let cursor = unsafe { + std::mem::transmute::<_, &'static mut QueryCursor>(query_cursor.deref_mut()) + }; + + cursor.set_byte_range(range.clone()); + let matches = cursor.matches(query, node, TextProvider(text)); + let grammar_index = result + .grammars + .iter() + .position(|g| g.id == grammar.id()) + .unwrap_or_else(|| { + result.grammars.push(grammar); + result.grammars.len() - 1 + }); + let mut layer = SyntaxMapMatchesLayer { + depth, + grammar_index, + matches, + next_pattern_index: 0, + next_captures: Vec::new(), + has_next: false, + _query_cursor: query_cursor, + }; + + layer.advance(); + if layer.has_next { + let key = layer.sort_key(); + let ix = match result.layers[..result.active_layer_count] + .binary_search_by_key(&key, |layer| layer.sort_key()) + { + Ok(ix) | Err(ix) => ix, + }; + result.layers.insert(ix, layer); + result.active_layer_count += 1; + } else { + result.layers.push(layer); + } + } + result + } + + pub fn grammars(&self) -> &[&'a Grammar] { + &self.grammars + } + + pub fn peek(&self) -> Option { + let layer = self.layers.first()?; + if !layer.has_next { + return None; + } + Some(SyntaxMapMatch { + depth: layer.depth, + grammar_index: layer.grammar_index, + pattern_index: layer.next_pattern_index, + captures: &layer.next_captures, + }) + } + + pub fn advance(&mut self) -> bool { + let layer = if let Some(layer) = self.layers.first_mut() { + layer + } else { + return false; + }; + + layer.advance(); + if layer.has_next { + let key = layer.sort_key(); + let i = 1 + self.layers[1..self.active_layer_count] + .iter() + .position(|later_layer| key < later_layer.sort_key()) + .unwrap_or(self.active_layer_count - 1); + self.layers[0..i].rotate_left(1); + } else { + self.layers[0..self.active_layer_count].rotate_left(1); + self.active_layer_count -= 1; + } + + true + } +} + +impl<'a> SyntaxMapCapturesLayer<'a> { + fn advance(&mut self) { + self.next_capture = self.captures.next().map(|(mat, ix)| mat.captures[ix]); + } + + fn sort_key(&self) -> (usize, Reverse, usize) { + if let Some(capture) = &self.next_capture { + let range = capture.node.byte_range(); + (range.start, Reverse(range.end), self.depth) + } else { + (usize::MAX, Reverse(0), usize::MAX) + } + } +} + +impl<'a> SyntaxMapMatchesLayer<'a> { + fn advance(&mut self) { + if let Some(mat) = self.matches.next() { + self.next_captures.clear(); + self.next_captures.extend_from_slice(&mat.captures); + self.next_pattern_index = mat.pattern_index; + self.has_next = true; + } else { + self.has_next = false; + } + } + + fn sort_key(&self) -> (usize, Reverse, usize) { + if self.has_next { + let captures = &self.next_captures; + if let Some((first, last)) = captures.first().zip(captures.last()) { + return ( + first.node.start_byte(), + Reverse(last.node.end_byte()), + self.depth, + ); + } + } + (usize::MAX, Reverse(0), usize::MAX) + } +} + +impl<'a> Iterator for SyntaxMapCaptures<'a> { + type Item = SyntaxMapCapture<'a>; + + fn next(&mut self) -> Option { + let result = self.peek(); + self.advance(); + result + } +} + +fn join_ranges( + a: impl Iterator>, + b: impl Iterator>, +) -> Vec> { + let mut result = Vec::>::new(); + let mut a = a.peekable(); + let mut b = b.peekable(); + loop { + let range = match (a.peek(), b.peek()) { + (Some(range_a), Some(range_b)) => { + if range_a.start < range_b.start { + a.next().unwrap() + } else { + b.next().unwrap() + } + } + (None, Some(_)) => b.next().unwrap(), + (Some(_), None) => a.next().unwrap(), + (None, None) => break, + }; + + if let Some(last) = result.last_mut() { + if range.start <= last.end { + last.end = last.end.max(range.end); + continue; + } + } + result.push(range); + } + result +} + +fn parse_text( + grammar: &Grammar, + text: &Rope, + old_tree: Option, + mut ranges: Vec, +) -> Tree { + let (start_byte, start_point) = ranges + .first() + .map(|range| (range.start_byte, Point::from_ts_point(range.start_point))) + .unwrap_or_default(); + + for range in &mut ranges { + range.start_byte -= start_byte; + range.end_byte -= start_byte; + range.start_point = (Point::from_ts_point(range.start_point) - start_point).to_ts_point(); + range.end_point = (Point::from_ts_point(range.end_point) - start_point).to_ts_point(); + } + + PARSER.with(|parser| { + let mut parser = parser.borrow_mut(); + let mut chunks = text.chunks_in_range(start_byte..text.len()); + parser + .set_included_ranges(&ranges) + .expect("overlapping ranges"); + parser + .set_language(grammar.ts_language) + .expect("incompatible grammar"); + parser + .parse_with( + &mut move |offset, _| { + chunks.seek(start_byte + offset); + chunks.next().unwrap_or("").as_bytes() + }, + old_tree.as_ref(), + ) + .expect("invalid language") + }) +} + +fn get_injections( + config: &InjectionConfig, + text: &BufferSnapshot, + node: Node, + language_registry: &LanguageRegistry, + depth: usize, + query_ranges: &[Range], + queue: &mut BinaryHeap, +) -> bool { + let mut result = false; + let mut query_cursor = QueryCursorHandle::new(); + let mut prev_match = None; + for query_range in query_ranges { + query_cursor.set_byte_range(query_range.start.saturating_sub(1)..query_range.end); + for mat in query_cursor.matches(&config.query, node, TextProvider(text.as_rope())) { + let content_ranges = mat + .nodes_for_capture_index(config.content_capture_ix) + .map(|node| node.range()) + .collect::>(); + if content_ranges.is_empty() { + continue; + } + + // Avoid duplicate matches if two changed ranges intersect the same injection. + let content_range = + content_ranges.first().unwrap().start_byte..content_ranges.last().unwrap().end_byte; + if let Some((last_pattern_ix, last_range)) = &prev_match { + if mat.pattern_index == *last_pattern_ix && content_range == *last_range { + continue; + } + } + prev_match = Some((mat.pattern_index, content_range.clone())); + + let language_name = config.languages_by_pattern_ix[mat.pattern_index] + .as_ref() + .map(|s| Cow::Borrowed(s.as_ref())) + .or_else(|| { + let ix = config.language_capture_ix?; + let node = mat.nodes_for_capture_index(ix).next()?; + Some(Cow::Owned(text.text_for_range(node.byte_range()).collect())) + }); + + if let Some(language_name) = language_name { + if let Some(language) = language_registry.get_language(language_name.as_ref()) { + result = true; + let range = text.anchor_before(content_range.start) + ..text.anchor_after(content_range.end); + queue.push(ReparseStep { + depth, + language, + ranges: content_ranges, + range, + }) + } + } + } + } + result +} + +impl std::ops::Deref for SyntaxMap { + type Target = SyntaxSnapshot; + + fn deref(&self) -> &Self::Target { + &self.snapshot + } +} + +impl PartialEq for ReparseStep { + fn eq(&self, _: &Self) -> bool { + false + } +} + +impl Eq for ReparseStep {} + +impl PartialOrd for ReparseStep { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(&other)) + } +} + +impl Ord for ReparseStep { + fn cmp(&self, other: &Self) -> Ordering { + let range_a = self.range(); + let range_b = other.range(); + Ord::cmp(&other.depth, &self.depth) + .then_with(|| Ord::cmp(&range_b.start, &range_a.start)) + .then_with(|| Ord::cmp(&range_a.end, &range_b.end)) + } +} + +impl ReparseStep { + fn range(&self) -> Range { + let start = self.ranges.first().map_or(0, |r| r.start_byte); + let end = self.ranges.last().map_or(0, |r| r.end_byte); + start..end + } +} + +impl ChangedRegion { + fn cmp(&self, other: &Self, buffer: &BufferSnapshot) -> Ordering { + let range_a = &self.range; + let range_b = &other.range; + Ord::cmp(&self.depth, &other.depth) + .then_with(|| range_a.start.cmp(&range_b.start, buffer)) + .then_with(|| range_b.end.cmp(&range_a.end, buffer)) + } +} + +impl ChangeRegionSet { + fn start_position(&self) -> DepthAndMaxPosition { + self.0 + .first() + .map_or(DepthAndMaxPosition(usize::MAX, Anchor::MAX), |region| { + DepthAndMaxPosition(region.depth, region.range.start) + }) + } + + fn intersects(&self, layer: &SyntaxLayer, text: &BufferSnapshot) -> bool { + for region in &self.0 { + if region.depth < layer.depth { + continue; + } + if region.depth > layer.depth { + break; + } + if region.range.end.cmp(&layer.range.start, text).is_le() { + continue; + } + if region.range.start.cmp(&layer.range.end, text).is_ge() { + break; + } + return true; + } + false + } + + fn insert(&mut self, region: ChangedRegion, text: &BufferSnapshot) { + if let Err(ix) = self.0.binary_search_by(|probe| probe.cmp(®ion, text)) { + self.0.insert(ix, region); + } + } + + fn prune(&mut self, summary: SyntaxLayerSummary, text: &BufferSnapshot) -> bool { + let prev_len = self.0.len(); + self.0.retain(|region| { + region.depth > summary.max_depth + || (region.depth == summary.max_depth + && region + .range + .end + .cmp(&summary.last_layer_range.start, text) + .is_gt()) + }); + self.0.len() < prev_len + } +} + +impl Default for SyntaxLayerSummary { + fn default() -> Self { + Self { + max_depth: 0, + min_depth: 0, + range: Anchor::MAX..Anchor::MIN, + last_layer_range: Anchor::MIN..Anchor::MAX, + } + } +} + +impl sum_tree::Summary for SyntaxLayerSummary { + type Context = BufferSnapshot; + + fn add_summary(&mut self, other: &Self, buffer: &Self::Context) { + if other.max_depth > self.max_depth { + self.max_depth = other.max_depth; + self.range = other.range.clone(); + } else { + if other.range.start.cmp(&self.range.start, buffer).is_lt() { + self.range.start = other.range.start; + } + if other.range.end.cmp(&self.range.end, buffer).is_gt() { + self.range.end = other.range.end; + } + } + self.last_layer_range = other.last_layer_range.clone(); + } +} + +impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary> for DepthAndRange { + fn cmp(&self, cursor_location: &SyntaxLayerSummary, buffer: &BufferSnapshot) -> Ordering { + Ord::cmp(&self.0, &cursor_location.max_depth) + .then_with(|| { + self.1 + .start + .cmp(&cursor_location.last_layer_range.start, buffer) + }) + .then_with(|| { + cursor_location + .last_layer_range + .end + .cmp(&self.1.end, buffer) + }) + } +} + +impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary> for DepthAndMaxPosition { + fn cmp(&self, cursor_location: &SyntaxLayerSummary, text: &BufferSnapshot) -> Ordering { + Ord::cmp(&self.0, &cursor_location.max_depth) + .then_with(|| self.1.cmp(&cursor_location.range.end, text)) + } +} + +impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary> for DepthAndRangeOrMaxPosition { + fn cmp(&self, cursor_location: &SyntaxLayerSummary, buffer: &BufferSnapshot) -> Ordering { + if self.1.cmp(cursor_location, buffer).is_le() { + return Ordering::Less; + } else { + self.0.cmp(cursor_location, buffer) + } + } +} + +impl sum_tree::Item for SyntaxLayer { + type Summary = SyntaxLayerSummary; + + fn summary(&self) -> Self::Summary { + SyntaxLayerSummary { + min_depth: self.depth, + max_depth: self.depth, + range: self.range.clone(), + last_layer_range: self.range.clone(), + } + } +} + +impl std::fmt::Debug for SyntaxLayer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SyntaxLayer") + .field("depth", &self.depth) + .field("range", &self.range) + .field("tree", &self.tree) + .finish() + } +} + +impl<'a> tree_sitter::TextProvider<'a> for TextProvider<'a> { + type I = ByteChunks<'a>; + + fn text(&mut self, node: tree_sitter::Node) -> Self::I { + ByteChunks(self.0.chunks_in_range(node.byte_range())) + } +} + +impl<'a> Iterator for ByteChunks<'a> { + type Item = &'a [u8]; + + fn next(&mut self) -> Option { + self.0.next().map(str::as_bytes) + } +} + +impl QueryCursorHandle { + pub(crate) fn new() -> Self { + let mut cursor = QUERY_CURSORS.lock().pop().unwrap_or_else(QueryCursor::new); + cursor.set_match_limit(64); + QueryCursorHandle(Some(cursor)) + } +} + +impl Deref for QueryCursorHandle { + type Target = QueryCursor; + + fn deref(&self) -> &Self::Target { + self.0.as_ref().unwrap() + } +} + +impl DerefMut for QueryCursorHandle { + fn deref_mut(&mut self) -> &mut Self::Target { + self.0.as_mut().unwrap() + } +} + +impl Drop for QueryCursorHandle { + fn drop(&mut self) { + let mut cursor = self.0.take().unwrap(); + cursor.set_byte_range(0..usize::MAX); + cursor.set_point_range(Point::zero().to_ts_point()..Point::MAX.to_ts_point()); + QUERY_CURSORS.lock().push(cursor) + } +} + +pub(crate) trait ToTreeSitterPoint { + fn to_ts_point(self) -> tree_sitter::Point; + fn from_ts_point(point: tree_sitter::Point) -> Self; +} + +impl ToTreeSitterPoint for Point { + fn to_ts_point(self) -> tree_sitter::Point { + tree_sitter::Point::new(self.row as usize, self.column as usize) + } + + fn from_ts_point(point: tree_sitter::Point) -> Self { + Point::new(point.row as u32, point.column as u32) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::LanguageConfig; + use rand::rngs::StdRng; + use std::env; + use text::{Buffer, Point}; + use unindent::Unindent as _; + use util::test::marked_text_ranges; + + #[gpui::test] + fn test_syntax_map_layers_for_range() { + let registry = Arc::new(LanguageRegistry::test()); + let language = Arc::new(rust_lang()); + registry.add(language.clone()); + + let mut buffer = Buffer::new( + 0, + 0, + r#" + fn a() { + assert_eq!( + b(vec![C {}]), + vec![d.e], + ); + println!("{}", f(|_| true)); + } + "# + .unindent(), + ); + + let mut syntax_map = SyntaxMap::new(); + syntax_map.set_language_registry(registry.clone()); + syntax_map.reparse(language.clone(), &buffer); + + assert_layers_for_range( + &syntax_map, + &buffer, + Point::new(2, 0)..Point::new(2, 0), + &[ + "...(function_item ... (block (expression_statement (macro_invocation...", + "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...", + ], + ); + assert_layers_for_range( + &syntax_map, + &buffer, + Point::new(2, 14)..Point::new(2, 16), + &[ + "...(function_item ...", + "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...", + "...(array_expression (struct_expression ...", + ], + ); + assert_layers_for_range( + &syntax_map, + &buffer, + Point::new(3, 14)..Point::new(3, 16), + &[ + "...(function_item ...", + "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...", + "...(array_expression (field_expression ...", + ], + ); + assert_layers_for_range( + &syntax_map, + &buffer, + Point::new(5, 12)..Point::new(5, 16), + &[ + "...(function_item ...", + "...(call_expression ... (arguments (closure_expression ...", + ], + ); + + // Replace a vec! macro invocation with a plain slice, removing a syntactic layer. + let macro_name_range = range_for_text(&buffer, "vec!"); + buffer.edit([(macro_name_range, "&")]); + syntax_map.interpolate(&buffer); + syntax_map.reparse(language.clone(), &buffer); + + assert_layers_for_range( + &syntax_map, + &buffer, + Point::new(2, 14)..Point::new(2, 16), + &[ + "...(function_item ...", + "...(tuple_expression (call_expression ... arguments: (arguments (reference_expression value: (array_expression...", + ], + ); + + // Put the vec! macro back, adding back the syntactic layer. + buffer.undo(); + syntax_map.interpolate(&buffer); + syntax_map.reparse(language.clone(), &buffer); + + assert_layers_for_range( + &syntax_map, + &buffer, + Point::new(2, 14)..Point::new(2, 16), + &[ + "...(function_item ...", + "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...", + "...(array_expression (struct_expression ...", + ], + ); + } + + #[gpui::test] + fn test_typing_multiple_new_injections() { + let (buffer, syntax_map) = test_edit_sequence(&[ + "fn a() { dbg }", + "fn a() { dbg«!» }", + "fn a() { dbg!«()» }", + "fn a() { dbg!(«b») }", + "fn a() { dbg!(b«.») }", + "fn a() { dbg!(b.«c») }", + "fn a() { dbg!(b.c«()») }", + "fn a() { dbg!(b.c(«vec»)) }", + "fn a() { dbg!(b.c(vec«!»)) }", + "fn a() { dbg!(b.c(vec!«[]»)) }", + "fn a() { dbg!(b.c(vec![«d»])) }", + "fn a() { dbg!(b.c(vec![d«.»])) }", + "fn a() { dbg!(b.c(vec![d.«e»])) }", + ]); + + assert_capture_ranges( + &syntax_map, + &buffer, + &["field"], + "fn a() { dbg!(b.«c»(vec![d.«e»])) }", + ); + } + + #[gpui::test] + fn test_pasting_new_injection_line_between_others() { + let (buffer, syntax_map) = test_edit_sequence(&[ + " + fn a() { + b!(B {}); + c!(C {}); + d!(D {}); + e!(E {}); + f!(F {}); + g!(G {}); + } + ", + " + fn a() { + b!(B {}); + c!(C {}); + d!(D {}); + « h!(H {}); + » e!(E {}); + f!(F {}); + g!(G {}); + } + ", + ]); + + assert_capture_ranges( + &syntax_map, + &buffer, + &["struct"], + " + fn a() { + b!(«B {}»); + c!(«C {}»); + d!(«D {}»); + h!(«H {}»); + e!(«E {}»); + f!(«F {}»); + g!(«G {}»); + } + ", + ); + } + + #[gpui::test] + fn test_joining_injections_with_child_injections() { + let (buffer, syntax_map) = test_edit_sequence(&[ + " + fn a() { + b!( + c![one.two.three], + d![four.five.six], + ); + e!( + f![seven.eight], + ); + } + ", + " + fn a() { + b!( + c![one.two.three], + d![four.five.six], + ˇ f![seven.eight], + ); + } + ", + ]); + + assert_capture_ranges( + &syntax_map, + &buffer, + &["field"], + " + fn a() { + b!( + c![one.«two».«three»], + d![four.«five».«six»], + f![seven.«eight»], + ); + } + ", + ); + } + + #[gpui::test] + fn test_editing_edges_of_injection() { + test_edit_sequence(&[ + " + fn a() { + b!(c!()) + } + ", + " + fn a() { + «d»!(c!()) + } + ", + " + fn a() { + «e»d!(c!()) + } + ", + " + fn a() { + ed!«[»c!()«]» + } + ", + ]); + } + + #[gpui::test] + fn test_edits_preceding_and_intersecting_injection() { + test_edit_sequence(&[ + // + "const aaaaaaaaaaaa: B = c!(d(e.f));", + "const aˇa: B = c!(d(eˇ));", + ]); + } + + #[gpui::test] + fn test_non_local_changes_create_injections() { + test_edit_sequence(&[ + " + // a! { + static B: C = d; + // } + ", + " + ˇa! { + static B: C = d; + ˇ} + ", + ]); + } + + #[gpui::test] + fn test_creating_many_injections_in_one_edit() { + test_edit_sequence(&[ + " + fn a() { + one(Two::three(3)); + four(Five::six(6)); + seven(Eight::nine(9)); + } + ", + " + fn a() { + one«!»(Two::three(3)); + four«!»(Five::six(6)); + seven«!»(Eight::nine(9)); + } + ", + " + fn a() { + one!(Two::three«!»(3)); + four!(Five::six«!»(6)); + seven!(Eight::nine«!»(9)); + } + ", + ]); + } + + #[gpui::test] + fn test_editing_across_injection_boundary() { + test_edit_sequence(&[ + " + fn one() { + two(); + three!( + three.four, + five.six, + ); + } + ", + " + fn one() { + two(); + th«irty_five![» + three.four, + five.six, + « seven.eight, + ];» + } + ", + ]); + } + + #[gpui::test] + fn test_removing_injection_by_replacing_across_boundary() { + test_edit_sequence(&[ + " + fn one() { + two!( + three.four, + ); + } + ", + " + fn one() { + t«en + .eleven( + twelve, + » + three.four, + ); + } + ", + ]); + } + + #[gpui::test(iterations = 100)] + fn test_random_syntax_map_edits(mut rng: StdRng) { + let operations = env::var("OPERATIONS") + .map(|i| i.parse().expect("invalid `OPERATIONS` variable")) + .unwrap_or(10); + + let text = r#" + fn test_something() { + let vec = vec![5, 1, 3, 8]; + assert_eq!( + vec + .into_iter() + .map(|i| i * 2) + .collect::>(), + vec![ + 5 * 2, 1 * 2, 3 * 2, 8 * 2 + ], + ); + } + "# + .unindent() + .repeat(2); + + let registry = Arc::new(LanguageRegistry::test()); + let language = Arc::new(rust_lang()); + registry.add(language.clone()); + let mut buffer = Buffer::new(0, 0, text); + + let mut syntax_map = SyntaxMap::new(); + syntax_map.set_language_registry(registry.clone()); + syntax_map.reparse(language.clone(), &buffer); + + let mut reference_syntax_map = SyntaxMap::new(); + reference_syntax_map.set_language_registry(registry.clone()); + + log::info!("initial text:\n{}", buffer.text()); + + for _ in 0..operations { + let prev_buffer = buffer.snapshot(); + let prev_syntax_map = syntax_map.snapshot(); + + buffer.randomly_edit(&mut rng, 3); + log::info!("text:\n{}", buffer.text()); + + syntax_map.interpolate(&buffer); + check_interpolation(&prev_syntax_map, &syntax_map, &prev_buffer, &buffer); + + syntax_map.reparse(language.clone(), &buffer); + + reference_syntax_map.clear(); + reference_syntax_map.reparse(language.clone(), &buffer); + } + + for i in 0..operations { + let i = operations - i - 1; + buffer.undo(); + log::info!("undoing operation {}", i); + log::info!("text:\n{}", buffer.text()); + + syntax_map.interpolate(&buffer); + syntax_map.reparse(language.clone(), &buffer); + + reference_syntax_map.clear(); + reference_syntax_map.reparse(language.clone(), &buffer); + assert_eq!( + syntax_map.layers(&buffer).len(), + reference_syntax_map.layers(&buffer).len(), + "wrong number of layers after undoing edit {i}" + ); + } + + let layers = syntax_map.layers(&buffer); + let reference_layers = reference_syntax_map.layers(&buffer); + for (edited_layer, reference_layer) in layers.into_iter().zip(reference_layers.into_iter()) + { + assert_eq!(edited_layer.2.to_sexp(), reference_layer.2.to_sexp()); + assert_eq!(edited_layer.2.range(), reference_layer.2.range()); + } + } + + fn check_interpolation( + old_syntax_map: &SyntaxSnapshot, + new_syntax_map: &SyntaxSnapshot, + old_buffer: &BufferSnapshot, + new_buffer: &BufferSnapshot, + ) { + let edits = new_buffer + .edits_since::(&old_buffer.version()) + .collect::>(); + + for (old_layer, new_layer) in old_syntax_map + .layers + .iter() + .zip(new_syntax_map.layers.iter()) + { + assert_eq!(old_layer.range, new_layer.range); + let old_start_byte = old_layer.range.start.to_offset(old_buffer); + let new_start_byte = new_layer.range.start.to_offset(new_buffer); + let old_start_point = old_layer.range.start.to_point(old_buffer).to_ts_point(); + let new_start_point = new_layer.range.start.to_point(new_buffer).to_ts_point(); + let old_node = old_layer + .tree + .root_node_with_offset(old_start_byte, old_start_point); + let new_node = new_layer + .tree + .root_node_with_offset(new_start_byte, new_start_point); + check_node_edits( + old_layer.depth, + &old_layer.range, + old_node, + new_node, + old_buffer, + new_buffer, + &edits, + ); + } + + fn check_node_edits( + depth: usize, + range: &Range, + old_node: Node, + new_node: Node, + old_buffer: &BufferSnapshot, + new_buffer: &BufferSnapshot, + edits: &[text::Edit], + ) { + assert_eq!(old_node.kind(), new_node.kind()); + + let old_range = old_node.byte_range(); + let new_range = new_node.byte_range(); + + let is_edited = edits + .iter() + .any(|edit| edit.new.start < new_range.end && edit.new.end > new_range.start); + if is_edited { + assert!( + new_node.has_changes(), + concat!( + "failed to mark node as edited.\n", + "layer depth: {}, old layer range: {:?}, new layer range: {:?},\n", + "node kind: {}, old node range: {:?}, new node range: {:?}", + ), + depth, + range.to_offset(old_buffer), + range.to_offset(new_buffer), + new_node.kind(), + old_range, + new_range, + ); + } + + if !new_node.has_changes() { + assert_eq!( + old_buffer + .text_for_range(old_range.clone()) + .collect::(), + new_buffer + .text_for_range(new_range.clone()) + .collect::(), + concat!( + "mismatched text for node\n", + "layer depth: {}, old layer range: {:?}, new layer range: {:?},\n", + "node kind: {}, old node range:{:?}, new node range:{:?}", + ), + depth, + range.to_offset(old_buffer), + range.to_offset(new_buffer), + new_node.kind(), + old_range, + new_range, + ); + } + + for i in 0..new_node.child_count() { + check_node_edits( + depth, + range, + old_node.child(i).unwrap(), + new_node.child(i).unwrap(), + old_buffer, + new_buffer, + edits, + ) + } + } + } + + fn test_edit_sequence(steps: &[&str]) -> (Buffer, SyntaxMap) { + let registry = Arc::new(LanguageRegistry::test()); + let language = Arc::new(rust_lang()); + registry.add(language.clone()); + let mut buffer = Buffer::new(0, 0, Default::default()); + + let mut mutated_syntax_map = SyntaxMap::new(); + mutated_syntax_map.set_language_registry(registry.clone()); + mutated_syntax_map.reparse(language.clone(), &buffer); + + for (i, marked_string) in steps.into_iter().enumerate() { + edit_buffer(&mut buffer, &marked_string.unindent()); + + // Reparse the syntax map + mutated_syntax_map.interpolate(&buffer); + mutated_syntax_map.reparse(language.clone(), &buffer); + + // Create a second syntax map from scratch + let mut reference_syntax_map = SyntaxMap::new(); + reference_syntax_map.set_language_registry(registry.clone()); + reference_syntax_map.reparse(language.clone(), &buffer); + + // Compare the mutated syntax map to the new syntax map + let mutated_layers = mutated_syntax_map.layers(&buffer); + let reference_layers = reference_syntax_map.layers(&buffer); + assert_eq!( + mutated_layers.len(), + reference_layers.len(), + "wrong number of layers at step {i}" + ); + for (edited_layer, reference_layer) in + mutated_layers.into_iter().zip(reference_layers.into_iter()) + { + assert_eq!( + edited_layer.2.to_sexp(), + reference_layer.2.to_sexp(), + "different layer at step {i}" + ); + assert_eq!( + edited_layer.2.range(), + reference_layer.2.range(), + "different layer at step {i}" + ); + } + } + + (buffer, mutated_syntax_map) + } + + fn rust_lang() -> Language { + Language::new( + LanguageConfig { + name: "Rust".into(), + path_suffixes: vec!["rs".to_string()], + ..Default::default() + }, + Some(tree_sitter_rust::language()), + ) + .with_highlights_query( + r#" + (field_identifier) @field + (struct_expression) @struct + "#, + ) + .unwrap() + .with_injection_query( + r#" + (macro_invocation + (token_tree) @content + (#set! "language" "rust")) + "#, + ) + .unwrap() + } + + fn range_for_text(buffer: &Buffer, text: &str) -> Range { + let start = buffer.as_rope().to_string().find(text).unwrap(); + start..start + text.len() + } + + fn assert_layers_for_range( + syntax_map: &SyntaxMap, + buffer: &BufferSnapshot, + range: Range, + expected_layers: &[&str], + ) { + let layers = syntax_map.layers_for_range(range, &buffer); + assert_eq!( + layers.len(), + expected_layers.len(), + "wrong number of layers" + ); + for (i, ((_, _, node), expected_s_exp)) in + layers.iter().zip(expected_layers.iter()).enumerate() + { + let actual_s_exp = node.to_sexp(); + assert!( + string_contains_sequence( + &actual_s_exp, + &expected_s_exp.split("...").collect::>() + ), + "layer {i}:\n\nexpected: {expected_s_exp}\nactual: {actual_s_exp}", + ); + } + } + + fn assert_capture_ranges( + syntax_map: &SyntaxMap, + buffer: &BufferSnapshot, + highlight_query_capture_names: &[&str], + marked_string: &str, + ) { + let mut actual_ranges = Vec::>::new(); + let captures = syntax_map.captures(0..buffer.len(), buffer, |grammar| { + grammar.highlights_query.as_ref() + }); + let queries = captures + .grammars() + .iter() + .map(|grammar| grammar.highlights_query.as_ref().unwrap()) + .collect::>(); + for capture in captures { + let name = &queries[capture.grammar_index].capture_names()[capture.index as usize]; + if highlight_query_capture_names.contains(&name.as_str()) { + actual_ranges.push(capture.node.byte_range()); + } + } + + let (text, expected_ranges) = marked_text_ranges(&marked_string.unindent(), false); + assert_eq!(text, buffer.text()); + assert_eq!(actual_ranges, expected_ranges); + } + + fn edit_buffer(buffer: &mut Buffer, marked_string: &str) { + let old_text = buffer.text(); + let (new_text, mut ranges) = marked_text_ranges(marked_string, false); + if ranges.is_empty() { + ranges.push(0..new_text.len()); + } + + assert_eq!( + old_text[..ranges[0].start], + new_text[..ranges[0].start], + "invalid edit" + ); + + let mut delta = 0; + let mut edits = Vec::new(); + let mut ranges = ranges.into_iter().peekable(); + + while let Some(inserted_range) = ranges.next() { + let new_start = inserted_range.start; + let old_start = (new_start as isize - delta) as usize; + + let following_text = if let Some(next_range) = ranges.peek() { + &new_text[inserted_range.end..next_range.start] + } else { + &new_text[inserted_range.end..] + }; + + let inserted_len = inserted_range.len(); + let deleted_len = old_text[old_start..] + .find(following_text) + .expect("invalid edit"); + + let old_range = old_start..old_start + deleted_len; + edits.push((old_range, new_text[inserted_range].to_string())); + delta += inserted_len as isize - deleted_len as isize; + } + + assert_eq!( + old_text.len() as isize + delta, + new_text.len() as isize, + "invalid edit" + ); + + buffer.edit(edits); + } + + pub fn string_contains_sequence(text: &str, parts: &[&str]) -> bool { + let mut last_part_end = 0; + for part in parts { + if let Some(start_ix) = text[last_part_end..].find(part) { + last_part_end = start_ix + part.len(); + } else { + return false; + } + } + true + } +} diff --git a/crates/language/src/tests.rs b/crates/language/src/tests.rs index 9c3d1a4509..06612347cd 100644 --- a/crates/language/src/tests.rs +++ b/crates/language/src/tests.rs @@ -999,6 +999,7 @@ fn test_autoindent_language_without_indents_query(cx: &mut MutableAppContext) { Arc::new(Language::new( LanguageConfig { name: "Markdown".into(), + auto_indent_using_last_non_empty_line: false, ..Default::default() }, Some(tree_sitter_json::language()), @@ -1437,7 +1438,9 @@ fn json_lang() -> Language { fn get_tree_sexp(buffer: &ModelHandle, cx: &gpui::TestAppContext) -> String { buffer.read_with(cx, |buffer, _| { - buffer.syntax_tree().unwrap().root_node().to_sexp() + let snapshot = buffer.snapshot(); + let layers = snapshot.syntax.layers(buffer.as_text_snapshot()); + layers[0].2.to_sexp() }) } diff --git a/crates/project/src/project.rs b/crates/project/src/project.rs index eb68b1a43f..09c5a72315 100644 --- a/crates/project/src/project.rs +++ b/crates/project/src/project.rs @@ -2067,6 +2067,7 @@ impl Project { let full_path = buffer.read(cx).file()?.full_path(cx); let language = self.languages.select_language(&full_path)?; buffer.update(cx, |buffer, cx| { + buffer.set_language_registry(self.languages.clone()); buffer.set_language(Some(language.clone()), cx); }); diff --git a/crates/sum_tree/src/cursor.rs b/crates/sum_tree/src/cursor.rs index 09f253d432..52200d64cf 100644 --- a/crates/sum_tree/src/cursor.rs +++ b/crates/sum_tree/src/cursor.rs @@ -608,9 +608,9 @@ where impl<'a, F, T, S, U> Iterator for FilterCursor<'a, F, T, U> where - F: Fn(&T::Summary) -> bool, + F: FnMut(&T::Summary) -> bool, T: Item, - S: Summary, + S: Summary, //Context for the summary must be unit type, as .next() doesn't take arguments U: Dimension<'a, T::Summary>, { type Item = &'a T; @@ -621,7 +621,7 @@ where } if let Some(item) = self.item() { - self.cursor.next_internal(&self.filter_node, &()); + self.cursor.next_internal(&mut self.filter_node, &()); Some(item) } else { None diff --git a/crates/sum_tree/src/sum_tree.rs b/crates/sum_tree/src/sum_tree.rs index fdfd5d9de2..cb05dff967 100644 --- a/crates/sum_tree/src/sum_tree.rs +++ b/crates/sum_tree/src/sum_tree.rs @@ -168,6 +168,8 @@ impl SumTree { Cursor::new(self) } + /// Note: If the summary type requires a non `()` context, then the filter cursor + /// that is returned cannot be used with Rust's iterators. pub fn filter<'a, F, U>(&'a self, filter_node: F) -> FilterCursor where F: FnMut(&T::Summary) -> bool, diff --git a/crates/text/src/text.rs b/crates/text/src/text.rs index 64b18ef813..0648e6341a 100644 --- a/crates/text/src/text.rs +++ b/crates/text/src/text.rs @@ -382,6 +382,7 @@ struct Edits<'a, D: TextDimension, F: FnMut(&FragmentSummary) -> bool> { old_end: D, new_end: D, range: Range<(&'a Locator, usize)>, + buffer_id: u64, } #[derive(Clone, Debug, Default, Eq, PartialEq)] @@ -1917,11 +1918,33 @@ impl BufferSnapshot { self.edits_since_in_range(since, Anchor::MIN..Anchor::MAX) } + pub fn anchored_edits_since<'a, D>( + &'a self, + since: &'a clock::Global, + ) -> impl 'a + Iterator, Range)> + where + D: TextDimension + Ord, + { + self.anchored_edits_since_in_range(since, Anchor::MIN..Anchor::MAX) + } + pub fn edits_since_in_range<'a, D>( &'a self, since: &'a clock::Global, range: Range, ) -> impl 'a + Iterator> + where + D: TextDimension + Ord, + { + self.anchored_edits_since_in_range(since, range) + .map(|item| item.0) + } + + pub fn anchored_edits_since_in_range<'a, D>( + &'a self, + since: &'a clock::Global, + range: Range, + ) -> impl 'a + Iterator, Range)> where D: TextDimension + Ord, { @@ -1961,6 +1984,7 @@ impl BufferSnapshot { old_end: Default::default(), new_end: Default::default(), range: (start_fragment_id, range.start.offset)..(end_fragment_id, range.end.offset), + buffer_id: self.remote_id, } } } @@ -2019,10 +2043,10 @@ impl<'a> RopeBuilder<'a> { } impl<'a, D: TextDimension + Ord, F: FnMut(&FragmentSummary) -> bool> Iterator for Edits<'a, D, F> { - type Item = Edit; + type Item = (Edit, Range); fn next(&mut self) -> Option { - let mut pending_edit: Option> = None; + let mut pending_edit: Option = None; let cursor = self.fragments_cursor.as_mut()?; while let Some(fragment) = cursor.item() { @@ -2041,11 +2065,25 @@ impl<'a, D: TextDimension + Ord, F: FnMut(&FragmentSummary) -> bool> Iterator fo if pending_edit .as_ref() - .map_or(false, |change| change.new.end < self.new_end) + .map_or(false, |(change, _)| change.new.end < self.new_end) { break; } + let timestamp = fragment.insertion_timestamp.local(); + let start_anchor = Anchor { + timestamp, + offset: fragment.insertion_offset, + bias: Bias::Right, + buffer_id: Some(self.buffer_id), + }; + let end_anchor = Anchor { + timestamp, + offset: fragment.insertion_offset + fragment.len, + bias: Bias::Left, + buffer_id: Some(self.buffer_id), + }; + if !fragment.was_visible(self.since, self.undos) && fragment.visible { let mut visible_end = cursor.end(&None).visible; if fragment.id == *self.range.end.0 { @@ -2058,13 +2096,17 @@ impl<'a, D: TextDimension + Ord, F: FnMut(&FragmentSummary) -> bool> Iterator fo let fragment_summary = self.visible_cursor.summary(visible_end); let mut new_end = self.new_end.clone(); new_end.add_assign(&fragment_summary); - if let Some(pending_edit) = pending_edit.as_mut() { - pending_edit.new.end = new_end.clone(); + if let Some((edit, range)) = pending_edit.as_mut() { + edit.new.end = new_end.clone(); + range.end = end_anchor; } else { - pending_edit = Some(Edit { - old: self.old_end.clone()..self.old_end.clone(), - new: self.new_end.clone()..new_end.clone(), - }); + pending_edit = Some(( + Edit { + old: self.old_end.clone()..self.old_end.clone(), + new: self.new_end.clone()..new_end.clone(), + }, + start_anchor..end_anchor, + )); } self.new_end = new_end; @@ -2083,13 +2125,17 @@ impl<'a, D: TextDimension + Ord, F: FnMut(&FragmentSummary) -> bool> Iterator fo let fragment_summary = self.deleted_cursor.summary(deleted_end); let mut old_end = self.old_end.clone(); old_end.add_assign(&fragment_summary); - if let Some(pending_edit) = pending_edit.as_mut() { - pending_edit.old.end = old_end.clone(); + if let Some((edit, range)) = pending_edit.as_mut() { + edit.old.end = old_end.clone(); + range.end = end_anchor; } else { - pending_edit = Some(Edit { - old: self.old_end.clone()..old_end.clone(), - new: self.new_end.clone()..self.new_end.clone(), - }); + pending_edit = Some(( + Edit { + old: self.old_end.clone()..old_end.clone(), + new: self.new_end.clone()..self.new_end.clone(), + }, + start_anchor..end_anchor, + )); } self.old_end = old_end; @@ -2435,7 +2481,7 @@ impl ToOffset for PointUtf16 { impl ToOffset for usize { fn to_offset<'a>(&self, snapshot: &BufferSnapshot) -> usize { - assert!(*self <= snapshot.len(), "offset is out of range"); + assert!(*self <= snapshot.len(), "offset {self} is out of range"); *self } } diff --git a/crates/zed/src/languages.rs b/crates/zed/src/languages.rs index 857783062a..6e57106e87 100644 --- a/crates/zed/src/languages.rs +++ b/crates/zed/src/languages.rs @@ -134,6 +134,11 @@ pub(crate) fn language( .with_outline_query(query.as_ref()) .expect("failed to load outline query"); } + if let Some(query) = load_query(name, "/injections") { + language = language + .with_injection_query(query.as_ref()) + .expect("failed to load injection query"); + } if let Some(lsp_adapter) = lsp_adapter { language = language.with_lsp_adapter(lsp_adapter) } diff --git a/crates/zed/src/languages/c/injections.scm b/crates/zed/src/languages/c/injections.scm new file mode 100644 index 0000000000..845a63bd1b --- /dev/null +++ b/crates/zed/src/languages/c/injections.scm @@ -0,0 +1,7 @@ +(preproc_def + value: (preproc_arg) @content + (#set! "language" "c")) + +(preproc_function_def + value: (preproc_arg) @content + (#set! "language" "c")) \ No newline at end of file diff --git a/crates/zed/src/languages/cpp/highlights.scm b/crates/zed/src/languages/cpp/highlights.scm index d579d70187..2dd9188308 100644 --- a/crates/zed/src/languages/cpp/highlights.scm +++ b/crates/zed/src/languages/cpp/highlights.scm @@ -1,3 +1,5 @@ +(identifier) @variable + (call_expression function: (qualified_identifier name: (identifier) @function)) @@ -34,8 +36,6 @@ (auto) @type (type_identifier) @type -(identifier) @variable - ((identifier) @constant (#match? @constant "^[A-Z][A-Z\\d_]*$")) diff --git a/crates/zed/src/languages/cpp/injections.scm b/crates/zed/src/languages/cpp/injections.scm new file mode 100644 index 0000000000..eca372d577 --- /dev/null +++ b/crates/zed/src/languages/cpp/injections.scm @@ -0,0 +1,7 @@ +(preproc_def + value: (preproc_arg) @content + (#set! "language" "c++")) + +(preproc_function_def + value: (preproc_arg) @content + (#set! "language" "c++")) \ No newline at end of file diff --git a/crates/zed/src/languages/rust/highlights.scm b/crates/zed/src/languages/rust/highlights.scm index d4f571dd52..72482b4073 100644 --- a/crates/zed/src/languages/rust/highlights.scm +++ b/crates/zed/src/languages/rust/highlights.scm @@ -1,6 +1,6 @@ (type_identifier) @type (primitive_type) @type.builtin - +(self) @variable.builtin (field_identifier) @property (call_expression @@ -15,6 +15,16 @@ (function_item name: (identifier) @function.definition) (function_signature_item name: (identifier) @function.definition) +(macro_invocation + macro: [ + (identifier) @function.special + (scoped_identifier + name: (identifier) @function.special) + ]) + +(macro_definition + name: (identifier) @function.special.definition) + ; Identifier conventions ; Assume uppercase names are enum constructors @@ -71,6 +81,7 @@ "mod" "move" "pub" + "ref" "return" "static" "struct" @@ -91,6 +102,13 @@ (char_literal) ] @string +[ + (integer_literal) + (float_literal) +] @number + +(boolean_literal) @constant + [ (line_comment) (block_comment) diff --git a/crates/zed/src/languages/rust/injections.scm b/crates/zed/src/languages/rust/injections.scm new file mode 100644 index 0000000000..57ebea8539 --- /dev/null +++ b/crates/zed/src/languages/rust/injections.scm @@ -0,0 +1,7 @@ +(macro_invocation + (token_tree) @content + (#set! "language" "rust")) + +(macro_rule + (token_tree) @content + (#set! "language" "rust")) \ No newline at end of file