@@ -4,10 +4,11 @@ use tree_sitter::{Query, QueryCursor, QueryPredicateArg};
5use crate::{
5use crate::{
6 chars::{char_is_line_ending, char_is_whitespace},
6 chars::{char_is_line_ending, char_is_whitespace},
7+ find_first_non_whitespace_char,
7 graphemes::{grapheme_width, tab_width_at},
8 graphemes::{grapheme_width, tab_width_at},
8− syntax::{LanguageConfiguration, RopeProvider, Syntax},
9+ syntax::{IndentationHeuristic, LanguageConfiguration, RopeProvider, Syntax},
9 tree_sitter::Node,
10 tree_sitter::Node,
10− Rope, RopeGraphemes, RopeSlice,
11+ Position, Rope, RopeGraphemes, RopeSlice,
13/// Enum representing indentation style.
14/// Enum representing indentation style.
@@ -21,7 +22,7 @@ pub enum IndentStyle {
22// 16 spaces
23// 16 spaces
23const INDENTS: &str = " ";
24const INDENTS: &str = " ";
24−const MAX_INDENT: u8 = 16;
25+pub const MAX_INDENT: u8 = 16;
26impl IndentStyle {
27impl IndentStyle {
27 /// Creates an `IndentStyle` from an indentation string.
28 /// Creates an `IndentStyle` from an indentation string.
@@ -196,6 +197,56 @@ pub fn indent_level_for_line(line: RopeSlice, tab_width: usize, indent_width: us
196 len / indent_width
197 len / indent_width
200+/// Create a string of tabs & spaces that has the same visual width as the given RopeSlice (independent of the tab width).
201+fn whitespace_with_same_width(text: RopeSlice) -> String {
202+ let mut s = String::new();
203+ for grapheme in RopeGraphemes::new(text) {
204+ if grapheme == "\t" {
207+ s.extend(std::iter::repeat(' ').take(grapheme_width(&Cow::from(grapheme))));
214+ mut base_indent: String,
215+ added_indent_level: isize,
216+ indent_style: &IndentStyle,
219+ if added_indent_level >= 0 {
220+ // Adding a non-negative indent is easy, we can simply append the indent string
221+ base_indent.push_str(&indent_style.as_str().repeat(added_indent_level as usize));
224+ // In this case, we want to return a prefix of `base_indent`.
225+ // Since the width of a tab depends on its offset, we cannot simply iterate over
226+ // the chars of `base_indent` in reverse until we have the desired indent reduction,
227+ // instead we iterate over them twice in forward direction.
228+ let base_indent_rope = RopeSlice::from(base_indent.as_str());
229+ #[allow(deprecated)]
230+ let base_indent_width =
231+ crate::visual_coords_at_pos(base_indent_rope, base_indent_rope.len_chars(), tab_width)
233+ let target_indent_width = base_indent_width
234+ .saturating_sub((-added_indent_level) as usize * indent_style.indent_width(tab_width));
235+ #[allow(deprecated)]
236+ let char_end_idx = crate::pos_at_visual_coords(
240+ col: target_indent_width,
244+ let byte_end_idx = base_indent_rope.char_to_byte(char_end_idx);
245+ base_indent.truncate(byte_end_idx);
199/// Computes for node and all ancestors whether they are the first node on their line.
250/// Computes for node and all ancestors whether they are the first node on their line.
200/// The first entry in the return value represents the root node, the last one the node itself
251/// The first entry in the return value represents the root node, the last one the node itself
201fn get_first_in_line(mut node: Node, new_line_byte_pos: Option<usize>) -> Vec<bool> {
252fn get_first_in_line(mut node: Node, new_line_byte_pos: Option<usize>) -> Vec<bool> {
@@ -241,21 +292,21 @@ fn get_first_in_line(mut node: Node, new_line_byte_pos: Option<usize>) -> Vec<bo
241/// - max(0, indent - outdent) tabs, if tabs are used for indentation
292/// - max(0, indent - outdent) tabs, if tabs are used for indentation
242/// - max(0, indent - outdent)*indent_width spaces, if spaces are used for indentation
293/// - max(0, indent - outdent)*indent_width spaces, if spaces are used for indentation
243#[derive(Default, Debug, PartialEq, Eq, Clone)]
294#[derive(Default, Debug, PartialEq, Eq, Clone)]
244−pub struct Indentation {
295+pub struct Indentation<'a> {
245 indent: usize,
296 indent: usize,
246 indent_always: usize,
297 indent_always: usize,
247 outdent: usize,
298 outdent: usize,
248 outdent_always: usize,
299 outdent_always: usize,
249 /// The alignment, as a string containing only tabs & spaces. Storing this as a string instead of e.g.
300 /// The alignment, as a string containing only tabs & spaces. Storing this as a string instead of e.g.
250 /// the (visual) width ensures that the alignment is preserved even if the tab width changes.
301 /// the (visual) width ensures that the alignment is preserved even if the tab width changes.
251− align: Option<String>,
302+ align: Option<RopeSlice<'a>>,
254−impl Indentation {
305+impl<'a> Indentation<'a> {
255 /// Add some other [Indentation] to this.
306 /// Add some other [Indentation] to this.
256 /// The added indent should be the total added indent from one line.
307 /// The added indent should be the total added indent from one line.
257 /// Indent should always be added starting from the bottom (or equivalently, the innermost tree-sitter node).
308 /// Indent should always be added starting from the bottom (or equivalently, the innermost tree-sitter node).
258− fn add_line(&mut self, added: Indentation) {
309+ fn add_line(&mut self, added: Indentation<'a>) {
259 // Align overrides the indent from outer scopes.
310 // Align overrides the indent from outer scopes.
260 if self.align.is_some() {
311 if self.align.is_some() {
@@ -274,7 +325,7 @@ impl Indentation {
274 /// Only captures that apply to the same line should be added together in this way (otherwise use `add_line`)
325 /// Only captures that apply to the same line should be added together in this way (otherwise use `add_line`)
275 /// and the captures should be added starting from the innermost tree-sitter node (currently this only matters
326 /// and the captures should be added starting from the innermost tree-sitter node (currently this only matters
276 /// if multiple `@align` patterns occur on the same line).
327 /// if multiple `@align` patterns occur on the same line).
277− fn add_capture(&mut self, added: IndentCaptureType) {
328+ fn add_capture(&mut self, added: IndentCaptureType<'a>) {
278 match added {
329 match added {
279 IndentCaptureType::Indent => {
330 IndentCaptureType::Indent => {
280 if self.indent_always == 0 {
331 if self.indent_always == 0 {
@@ -303,43 +354,62 @@ impl Indentation {
306− fn into_string(self, indent_style: &IndentStyle) -> String {
357+ fn net_indent(&self) -> isize {
307− let indent = self.indent_always + self.indent;
358+ (self.indent + self.indent_always) as isize
308− let outdent = self.outdent_always + self.outdent;
359+ - ((self.outdent + self.outdent_always) as isize)
310− let indent_level = if indent >= outdent {
361+ /// Convert `self` into a string, taking into account the computed and actual indentation of some other line.
311− indent - outdent
362+ fn relative_indent(
313− log::warn!("Encountered more outdent than indent nodes while calculating indentation: {} outdent, {} indent", self.outdent, self.indent);
364+ other_computed_indent: &Self,
314− 0
365+ other_leading_whitespace: RopeSlice,
315− };
366+ indent_style: &IndentStyle,
316− let mut indent_string = if let Some(align) = self.align {
367+ tab_width: usize,
317− align
368+ ) -> Option<String> {
369+ if self.align == other_computed_indent.align {
370+ // If self and baseline are either not aligned to anything or both aligned the same way,
371+ // we can simply take `other_leading_whitespace` and add some indent / outdent to it (in the second
372+ // case, the alignment should already be accounted for in `other_leading_whitespace`).
373+ let indent_diff = self.net_indent() - other_computed_indent.net_indent();
374+ Some(add_indent_level(
375+ String::from(other_leading_whitespace),
319− String::new()
381+ // If the alignment of both lines is different, we cannot compare their indentation in any meaningful way
321− indent_string.push_str(&indent_style.as_str().repeat(indent_level));
383+ }
385+ pub fn to_string(&self, indent_style: &IndentStyle, tab_width: usize) -> String {
388+ .map_or_else(String::new, whitespace_with_same_width),
326/// An indent definition which corresponds to a capture from the indent query
396/// An indent definition which corresponds to a capture from the indent query
327#[derive(Debug)]
397#[derive(Debug)]
328−struct IndentCapture {
398+struct IndentCapture<'a> {
329− capture_type: IndentCaptureType,
399+ capture_type: IndentCaptureType<'a>,
330 scope: IndentScope,
400 scope: IndentScope,
332#[derive(Debug, Clone, PartialEq)]
402#[derive(Debug, Clone, PartialEq)]
333−enum IndentCaptureType {
403+enum IndentCaptureType<'a> {
335 IndentAlways,
405 IndentAlways,
337 OutdentAlways,
407 OutdentAlways,
338 /// Alignment given as a string of whitespace
408 /// Alignment given as a string of whitespace
339− Align(String),
409+ Align(RopeSlice<'a>),
342−impl IndentCaptureType {
412+impl<'a> IndentCaptureType<'a> {
343 fn default_scope(&self) -> IndentScope {
413 fn default_scope(&self) -> IndentScope {
344 match self {
414 match self {
345 IndentCaptureType::Indent | IndentCaptureType::IndentAlways => IndentScope::Tail,
415 IndentCaptureType::Indent | IndentCaptureType::IndentAlways => IndentScope::Tail,
@@ -371,8 +441,8 @@ enum ExtendCapture {
371/// each node (identified by its ID) the relevant captures (already filtered
441/// each node (identified by its ID) the relevant captures (already filtered
372/// by predicates).
442/// by predicates).
373#[derive(Debug)]
443#[derive(Debug)]
374−struct IndentQueryResult {
444+struct IndentQueryResult<'a> {
375− indent_captures: HashMap<usize, Vec<IndentCapture>>,
445+ indent_captures: HashMap<usize, Vec<IndentCapture<'a>>>,
376 extend_captures: HashMap<usize, Vec<ExtendCapture>>,
446 extend_captures: HashMap<usize, Vec<ExtendCapture>>,
@@ -393,14 +463,14 @@ fn get_node_end_line(node: Node, new_line_byte_pos: Option<usize>) -> usize {
393 node_line
463 node_line
396−fn query_indents(
466+fn query_indents<'a>(
397 query: &Query,
467 query: &Query,
398 syntax: &Syntax,
468 syntax: &Syntax,
399 cursor: &mut QueryCursor,
469 cursor: &mut QueryCursor,
400− text: RopeSlice,
470+ text: RopeSlice<'a>,
401 range: std::ops::Range<usize>,
471 range: std::ops::Range<usize>,
402 new_line_byte_pos: Option<usize>,
472 new_line_byte_pos: Option<usize>,
403−) -> IndentQueryResult {
473+) -> IndentQueryResult<'a> {
404 let mut indent_captures: HashMap<usize, Vec<IndentCapture>> = HashMap::new();
474 let mut indent_captures: HashMap<usize, Vec<IndentCapture>> = HashMap::new();
405 let mut extend_captures: HashMap<usize, Vec<ExtendCapture>> = HashMap::new();
475 let mut extend_captures: HashMap<usize, Vec<ExtendCapture>> = HashMap::new();
406 cursor.set_byte_range(range);
476 cursor.set_byte_range(range);
@@ -488,7 +558,7 @@ fn query_indents(
488 "outdent" => IndentCaptureType::Outdent,
558 "outdent" => IndentCaptureType::Outdent,
489 "outdent.always" => IndentCaptureType::OutdentAlways,
559 "outdent.always" => IndentCaptureType::OutdentAlways,
490 // The alignment will be updated to the correct value at the end, when the anchor is known.
560 // The alignment will be updated to the correct value at the end, when the anchor is known.
491− "align" => IndentCaptureType::Align(String::from("")),
561+ "align" => IndentCaptureType::Align(RopeSlice::from("")),
492 "anchor" => {
562 "anchor" => {
493 if anchor.is_some() {
563 if anchor.is_some() {
494 log::error!("Invalid indent query: Encountered more than one @anchor in the same match.")
564 log::error!("Invalid indent query: Encountered more than one @anchor in the same match.")
@@ -560,22 +630,10 @@ fn query_indents(
561 Some(anchor) => anchor,
631 Some(anchor) => anchor,
563− // Create a string of tabs & spaces that should have the same width
633+ capture.capture_type = IndentCaptureType::Align(
564− // as the string that precedes the anchor (independent of the tab width).
565− let mut align = String::new();
566− for grapheme in RopeGraphemes::new(
567 text.line(anchor.start_position().row)
634 text.line(anchor.start_position().row)
568 .byte_slice(0..anchor.start_position().column),
635 .byte_slice(0..anchor.start_position().column),
570− if grapheme == "\t" {
574− std::iter::repeat(' ').take(grapheme_width(&Cow::from(grapheme))),
578− capture.capture_type = IndentCaptureType::Align(align);
580 indent_captures
638 indent_captures
581 .entry(node_id)
639 .entry(node_id)
@@ -661,56 +719,20 @@ fn extend_nodes<'a>(
664−/// Use the syntax tree to determine the indentation for a given position.
722+/// Prepare an indent query by computing:
665−/// This can be used in 2 ways:
723+/// - The node from which to start the query (this is non-trivial due to `@extend` captures)
666−///
724+/// - The indent captures for all relevant nodes.
667−/// - To get the correct indentation for an existing line (new_line=false), not necessarily equal to the current indentation.
668−/// - In this case, pos should be inside the first tree-sitter node on that line.
669−/// In most cases, this can just be the first non-whitespace on that line.
670−/// - To get the indentation for a new line (new_line=true). This behaves like the first usecase if the part of the current line
671−/// after pos were moved to a new line.
673−/// The indentation is determined by traversing all the tree-sitter nodes containing the position.
674−/// Each of these nodes produces some [Indentation] for:
676−/// - The line of the (beginning of the) node. This is defined by the scope `all` if this is the first node on its line.
677−/// - The line after the node. This is defined by:
678−/// - The scope `tail`.
679−/// - The scope `all` if this node is not the first node on its line.
680−/// Intuitively, `all` applies to everything contained in this node while `tail` applies to everything except for the first line of the node.
681−/// The indents from different nodes for the same line are then combined.
682−/// The result [Indentation] is simply the sum of the [Indentation] for all lines.
684−/// Specifying which line exactly an [Indentation] applies to is important because indents on the same line combine differently than indents on different lines:
686−/// some_function(|| {
687−/// // Both the function parameters as well as the contained block should be indented.
688−/// // Because they are on the same line, this only yields one indent level
696−/// // Here we get 2 indent levels because the 'parameters' and the 'block' node begin on different lines
700#[allow(clippy::too_many_arguments)]
725#[allow(clippy::too_many_arguments)]
701−pub fn treesitter_indent_for_pos(
726+fn init_indent_query<'a, 'b>(
702 query: &Query,
727 query: &Query,
703− syntax: &Syntax,
728+ syntax: &'a Syntax,
704− indent_style: &IndentStyle,
729+ text: RopeSlice<'b>,
705 tab_width: usize,
730 tab_width: usize,
706 indent_width: usize,
731 indent_width: usize,
708 line: usize,
732 line: usize,
709− pos: usize,
733+ byte_pos: usize,
710− new_line: bool,
734+ new_line_byte_pos: Option<usize>,
711−) -> Option<String> {
735+) -> Option<(Node<'a>, HashMap<usize, Vec<IndentCapture<'b>>>)> {
712− let byte_pos = text.char_to_byte(pos);
713− let new_line_byte_pos = new_line.then_some(byte_pos);
714 // The innermost tree-sitter node which is considered for the indent
736 // The innermost tree-sitter node which is considered for the indent
715 // computation. It may change if some predeceding node is extended
737 // computation. It may change if some predeceding node is extended
716 let mut node = syntax
738 let mut node = syntax
@@ -754,7 +776,6 @@ pub fn treesitter_indent_for_pos(
754 (query_result, deepest_preceding)
776 (query_result, deepest_preceding)
757− let mut indent_captures = query_result.indent_captures;
758 let extend_captures = query_result.extend_captures;
779 let extend_captures = query_result.extend_captures;
760 // Check for extend captures, potentially changing the node that the indent calculation starts with
781 // Check for extend captures, potentially changing the node that the indent calculation starts with
@@ -769,6 +790,68 @@ pub fn treesitter_indent_for_pos(
769 indent_width,
790 indent_width,
793+ Some((node, query_result.indent_captures))
796+/// Use the syntax tree to determine the indentation for a given position.
797+/// This can be used in 2 ways:
799+/// - To get the correct indentation for an existing line (new_line=false), not necessarily equal to the current indentation.
800+/// - In this case, pos should be inside the first tree-sitter node on that line.
801+/// In most cases, this can just be the first non-whitespace on that line.
802+/// - To get the indentation for a new line (new_line=true). This behaves like the first usecase if the part of the current line
803+/// after pos were moved to a new line.
805+/// The indentation is determined by traversing all the tree-sitter nodes containing the position.
806+/// Each of these nodes produces some [Indentation] for:
808+/// - The line of the (beginning of the) node. This is defined by the scope `all` if this is the first node on its line.
809+/// - The line after the node. This is defined by:
810+/// - The scope `tail`.
811+/// - The scope `all` if this node is not the first node on its line.
812+/// Intuitively, `all` applies to everything contained in this node while `tail` applies to everything except for the first line of the node.
813+/// The indents from different nodes for the same line are then combined.
814+/// The result [Indentation] is simply the sum of the [Indentation] for all lines.
816+/// Specifying which line exactly an [Indentation] applies to is important because indents on the same line combine differently than indents on different lines:
818+/// some_function(|| {
819+/// // Both the function parameters as well as the contained block should be indented.
820+/// // Because they are on the same line, this only yields one indent level
828+/// // Here we get 2 indent levels because the 'parameters' and the 'block' node begin on different lines
832+#[allow(clippy::too_many_arguments)]
833+pub fn treesitter_indent_for_pos<'a>(
837+ indent_width: usize,
838+ text: RopeSlice<'a>,
842+) -> Option<Indentation<'a>> {
843+ let byte_pos = text.char_to_byte(pos);
844+ let new_line_byte_pos = new_line.then_some(byte_pos);
845+ let (mut node, mut indent_captures) = init_indent_query(
772 let mut first_in_line = get_first_in_line(node, new_line.then_some(byte_pos));
855 let mut first_in_line = get_first_in_line(node, new_line.then_some(byte_pos));
774 let mut result = Indentation::default();
857 let mut result = Indentation::default();
@@ -836,7 +919,7 @@ pub fn treesitter_indent_for_pos(
839− Some(result.into_string(indent_style))
922+ Some(result)
842/// Returns the indentation for a new line.
925/// Returns the indentation for a new line.
@@ -845,6 +928,7 @@ pub fn treesitter_indent_for_pos(
845pub fn indent_for_newline(
928pub fn indent_for_newline(
846 language_config: Option<&LanguageConfiguration>,
929 language_config: Option<&LanguageConfiguration>,
847 syntax: Option<&Syntax>,
930 syntax: Option<&Syntax>,
931+ indent_heuristic: &IndentationHeuristic,
848 indent_style: &IndentStyle,
932 indent_style: &IndentStyle,
849 tab_width: usize,
933 tab_width: usize,
850 text: RopeSlice,
934 text: RopeSlice,
@@ -853,14 +937,18 @@ pub fn indent_for_newline(
853 current_line: usize,
937 current_line: usize,
854) -> String {
938) -> String {
855 let indent_width = indent_style.indent_width(tab_width);
939 let indent_width = indent_style.indent_width(tab_width);
941+ IndentationHeuristic::TreeSitter | IndentationHeuristic::Hybrid,
856− if let (Some(query), Some(syntax)) = (
943+ Some(syntax),
857 language_config.and_then(|config| config.indent_query()),
946 language_config.and_then(|config| config.indent_query()),
860 if let Some(indent) = treesitter_indent_for_pos(
949 if let Some(indent) = treesitter_indent_for_pos(
864 tab_width,
952 tab_width,
865 indent_width,
953 indent_width,
@@ -868,9 +956,57 @@ pub fn indent_for_newline(
868 line_before_end_pos,
956 line_before_end_pos,
959+ if *indent_heuristic == IndentationHeuristic::Hybrid {
960+ // We want to compute the indentation not only based on the
961+ // syntax tree but also on the actual indentation of a previous
962+ // line. This makes indentation computation more resilient to
963+ // incomplete queries, incomplete source code & differing indentation
964+ // styles for the same language.
965+ // However, using the indent of a previous line as a baseline may not
966+ // make sense, e.g. if it has a different alignment than the new line.
967+ // In order to prevent edge cases with long running times, we only try
968+ // a constant number of (non-empty) lines.
969+ const MAX_ATTEMPTS: usize = 4;
970+ let mut num_attempts = 0;
971+ for line_idx in (0..=line_before).rev() {
972+ let line = text.line(line_idx);
973+ let first_non_whitespace_char = match find_first_non_whitespace_char(line) {
979+ if let Some(indent) = (|| {
980+ let computed_indent = treesitter_indent_for_pos(
987+ text.line_to_char(line_idx) + first_non_whitespace_char,
990+ let leading_whitespace = line.slice(0..first_non_whitespace_char);
991+ indent.relative_indent(
871− return indent;
998+ return indent;
1001+ if num_attempts == MAX_ATTEMPTS {
1006+ return indent.to_string(indent_style, tab_width);
1009+ // Fallback in case we either don't have indent queries or they failed for some reason
874 let indent_level = indent_level_for_line(text.line(current_line), tab_width, indent_width);
1010 let indent_level = indent_level_for_line(text.line(current_line), tab_width, indent_width);
875 indent_style.as_str().repeat(indent_level)
1011 indent_style.as_str().repeat(indent_level)
@@ -962,10 +1098,13 @@ mod test {
962 ..Default::default()
1098 ..Default::default()
1101+ fn add_capture<'a>(
965− let add_capture = |mut indent: Indentation, capture| {
1102+ mut indent: Indentation<'a>,
1103+ capture: IndentCaptureType<'a>,
1104+ ) -> Indentation<'a> {
966 indent.add_capture(capture);
1105 indent.add_capture(capture);
970 // adding an indent to no indent makes an indent
1109 // adding an indent to no indent makes an indent
971 assert_eq!(
1110 assert_eq!(
@@ -1060,4 +1199,74 @@ mod test {
1204+ fn test_relative_indent() {
1205+ let indent_style = IndentStyle::Spaces(4);
1206+ let tab_width: usize = 4;
1208+ Indentation::default(),
1211+ ..Default::default()
1216+ ..Default::default()
1219+ let align = no_align.clone().map(|indent| Indentation {
1220+ align: Some(RopeSlice::from("12345")),
1223+ let different_align = Indentation {
1224+ align: Some(RopeSlice::from("123456")),
1225+ ..Default::default()
1228+ // Check that relative and absolute indentation computation are the same when the line we compare to is
1229+ // indented as we expect.
1230+ let check_consistency = |indent: &Indentation, other: &Indentation| {
1232+ indent.relative_indent(
1234+ RopeSlice::from(other.to_string(&indent_style, tab_width).as_str()),
1238+ Some(indent.to_string(&indent_style, tab_width))
1241+ for a in &no_align {
1242+ for b in &no_align {
1243+ check_consistency(a, b);
1248+ check_consistency(a, b);
1252+ // Relative indent computation makes no sense if the alignment differs
1254+ align[0].relative_indent(
1256+ RopeSlice::from(" "),
1263+ align[0].relative_indent(
1265+ RopeSlice::from(" "),