git.iko.soy/helix
on 6c11708 help sign in
helix/helix-core/src/syntax.rs
1928 lines 72 kB history raw
1use crate::{
2 chars::char_is_line_ending,
3 diagnostic::Severity,
4 regex::Regex,
5 transaction::{ChangeSet, Operation},
6 Rope, RopeSlice, Tendril,
7};
9pub use helix_syntax::get_language;
11use arc_swap::{ArcSwap, Guard};
12use slotmap::{DefaultKey as LayerId, HopSlotMap};
14use std::{
15 borrow::Cow,
16 cell::RefCell,
17 collections::{HashMap, HashSet, VecDeque},
18 fmt,
19 path::Path,
20 sync::Arc,
21};
23use once_cell::sync::{Lazy, OnceCell};
24use serde::{Deserialize, Serialize};
26fn deserialize_regex<'de, D>(deserializer: D) -> Result<Option<Regex>, D::Error>
27where
28 D: serde::Deserializer<'de>,
30 Option::<String>::deserialize(deserializer)?
31 .map(|buf| Regex::new(&buf).map_err(serde::de::Error::custom))
32 .transpose()
35fn deserialize_lsp_config<'de, D>(deserializer: D) -> Result<Option<serde_json::Value>, D::Error>
36where
37 D: serde::Deserializer<'de>,
39 Option::<toml::Value>::deserialize(deserializer)?
40 .map(|toml| toml.try_into().map_err(serde::de::Error::custom))
41 .transpose()
44#[derive(Debug, Serialize, Deserialize)]
45#[serde(deny_unknown_fields)]
46pub struct Configuration {
47 pub language: Vec<LanguageConfiguration>,
50// largely based on tree-sitter/cli/src/loader.rs
51#[derive(Debug, Serialize, Deserialize)]
52#[serde(rename_all = "kebab-case", deny_unknown_fields)]
53pub struct LanguageConfiguration {
54 #[serde(rename = "name")]
55 pub language_id: String, // c-sharp, rust
56 pub scope: String, // source.rust
57 pub file_types: Vec<String>, // filename ends_with? <Gemfile, rb, etc>
58 #[serde(default)]
59 pub shebangs: Vec<String>, // interpreter(s) associated with language
60 pub roots: Vec<String>, // these indicate project roots <.git, Cargo.toml>
61 pub comment_token: Option<String>,
63 #[serde(default, skip_serializing, deserialize_with = "deserialize_lsp_config")]
64 pub config: Option<serde_json::Value>,
66 #[serde(default)]
67 pub auto_format: bool,
68 #[serde(default)]
69 pub diagnostic_severity: Severity,
71 pub tree_sitter_library: Option<String>, // tree-sitter library name, defaults to language_id
73 // content_regex
74 #[serde(default, skip_serializing, deserialize_with = "deserialize_regex")]
75 pub injection_regex: Option<Regex>,
76 // first_line_regex
77 //
78 #[serde(skip)]
79 pub(crate) highlight_config: OnceCell<Option<Arc<HighlightConfiguration>>>,
80 // tags_config OnceCell<> https://github.com/tree-sitter/tree-sitter/pull/583
81 #[serde(skip_serializing_if = "Option::is_none")]
82 pub language_server: Option<LanguageServerConfiguration>,
83 #[serde(skip_serializing_if = "Option::is_none")]
84 pub indent: Option<IndentationConfiguration>,
86 #[serde(skip)]
87 pub(crate) indent_query: OnceCell<Option<IndentQuery>>,
88 #[serde(skip)]
89 pub(crate) textobject_query: OnceCell<Option<TextObjectQuery>>,
92#[derive(Debug, Serialize, Deserialize)]
93#[serde(rename_all = "kebab-case")]
94pub struct LanguageServerConfiguration {
95 pub command: String,
96 #[serde(default)]
97 #[serde(skip_serializing_if = "Vec::is_empty")]
98 pub args: Vec<String>,
99 pub language_id: Option<String>,
102#[derive(Debug, Serialize, Deserialize)]
103#[serde(rename_all = "kebab-case")]
104pub struct IndentationConfiguration {
105 pub tab_width: usize,
106 pub unit: String,
109#[derive(Debug, Serialize, Deserialize)]
110#[serde(rename_all = "kebab-case")]
111pub struct IndentQuery {
112 #[serde(default)]
113 #[serde(skip_serializing_if = "HashSet::is_empty")]
114 pub indent: HashSet<String>,
115 #[serde(default)]
116 #[serde(skip_serializing_if = "HashSet::is_empty")]
117 pub outdent: HashSet<String>,
120#[derive(Debug)]
121pub struct TextObjectQuery {
122 pub query: Query,
125impl TextObjectQuery {
126 /// Run the query on the given node and return sub nodes which match given
127 /// capture ("function.inside", "class.around", etc).
128 pub fn capture_nodes<'a>(
129 &'a self,
130 capture_name: &str,
131 node: Node<'a>,
132 slice: RopeSlice<'a>,
133 cursor: &'a mut QueryCursor,
134 ) -> Option<impl Iterator<Item = Node<'a>>> {
135 let capture_idx = self.query.capture_index_for_name(capture_name)?;
136 let captures = cursor.captures(&self.query, node, RopeProvider(slice));
138 captures
139 .filter_map(move |(mat, idx)| {
140 (mat.captures[idx].index == capture_idx).then(|| mat.captures[idx].node)
141 })
142 .into()
143 }
146fn load_runtime_file(language: &str, filename: &str) -> Result<String, std::io::Error> {
147 let path = crate::RUNTIME_DIR
148 .join("queries")
149 .join(language)
150 .join(filename);
151 std::fs::read_to_string(&path)
154fn read_query(language: &str, filename: &str) -> String {
155 static INHERITS_REGEX: Lazy<Regex> =
156 Lazy::new(|| Regex::new(r";+\s*inherits\s*:?\s*([a-z_,()]+)\s*").unwrap());
158 let query = load_runtime_file(language, filename).unwrap_or_default();
160 // TODO: the collect() is not ideal
161 let inherits = INHERITS_REGEX
162 .captures_iter(&query)
163 .flat_map(|captures| {
164 captures[1]
165 .split(',')
166 .map(str::to_owned)
167 .collect::<Vec<_>>()
168 })
169 .collect::<Vec<_>>();
171 if inherits.is_empty() {
172 return query;
173 }
175 let mut queries = inherits
176 .iter()
177 .map(|language| read_query(language, filename))
178 .collect::<Vec<_>>();
180 queries.push(query);
182 queries.concat()
185impl LanguageConfiguration {
186 fn initialize_highlight(&self, scopes: &[String]) -> Option<Arc<HighlightConfiguration>> {
187 let language = self.language_id.to_ascii_lowercase();
189 let highlights_query = read_query(&language, "highlights.scm");
190 // always highlight syntax errors
191 // highlights_query += "\n(ERROR) @error";
193 let injections_query = read_query(&language, "injections.scm");
194 let locals_query = read_query(&language, "locals.scm");
196 if highlights_query.is_empty() {
197 None
198 } else {
199 let language = get_language(
200 &crate::RUNTIME_DIR,
201 self.tree_sitter_library
202 .as_deref()
203 .unwrap_or(&self.language_id),
204 )
205 .map_err(|e| log::info!("{}", e))
206 .ok()?;
207 let config = HighlightConfiguration::new(
208 language,
209 &highlights_query,
210 &injections_query,
211 &locals_query,
212 )
213 .unwrap(); // TODO: avoid panic
215 config.configure(scopes);
216 Some(Arc::new(config))
217 }
218 }
220 pub fn reconfigure(&self, scopes: &[String]) {
221 if let Some(Some(config)) = self.highlight_config.get() {
222 config.configure(scopes);
223 }
224 }
226 pub fn highlight_config(&self, scopes: &[String]) -> Option<Arc<HighlightConfiguration>> {
227 self.highlight_config
228 .get_or_init(|| self.initialize_highlight(scopes))
229 .clone()
230 }
232 pub fn is_highlight_initialized(&self) -> bool {
233 self.highlight_config.get().is_some()
234 }
236 pub fn indent_query(&self) -> Option<&IndentQuery> {
237 self.indent_query
238 .get_or_init(|| {
239 let language = self.language_id.to_ascii_lowercase();
241 let toml = load_runtime_file(&language, "indents.toml").ok()?;
242 toml::from_slice(toml.as_bytes()).ok()
243 })
244 .as_ref()
245 }
247 pub fn textobject_query(&self) -> Option<&TextObjectQuery> {
248 self.textobject_query
249 .get_or_init(|| -> Option<TextObjectQuery> {
250 let lang_name = self.language_id.to_ascii_lowercase();
251 let query_text = read_query(&lang_name, "textobjects.scm");
252 let lang = self.highlight_config.get()?.as_ref()?.language;
253 let query = Query::new(lang, &query_text).ok()?;
254 Some(TextObjectQuery { query })
255 })
256 .as_ref()
257 }
259 pub fn scope(&self) -> &str {
260 &self.scope
261 }
264// Expose loader as Lazy<> global since it's always static?
266#[derive(Debug)]
267pub struct Loader {
268 // highlight_names ?
269 language_configs: Vec<Arc<LanguageConfiguration>>,
270 language_config_ids_by_file_type: HashMap<String, usize>, // Vec<usize>
271 language_config_ids_by_shebang: HashMap<String, usize>,
273 scopes: ArcSwap<Vec<String>>,
276impl Loader {
277 pub fn new(config: Configuration) -> Self {
278 let mut loader = Self {
279 language_configs: Vec::new(),
280 language_config_ids_by_file_type: HashMap::new(),
281 language_config_ids_by_shebang: HashMap::new(),
282 scopes: ArcSwap::from_pointee(Vec::new()),
283 };
285 for config in config.language {
286 // get the next id
287 let language_id = loader.language_configs.len();
289 for file_type in &config.file_types {
290 // entry().or_insert(Vec::new).push(language_id);
291 loader
292 .language_config_ids_by_file_type
293 .insert(file_type.clone(), language_id);
294 }
295 for shebang in &config.shebangs {
296 loader
297 .language_config_ids_by_shebang
298 .insert(shebang.clone(), language_id);
299 }
301 loader.language_configs.push(Arc::new(config));
302 }
304 loader
305 }
307 pub fn language_config_for_file_name(&self, path: &Path) -> Option<Arc<LanguageConfiguration>> {
308 // Find all the language configurations that match this file name
309 // or a suffix of the file name.
310 let configuration_id = path
311 .file_name()
312 .and_then(|n| n.to_str())
313 .and_then(|file_name| self.language_config_ids_by_file_type.get(file_name))
314 .or_else(|| {
315 path.extension()
316 .and_then(|extension| extension.to_str())
317 .and_then(|extension| self.language_config_ids_by_file_type.get(extension))
318 });
320 configuration_id.and_then(|&id| self.language_configs.get(id).cloned())
322 // TODO: content_regex handling conflict resolution
323 }
325 pub fn language_config_for_shebang(&self, source: &Rope) -> Option<Arc<LanguageConfiguration>> {
326 let line = Cow::from(source.line(0));
327 static SHEBANG_REGEX: Lazy<Regex> = Lazy::new(|| {
328 Regex::new(r"^#!\s*(?:\S*[/\\](?:env\s+(?:\-\S+\s+)*)?)?([^\s\.\d]+)").unwrap()
329 });
330 let configuration_id = SHEBANG_REGEX
331 .captures(&line)
332 .and_then(|cap| self.language_config_ids_by_shebang.get(&cap[1]));
334 configuration_id.and_then(|&id| self.language_configs.get(id).cloned())
335 }
337 pub fn language_config_for_scope(&self, scope: &str) -> Option<Arc<LanguageConfiguration>> {
338 self.language_configs
339 .iter()
340 .find(|config| config.scope == scope)
341 .cloned()
342 }
344 pub fn language_configuration_for_injection_string(
345 &self,
346 string: &str,
347 ) -> Option<Arc<LanguageConfiguration>> {
348 let mut best_match_length = 0;
349 let mut best_match_position = None;
350 for (i, configuration) in self.language_configs.iter().enumerate() {
351 if let Some(injection_regex) = &configuration.injection_regex {
352 if let Some(mat) = injection_regex.find(string) {
353 let length = mat.end() - mat.start();
354 if length > best_match_length {
355 best_match_position = Some(i);
356 best_match_length = length;
357 }
358 }
359 }
360 }
362 if let Some(i) = best_match_position {
363 let configuration = &self.language_configs[i];
364 return Some(configuration.clone());
365 }
366 None
367 }
369 pub fn set_scopes(&self, scopes: Vec<String>) {
370 self.scopes.store(Arc::new(scopes));
372 // Reconfigure existing grammars
373 for config in self
374 .language_configs
375 .iter()
376 .filter(|cfg| cfg.is_highlight_initialized())
377 {
378 config.reconfigure(&self.scopes());
379 }
380 }
382 pub fn scopes(&self) -> Guard<Arc<Vec<String>>> {
383 self.scopes.load()
384 }
387pub struct TsParser {
388 parser: tree_sitter::Parser,
389 cursors: Vec<QueryCursor>,
392// could also just use a pool, or a single instance?
393thread_local! {
394 pub static PARSER: RefCell<TsParser> = RefCell::new(TsParser {
395 parser: Parser::new(),
396 cursors: Vec::new(),
397 })
400#[derive(Debug)]
401pub struct Syntax {
402 layers: HopSlotMap<LayerId, LanguageLayer>,
403 root: LayerId,
404 loader: Arc<Loader>,
407fn byte_range_to_str(range: std::ops::Range<usize>, source: RopeSlice) -> Cow<str> {
408 let start_char = source.byte_to_char(range.start);
409 let end_char = source.byte_to_char(range.end);
410 Cow::from(source.slice(start_char..end_char))
413impl Syntax {
414 pub fn new(source: &Rope, config: Arc<HighlightConfiguration>, loader: Arc<Loader>) -> Self {
415 let root_layer = LanguageLayer {
416 tree: None,
417 config,
418 depth: 0,
419 ranges: vec![Range {
420 start_byte: 0,
421 end_byte: usize::MAX,
422 start_point: Point::new(0, 0),
423 end_point: Point::new(usize::MAX, usize::MAX),
424 }],
425 };
427 // track scope_descriptor: a Vec of scopes for item in tree
429 let mut layers = HopSlotMap::default();
430 let root = layers.insert(root_layer);
432 let mut syntax = Self {
433 root,
434 layers,
435 loader,
436 };
438 syntax
439 .update(source, source, &ChangeSet::new(source))
440 .unwrap();
442 syntax
443 }
445 pub fn update(
446 &mut self,
447 old_source: &Rope,
448 source: &Rope,
449 changeset: &ChangeSet,
450 ) -> Result<(), Error> {
451 let mut queue = VecDeque::new();
452 queue.push_back(self.root);
454 let scopes = self.loader.scopes.load();
455 let injection_callback = |language: &str| {
456 self.loader
457 .language_configuration_for_injection_string(language)
458 .and_then(|language_config| language_config.highlight_config(&scopes))
459 };
461 // Convert the changeset into tree sitter edits.
462 let edits = generate_edits(old_source, changeset);
464 // Use the edits to update all layers markers
465 if !edits.is_empty() {
466 fn point_add(a: Point, b: Point) -> Point {
467 if b.row > 0 {
468 Point::new(a.row.saturating_add(b.row), b.column)
469 } else {
470 Point::new(0, a.column.saturating_add(b.column))
471 }
472 }
473 fn point_sub(a: Point, b: Point) -> Point {
474 if a.row > b.row {
475 Point::new(a.row.saturating_sub(b.row), a.column)
476 } else {
477 Point::new(0, a.column.saturating_sub(b.column))
478 }
479 }
481 for layer in &mut self.layers.values_mut() {
482 // The root layer always covers the whole range (0..usize::MAX)
483 if layer.depth == 0 {
484 continue;
485 }
487 for range in &mut layer.ranges {
488 // Roughly based on https://github.com/tree-sitter/tree-sitter/blob/ddeaa0c7f534268b35b4f6cb39b52df082754413/lib/src/subtree.c#L691-L720
489 for edit in edits.iter().rev() {
490 let is_pure_insertion = edit.old_end_byte == edit.start_byte;
492 // if edit is after range, skip
493 if edit.start_byte > range.end_byte {
494 // TODO: || (is_noop && edit.start_byte == range.end_byte)
495 continue;
496 }
498 // if edit is before range, shift entire range by len
499 if edit.old_end_byte < range.start_byte {
500 range.start_byte =
501 edit.new_end_byte + (range.start_byte - edit.old_end_byte);
502 range.start_point = point_add(
503 edit.new_end_position,
504 point_sub(range.start_point, edit.old_end_position),
505 );
507 range.end_byte = edit
508 .new_end_byte
509 .saturating_add(range.end_byte - edit.old_end_byte);
510 range.end_point = point_add(
511 edit.new_end_position,
512 point_sub(range.end_point, edit.old_end_position),
513 );
514 }
515 // if the edit starts in the space before and extends into the range
516 else if edit.start_byte < range.start_byte {
517 range.start_byte = edit.new_end_byte;
518 range.start_point = edit.new_end_position;
520 range.end_byte = range
521 .end_byte
522 .saturating_sub(edit.old_end_byte)
523 .saturating_add(edit.new_end_byte);
524 range.end_point = point_add(
525 edit.new_end_position,
526 point_sub(range.end_point, edit.old_end_position),
527 );
528 }
529 // If the edit is an insertion at the start of the tree, shift
530 else if edit.start_byte == range.start_byte && is_pure_insertion {
531 range.start_byte = edit.new_end_byte;
532 range.start_point = edit.new_end_position;
533 } else {
534 range.end_byte = range
535 .end_byte
536 .saturating_sub(edit.old_end_byte)
537 .saturating_add(edit.new_end_byte);
538 range.end_point = point_add(
539 edit.new_end_position,
540 point_sub(range.end_point, edit.old_end_position),
541 );
542 }
543 }
544 }
545 }
546 }
548 PARSER.with(|ts_parser| {
549 let ts_parser = &mut ts_parser.borrow_mut();
550 let mut cursor = ts_parser.cursors.pop().unwrap_or_else(QueryCursor::new);
551 // TODO: might need to set cursor range
552 cursor.set_byte_range(0..usize::MAX);
554 let source_slice = source.slice(..);
556 let mut touched = HashSet::new();
558 // TODO: we should be able to avoid editing & parsing layers with ranges earlier in the document before the edit
560 while let Some(layer_id) = queue.pop_front() {
561 // Mark the layer as touched
562 touched.insert(layer_id);
564 let layer = &mut self.layers[layer_id];
566 // If a tree already exists, notify it of changes.
567 if let Some(tree) = &mut layer.tree {
568 for edit in edits.iter().rev() {
569 // Apply the edits in reverse.
570 // If we applied them in order then edit 1 would disrupt the positioning of edit 2.
571 tree.edit(edit);
572 }
573 }
575 // Re-parse the tree.
576 layer.parse(&mut ts_parser.parser, source)?;
578 // Switch to an immutable borrow.
579 let layer = &self.layers[layer_id];
581 // Process injections.
582 let matches = cursor.matches(
583 &layer.config.injections_query,
584 layer.tree().root_node(),
585 RopeProvider(source_slice),
586 );
587 let mut injections = Vec::new();
588 for mat in matches {
589 let (language_name, content_node, include_children) = injection_for_match(
590 &layer.config,
591 &layer.config.injections_query,
592 &mat,
593 source_slice,
594 );
596 // Explicitly remove this match so that none of its other captures will remain
597 // in the stream of captures.
598 mat.remove();
600 // If a language is found with the given name, then add a new language layer
601 // to the highlighted document.
602 if let (Some(language_name), Some(content_node)) = (language_name, content_node)
603 {
604 if let Some(config) = (injection_callback)(&language_name) {
605 let ranges =
606 intersect_ranges(&layer.ranges, &[content_node], include_children);
608 if !ranges.is_empty() {
609 injections.push((config, ranges));
610 }
611 }
612 }
613 }
615 // Process combined injections.
616 if let Some(combined_injections_query) = &layer.config.combined_injections_query {
617 let mut injections_by_pattern_index =
618 vec![(None, Vec::new(), false); combined_injections_query.pattern_count()];
619 let matches = cursor.matches(
620 combined_injections_query,
621 layer.tree().root_node(),
622 RopeProvider(source_slice),
623 );
624 for mat in matches {
625 let entry = &mut injections_by_pattern_index[mat.pattern_index];
626 let (language_name, content_node, include_children) = injection_for_match(
627 &layer.config,
628 combined_injections_query,
629 &mat,
630 source_slice,
631 );
632 if language_name.is_some() {
633 entry.0 = language_name;
634 }
635 if let Some(content_node) = content_node {
636 entry.1.push(content_node);
637 }
638 entry.2 = include_children;
639 }
640 for (lang_name, content_nodes, includes_children) in injections_by_pattern_index
641 {
642 if let (Some(lang_name), false) = (lang_name, content_nodes.is_empty()) {
643 if let Some(config) = (injection_callback)(&lang_name) {
644 let ranges = intersect_ranges(
645 &layer.ranges,
646 &content_nodes,
647 includes_children,
648 );
649 if !ranges.is_empty() {
650 injections.push((config, ranges));
651 }
652 }
653 }
654 }
655 }
657 let depth = layer.depth + 1;
658 // TODO: can't inline this since matches borrows self.layers
659 for (config, ranges) in injections {
660 // Find an existing layer
661 let layer = self
662 .layers
663 .iter_mut()
664 .find(|(_, layer)| {
665 layer.depth == depth && // TODO: track parent id instead
666 layer.config.language == config.language && layer.ranges == ranges
667 })
668 .map(|(id, _layer)| id);
670 // ...or insert a new one.
671 let layer_id = layer.unwrap_or_else(|| {
672 self.layers.insert(LanguageLayer {
673 tree: None,
674 config,
675 depth,
676 ranges,
677 })
678 });
680 queue.push_back(layer_id);
681 }
683 // TODO: pre-process local scopes at this time, rather than highlight?
684 // would solve problems with locals not working across boundaries
685 }
687 // Return the cursor back in the pool.
688 ts_parser.cursors.push(cursor);
690 // Remove all untouched layers
691 self.layers.retain(|id, _| touched.contains(&id));
693 Ok(())
694 })
695 }
697 pub fn tree(&self) -> &Tree {
698 self.layers[self.root].tree()
699 }
701 /// Iterate over the highlighted regions for a given slice of source code.
702 pub fn highlight_iter<'a>(
703 &'a self,
704 source: RopeSlice<'a>,
705 range: Option<std::ops::Range<usize>>,
706 cancellation_flag: Option<&'a AtomicUsize>,
707 ) -> impl Iterator<Item = Result<HighlightEvent, Error>> + 'a {
708 let mut layers = self
709 .layers
710 .iter()
711 .filter_map(|(_, layer)| {
712 // TODO: if range doesn't overlap layer range, skip it
714 // Reuse a cursor from the pool if available.
715 let mut cursor = PARSER.with(|ts_parser| {
716 let highlighter = &mut ts_parser.borrow_mut();
717 highlighter.cursors.pop().unwrap_or_else(QueryCursor::new)
718 });
720 // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which
721 // prevents them from being moved. But both of these values are really just
722 // pointers, so it's actually ok to move them.
723 let cursor_ref =
724 unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) };
726 // if reusing cursors & no range this resets to whole range
727 cursor_ref.set_byte_range(range.clone().unwrap_or(0..usize::MAX));
729 let mut captures = cursor_ref
730 .captures(
731 &layer.config.query,
732 layer.tree().root_node(),
733 RopeProvider(source),
734 )
735 .peekable();
737 // If there's no captures, skip the layer
738 captures.peek()?;
740 Some(HighlightIterLayer {
741 highlight_end_stack: Vec::new(),
742 scope_stack: vec![LocalScope {
743 inherits: false,
744 range: 0..usize::MAX,
745 local_defs: Vec::new(),
746 }],
747 cursor,
748 _tree: None,
749 captures,
750 config: layer.config.as_ref(), // TODO: just reuse `layer`
751 depth: layer.depth, // TODO: just reuse `layer`
752 ranges: &layer.ranges, // TODO: temp
753 })
754 })
755 .collect::<Vec<_>>();
757 // HAXX: arrange layers by byte range, with deeper layers positioned first
758 layers.sort_by_key(|layer| {
759 (
760 layer.ranges.first().cloned(),
761 std::cmp::Reverse(layer.depth),
762 )
763 });
765 let mut result = HighlightIter {
766 source,
767 byte_offset: range.map_or(0, |r| r.start),
768 cancellation_flag,
769 iter_count: 0,
770 layers,
771 next_event: None,
772 last_highlight_range: None,
773 };
774 result.sort_layers();
775 result
776 }
778 // Commenting
779 // comment_strings_for_pos
780 // is_commented
782 // Indentation
783 // suggested_indent_for_line_at_buffer_row
784 // suggested_indent_for_buffer_row
785 // indent_level_for_line
787 // TODO: Folding
790#[derive(Debug)]
791pub struct LanguageLayer {
792 // mode
793 // grammar
794 pub config: Arc<HighlightConfiguration>,
795 pub(crate) tree: Option<Tree>,
796 pub ranges: Vec<Range>,
797 pub depth: usize,
800impl LanguageLayer {
801 pub fn tree(&self) -> &Tree {
802 // TODO: no unwrap
803 self.tree.as_ref().unwrap()
804 }
806 fn parse(&mut self, parser: &mut Parser, source: &Rope) -> Result<(), Error> {
807 parser.set_included_ranges(&self.ranges).unwrap();
809 parser
810 .set_language(self.config.language)
811 .map_err(|_| Error::InvalidLanguage)?;
813 // unsafe { syntax.parser.set_cancellation_flag(cancellation_flag) };
814 let tree = parser
815 .parse_with(
816 &mut |byte, _| {
817 if byte <= source.len_bytes() {
818 let (chunk, start_byte, _, _) = source.chunk_at_byte(byte);
819 chunk[byte - start_byte..].as_bytes()
820 } else {
821 // out of range
822 &[]
823 }
824 },
825 self.tree.as_ref(),
826 )
827 .ok_or(Error::Cancelled)?;
828 // unsafe { ts_parser.parser.set_cancellation_flag(None) };
829 self.tree = Some(tree);
830 Ok(())
831 }
834pub(crate) fn generate_edits(
835 old_text: &Rope,
836 changeset: &ChangeSet,
837) -> Vec<tree_sitter::InputEdit> {
838 use Operation::*;
839 let mut old_pos = 0;
841 let mut edits = Vec::new();
843 if changeset.changes.is_empty() {
844 return edits;
845 }
847 let mut iter = changeset.changes.iter().peekable();
849 // TODO; this is a lot easier with Change instead of Operation.
851 fn point_at_pos(text: &Rope, pos: usize) -> (usize, Point) {
852 let byte = text.char_to_byte(pos); // <- attempted to index past end
853 let line = text.char_to_line(pos);
854 let line_start_byte = text.line_to_byte(line);
855 let col = byte - line_start_byte;
857 (byte, Point::new(line, col))
858 }
860 fn traverse(point: Point, text: &Tendril) -> Point {
861 let Point {
862 mut row,
863 mut column,
864 } = point;
866 // TODO: there should be a better way here.
867 let mut chars = text.chars().peekable();
868 while let Some(ch) = chars.next() {
869 if char_is_line_ending(ch) && !(ch == '\r' && chars.peek() == Some(&'\n')) {
870 row += 1;
871 column = 0;
872 } else {
873 column += 1;
874 }
875 }
876 Point { row, column }
877 }
879 while let Some(change) = iter.next() {
880 let len = match change {
881 Delete(i) | Retain(i) => *i,
882 Insert(_) => 0,
883 };
884 let mut old_end = old_pos + len;
886 match change {
887 Retain(_) => {}
888 Delete(_) => {
889 let (start_byte, start_position) = point_at_pos(old_text, old_pos);
890 let (old_end_byte, old_end_position) = point_at_pos(old_text, old_end);
892 // deletion
893 edits.push(tree_sitter::InputEdit {
894 start_byte, // old_pos to byte
895 old_end_byte, // old_end to byte
896 new_end_byte: start_byte, // old_pos to byte
897 start_position, // old pos to coords
898 old_end_position, // old_end to coords
899 new_end_position: start_position, // old pos to coords
900 });
901 }
902 Insert(s) => {
903 let (start_byte, start_position) = point_at_pos(old_text, old_pos);
905 // a subsequent delete means a replace, consume it
906 if let Some(Delete(len)) = iter.peek() {
907 old_end = old_pos + len;
908 let (old_end_byte, old_end_position) = point_at_pos(old_text, old_end);
910 iter.next();
912 // replacement
913 edits.push(tree_sitter::InputEdit {
914 start_byte, // old_pos to byte
915 old_end_byte, // old_end to byte
916 new_end_byte: start_byte + s.len(), // old_pos to byte + s.len()
917 start_position, // old pos to coords
918 old_end_position, // old_end to coords
919 new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)
920 });
921 } else {
922 // insert
923 edits.push(tree_sitter::InputEdit {
924 start_byte, // old_pos to byte
925 old_end_byte: start_byte, // same
926 new_end_byte: start_byte + s.len(), // old_pos + s.len()
927 start_position, // old pos to coords
928 old_end_position: start_position, // same
929 new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)
930 });
931 }
932 }
933 }
934 old_pos = old_end;
935 }
936 edits
939use std::sync::atomic::{AtomicUsize, Ordering};
940use std::{iter, mem, ops, str, usize};
941use tree_sitter::{
942 Language as Grammar, Node, Parser, Point, Query, QueryCaptures, QueryCursor, QueryError,
943 QueryMatch, Range, TextProvider, Tree,
944};
946const CANCELLATION_CHECK_INTERVAL: usize = 100;
948/// Indicates which highlight should be applied to a region of source code.
949#[derive(Copy, Clone, Debug, PartialEq, Eq)]
950pub struct Highlight(pub usize);
952/// Represents the reason why syntax highlighting failed.
953#[derive(Debug, PartialEq, Eq)]
954pub enum Error {
955 Cancelled,
956 InvalidLanguage,
957 Unknown,
960/// Represents a single step in rendering a syntax-highlighted document.
961#[derive(Copy, Clone, Debug)]
962pub enum HighlightEvent {
963 Source { start: usize, end: usize },
964 HighlightStart(Highlight),
965 HighlightEnd,
968/// Contains the data neeeded to higlight code written in a particular language.
969///
970/// This struct is immutable and can be shared between threads.
971#[derive(Debug)]
972pub struct HighlightConfiguration {
973 pub language: Grammar,
974 pub query: Query,
975 injections_query: Query,
976 combined_injections_query: Option<Query>,
977 highlights_pattern_index: usize,
978 highlight_indices: ArcSwap<Vec<Option<Highlight>>>,
979 non_local_variable_patterns: Vec<bool>,
980 injection_content_capture_index: Option<u32>,
981 injection_language_capture_index: Option<u32>,
982 local_scope_capture_index: Option<u32>,
983 local_def_capture_index: Option<u32>,
984 local_def_value_capture_index: Option<u32>,
985 local_ref_capture_index: Option<u32>,
988#[derive(Debug)]
989struct LocalDef<'a> {
990 name: Cow<'a, str>,
991 value_range: ops::Range<usize>,
992 highlight: Option<Highlight>,
995#[derive(Debug)]
996struct LocalScope<'a> {
997 inherits: bool,
998 range: ops::Range<usize>,
999 local_defs: Vec<LocalDef<'a>>,
1002#[derive(Debug)]
1003struct HighlightIter<'a> {
1004 source: RopeSlice<'a>,
1005 byte_offset: usize,
1006 cancellation_flag: Option<&'a AtomicUsize>,
1007 layers: Vec<HighlightIterLayer<'a>>,
1008 iter_count: usize,
1009 next_event: Option<HighlightEvent>,
1010 last_highlight_range: Option<(usize, usize, usize)>,
1013// Adapter to convert rope chunks to bytes
1014struct ChunksBytes<'a> {
1015 chunks: ropey::iter::Chunks<'a>,
1017impl<'a> Iterator for ChunksBytes<'a> {
1018 type Item = &'a [u8];
1019 fn next(&mut self) -> Option<Self::Item> {
1020 self.chunks.next().map(str::as_bytes)
1024struct RopeProvider<'a>(RopeSlice<'a>);
1025impl<'a> TextProvider<'a> for RopeProvider<'a> {
1026 type I = ChunksBytes<'a>;
1028 fn text(&mut self, node: Node) -> Self::I {
1029 let start_char = self.0.byte_to_char(node.start_byte());
1030 let end_char = self.0.byte_to_char(node.end_byte());
1031 let fragment = self.0.slice(start_char..end_char);
1032 ChunksBytes {
1033 chunks: fragment.chunks(),
1038struct HighlightIterLayer<'a> {
1039 _tree: Option<Tree>,
1040 cursor: QueryCursor,
1041 captures: iter::Peekable<QueryCaptures<'a, 'a, RopeProvider<'a>>>,
1042 config: &'a HighlightConfiguration,
1043 highlight_end_stack: Vec<usize>,
1044 scope_stack: Vec<LocalScope<'a>>,
1045 depth: usize,
1046 ranges: &'a [Range],
1049impl<'a> fmt::Debug for HighlightIterLayer<'a> {
1050 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1051 f.debug_struct("HighlightIterLayer").finish()
1055impl HighlightConfiguration {
1056 /// Creates a `HighlightConfiguration` for a given `Grammar` and set of highlighting
1057 /// queries.
1058 ///
1059 /// # Parameters
1060 ///
1061 /// * `language` - The Tree-sitter `Grammar` that should be used for parsing.
1062 /// * `highlights_query` - A string containing tree patterns for syntax highlighting. This
1063 /// should be non-empty, otherwise no syntax highlights will be added.
1064 /// * `injections_query` - A string containing tree patterns for injecting other languages
1065 /// into the document. This can be empty if no injections are desired.
1066 /// * `locals_query` - A string containing tree patterns for tracking local variable
1067 /// definitions and references. This can be empty if local variable tracking is not needed.
1068 ///
1069 /// Returns a `HighlightConfiguration` that can then be used with the `highlight` method.
1070 pub fn new(
1071 language: Grammar,
1072 highlights_query: &str,
1073 injection_query: &str,
1074 locals_query: &str,
1075 ) -> Result<Self, QueryError> {
1076 // Concatenate the query strings, keeping track of the start offset of each section.
1077 let mut query_source = String::new();
1078 query_source.push_str(locals_query);
1079 let highlights_query_offset = query_source.len();
1080 query_source.push_str(highlights_query);
1082 // Construct a single query by concatenating the three query strings, but record the
1083 // range of pattern indices that belong to each individual string.
1084 let query = Query::new(language, &query_source)?;
1085 let mut highlights_pattern_index = 0;
1086 for i in 0..(query.pattern_count()) {
1087 let pattern_offset = query.start_byte_for_pattern(i);
1088 if pattern_offset < highlights_query_offset {
1089 highlights_pattern_index += 1;
1093 let mut injections_query = Query::new(language, injection_query)?;
1095 // Construct a separate query just for dealing with the 'combined injections'.
1096 // Disable the combined injection patterns in the main query.
1097 let mut combined_injections_query = Query::new(language, injection_query)?;
1098 let mut has_combined_queries = false;
1099 for pattern_index in 0..injections_query.pattern_count() {
1100 let settings = injections_query.property_settings(pattern_index);
1101 if settings.iter().any(|s| &*s.key == "injection.combined") {
1102 has_combined_queries = true;
1103 injections_query.disable_pattern(pattern_index);
1104 } else {
1105 combined_injections_query.disable_pattern(pattern_index);
1108 let combined_injections_query = if has_combined_queries {
1109 Some(combined_injections_query)
1110 } else {
1111 None
1112 };
1114 // Find all of the highlighting patterns that are disabled for nodes that
1115 // have been identified as local variables.
1116 let non_local_variable_patterns = (0..query.pattern_count())
1117 .map(|i| {
1118 query
1119 .property_predicates(i)
1120 .iter()
1121 .any(|(prop, positive)| !*positive && prop.key.as_ref() == "local")
1122 })
1123 .collect();
1125 // Store the numeric ids for all of the special captures.
1126 let mut injection_content_capture_index = None;
1127 let mut injection_language_capture_index = None;
1128 let mut local_def_capture_index = None;
1129 let mut local_def_value_capture_index = None;
1130 let mut local_ref_capture_index = None;
1131 let mut local_scope_capture_index = None;
1132 for (i, name) in query.capture_names().iter().enumerate() {
1133 let i = Some(i as u32);
1134 match name.as_str() {
1135 "local.definition" => local_def_capture_index = i,
1136 "local.definition-value" => local_def_value_capture_index = i,
1137 "local.reference" => local_ref_capture_index = i,
1138 "local.scope" => local_scope_capture_index = i,
1139 _ => {}
1143 for (i, name) in injections_query.capture_names().iter().enumerate() {
1144 let i = Some(i as u32);
1145 match name.as_str() {
1146 "injection.content" => injection_content_capture_index = i,
1147 "injection.language" => injection_language_capture_index = i,
1148 _ => {}
1152 let highlight_indices = ArcSwap::from_pointee(vec![None; query.capture_names().len()]);
1153 Ok(Self {
1154 language,
1155 query,
1156 injections_query,
1157 combined_injections_query,
1158 highlights_pattern_index,
1159 highlight_indices,
1160 non_local_variable_patterns,
1161 injection_content_capture_index,
1162 injection_language_capture_index,
1163 local_scope_capture_index,
1164 local_def_capture_index,
1165 local_def_value_capture_index,
1166 local_ref_capture_index,
1167 })
1170 /// Get a slice containing all of the highlight names used in the configuration.
1171 pub fn names(&self) -> &[String] {
1172 self.query.capture_names()
1175 /// Set the list of recognized highlight names.
1176 ///
1177 /// Tree-sitter syntax-highlighting queries specify highlights in the form of dot-separated
1178 /// highlight names like `punctuation.bracket` and `function.method.builtin`. Consumers of
1179 /// these queries can choose to recognize highlights with different levels of specificity.
1180 /// For example, the string `function.builtin` will match against `function.method.builtin`
1181 /// and `function.builtin.constructor`, but will not match `function.method`.
1182 ///
1183 /// When highlighting, results are returned as `Highlight` values, which contain the index
1184 /// of the matched highlight this list of highlight names.
1185 pub fn configure(&self, recognized_names: &[String]) {
1186 let mut capture_parts = Vec::new();
1187 let indices: Vec<_> = self
1188 .query
1189 .capture_names()
1190 .iter()
1191 .map(move |capture_name| {
1192 capture_parts.clear();
1193 capture_parts.extend(capture_name.split('.'));
1195 let mut best_index = None;
1196 let mut best_match_len = 0;
1197 for (i, recognized_name) in recognized_names.iter().enumerate() {
1198 let recognized_name = recognized_name;
1199 let mut len = 0;
1200 let mut matches = true;
1201 for part in recognized_name.split('.') {
1202 len += 1;
1203 if !capture_parts.contains(&part) {
1204 matches = false;
1205 break;
1208 if matches && len > best_match_len {
1209 best_index = Some(i);
1210 best_match_len = len;
1213 best_index.map(Highlight)
1214 })
1215 .collect();
1217 self.highlight_indices.store(Arc::new(indices));
1221impl<'a> HighlightIterLayer<'a> {
1222 // First, sort scope boundaries by their byte offset in the document. At a
1223 // given position, emit scope endings before scope beginnings. Finally, emit
1224 // scope boundaries from deeper layers first.
1225 fn sort_key(&mut self) -> Option<(usize, bool, isize)> {
1226 let depth = -(self.depth as isize);
1227 let next_start = self
1228 .captures
1229 .peek()
1230 .map(|(m, i)| m.captures[*i].node.start_byte());
1231 let next_end = self.highlight_end_stack.last().cloned();
1232 match (next_start, next_end) {
1233 (Some(start), Some(end)) => {
1234 if start < end {
1235 Some((start, true, depth))
1236 } else {
1237 Some((end, false, depth))
1240 (Some(i), None) => Some((i, true, depth)),
1241 (None, Some(j)) => Some((j, false, depth)),
1242 _ => None,
1247// Compute the ranges that should be included when parsing an injection.
1248// This takes into account three things:
1249// * `parent_ranges` - The ranges must all fall within the *current* layer's ranges.
1250// * `nodes` - Every injection takes place within a set of nodes. The injection ranges
1251// are the ranges of those nodes.
1252// * `includes_children` - For some injections, the content nodes' children should be
1253// excluded from the nested document, so that only the content nodes' *own* content
1254// is reparsed. For other injections, the content nodes' entire ranges should be
1255// reparsed, including the ranges of their children.
1256fn intersect_ranges(
1257 parent_ranges: &[Range],
1258 nodes: &[Node],
1259 includes_children: bool,
1260) -> Vec<Range> {
1261 let mut cursor = nodes[0].walk();
1262 let mut result = Vec::new();
1263 let mut parent_range_iter = parent_ranges.iter();
1264 let mut parent_range = parent_range_iter
1265 .next()
1266 .expect("Layers should only be constructed with non-empty ranges vectors");
1267 for node in nodes.iter() {
1268 let mut preceding_range = Range {
1269 start_byte: 0,
1270 start_point: Point::new(0, 0),
1271 end_byte: node.start_byte(),
1272 end_point: node.start_position(),
1273 };
1274 let following_range = Range {
1275 start_byte: node.end_byte(),
1276 start_point: node.end_position(),
1277 end_byte: usize::MAX,
1278 end_point: Point::new(usize::MAX, usize::MAX),
1279 };
1281 for excluded_range in node
1282 .children(&mut cursor)
1283 .filter_map(|child| {
1284 if includes_children {
1285 None
1286 } else {
1287 Some(child.range())
1289 })
1290 .chain([following_range].iter().cloned())
1292 let mut range = Range {
1293 start_byte: preceding_range.end_byte,
1294 start_point: preceding_range.end_point,
1295 end_byte: excluded_range.start_byte,
1296 end_point: excluded_range.start_point,
1297 };
1298 preceding_range = excluded_range;
1300 if range.end_byte < parent_range.start_byte {
1301 continue;
1304 while parent_range.start_byte <= range.end_byte {
1305 if parent_range.end_byte > range.start_byte {
1306 if range.start_byte < parent_range.start_byte {
1307 range.start_byte = parent_range.start_byte;
1308 range.start_point = parent_range.start_point;
1311 if parent_range.end_byte < range.end_byte {
1312 if range.start_byte < parent_range.end_byte {
1313 result.push(Range {
1314 start_byte: range.start_byte,
1315 start_point: range.start_point,
1316 end_byte: parent_range.end_byte,
1317 end_point: parent_range.end_point,
1318 });
1320 range.start_byte = parent_range.end_byte;
1321 range.start_point = parent_range.end_point;
1322 } else {
1323 if range.start_byte < range.end_byte {
1324 result.push(range);
1326 break;
1330 if let Some(next_range) = parent_range_iter.next() {
1331 parent_range = next_range;
1332 } else {
1333 return result;
1338 result
1341impl<'a> HighlightIter<'a> {
1342 fn emit_event(
1343 &mut self,
1344 offset: usize,
1345 event: Option<HighlightEvent>,
1346 ) -> Option<Result<HighlightEvent, Error>> {
1347 let result;
1348 if self.byte_offset < offset {
1349 result = Some(Ok(HighlightEvent::Source {
1350 start: self.byte_offset,
1351 end: offset,
1352 }));
1353 self.byte_offset = offset;
1354 self.next_event = event;
1355 } else {
1356 result = event.map(Ok);
1358 self.sort_layers();
1359 result
1362 fn sort_layers(&mut self) {
1363 while !self.layers.is_empty() {
1364 if let Some(sort_key) = self.layers[0].sort_key() {
1365 let mut i = 0;
1366 while i + 1 < self.layers.len() {
1367 if let Some(next_offset) = self.layers[i + 1].sort_key() {
1368 if next_offset < sort_key {
1369 i += 1;
1370 continue;
1372 } else {
1373 let layer = self.layers.remove(i + 1);
1374 PARSER.with(|ts_parser| {
1375 let highlighter = &mut ts_parser.borrow_mut();
1376 highlighter.cursors.push(layer.cursor);
1377 });
1379 break;
1381 if i > 0 {
1382 self.layers[0..(i + 1)].rotate_left(1);
1384 break;
1385 } else {
1386 let layer = self.layers.remove(0);
1387 PARSER.with(|ts_parser| {
1388 let highlighter = &mut ts_parser.borrow_mut();
1389 highlighter.cursors.push(layer.cursor);
1390 });
1396impl<'a> Iterator for HighlightIter<'a> {
1397 type Item = Result<HighlightEvent, Error>;
1399 fn next(&mut self) -> Option<Self::Item> {
1400 'main: loop {
1401 // If we've already determined the next highlight boundary, just return it.
1402 if let Some(e) = self.next_event.take() {
1403 return Some(Ok(e));
1406 // Periodically check for cancellation, returning `Cancelled` error if the
1407 // cancellation flag was flipped.
1408 if let Some(cancellation_flag) = self.cancellation_flag {
1409 self.iter_count += 1;
1410 if self.iter_count >= CANCELLATION_CHECK_INTERVAL {
1411 self.iter_count = 0;
1412 if cancellation_flag.load(Ordering::Relaxed) != 0 {
1413 return Some(Err(Error::Cancelled));
1418 // If none of the layers have any more highlight boundaries, terminate.
1419 if self.layers.is_empty() {
1420 let len = self.source.len_bytes();
1421 return if self.byte_offset < len {
1422 let result = Some(Ok(HighlightEvent::Source {
1423 start: self.byte_offset,
1424 end: len,
1425 }));
1426 self.byte_offset = len;
1427 result
1428 } else {
1429 None
1430 };
1433 // Get the next capture from whichever layer has the earliest highlight boundary.
1434 let range;
1435 let layer = &mut self.layers[0];
1436 if let Some((next_match, capture_index)) = layer.captures.peek() {
1437 let next_capture = next_match.captures[*capture_index];
1438 range = next_capture.node.byte_range();
1440 // If any previous highlight ends before this node starts, then before
1441 // processing this capture, emit the source code up until the end of the
1442 // previous highlight, and an end event for that highlight.
1443 if let Some(end_byte) = layer.highlight_end_stack.last().cloned() {
1444 if end_byte <= range.start {
1445 layer.highlight_end_stack.pop();
1446 return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd));
1450 // If there are no more captures, then emit any remaining highlight end events.
1451 // And if there are none of those, then just advance to the end of the document.
1452 else if let Some(end_byte) = layer.highlight_end_stack.last().cloned() {
1453 layer.highlight_end_stack.pop();
1454 return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd));
1455 } else {
1456 return self.emit_event(self.source.len_bytes(), None);
1457 };
1459 let (mut match_, capture_index) = layer.captures.next().unwrap();
1460 let mut capture = match_.captures[capture_index];
1462 // Remove from the local scope stack any local scopes that have already ended.
1463 while range.start > layer.scope_stack.last().unwrap().range.end {
1464 layer.scope_stack.pop();
1467 // If this capture is for tracking local variables, then process the
1468 // local variable info.
1469 let mut reference_highlight = None;
1470 let mut definition_highlight = None;
1471 while match_.pattern_index < layer.config.highlights_pattern_index {
1472 // If the node represents a local scope, push a new local scope onto
1473 // the scope stack.
1474 if Some(capture.index) == layer.config.local_scope_capture_index {
1475 definition_highlight = None;
1476 let mut scope = LocalScope {
1477 inherits: true,
1478 range: range.clone(),
1479 local_defs: Vec::new(),
1480 };
1481 for prop in layer.config.query.property_settings(match_.pattern_index) {
1482 if let "local.scope-inherits" = prop.key.as_ref() {
1483 scope.inherits =
1484 prop.value.as_ref().map_or(true, |r| r.as_ref() == "true");
1487 layer.scope_stack.push(scope);
1489 // If the node represents a definition, add a new definition to the
1490 // local scope at the top of the scope stack.
1491 else if Some(capture.index) == layer.config.local_def_capture_index {
1492 reference_highlight = None;
1493 let scope = layer.scope_stack.last_mut().unwrap();
1495 let mut value_range = 0..0;
1496 for capture in match_.captures {
1497 if Some(capture.index) == layer.config.local_def_value_capture_index {
1498 value_range = capture.node.byte_range();
1502 let name = byte_range_to_str(range.clone(), self.source);
1503 scope.local_defs.push(LocalDef {
1504 name,
1505 value_range,
1506 highlight: None,
1507 });
1508 definition_highlight = scope.local_defs.last_mut().map(|s| &mut s.highlight);
1510 // If the node represents a reference, then try to find the corresponding
1511 // definition in the scope stack.
1512 else if Some(capture.index) == layer.config.local_ref_capture_index
1513 && definition_highlight.is_none()
1515 definition_highlight = None;
1516 let name = byte_range_to_str(range.clone(), self.source);
1517 for scope in layer.scope_stack.iter().rev() {
1518 if let Some(highlight) = scope.local_defs.iter().rev().find_map(|def| {
1519 if def.name == name && range.start >= def.value_range.end {
1520 Some(def.highlight)
1521 } else {
1522 None
1524 }) {
1525 reference_highlight = highlight;
1526 break;
1528 if !scope.inherits {
1529 break;
1534 // Continue processing any additional matches for the same node.
1535 if let Some((next_match, next_capture_index)) = layer.captures.peek() {
1536 let next_capture = next_match.captures[*next_capture_index];
1537 if next_capture.node == capture.node {
1538 capture = next_capture;
1539 match_ = layer.captures.next().unwrap().0;
1540 continue;
1544 self.sort_layers();
1545 continue 'main;
1548 // Otherwise, this capture must represent a highlight.
1549 // If this exact range has already been highlighted by an earlier pattern, or by
1550 // a different layer, then skip over this one.
1551 if let Some((last_start, last_end, last_depth)) = self.last_highlight_range {
1552 if range.start == last_start && range.end == last_end && layer.depth < last_depth {
1553 self.sort_layers();
1554 continue 'main;
1558 // If the current node was found to be a local variable, then skip over any
1559 // highlighting patterns that are disabled for local variables.
1560 if definition_highlight.is_some() || reference_highlight.is_some() {
1561 while layer.config.non_local_variable_patterns[match_.pattern_index] {
1562 if let Some((next_match, next_capture_index)) = layer.captures.peek() {
1563 let next_capture = next_match.captures[*next_capture_index];
1564 if next_capture.node == capture.node {
1565 capture = next_capture;
1566 match_ = layer.captures.next().unwrap().0;
1567 continue;
1571 self.sort_layers();
1572 continue 'main;
1576 // Once a highlighting pattern is found for the current node, skip over
1577 // any later highlighting patterns that also match this node. Captures
1578 // for a given node are ordered by pattern index, so these subsequent
1579 // captures are guaranteed to be for highlighting, not injections or
1580 // local variables.
1581 while let Some((next_match, next_capture_index)) = layer.captures.peek() {
1582 let next_capture = next_match.captures[*next_capture_index];
1583 if next_capture.node == capture.node {
1584 layer.captures.next();
1585 } else {
1586 break;
1590 let current_highlight = layer.config.highlight_indices.load()[capture.index as usize];
1592 // If this node represents a local definition, then store the current
1593 // highlight value on the local scope entry representing this node.
1594 if let Some(definition_highlight) = definition_highlight {
1595 *definition_highlight = current_highlight;
1598 // Emit a scope start event and push the node's end position to the stack.
1599 if let Some(highlight) = reference_highlight.or(current_highlight) {
1600 self.last_highlight_range = Some((range.start, range.end, layer.depth));
1601 layer.highlight_end_stack.push(range.end);
1602 return self
1603 .emit_event(range.start, Some(HighlightEvent::HighlightStart(highlight)));
1606 self.sort_layers();
1611fn injection_for_match<'a>(
1612 config: &HighlightConfiguration,
1613 query: &'a Query,
1614 query_match: &QueryMatch<'a, 'a>,
1615 source: RopeSlice<'a>,
1616) -> (Option<Cow<'a, str>>, Option<Node<'a>>, bool) {
1617 let content_capture_index = config.injection_content_capture_index;
1618 let language_capture_index = config.injection_language_capture_index;
1620 let mut language_name = None;
1621 let mut content_node = None;
1622 for capture in query_match.captures {
1623 let index = Some(capture.index);
1624 if index == language_capture_index {
1625 let name = byte_range_to_str(capture.node.byte_range(), source);
1626 language_name = Some(name);
1627 } else if index == content_capture_index {
1628 content_node = Some(capture.node);
1632 let mut include_children = false;
1633 for prop in query.property_settings(query_match.pattern_index) {
1634 match prop.key.as_ref() {
1635 // In addition to specifying the language name via the text of a
1636 // captured node, it can also be hard-coded via a `#set!` predicate
1637 // that sets the injection.language key.
1638 "injection.language" => {
1639 if language_name.is_none() {
1640 language_name = prop.value.as_ref().map(|s| s.as_ref().into())
1644 // By default, injections do not include the *children* of an
1645 // `injection.content` node - only the ranges that belong to the
1646 // node itself. This can be changed using a `#set!` predicate that
1647 // sets the `injection.include-children` key.
1648 "injection.include-children" => include_children = true,
1649 _ => {}
1653 (language_name, content_node, include_children)
1656pub struct Merge<I> {
1657 iter: I,
1658 spans: Box<dyn Iterator<Item = (usize, std::ops::Range<usize>)>>,
1660 next_event: Option<HighlightEvent>,
1661 next_span: Option<(usize, std::ops::Range<usize>)>,
1663 queue: Vec<HighlightEvent>,
1666/// Merge a list of spans into the highlight event stream.
1667pub fn merge<I: Iterator<Item = HighlightEvent>>(
1668 iter: I,
1669 spans: Vec<(usize, std::ops::Range<usize>)>,
1670) -> Merge<I> {
1671 let spans = Box::new(spans.into_iter());
1672 let mut merge = Merge {
1673 iter,
1674 spans,
1675 next_event: None,
1676 next_span: None,
1677 queue: Vec::new(),
1678 };
1679 merge.next_event = merge.iter.next();
1680 merge.next_span = merge.spans.next();
1681 merge
1684impl<I: Iterator<Item = HighlightEvent>> Iterator for Merge<I> {
1685 type Item = HighlightEvent;
1686 fn next(&mut self) -> Option<Self::Item> {
1687 use HighlightEvent::*;
1688 if let Some(event) = self.queue.pop() {
1689 return Some(event);
1692 loop {
1693 match (self.next_event, &self.next_span) {
1694 // this happens when range is partially or fully offscreen
1695 (Some(Source { start, .. }), Some((span, range))) if start > range.start => {
1696 if start > range.end {
1697 self.next_span = self.spans.next();
1698 } else {
1699 self.next_span = Some((*span, start..range.end));
1700 };
1702 _ => break,
1706 match (self.next_event, &self.next_span) {
1707 (Some(HighlightStart(i)), _) => {
1708 self.next_event = self.iter.next();
1709 Some(HighlightStart(i))
1711 (Some(HighlightEnd), _) => {
1712 self.next_event = self.iter.next();
1713 Some(HighlightEnd)
1715 (Some(Source { start, end }), Some((_, range))) if start < range.start => {
1716 let intersect = range.start.min(end);
1717 let event = Source {
1718 start,
1719 end: intersect,
1720 };
1722 if end == intersect {
1723 // the event is complete
1724 self.next_event = self.iter.next();
1725 } else {
1726 // subslice the event
1727 self.next_event = Some(Source {
1728 start: intersect,
1729 end,
1730 });
1731 };
1733 Some(event)
1735 (Some(Source { start, end }), Some((span, range))) if start == range.start => {
1736 let intersect = range.end.min(end);
1737 let event = HighlightStart(Highlight(*span));
1739 // enqueue in reverse order
1740 self.queue.push(HighlightEnd);
1741 self.queue.push(Source {
1742 start,
1743 end: intersect,
1744 });
1746 if end == intersect {
1747 // the event is complete
1748 self.next_event = self.iter.next();
1749 } else {
1750 // subslice the event
1751 self.next_event = Some(Source {
1752 start: intersect,
1753 end,
1754 });
1755 };
1757 if intersect == range.end {
1758 self.next_span = self.spans.next();
1759 } else {
1760 self.next_span = Some((*span, intersect..range.end));
1763 Some(event)
1765 (Some(event), None) => {
1766 self.next_event = self.iter.next();
1767 Some(event)
1769 // Can happen if cursor at EOF and/or diagnostic reaches past the end.
1770 // We need to actually emit events for the cursor-at-EOF situation,
1771 // even though the range is past the end of the text. This needs to be
1772 // handled appropriately by the drawing code by not assuming that
1773 // all `Source` events point to valid indices in the rope.
1774 (None, Some((span, range))) => {
1775 let event = HighlightStart(Highlight(*span));
1776 self.queue.push(HighlightEnd);
1777 self.queue.push(Source {
1778 start: range.start,
1779 end: range.end,
1780 });
1781 self.next_span = self.spans.next();
1782 Some(event)
1784 (None, None) => None,
1785 e => unreachable!("{:?}", e),
1790#[cfg(test)]
1791mod test {
1792 use super::*;
1793 use crate::{Rope, Transaction};
1795 #[test]
1796 fn test_parser() {
1797 let highlight_names: Vec<String> = [
1798 "attribute",
1799 "constant",
1800 "function.builtin",
1801 "function",
1802 "keyword",
1803 "operator",
1804 "property",
1805 "punctuation",
1806 "punctuation.bracket",
1807 "punctuation.delimiter",
1808 "string",
1809 "string.special",
1810 "tag",
1811 "type",
1812 "type.builtin",
1813 "variable",
1814 "variable.builtin",
1815 "variable.parameter",
1817 .iter()
1818 .cloned()
1819 .map(String::from)
1820 .collect();
1822 let loader = Loader::new(Configuration { language: vec![] });
1824 let language = get_language(&crate::RUNTIME_DIR, "Rust").unwrap();
1825 let config = HighlightConfiguration::new(
1826 language,
1827 &std::fs::read_to_string(
1828 "../helix-syntax/languages/tree-sitter-rust/queries/highlights.scm",
1830 .unwrap(),
1831 &std::fs::read_to_string(
1832 "../helix-syntax/languages/tree-sitter-rust/queries/injections.scm",
1834 .unwrap(),
1835 "", // locals.scm
1837 .unwrap();
1838 config.configure(&highlight_names);
1840 let source = Rope::from_str(
1842 struct Stuff {}
1843 fn main() {}
1844 ",
1845 );
1846 let syntax = Syntax::new(&source, Arc::new(config), Arc::new(loader));
1847 let tree = syntax.tree();
1848 let root = tree.root_node();
1849 assert_eq!(root.kind(), "source_file");
1851 assert_eq!(
1852 root.to_sexp(),
1853 concat!(
1854 "(source_file ",
1855 "(struct_item name: (type_identifier) body: (field_declaration_list)) ",
1856 "(function_item name: (identifier) parameters: (parameters) body: (block)))"
1858 );
1860 let struct_node = root.child(0).unwrap();
1861 assert_eq!(struct_node.kind(), "struct_item");
1864 #[test]
1865 fn test_input_edits() {
1866 use tree_sitter::InputEdit;
1868 let doc = Rope::from("hello world!\ntest 123");
1869 let transaction = Transaction::change(
1870 &doc,
1871 vec![(6, 11, Some("test".into())), (12, 17, None)].into_iter(),
1872 );
1873 let edits = generate_edits(&doc, transaction.changes());
1874 // transaction.apply(&mut state);
1876 assert_eq!(
1877 edits,
1878 &[
1879 InputEdit {
1880 start_byte: 6,
1881 old_end_byte: 11,
1882 new_end_byte: 10,
1883 start_position: Point { row: 0, column: 6 },
1884 old_end_position: Point { row: 0, column: 11 },
1885 new_end_position: Point { row: 0, column: 10 }
1886 },
1887 InputEdit {
1888 start_byte: 12,
1889 old_end_byte: 17,
1890 new_end_byte: 12,
1891 start_position: Point { row: 0, column: 12 },
1892 old_end_position: Point { row: 1, column: 4 },
1893 new_end_position: Point { row: 0, column: 12 }
1896 );
1898 // Testing with the official example from tree-sitter
1899 let mut doc = Rope::from("fn test() {}");
1900 let transaction =
1901 Transaction::change(&doc, vec![(8, 8, Some("a: u32".into()))].into_iter());
1902 let edits = generate_edits(&doc, transaction.changes());
1903 transaction.apply(&mut doc);
1905 assert_eq!(doc, "fn test(a: u32) {}");
1906 assert_eq!(
1907 edits,
1908 &[InputEdit {
1909 start_byte: 8,
1910 old_end_byte: 8,
1911 new_end_byte: 14,
1912 start_position: Point { row: 0, column: 8 },
1913 old_end_position: Point { row: 0, column: 8 },
1914 new_end_position: Point { row: 0, column: 14 }
1915 }]
1916 );
1919 #[test]
1920 fn test_load_runtime_file() {
1921 // Test to make sure we can load some data from the runtime directory.
1922 let contents = load_runtime_file("rust", "indents.toml").unwrap();
1923 assert!(!contents.is_empty());
1925 let results = load_runtime_file("rust", "does-not-exist");
1926 assert!(results.is_err());