git.iko.soy/helix
on aef00a3 help sign in
helix/helix-core/src/syntax.rs
2173 lines 80 kB history raw
1use crate::{
2 auto_pairs::AutoPairs,
3 chars::char_is_line_ending,
4 diagnostic::Severity,
5 regex::Regex,
6 transaction::{ChangeSet, Operation},
7 Rope, RopeSlice, Tendril,
8};
10use arc_swap::{ArcSwap, Guard};
11use slotmap::{DefaultKey as LayerId, HopSlotMap};
13use std::{
14 borrow::Cow,
15 cell::RefCell,
16 collections::{HashMap, HashSet, VecDeque},
17 fmt,
18 path::Path,
19 str::FromStr,
20 sync::Arc,
21};
23use once_cell::sync::{Lazy, OnceCell};
24use serde::{Deserialize, Serialize};
26use helix_loader::grammar::{get_language, load_runtime_file};
28fn deserialize_regex<'de, D>(deserializer: D) -> Result<Option<Regex>, D::Error>
29where
30 D: serde::Deserializer<'de>,
32 Option::<String>::deserialize(deserializer)?
33 .map(|buf| Regex::new(&buf).map_err(serde::de::Error::custom))
34 .transpose()
37fn deserialize_lsp_config<'de, D>(deserializer: D) -> Result<Option<serde_json::Value>, D::Error>
38where
39 D: serde::Deserializer<'de>,
41 Option::<toml::Value>::deserialize(deserializer)?
42 .map(|toml| toml.try_into().map_err(serde::de::Error::custom))
43 .transpose()
46pub fn deserialize_auto_pairs<'de, D>(deserializer: D) -> Result<Option<AutoPairs>, D::Error>
47where
48 D: serde::Deserializer<'de>,
50 Ok(Option::<AutoPairConfig>::deserialize(deserializer)?.and_then(AutoPairConfig::into))
53fn default_timeout() -> u64 {
54 20
57#[derive(Debug, Serialize, Deserialize)]
58pub struct Configuration {
59 pub language: Vec<LanguageConfiguration>,
62// largely based on tree-sitter/cli/src/loader.rs
63#[derive(Debug, Serialize, Deserialize)]
64#[serde(rename_all = "kebab-case", deny_unknown_fields)]
65pub struct LanguageConfiguration {
66 #[serde(rename = "name")]
67 pub language_id: String, // c-sharp, rust
68 pub scope: String, // source.rust
69 pub file_types: Vec<String>, // filename ends_with? <Gemfile, rb, etc>
70 #[serde(default)]
71 pub shebangs: Vec<String>, // interpreter(s) associated with language
72 pub roots: Vec<String>, // these indicate project roots <.git, Cargo.toml>
73 pub comment_token: Option<String>,
74 pub max_line_length: Option<usize>,
76 #[serde(default, skip_serializing, deserialize_with = "deserialize_lsp_config")]
77 pub config: Option<serde_json::Value>,
79 #[serde(default)]
80 pub auto_format: bool,
82 #[serde(skip_serializing_if = "Option::is_none")]
83 pub formatter: Option<FormatterConfiguration>,
85 #[serde(default)]
86 pub diagnostic_severity: Severity,
88 pub grammar: Option<String>, // tree-sitter grammar name, defaults to language_id
90 // content_regex
91 #[serde(default, skip_serializing, deserialize_with = "deserialize_regex")]
92 pub injection_regex: Option<Regex>,
93 // first_line_regex
94 //
95 #[serde(skip)]
96 pub(crate) highlight_config: OnceCell<Option<Arc<HighlightConfiguration>>>,
97 // tags_config OnceCell<> https://github.com/tree-sitter/tree-sitter/pull/583
98 #[serde(skip_serializing_if = "Option::is_none")]
99 pub language_server: Option<LanguageServerConfiguration>,
100 #[serde(skip_serializing_if = "Option::is_none")]
101 pub indent: Option<IndentationConfiguration>,
103 #[serde(skip)]
104 pub(crate) indent_query: OnceCell<Option<Query>>,
105 #[serde(skip)]
106 pub(crate) textobject_query: OnceCell<Option<TextObjectQuery>>,
107 #[serde(skip_serializing_if = "Option::is_none")]
108 pub debugger: Option<DebugAdapterConfig>,
110 /// Automatic insertion of pairs to parentheses, brackets,
111 /// etc. Defaults to true. Optionally, this can be a list of 2-tuples
112 /// to specify a list of characters to pair. This overrides the
113 /// global setting.
114 #[serde(default, skip_serializing, deserialize_with = "deserialize_auto_pairs")]
115 pub auto_pairs: Option<AutoPairs>,
117 pub rulers: Option<Vec<u16>>, // if set, override editor's rulers
120#[derive(Debug, Serialize, Deserialize)]
121#[serde(rename_all = "kebab-case")]
122pub struct LanguageServerConfiguration {
123 pub command: String,
124 #[serde(default)]
125 #[serde(skip_serializing_if = "Vec::is_empty")]
126 pub args: Vec<String>,
127 #[serde(default = "default_timeout")]
128 pub timeout: u64,
129 pub language_id: Option<String>,
132#[derive(Debug, Clone, Serialize, Deserialize)]
133#[serde(rename_all = "kebab-case")]
134pub struct FormatterConfiguration {
135 pub command: String,
136 #[serde(default)]
137 #[serde(skip_serializing_if = "Vec::is_empty")]
138 pub args: Vec<String>,
141#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
142#[serde(rename_all = "kebab-case")]
143pub struct AdvancedCompletion {
144 pub name: Option<String>,
145 pub completion: Option<String>,
146 pub default: Option<String>,
149#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
150#[serde(rename_all = "kebab-case", untagged)]
151pub enum DebugConfigCompletion {
152 Named(String),
153 Advanced(AdvancedCompletion),
156#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
157#[serde(untagged)]
158pub enum DebugArgumentValue {
159 String(String),
160 Array(Vec<String>),
161 Boolean(bool),
164#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
165#[serde(rename_all = "kebab-case")]
166pub struct DebugTemplate {
167 pub name: String,
168 pub request: String,
169 pub completion: Vec<DebugConfigCompletion>,
170 pub args: HashMap<String, DebugArgumentValue>,
173#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
174#[serde(rename_all = "kebab-case")]
175pub struct DebugAdapterConfig {
176 pub name: String,
177 pub transport: String,
178 #[serde(default)]
179 pub command: String,
180 #[serde(default)]
181 pub args: Vec<String>,
182 pub port_arg: Option<String>,
183 pub templates: Vec<DebugTemplate>,
184 #[serde(default)]
185 pub quirks: DebuggerQuirks,
188// Different workarounds for adapters' differences
189#[derive(Debug, Default, PartialEq, Clone, Serialize, Deserialize)]
190pub struct DebuggerQuirks {
191 #[serde(default)]
192 pub absolute_paths: bool,
195#[derive(Debug, Serialize, Deserialize)]
196#[serde(rename_all = "kebab-case")]
197pub struct IndentationConfiguration {
198 pub tab_width: usize,
199 pub unit: String,
202/// Configuration for auto pairs
203#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
204#[serde(rename_all = "kebab-case", deny_unknown_fields, untagged)]
205pub enum AutoPairConfig {
206 /// Enables or disables auto pairing. False means disabled. True means to use the default pairs.
207 Enable(bool),
209 /// The mappings of pairs.
210 Pairs(HashMap<char, char>),
213impl Default for AutoPairConfig {
214 fn default() -> Self {
215 AutoPairConfig::Enable(true)
216 }
219impl From<&AutoPairConfig> for Option<AutoPairs> {
220 fn from(auto_pair_config: &AutoPairConfig) -> Self {
221 match auto_pair_config {
222 AutoPairConfig::Enable(false) => None,
223 AutoPairConfig::Enable(true) => Some(AutoPairs::default()),
224 AutoPairConfig::Pairs(pairs) => Some(AutoPairs::new(pairs.iter())),
225 }
226 }
229impl From<AutoPairConfig> for Option<AutoPairs> {
230 fn from(auto_pairs_config: AutoPairConfig) -> Self {
231 (&auto_pairs_config).into()
232 }
235impl FromStr for AutoPairConfig {
236 type Err = std::str::ParseBoolError;
238 // only do bool parsing for runtime setting
239 fn from_str(s: &str) -> Result<Self, Self::Err> {
240 let enable: bool = s.parse()?;
241 Ok(AutoPairConfig::Enable(enable))
242 }
245#[derive(Debug)]
246pub struct TextObjectQuery {
247 pub query: Query,
250#[derive(Debug)]
251pub enum CapturedNode<'a> {
252 Single(Node<'a>),
253 /// Guaranteed to be not empty
254 Grouped(Vec<Node<'a>>),
257impl<'a> CapturedNode<'a> {
258 pub fn start_byte(&self) -> usize {
259 match self {
260 Self::Single(n) => n.start_byte(),
261 Self::Grouped(ns) => ns[0].start_byte(),
262 }
263 }
265 pub fn end_byte(&self) -> usize {
266 match self {
267 Self::Single(n) => n.end_byte(),
268 Self::Grouped(ns) => ns.last().unwrap().end_byte(),
269 }
270 }
272 pub fn byte_range(&self) -> std::ops::Range<usize> {
273 self.start_byte()..self.end_byte()
274 }
277impl TextObjectQuery {
278 /// Run the query on the given node and return sub nodes which match given
279 /// capture ("function.inside", "class.around", etc).
280 ///
281 /// Captures may contain multiple nodes by using quantifiers (+, *, etc),
282 /// and support for this is partial and could use improvement.
283 ///
284 /// ```query
285 /// (comment)+ @capture
286 ///
287 /// ; OR
288 /// (
289 /// (comment)*
290 /// .
291 /// (function)
292 /// ) @capture
293 /// ```
294 pub fn capture_nodes<'a>(
295 &'a self,
296 capture_name: &str,
297 node: Node<'a>,
298 slice: RopeSlice<'a>,
299 cursor: &'a mut QueryCursor,
300 ) -> Option<impl Iterator<Item = CapturedNode<'a>>> {
301 self.capture_nodes_any(&[capture_name], node, slice, cursor)
302 }
304 /// Find the first capture that exists out of all given `capture_names`
305 /// and return sub nodes that match this capture.
306 pub fn capture_nodes_any<'a>(
307 &'a self,
308 capture_names: &[&str],
309 node: Node<'a>,
310 slice: RopeSlice<'a>,
311 cursor: &'a mut QueryCursor,
312 ) -> Option<impl Iterator<Item = CapturedNode<'a>>> {
313 let capture_idx = capture_names
314 .iter()
315 .find_map(|cap| self.query.capture_index_for_name(cap))?;
317 let nodes = cursor
318 .captures(&self.query, node, RopeProvider(slice))
319 .filter_map(move |(mat, _)| {
320 let nodes: Vec<_> = mat
321 .captures
322 .iter()
323 .filter_map(|cap| (cap.index == capture_idx).then(|| cap.node))
324 .collect();
326 if nodes.len() > 1 {
327 Some(CapturedNode::Grouped(nodes))
328 } else {
329 nodes.into_iter().map(CapturedNode::Single).next()
330 }
331 });
333 Some(nodes)
334 }
337fn read_query(language: &str, filename: &str) -> String {
338 static INHERITS_REGEX: Lazy<Regex> =
339 Lazy::new(|| Regex::new(r";+\s*inherits\s*:?\s*([a-z_,()-]+)\s*").unwrap());
341 let query = load_runtime_file(language, filename).unwrap_or_default();
343 // replaces all "; inherits <language>(,<language>)*" with the queries of the given language(s)
344 INHERITS_REGEX
345 .replace_all(&query, |captures: &regex::Captures| {
346 captures[1]
347 .split(',')
348 .map(|language| format!("\n{}\n", read_query(language, filename)))
349 .collect::<String>()
350 })
351 .to_string()
354impl LanguageConfiguration {
355 fn initialize_highlight(&self, scopes: &[String]) -> Option<Arc<HighlightConfiguration>> {
356 let language = self.language_id.to_ascii_lowercase();
358 let highlights_query = read_query(&language, "highlights.scm");
359 // always highlight syntax errors
360 // highlights_query += "\n(ERROR) @error";
362 let injections_query = read_query(&language, "injections.scm");
363 let locals_query = read_query(&language, "locals.scm");
365 if highlights_query.is_empty() {
366 None
367 } else {
368 let language = get_language(self.grammar.as_deref().unwrap_or(&self.language_id))
369 .map_err(|e| log::info!("{}", e))
370 .ok()?;
371 let config = HighlightConfiguration::new(
372 language,
373 &highlights_query,
374 &injections_query,
375 &locals_query,
376 )
377 .unwrap_or_else(|query_error| panic!("Could not parse queries for language {:?}. Are your grammars out of sync? Try running 'hx --grammar fetch' and 'hx --grammar build'. This query could not be parsed: {:?}", self.language_id, query_error));
379 config.configure(scopes);
380 Some(Arc::new(config))
381 }
382 }
384 pub fn reconfigure(&self, scopes: &[String]) {
385 if let Some(Some(config)) = self.highlight_config.get() {
386 config.configure(scopes);
387 }
388 }
390 pub fn highlight_config(&self, scopes: &[String]) -> Option<Arc<HighlightConfiguration>> {
391 self.highlight_config
392 .get_or_init(|| self.initialize_highlight(scopes))
393 .clone()
394 }
396 pub fn is_highlight_initialized(&self) -> bool {
397 self.highlight_config.get().is_some()
398 }
400 pub fn indent_query(&self) -> Option<&Query> {
401 self.indent_query
402 .get_or_init(|| {
403 let lang_name = self.language_id.to_ascii_lowercase();
404 let query_text = read_query(&lang_name, "indents.scm");
405 if query_text.is_empty() {
406 return None;
407 }
408 let lang = self.highlight_config.get()?.as_ref()?.language;
409 Query::new(lang, &query_text).ok()
410 })
411 .as_ref()
412 }
414 pub fn textobject_query(&self) -> Option<&TextObjectQuery> {
415 self.textobject_query
416 .get_or_init(|| -> Option<TextObjectQuery> {
417 let lang_name = self.language_id.to_ascii_lowercase();
418 let query_text = read_query(&lang_name, "textobjects.scm");
419 let lang = self.highlight_config.get()?.as_ref()?.language;
420 let query = Query::new(lang, &query_text)
421 .map_err(|e| log::error!("Failed to parse textobjects.scm queries: {}", e))
422 .ok()?;
423 Some(TextObjectQuery { query })
424 })
425 .as_ref()
426 }
428 pub fn scope(&self) -> &str {
429 &self.scope
430 }
433// Expose loader as Lazy<> global since it's always static?
435#[derive(Debug)]
436pub struct Loader {
437 // highlight_names ?
438 language_configs: Vec<Arc<LanguageConfiguration>>,
439 language_config_ids_by_file_type: HashMap<String, usize>, // Vec<usize>
440 language_config_ids_by_shebang: HashMap<String, usize>,
442 scopes: ArcSwap<Vec<String>>,
445impl Loader {
446 pub fn new(config: Configuration) -> Self {
447 let mut loader = Self {
448 language_configs: Vec::new(),
449 language_config_ids_by_file_type: HashMap::new(),
450 language_config_ids_by_shebang: HashMap::new(),
451 scopes: ArcSwap::from_pointee(Vec::new()),
452 };
454 for config in config.language {
455 // get the next id
456 let language_id = loader.language_configs.len();
458 for file_type in &config.file_types {
459 // entry().or_insert(Vec::new).push(language_id);
460 loader
461 .language_config_ids_by_file_type
462 .insert(file_type.clone(), language_id);
463 }
464 for shebang in &config.shebangs {
465 loader
466 .language_config_ids_by_shebang
467 .insert(shebang.clone(), language_id);
468 }
470 loader.language_configs.push(Arc::new(config));
471 }
473 loader
474 }
476 pub fn language_config_for_file_name(&self, path: &Path) -> Option<Arc<LanguageConfiguration>> {
477 // Find all the language configurations that match this file name
478 // or a suffix of the file name.
479 let configuration_id = path
480 .file_name()
481 .and_then(|n| n.to_str())
482 .and_then(|file_name| self.language_config_ids_by_file_type.get(file_name))
483 .or_else(|| {
484 path.extension()
485 .and_then(|extension| extension.to_str())
486 .and_then(|extension| self.language_config_ids_by_file_type.get(extension))
487 });
489 configuration_id.and_then(|&id| self.language_configs.get(id).cloned())
491 // TODO: content_regex handling conflict resolution
492 }
494 pub fn language_config_for_shebang(&self, source: &Rope) -> Option<Arc<LanguageConfiguration>> {
495 let line = Cow::from(source.line(0));
496 static SHEBANG_REGEX: Lazy<Regex> = Lazy::new(|| {
497 Regex::new(r"^#!\s*(?:\S*[/\\](?:env\s+(?:\-\S+\s+)*)?)?([^\s\.\d]+)").unwrap()
498 });
499 let configuration_id = SHEBANG_REGEX
500 .captures(&line)
501 .and_then(|cap| self.language_config_ids_by_shebang.get(&cap[1]));
503 configuration_id.and_then(|&id| self.language_configs.get(id).cloned())
504 }
506 pub fn language_config_for_scope(&self, scope: &str) -> Option<Arc<LanguageConfiguration>> {
507 self.language_configs
508 .iter()
509 .find(|config| config.scope == scope)
510 .cloned()
511 }
513 pub fn language_config_for_language_id(&self, id: &str) -> Option<Arc<LanguageConfiguration>> {
514 self.language_configs
515 .iter()
516 .find(|config| config.language_id == id)
517 .cloned()
518 }
520 pub fn language_configuration_for_injection_string(
521 &self,
522 string: &str,
523 ) -> Option<Arc<LanguageConfiguration>> {
524 let mut best_match_length = 0;
525 let mut best_match_position = None;
526 for (i, configuration) in self.language_configs.iter().enumerate() {
527 if let Some(injection_regex) = &configuration.injection_regex {
528 if let Some(mat) = injection_regex.find(string) {
529 let length = mat.end() - mat.start();
530 if length > best_match_length {
531 best_match_position = Some(i);
532 best_match_length = length;
533 }
534 }
535 }
536 }
538 if let Some(i) = best_match_position {
539 let configuration = &self.language_configs[i];
540 return Some(configuration.clone());
541 }
542 None
543 }
545 pub fn language_configs(&self) -> impl Iterator<Item = &Arc<LanguageConfiguration>> {
546 self.language_configs.iter()
547 }
549 pub fn set_scopes(&self, scopes: Vec<String>) {
550 self.scopes.store(Arc::new(scopes));
552 // Reconfigure existing grammars
553 for config in self
554 .language_configs
555 .iter()
556 .filter(|cfg| cfg.is_highlight_initialized())
557 {
558 config.reconfigure(&self.scopes());
559 }
560 }
562 pub fn scopes(&self) -> Guard<Arc<Vec<String>>> {
563 self.scopes.load()
564 }
567pub struct TsParser {
568 parser: tree_sitter::Parser,
569 pub cursors: Vec<QueryCursor>,
572// could also just use a pool, or a single instance?
573thread_local! {
574 pub static PARSER: RefCell<TsParser> = RefCell::new(TsParser {
575 parser: Parser::new(),
576 cursors: Vec::new(),
577 })
580#[derive(Debug)]
581pub struct Syntax {
582 layers: HopSlotMap<LayerId, LanguageLayer>,
583 root: LayerId,
584 loader: Arc<Loader>,
587fn byte_range_to_str(range: std::ops::Range<usize>, source: RopeSlice) -> Cow<str> {
588 Cow::from(source.byte_slice(range))
591impl Syntax {
592 pub fn new(source: &Rope, config: Arc<HighlightConfiguration>, loader: Arc<Loader>) -> Self {
593 let root_layer = LanguageLayer {
594 tree: None,
595 config,
596 depth: 0,
597 ranges: vec![Range {
598 start_byte: 0,
599 end_byte: usize::MAX,
600 start_point: Point::new(0, 0),
601 end_point: Point::new(usize::MAX, usize::MAX),
602 }],
603 };
605 // track scope_descriptor: a Vec of scopes for item in tree
607 let mut layers = HopSlotMap::default();
608 let root = layers.insert(root_layer);
610 let mut syntax = Self {
611 root,
612 layers,
613 loader,
614 };
616 syntax
617 .update(source, source, &ChangeSet::new(source))
618 .unwrap();
620 syntax
621 }
623 pub fn update(
624 &mut self,
625 old_source: &Rope,
626 source: &Rope,
627 changeset: &ChangeSet,
628 ) -> Result<(), Error> {
629 let mut queue = VecDeque::new();
630 queue.push_back(self.root);
632 let scopes = self.loader.scopes.load();
633 let injection_callback = |language: &str| {
634 self.loader
635 .language_configuration_for_injection_string(language)
636 .and_then(|language_config| language_config.highlight_config(&scopes))
637 };
639 // Convert the changeset into tree sitter edits.
640 let edits = generate_edits(old_source, changeset);
642 // Use the edits to update all layers markers
643 if !edits.is_empty() {
644 fn point_add(a: Point, b: Point) -> Point {
645 if b.row > 0 {
646 Point::new(a.row.saturating_add(b.row), b.column)
647 } else {
648 Point::new(0, a.column.saturating_add(b.column))
649 }
650 }
651 fn point_sub(a: Point, b: Point) -> Point {
652 if a.row > b.row {
653 Point::new(a.row.saturating_sub(b.row), a.column)
654 } else {
655 Point::new(0, a.column.saturating_sub(b.column))
656 }
657 }
659 for layer in &mut self.layers.values_mut() {
660 // The root layer always covers the whole range (0..usize::MAX)
661 if layer.depth == 0 {
662 continue;
663 }
665 for range in &mut layer.ranges {
666 // Roughly based on https://github.com/tree-sitter/tree-sitter/blob/ddeaa0c7f534268b35b4f6cb39b52df082754413/lib/src/subtree.c#L691-L720
667 for edit in edits.iter().rev() {
668 let is_pure_insertion = edit.old_end_byte == edit.start_byte;
670 // if edit is after range, skip
671 if edit.start_byte > range.end_byte {
672 // TODO: || (is_noop && edit.start_byte == range.end_byte)
673 continue;
674 }
676 // if edit is before range, shift entire range by len
677 if edit.old_end_byte < range.start_byte {
678 range.start_byte =
679 edit.new_end_byte + (range.start_byte - edit.old_end_byte);
680 range.start_point = point_add(
681 edit.new_end_position,
682 point_sub(range.start_point, edit.old_end_position),
683 );
685 range.end_byte = edit
686 .new_end_byte
687 .saturating_add(range.end_byte - edit.old_end_byte);
688 range.end_point = point_add(
689 edit.new_end_position,
690 point_sub(range.end_point, edit.old_end_position),
691 );
692 }
693 // if the edit starts in the space before and extends into the range
694 else if edit.start_byte < range.start_byte {
695 range.start_byte = edit.new_end_byte;
696 range.start_point = edit.new_end_position;
698 range.end_byte = range
699 .end_byte
700 .saturating_sub(edit.old_end_byte)
701 .saturating_add(edit.new_end_byte);
702 range.end_point = point_add(
703 edit.new_end_position,
704 point_sub(range.end_point, edit.old_end_position),
705 );
706 }
707 // If the edit is an insertion at the start of the tree, shift
708 else if edit.start_byte == range.start_byte && is_pure_insertion {
709 range.start_byte = edit.new_end_byte;
710 range.start_point = edit.new_end_position;
711 } else {
712 range.end_byte = range
713 .end_byte
714 .saturating_sub(edit.old_end_byte)
715 .saturating_add(edit.new_end_byte);
716 range.end_point = point_add(
717 edit.new_end_position,
718 point_sub(range.end_point, edit.old_end_position),
719 );
720 }
721 }
722 }
723 }
724 }
726 PARSER.with(|ts_parser| {
727 let ts_parser = &mut ts_parser.borrow_mut();
728 let mut cursor = ts_parser.cursors.pop().unwrap_or_else(QueryCursor::new);
729 // TODO: might need to set cursor range
730 cursor.set_byte_range(0..usize::MAX);
732 let source_slice = source.slice(..);
734 let mut touched = HashSet::new();
736 // TODO: we should be able to avoid editing & parsing layers with ranges earlier in the document before the edit
738 while let Some(layer_id) = queue.pop_front() {
739 // Mark the layer as touched
740 touched.insert(layer_id);
742 let layer = &mut self.layers[layer_id];
744 // If a tree already exists, notify it of changes.
745 if let Some(tree) = &mut layer.tree {
746 for edit in edits.iter().rev() {
747 // Apply the edits in reverse.
748 // If we applied them in order then edit 1 would disrupt the positioning of edit 2.
749 tree.edit(edit);
750 }
751 }
753 // Re-parse the tree.
754 layer.parse(&mut ts_parser.parser, source)?;
756 // Switch to an immutable borrow.
757 let layer = &self.layers[layer_id];
759 // Process injections.
760 let matches = cursor.matches(
761 &layer.config.injections_query,
762 layer.tree().root_node(),
763 RopeProvider(source_slice),
764 );
765 let mut injections = Vec::new();
766 for mat in matches {
767 let (language_name, content_node, included_children) = injection_for_match(
768 &layer.config,
769 &layer.config.injections_query,
770 &mat,
771 source_slice,
772 );
774 // Explicitly remove this match so that none of its other captures will remain
775 // in the stream of captures.
776 mat.remove();
778 // If a language is found with the given name, then add a new language layer
779 // to the highlighted document.
780 if let (Some(language_name), Some(content_node)) = (language_name, content_node)
781 {
782 if let Some(config) = (injection_callback)(&language_name) {
783 let ranges =
784 intersect_ranges(&layer.ranges, &[content_node], included_children);
786 if !ranges.is_empty() {
787 injections.push((config, ranges));
788 }
789 }
790 }
791 }
793 // Process combined injections.
794 if let Some(combined_injections_query) = &layer.config.combined_injections_query {
795 let mut injections_by_pattern_index =
796 vec![
797 (None, Vec::new(), IncludedChildren::default());
798 combined_injections_query.pattern_count()
799 ];
800 let matches = cursor.matches(
801 combined_injections_query,
802 layer.tree().root_node(),
803 RopeProvider(source_slice),
804 );
805 for mat in matches {
806 let entry = &mut injections_by_pattern_index[mat.pattern_index];
807 let (language_name, content_node, included_children) = injection_for_match(
808 &layer.config,
809 combined_injections_query,
810 &mat,
811 source_slice,
812 );
813 if language_name.is_some() {
814 entry.0 = language_name;
815 }
816 if let Some(content_node) = content_node {
817 entry.1.push(content_node);
818 }
819 entry.2 = included_children;
820 }
821 for (lang_name, content_nodes, included_children) in injections_by_pattern_index
822 {
823 if let (Some(lang_name), false) = (lang_name, content_nodes.is_empty()) {
824 if let Some(config) = (injection_callback)(&lang_name) {
825 let ranges = intersect_ranges(
826 &layer.ranges,
827 &content_nodes,
828 included_children,
829 );
830 if !ranges.is_empty() {
831 injections.push((config, ranges));
832 }
833 }
834 }
835 }
836 }
838 let depth = layer.depth + 1;
839 // TODO: can't inline this since matches borrows self.layers
840 for (config, ranges) in injections {
841 // Find an existing layer
842 let layer = self
843 .layers
844 .iter_mut()
845 .find(|(_, layer)| {
846 layer.depth == depth && // TODO: track parent id instead
847 layer.config.language == config.language && layer.ranges == ranges
848 })
849 .map(|(id, _layer)| id);
851 // ...or insert a new one.
852 let layer_id = layer.unwrap_or_else(|| {
853 self.layers.insert(LanguageLayer {
854 tree: None,
855 config,
856 depth,
857 ranges,
858 })
859 });
861 queue.push_back(layer_id);
862 }
864 // TODO: pre-process local scopes at this time, rather than highlight?
865 // would solve problems with locals not working across boundaries
866 }
868 // Return the cursor back in the pool.
869 ts_parser.cursors.push(cursor);
871 // Remove all untouched layers
872 self.layers.retain(|id, _| touched.contains(&id));
874 Ok(())
875 })
876 }
878 pub fn tree(&self) -> &Tree {
879 self.layers[self.root].tree()
880 }
882 /// Iterate over the highlighted regions for a given slice of source code.
883 pub fn highlight_iter<'a>(
884 &'a self,
885 source: RopeSlice<'a>,
886 range: Option<std::ops::Range<usize>>,
887 cancellation_flag: Option<&'a AtomicUsize>,
888 ) -> impl Iterator<Item = Result<HighlightEvent, Error>> + 'a {
889 let mut layers = self
890 .layers
891 .iter()
892 .filter_map(|(_, layer)| {
893 // TODO: if range doesn't overlap layer range, skip it
895 // Reuse a cursor from the pool if available.
896 let mut cursor = PARSER.with(|ts_parser| {
897 let highlighter = &mut ts_parser.borrow_mut();
898 highlighter.cursors.pop().unwrap_or_else(QueryCursor::new)
899 });
901 // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which
902 // prevents them from being moved. But both of these values are really just
903 // pointers, so it's actually ok to move them.
904 let cursor_ref =
905 unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) };
907 // if reusing cursors & no range this resets to whole range
908 cursor_ref.set_byte_range(range.clone().unwrap_or(0..usize::MAX));
910 let mut captures = cursor_ref
911 .captures(
912 &layer.config.query,
913 layer.tree().root_node(),
914 RopeProvider(source),
915 )
916 .peekable();
918 // If there's no captures, skip the layer
919 captures.peek()?;
921 Some(HighlightIterLayer {
922 highlight_end_stack: Vec::new(),
923 scope_stack: vec![LocalScope {
924 inherits: false,
925 range: 0..usize::MAX,
926 local_defs: Vec::new(),
927 }],
928 cursor,
929 _tree: None,
930 captures,
931 config: layer.config.as_ref(), // TODO: just reuse `layer`
932 depth: layer.depth, // TODO: just reuse `layer`
933 ranges: &layer.ranges, // TODO: temp
934 })
935 })
936 .collect::<Vec<_>>();
938 // HAXX: arrange layers by byte range, with deeper layers positioned first
939 layers.sort_by_key(|layer| {
940 (
941 layer.ranges.first().cloned(),
942 std::cmp::Reverse(layer.depth),
943 )
944 });
946 let mut result = HighlightIter {
947 source,
948 byte_offset: range.map_or(0, |r| r.start),
949 cancellation_flag,
950 iter_count: 0,
951 layers,
952 next_event: None,
953 last_highlight_range: None,
954 };
955 result.sort_layers();
956 result
957 }
959 // Commenting
960 // comment_strings_for_pos
961 // is_commented
963 // Indentation
964 // suggested_indent_for_line_at_buffer_row
965 // suggested_indent_for_buffer_row
966 // indent_level_for_line
968 // TODO: Folding
971#[derive(Debug)]
972pub struct LanguageLayer {
973 // mode
974 // grammar
975 pub config: Arc<HighlightConfiguration>,
976 pub(crate) tree: Option<Tree>,
977 pub ranges: Vec<Range>,
978 pub depth: usize,
981impl LanguageLayer {
982 pub fn tree(&self) -> &Tree {
983 // TODO: no unwrap
984 self.tree.as_ref().unwrap()
985 }
987 fn parse(&mut self, parser: &mut Parser, source: &Rope) -> Result<(), Error> {
988 parser.set_included_ranges(&self.ranges).unwrap();
990 parser
991 .set_language(self.config.language)
992 .map_err(|_| Error::InvalidLanguage)?;
994 // unsafe { syntax.parser.set_cancellation_flag(cancellation_flag) };
995 let tree = parser
996 .parse_with(
997 &mut |byte, _| {
998 if byte <= source.len_bytes() {
999 let (chunk, start_byte, _, _) = source.chunk_at_byte(byte);
1000 chunk[byte - start_byte..].as_bytes()
1001 } else {
1002 // out of range
1003 &[]
1005 },
1006 self.tree.as_ref(),
1008 .ok_or(Error::Cancelled)?;
1009 // unsafe { ts_parser.parser.set_cancellation_flag(None) };
1010 self.tree = Some(tree);
1011 Ok(())
1015pub(crate) fn generate_edits(
1016 old_text: &Rope,
1017 changeset: &ChangeSet,
1018) -> Vec<tree_sitter::InputEdit> {
1019 use Operation::*;
1020 let mut old_pos = 0;
1022 let mut edits = Vec::new();
1024 if changeset.changes.is_empty() {
1025 return edits;
1028 let mut iter = changeset.changes.iter().peekable();
1030 // TODO; this is a lot easier with Change instead of Operation.
1032 fn point_at_pos(text: &Rope, pos: usize) -> (usize, Point) {
1033 let byte = text.char_to_byte(pos); // <- attempted to index past end
1034 let line = text.char_to_line(pos);
1035 let line_start_byte = text.line_to_byte(line);
1036 let col = byte - line_start_byte;
1038 (byte, Point::new(line, col))
1041 fn traverse(point: Point, text: &Tendril) -> Point {
1042 let Point {
1043 mut row,
1044 mut column,
1045 } = point;
1047 // TODO: there should be a better way here.
1048 let mut chars = text.chars().peekable();
1049 while let Some(ch) = chars.next() {
1050 if char_is_line_ending(ch) && !(ch == '\r' && chars.peek() == Some(&'\n')) {
1051 row += 1;
1052 column = 0;
1053 } else {
1054 column += 1;
1057 Point { row, column }
1060 while let Some(change) = iter.next() {
1061 let len = match change {
1062 Delete(i) | Retain(i) => *i,
1063 Insert(_) => 0,
1064 };
1065 let mut old_end = old_pos + len;
1067 match change {
1068 Retain(_) => {}
1069 Delete(_) => {
1070 let (start_byte, start_position) = point_at_pos(old_text, old_pos);
1071 let (old_end_byte, old_end_position) = point_at_pos(old_text, old_end);
1073 // deletion
1074 edits.push(tree_sitter::InputEdit {
1075 start_byte, // old_pos to byte
1076 old_end_byte, // old_end to byte
1077 new_end_byte: start_byte, // old_pos to byte
1078 start_position, // old pos to coords
1079 old_end_position, // old_end to coords
1080 new_end_position: start_position, // old pos to coords
1081 });
1083 Insert(s) => {
1084 let (start_byte, start_position) = point_at_pos(old_text, old_pos);
1086 // a subsequent delete means a replace, consume it
1087 if let Some(Delete(len)) = iter.peek() {
1088 old_end = old_pos + len;
1089 let (old_end_byte, old_end_position) = point_at_pos(old_text, old_end);
1091 iter.next();
1093 // replacement
1094 edits.push(tree_sitter::InputEdit {
1095 start_byte, // old_pos to byte
1096 old_end_byte, // old_end to byte
1097 new_end_byte: start_byte + s.len(), // old_pos to byte + s.len()
1098 start_position, // old pos to coords
1099 old_end_position, // old_end to coords
1100 new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)
1101 });
1102 } else {
1103 // insert
1104 edits.push(tree_sitter::InputEdit {
1105 start_byte, // old_pos to byte
1106 old_end_byte: start_byte, // same
1107 new_end_byte: start_byte + s.len(), // old_pos + s.len()
1108 start_position, // old pos to coords
1109 old_end_position: start_position, // same
1110 new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)
1111 });
1115 old_pos = old_end;
1117 edits
1120use std::sync::atomic::{AtomicUsize, Ordering};
1121use std::{iter, mem, ops, str, usize};
1122use tree_sitter::{
1123 Language as Grammar, Node, Parser, Point, Query, QueryCaptures, QueryCursor, QueryError,
1124 QueryMatch, Range, TextProvider, Tree,
1127const CANCELLATION_CHECK_INTERVAL: usize = 100;
1129/// Indicates which highlight should be applied to a region of source code.
1130#[derive(Copy, Clone, Debug, PartialEq, Eq)]
1131pub struct Highlight(pub usize);
1133/// Represents the reason why syntax highlighting failed.
1134#[derive(Debug, PartialEq, Eq)]
1135pub enum Error {
1136 Cancelled,
1137 InvalidLanguage,
1138 Unknown,
1141/// Represents a single step in rendering a syntax-highlighted document.
1142#[derive(Copy, Clone, Debug)]
1143pub enum HighlightEvent {
1144 Source { start: usize, end: usize },
1145 HighlightStart(Highlight),
1146 HighlightEnd,
1149/// Contains the data needed to highlight code written in a particular language.
1150///
1151/// This struct is immutable and can be shared between threads.
1152#[derive(Debug)]
1153pub struct HighlightConfiguration {
1154 pub language: Grammar,
1155 pub query: Query,
1156 injections_query: Query,
1157 combined_injections_query: Option<Query>,
1158 highlights_pattern_index: usize,
1159 highlight_indices: ArcSwap<Vec<Option<Highlight>>>,
1160 non_local_variable_patterns: Vec<bool>,
1161 injection_content_capture_index: Option<u32>,
1162 injection_language_capture_index: Option<u32>,
1163 local_scope_capture_index: Option<u32>,
1164 local_def_capture_index: Option<u32>,
1165 local_def_value_capture_index: Option<u32>,
1166 local_ref_capture_index: Option<u32>,
1169#[derive(Debug)]
1170struct LocalDef<'a> {
1171 name: Cow<'a, str>,
1172 value_range: ops::Range<usize>,
1173 highlight: Option<Highlight>,
1176#[derive(Debug)]
1177struct LocalScope<'a> {
1178 inherits: bool,
1179 range: ops::Range<usize>,
1180 local_defs: Vec<LocalDef<'a>>,
1183#[derive(Debug)]
1184struct HighlightIter<'a> {
1185 source: RopeSlice<'a>,
1186 byte_offset: usize,
1187 cancellation_flag: Option<&'a AtomicUsize>,
1188 layers: Vec<HighlightIterLayer<'a>>,
1189 iter_count: usize,
1190 next_event: Option<HighlightEvent>,
1191 last_highlight_range: Option<(usize, usize, usize)>,
1194// Adapter to convert rope chunks to bytes
1195pub struct ChunksBytes<'a> {
1196 chunks: ropey::iter::Chunks<'a>,
1198impl<'a> Iterator for ChunksBytes<'a> {
1199 type Item = &'a [u8];
1200 fn next(&mut self) -> Option<Self::Item> {
1201 self.chunks.next().map(str::as_bytes)
1205pub struct RopeProvider<'a>(pub RopeSlice<'a>);
1206impl<'a> TextProvider<'a> for RopeProvider<'a> {
1207 type I = ChunksBytes<'a>;
1209 fn text(&mut self, node: Node) -> Self::I {
1210 let fragment = self.0.byte_slice(node.start_byte()..node.end_byte());
1211 ChunksBytes {
1212 chunks: fragment.chunks(),
1217struct HighlightIterLayer<'a> {
1218 _tree: Option<Tree>,
1219 cursor: QueryCursor,
1220 captures: iter::Peekable<QueryCaptures<'a, 'a, RopeProvider<'a>>>,
1221 config: &'a HighlightConfiguration,
1222 highlight_end_stack: Vec<usize>,
1223 scope_stack: Vec<LocalScope<'a>>,
1224 depth: usize,
1225 ranges: &'a [Range],
1228impl<'a> fmt::Debug for HighlightIterLayer<'a> {
1229 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1230 f.debug_struct("HighlightIterLayer").finish()
1234impl HighlightConfiguration {
1235 /// Creates a `HighlightConfiguration` for a given `Grammar` and set of highlighting
1236 /// queries.
1237 ///
1238 /// # Parameters
1239 ///
1240 /// * `language` - The Tree-sitter `Grammar` that should be used for parsing.
1241 /// * `highlights_query` - A string containing tree patterns for syntax highlighting. This
1242 /// should be non-empty, otherwise no syntax highlights will be added.
1243 /// * `injections_query` - A string containing tree patterns for injecting other languages
1244 /// into the document. This can be empty if no injections are desired.
1245 /// * `locals_query` - A string containing tree patterns for tracking local variable
1246 /// definitions and references. This can be empty if local variable tracking is not needed.
1247 ///
1248 /// Returns a `HighlightConfiguration` that can then be used with the `highlight` method.
1249 pub fn new(
1250 language: Grammar,
1251 highlights_query: &str,
1252 injection_query: &str,
1253 locals_query: &str,
1254 ) -> Result<Self, QueryError> {
1255 // Concatenate the query strings, keeping track of the start offset of each section.
1256 let mut query_source = String::new();
1257 query_source.push_str(locals_query);
1258 let highlights_query_offset = query_source.len();
1259 query_source.push_str(highlights_query);
1261 // Construct a single query by concatenating the three query strings, but record the
1262 // range of pattern indices that belong to each individual string.
1263 let query = Query::new(language, &query_source)?;
1264 let mut highlights_pattern_index = 0;
1265 for i in 0..(query.pattern_count()) {
1266 let pattern_offset = query.start_byte_for_pattern(i);
1267 if pattern_offset < highlights_query_offset {
1268 highlights_pattern_index += 1;
1272 let mut injections_query = Query::new(language, injection_query)?;
1274 // Construct a separate query just for dealing with the 'combined injections'.
1275 // Disable the combined injection patterns in the main query.
1276 let mut combined_injections_query = Query::new(language, injection_query)?;
1277 let mut has_combined_queries = false;
1278 for pattern_index in 0..injections_query.pattern_count() {
1279 let settings = injections_query.property_settings(pattern_index);
1280 if settings.iter().any(|s| &*s.key == "injection.combined") {
1281 has_combined_queries = true;
1282 injections_query.disable_pattern(pattern_index);
1283 } else {
1284 combined_injections_query.disable_pattern(pattern_index);
1287 let combined_injections_query = if has_combined_queries {
1288 Some(combined_injections_query)
1289 } else {
1290 None
1291 };
1293 // Find all of the highlighting patterns that are disabled for nodes that
1294 // have been identified as local variables.
1295 let non_local_variable_patterns = (0..query.pattern_count())
1296 .map(|i| {
1297 query
1298 .property_predicates(i)
1299 .iter()
1300 .any(|(prop, positive)| !*positive && prop.key.as_ref() == "local")
1301 })
1302 .collect();
1304 // Store the numeric ids for all of the special captures.
1305 let mut injection_content_capture_index = None;
1306 let mut injection_language_capture_index = None;
1307 let mut local_def_capture_index = None;
1308 let mut local_def_value_capture_index = None;
1309 let mut local_ref_capture_index = None;
1310 let mut local_scope_capture_index = None;
1311 for (i, name) in query.capture_names().iter().enumerate() {
1312 let i = Some(i as u32);
1313 match name.as_str() {
1314 "local.definition" => local_def_capture_index = i,
1315 "local.definition-value" => local_def_value_capture_index = i,
1316 "local.reference" => local_ref_capture_index = i,
1317 "local.scope" => local_scope_capture_index = i,
1318 _ => {}
1322 for (i, name) in injections_query.capture_names().iter().enumerate() {
1323 let i = Some(i as u32);
1324 match name.as_str() {
1325 "injection.content" => injection_content_capture_index = i,
1326 "injection.language" => injection_language_capture_index = i,
1327 _ => {}
1331 let highlight_indices = ArcSwap::from_pointee(vec![None; query.capture_names().len()]);
1332 Ok(Self {
1333 language,
1334 query,
1335 injections_query,
1336 combined_injections_query,
1337 highlights_pattern_index,
1338 highlight_indices,
1339 non_local_variable_patterns,
1340 injection_content_capture_index,
1341 injection_language_capture_index,
1342 local_scope_capture_index,
1343 local_def_capture_index,
1344 local_def_value_capture_index,
1345 local_ref_capture_index,
1346 })
1349 /// Get a slice containing all of the highlight names used in the configuration.
1350 pub fn names(&self) -> &[String] {
1351 self.query.capture_names()
1354 /// Set the list of recognized highlight names.
1355 ///
1356 /// Tree-sitter syntax-highlighting queries specify highlights in the form of dot-separated
1357 /// highlight names like `punctuation.bracket` and `function.method.builtin`. Consumers of
1358 /// these queries can choose to recognize highlights with different levels of specificity.
1359 /// For example, the string `function.builtin` will match against `function.builtin.constructor`
1360 /// but will not match `function.method.builtin` and `function.method`.
1361 ///
1362 /// When highlighting, results are returned as `Highlight` values, which contain the index
1363 /// of the matched highlight this list of highlight names.
1364 pub fn configure(&self, recognized_names: &[String]) {
1365 let mut capture_parts = Vec::new();
1366 let indices: Vec<_> = self
1367 .query
1368 .capture_names()
1369 .iter()
1370 .map(move |capture_name| {
1371 capture_parts.clear();
1372 capture_parts.extend(capture_name.split('.'));
1374 let mut best_index = None;
1375 let mut best_match_len = 0;
1376 for (i, recognized_name) in recognized_names.iter().enumerate() {
1377 let recognized_name = recognized_name;
1378 let mut len = 0;
1379 let mut matches = true;
1380 for (i, part) in recognized_name.split('.').enumerate() {
1381 match capture_parts.get(i) {
1382 Some(capture_part) if *capture_part == part => len += 1,
1383 _ => {
1384 matches = false;
1385 break;
1389 if matches && len > best_match_len {
1390 best_index = Some(i);
1391 best_match_len = len;
1394 best_index.map(Highlight)
1395 })
1396 .collect();
1398 self.highlight_indices.store(Arc::new(indices));
1402impl<'a> HighlightIterLayer<'a> {
1403 // First, sort scope boundaries by their byte offset in the document. At a
1404 // given position, emit scope endings before scope beginnings. Finally, emit
1405 // scope boundaries from deeper layers first.
1406 fn sort_key(&mut self) -> Option<(usize, bool, isize)> {
1407 let depth = -(self.depth as isize);
1408 let next_start = self
1409 .captures
1410 .peek()
1411 .map(|(m, i)| m.captures[*i].node.start_byte());
1412 let next_end = self.highlight_end_stack.last().cloned();
1413 match (next_start, next_end) {
1414 (Some(start), Some(end)) => {
1415 if start < end {
1416 Some((start, true, depth))
1417 } else {
1418 Some((end, false, depth))
1421 (Some(i), None) => Some((i, true, depth)),
1422 (None, Some(j)) => Some((j, false, depth)),
1423 _ => None,
1428#[derive(Clone)]
1429enum IncludedChildren {
1430 None,
1431 All,
1432 Unnamed,
1435impl Default for IncludedChildren {
1436 fn default() -> Self {
1437 Self::None
1441// Compute the ranges that should be included when parsing an injection.
1442// This takes into account three things:
1443// * `parent_ranges` - The ranges must all fall within the *current* layer's ranges.
1444// * `nodes` - Every injection takes place within a set of nodes. The injection ranges
1445// are the ranges of those nodes.
1446// * `includes_children` - For some injections, the content nodes' children should be
1447// excluded from the nested document, so that only the content nodes' *own* content
1448// is reparsed. For other injections, the content nodes' entire ranges should be
1449// reparsed, including the ranges of their children.
1450fn intersect_ranges(
1451 parent_ranges: &[Range],
1452 nodes: &[Node],
1453 included_children: IncludedChildren,
1454) -> Vec<Range> {
1455 let mut cursor = nodes[0].walk();
1456 let mut result = Vec::new();
1457 let mut parent_range_iter = parent_ranges.iter();
1458 let mut parent_range = parent_range_iter
1459 .next()
1460 .expect("Layers should only be constructed with non-empty ranges vectors");
1461 for node in nodes.iter() {
1462 let mut preceding_range = Range {
1463 start_byte: 0,
1464 start_point: Point::new(0, 0),
1465 end_byte: node.start_byte(),
1466 end_point: node.start_position(),
1467 };
1468 let following_range = Range {
1469 start_byte: node.end_byte(),
1470 start_point: node.end_position(),
1471 end_byte: usize::MAX,
1472 end_point: Point::new(usize::MAX, usize::MAX),
1473 };
1475 for excluded_range in node
1476 .children(&mut cursor)
1477 .filter_map(|child| match included_children {
1478 IncludedChildren::None => Some(child.range()),
1479 IncludedChildren::All => None,
1480 IncludedChildren::Unnamed => {
1481 if child.is_named() {
1482 Some(child.range())
1483 } else {
1484 None
1487 })
1488 .chain([following_range].iter().cloned())
1490 let mut range = Range {
1491 start_byte: preceding_range.end_byte,
1492 start_point: preceding_range.end_point,
1493 end_byte: excluded_range.start_byte,
1494 end_point: excluded_range.start_point,
1495 };
1496 preceding_range = excluded_range;
1498 if range.end_byte < parent_range.start_byte {
1499 continue;
1502 while parent_range.start_byte <= range.end_byte {
1503 if parent_range.end_byte > range.start_byte {
1504 if range.start_byte < parent_range.start_byte {
1505 range.start_byte = parent_range.start_byte;
1506 range.start_point = parent_range.start_point;
1509 if parent_range.end_byte < range.end_byte {
1510 if range.start_byte < parent_range.end_byte {
1511 result.push(Range {
1512 start_byte: range.start_byte,
1513 start_point: range.start_point,
1514 end_byte: parent_range.end_byte,
1515 end_point: parent_range.end_point,
1516 });
1518 range.start_byte = parent_range.end_byte;
1519 range.start_point = parent_range.end_point;
1520 } else {
1521 if range.start_byte < range.end_byte {
1522 result.push(range);
1524 break;
1528 if let Some(next_range) = parent_range_iter.next() {
1529 parent_range = next_range;
1530 } else {
1531 return result;
1536 result
1539impl<'a> HighlightIter<'a> {
1540 fn emit_event(
1541 &mut self,
1542 offset: usize,
1543 event: Option<HighlightEvent>,
1544 ) -> Option<Result<HighlightEvent, Error>> {
1545 let result;
1546 if self.byte_offset < offset {
1547 result = Some(Ok(HighlightEvent::Source {
1548 start: self.byte_offset,
1549 end: offset,
1550 }));
1551 self.byte_offset = offset;
1552 self.next_event = event;
1553 } else {
1554 result = event.map(Ok);
1556 self.sort_layers();
1557 result
1560 fn sort_layers(&mut self) {
1561 while !self.layers.is_empty() {
1562 if let Some(sort_key) = self.layers[0].sort_key() {
1563 let mut i = 0;
1564 while i + 1 < self.layers.len() {
1565 if let Some(next_offset) = self.layers[i + 1].sort_key() {
1566 if next_offset < sort_key {
1567 i += 1;
1568 continue;
1570 } else {
1571 let layer = self.layers.remove(i + 1);
1572 PARSER.with(|ts_parser| {
1573 let highlighter = &mut ts_parser.borrow_mut();
1574 highlighter.cursors.push(layer.cursor);
1575 });
1577 break;
1579 if i > 0 {
1580 self.layers[0..(i + 1)].rotate_left(1);
1582 break;
1583 } else {
1584 let layer = self.layers.remove(0);
1585 PARSER.with(|ts_parser| {
1586 let highlighter = &mut ts_parser.borrow_mut();
1587 highlighter.cursors.push(layer.cursor);
1588 });
1594impl<'a> Iterator for HighlightIter<'a> {
1595 type Item = Result<HighlightEvent, Error>;
1597 fn next(&mut self) -> Option<Self::Item> {
1598 'main: loop {
1599 // If we've already determined the next highlight boundary, just return it.
1600 if let Some(e) = self.next_event.take() {
1601 return Some(Ok(e));
1604 // Periodically check for cancellation, returning `Cancelled` error if the
1605 // cancellation flag was flipped.
1606 if let Some(cancellation_flag) = self.cancellation_flag {
1607 self.iter_count += 1;
1608 if self.iter_count >= CANCELLATION_CHECK_INTERVAL {
1609 self.iter_count = 0;
1610 if cancellation_flag.load(Ordering::Relaxed) != 0 {
1611 return Some(Err(Error::Cancelled));
1616 // If none of the layers have any more highlight boundaries, terminate.
1617 if self.layers.is_empty() {
1618 let len = self.source.len_bytes();
1619 return if self.byte_offset < len {
1620 let result = Some(Ok(HighlightEvent::Source {
1621 start: self.byte_offset,
1622 end: len,
1623 }));
1624 self.byte_offset = len;
1625 result
1626 } else {
1627 None
1628 };
1631 // Get the next capture from whichever layer has the earliest highlight boundary.
1632 let range;
1633 let layer = &mut self.layers[0];
1634 if let Some((next_match, capture_index)) = layer.captures.peek() {
1635 let next_capture = next_match.captures[*capture_index];
1636 range = next_capture.node.byte_range();
1638 // If any previous highlight ends before this node starts, then before
1639 // processing this capture, emit the source code up until the end of the
1640 // previous highlight, and an end event for that highlight.
1641 if let Some(end_byte) = layer.highlight_end_stack.last().cloned() {
1642 if end_byte <= range.start {
1643 layer.highlight_end_stack.pop();
1644 return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd));
1648 // If there are no more captures, then emit any remaining highlight end events.
1649 // And if there are none of those, then just advance to the end of the document.
1650 else if let Some(end_byte) = layer.highlight_end_stack.last().cloned() {
1651 layer.highlight_end_stack.pop();
1652 return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd));
1653 } else {
1654 return self.emit_event(self.source.len_bytes(), None);
1655 };
1657 let (mut match_, capture_index) = layer.captures.next().unwrap();
1658 let mut capture = match_.captures[capture_index];
1660 // Remove from the local scope stack any local scopes that have already ended.
1661 while range.start > layer.scope_stack.last().unwrap().range.end {
1662 layer.scope_stack.pop();
1665 // If this capture is for tracking local variables, then process the
1666 // local variable info.
1667 let mut reference_highlight = None;
1668 let mut definition_highlight = None;
1669 while match_.pattern_index < layer.config.highlights_pattern_index {
1670 // If the node represents a local scope, push a new local scope onto
1671 // the scope stack.
1672 if Some(capture.index) == layer.config.local_scope_capture_index {
1673 definition_highlight = None;
1674 let mut scope = LocalScope {
1675 inherits: true,
1676 range: range.clone(),
1677 local_defs: Vec::new(),
1678 };
1679 for prop in layer.config.query.property_settings(match_.pattern_index) {
1680 if let "local.scope-inherits" = prop.key.as_ref() {
1681 scope.inherits =
1682 prop.value.as_ref().map_or(true, |r| r.as_ref() == "true");
1685 layer.scope_stack.push(scope);
1687 // If the node represents a definition, add a new definition to the
1688 // local scope at the top of the scope stack.
1689 else if Some(capture.index) == layer.config.local_def_capture_index {
1690 reference_highlight = None;
1691 let scope = layer.scope_stack.last_mut().unwrap();
1693 let mut value_range = 0..0;
1694 for capture in match_.captures {
1695 if Some(capture.index) == layer.config.local_def_value_capture_index {
1696 value_range = capture.node.byte_range();
1700 let name = byte_range_to_str(range.clone(), self.source);
1701 scope.local_defs.push(LocalDef {
1702 name,
1703 value_range,
1704 highlight: None,
1705 });
1706 definition_highlight = scope.local_defs.last_mut().map(|s| &mut s.highlight);
1708 // If the node represents a reference, then try to find the corresponding
1709 // definition in the scope stack.
1710 else if Some(capture.index) == layer.config.local_ref_capture_index
1711 && definition_highlight.is_none()
1713 definition_highlight = None;
1714 let name = byte_range_to_str(range.clone(), self.source);
1715 for scope in layer.scope_stack.iter().rev() {
1716 if let Some(highlight) = scope.local_defs.iter().rev().find_map(|def| {
1717 if def.name == name && range.start >= def.value_range.end {
1718 Some(def.highlight)
1719 } else {
1720 None
1722 }) {
1723 reference_highlight = highlight;
1724 break;
1726 if !scope.inherits {
1727 break;
1732 // Continue processing any additional matches for the same node.
1733 if let Some((next_match, next_capture_index)) = layer.captures.peek() {
1734 let next_capture = next_match.captures[*next_capture_index];
1735 if next_capture.node == capture.node {
1736 capture = next_capture;
1737 match_ = layer.captures.next().unwrap().0;
1738 continue;
1742 self.sort_layers();
1743 continue 'main;
1746 // Otherwise, this capture must represent a highlight.
1747 // If this exact range has already been highlighted by an earlier pattern, or by
1748 // a different layer, then skip over this one.
1749 if let Some((last_start, last_end, last_depth)) = self.last_highlight_range {
1750 if range.start == last_start && range.end == last_end && layer.depth < last_depth {
1751 self.sort_layers();
1752 continue 'main;
1756 // If the current node was found to be a local variable, then skip over any
1757 // highlighting patterns that are disabled for local variables.
1758 if definition_highlight.is_some() || reference_highlight.is_some() {
1759 while layer.config.non_local_variable_patterns[match_.pattern_index] {
1760 if let Some((next_match, next_capture_index)) = layer.captures.peek() {
1761 let next_capture = next_match.captures[*next_capture_index];
1762 if next_capture.node == capture.node {
1763 capture = next_capture;
1764 match_ = layer.captures.next().unwrap().0;
1765 continue;
1769 self.sort_layers();
1770 continue 'main;
1774 // Once a highlighting pattern is found for the current node, skip over
1775 // any later highlighting patterns that also match this node. Captures
1776 // for a given node are ordered by pattern index, so these subsequent
1777 // captures are guaranteed to be for highlighting, not injections or
1778 // local variables.
1779 while let Some((next_match, next_capture_index)) = layer.captures.peek() {
1780 let next_capture = next_match.captures[*next_capture_index];
1781 if next_capture.node == capture.node {
1782 layer.captures.next();
1783 } else {
1784 break;
1788 let current_highlight = layer.config.highlight_indices.load()[capture.index as usize];
1790 // If this node represents a local definition, then store the current
1791 // highlight value on the local scope entry representing this node.
1792 if let Some(definition_highlight) = definition_highlight {
1793 *definition_highlight = current_highlight;
1796 // Emit a scope start event and push the node's end position to the stack.
1797 if let Some(highlight) = reference_highlight.or(current_highlight) {
1798 self.last_highlight_range = Some((range.start, range.end, layer.depth));
1799 layer.highlight_end_stack.push(range.end);
1800 return self
1801 .emit_event(range.start, Some(HighlightEvent::HighlightStart(highlight)));
1804 self.sort_layers();
1809fn injection_for_match<'a>(
1810 config: &HighlightConfiguration,
1811 query: &'a Query,
1812 query_match: &QueryMatch<'a, 'a>,
1813 source: RopeSlice<'a>,
1814) -> (Option<Cow<'a, str>>, Option<Node<'a>>, IncludedChildren) {
1815 let content_capture_index = config.injection_content_capture_index;
1816 let language_capture_index = config.injection_language_capture_index;
1818 let mut language_name = None;
1819 let mut content_node = None;
1820 for capture in query_match.captures {
1821 let index = Some(capture.index);
1822 if index == language_capture_index {
1823 let name = byte_range_to_str(capture.node.byte_range(), source);
1824 language_name = Some(name);
1825 } else if index == content_capture_index {
1826 content_node = Some(capture.node);
1830 let mut included_children = IncludedChildren::default();
1831 for prop in query.property_settings(query_match.pattern_index) {
1832 match prop.key.as_ref() {
1833 // In addition to specifying the language name via the text of a
1834 // captured node, it can also be hard-coded via a `#set!` predicate
1835 // that sets the injection.language key.
1836 "injection.language" => {
1837 if language_name.is_none() {
1838 language_name = prop.value.as_ref().map(|s| s.as_ref().into())
1842 // By default, injections do not include the *children* of an
1843 // `injection.content` node - only the ranges that belong to the
1844 // node itself. This can be changed using a `#set!` predicate that
1845 // sets the `injection.include-children` key.
1846 "injection.include-children" => included_children = IncludedChildren::All,
1848 // Some queries might only exclude named children but include unnamed
1849 // children in their `injection.content` node. This can be enabled using
1850 // a `#set!` predicate that sets the `injection.include-unnamed-children` key.
1851 "injection.include-unnamed-children" => included_children = IncludedChildren::Unnamed,
1852 _ => {}
1856 (language_name, content_node, included_children)
1859pub struct Merge<I> {
1860 iter: I,
1861 spans: Box<dyn Iterator<Item = (usize, std::ops::Range<usize>)>>,
1863 next_event: Option<HighlightEvent>,
1864 next_span: Option<(usize, std::ops::Range<usize>)>,
1866 queue: Vec<HighlightEvent>,
1869/// Merge a list of spans into the highlight event stream.
1870pub fn merge<I: Iterator<Item = HighlightEvent>>(
1871 iter: I,
1872 spans: Vec<(usize, std::ops::Range<usize>)>,
1873) -> Merge<I> {
1874 let spans = Box::new(spans.into_iter());
1875 let mut merge = Merge {
1876 iter,
1877 spans,
1878 next_event: None,
1879 next_span: None,
1880 queue: Vec::new(),
1881 };
1882 merge.next_event = merge.iter.next();
1883 merge.next_span = merge.spans.next();
1884 merge
1887impl<I: Iterator<Item = HighlightEvent>> Iterator for Merge<I> {
1888 type Item = HighlightEvent;
1889 fn next(&mut self) -> Option<Self::Item> {
1890 use HighlightEvent::*;
1891 if let Some(event) = self.queue.pop() {
1892 return Some(event);
1895 loop {
1896 match (self.next_event, &self.next_span) {
1897 // this happens when range is partially or fully offscreen
1898 (Some(Source { start, .. }), Some((span, range))) if start > range.start => {
1899 if start > range.end {
1900 self.next_span = self.spans.next();
1901 } else {
1902 self.next_span = Some((*span, start..range.end));
1903 };
1905 _ => break,
1909 match (self.next_event, &self.next_span) {
1910 (Some(HighlightStart(i)), _) => {
1911 self.next_event = self.iter.next();
1912 Some(HighlightStart(i))
1914 (Some(HighlightEnd), _) => {
1915 self.next_event = self.iter.next();
1916 Some(HighlightEnd)
1918 (Some(Source { start, end }), Some((_, range))) if start < range.start => {
1919 let intersect = range.start.min(end);
1920 let event = Source {
1921 start,
1922 end: intersect,
1923 };
1925 if end == intersect {
1926 // the event is complete
1927 self.next_event = self.iter.next();
1928 } else {
1929 // subslice the event
1930 self.next_event = Some(Source {
1931 start: intersect,
1932 end,
1933 });
1934 };
1936 Some(event)
1938 (Some(Source { start, end }), Some((span, range))) if start == range.start => {
1939 let intersect = range.end.min(end);
1940 let event = HighlightStart(Highlight(*span));
1942 // enqueue in reverse order
1943 self.queue.push(HighlightEnd);
1944 self.queue.push(Source {
1945 start,
1946 end: intersect,
1947 });
1949 if end == intersect {
1950 // the event is complete
1951 self.next_event = self.iter.next();
1952 } else {
1953 // subslice the event
1954 self.next_event = Some(Source {
1955 start: intersect,
1956 end,
1957 });
1958 };
1960 if intersect == range.end {
1961 self.next_span = self.spans.next();
1962 } else {
1963 self.next_span = Some((*span, intersect..range.end));
1966 Some(event)
1968 (Some(event), None) => {
1969 self.next_event = self.iter.next();
1970 Some(event)
1972 // Can happen if cursor at EOF and/or diagnostic reaches past the end.
1973 // We need to actually emit events for the cursor-at-EOF situation,
1974 // even though the range is past the end of the text. This needs to be
1975 // handled appropriately by the drawing code by not assuming that
1976 // all `Source` events point to valid indices in the rope.
1977 (None, Some((span, range))) => {
1978 let event = HighlightStart(Highlight(*span));
1979 self.queue.push(HighlightEnd);
1980 self.queue.push(Source {
1981 start: range.start,
1982 end: range.end,
1983 });
1984 self.next_span = self.spans.next();
1985 Some(event)
1987 (None, None) => None,
1988 e => unreachable!("{:?}", e),
1993#[cfg(test)]
1994mod test {
1995 use super::*;
1996 use crate::{Rope, Transaction};
1998 #[test]
1999 fn test_textobject_queries() {
2000 let query_str = r#"
2001 (line_comment)+ @quantified_nodes
2002 ((line_comment)+) @quantified_nodes_grouped
2003 ((line_comment) (line_comment)) @multiple_nodes_grouped
2004 "#;
2005 let source = Rope::from_str(
2006 r#"
2007/// a comment on
2008/// multiple lines
2009 "#,
2010 );
2012 let loader = Loader::new(Configuration { language: vec![] });
2013 let language = get_language("Rust").unwrap();
2015 let query = Query::new(language, query_str).unwrap();
2016 let textobject = TextObjectQuery { query };
2017 let mut cursor = QueryCursor::new();
2019 let config = HighlightConfiguration::new(language, "", "", "").unwrap();
2020 let syntax = Syntax::new(&source, Arc::new(config), Arc::new(loader));
2022 let root = syntax.tree().root_node();
2023 let mut test = |capture, range| {
2024 let matches: Vec<_> = textobject
2025 .capture_nodes(capture, root, source.slice(..), &mut cursor)
2026 .unwrap()
2027 .collect();
2029 assert_eq!(
2030 matches[0].byte_range(),
2031 range,
2032 "@{} expected {:?}",
2033 capture,
2034 range
2036 };
2038 test("quantified_nodes", 1..36);
2039 // NOTE: Enable after implementing proper node group capturing
2040 // test("quantified_nodes_grouped", 1..36);
2041 // test("multiple_nodes_grouped", 1..36);
2044 #[test]
2045 fn test_parser() {
2046 let highlight_names: Vec<String> = [
2047 "attribute",
2048 "constant",
2049 "function.builtin",
2050 "function",
2051 "keyword",
2052 "operator",
2053 "property",
2054 "punctuation",
2055 "punctuation.bracket",
2056 "punctuation.delimiter",
2057 "string",
2058 "string.special",
2059 "tag",
2060 "type",
2061 "type.builtin",
2062 "variable",
2063 "variable.builtin",
2064 "variable.parameter",
2066 .iter()
2067 .cloned()
2068 .map(String::from)
2069 .collect();
2071 let loader = Loader::new(Configuration { language: vec![] });
2073 let language = get_language("Rust").unwrap();
2074 let config = HighlightConfiguration::new(
2075 language,
2076 &std::fs::read_to_string("../runtime/grammars/sources/rust/queries/highlights.scm")
2077 .unwrap(),
2078 &std::fs::read_to_string("../runtime/grammars/sources/rust/queries/injections.scm")
2079 .unwrap(),
2080 "", // locals.scm
2082 .unwrap();
2083 config.configure(&highlight_names);
2085 let source = Rope::from_str(
2087 struct Stuff {}
2088 fn main() {}
2089 ",
2090 );
2091 let syntax = Syntax::new(&source, Arc::new(config), Arc::new(loader));
2092 let tree = syntax.tree();
2093 let root = tree.root_node();
2094 assert_eq!(root.kind(), "source_file");
2096 assert_eq!(
2097 root.to_sexp(),
2098 concat!(
2099 "(source_file ",
2100 "(struct_item name: (type_identifier) body: (field_declaration_list)) ",
2101 "(function_item name: (identifier) parameters: (parameters) body: (block)))"
2103 );
2105 let struct_node = root.child(0).unwrap();
2106 assert_eq!(struct_node.kind(), "struct_item");
2109 #[test]
2110 fn test_input_edits() {
2111 use tree_sitter::InputEdit;
2113 let doc = Rope::from("hello world!\ntest 123");
2114 let transaction = Transaction::change(
2115 &doc,
2116 vec![(6, 11, Some("test".into())), (12, 17, None)].into_iter(),
2117 );
2118 let edits = generate_edits(&doc, transaction.changes());
2119 // transaction.apply(&mut state);
2121 assert_eq!(
2122 edits,
2123 &[
2124 InputEdit {
2125 start_byte: 6,
2126 old_end_byte: 11,
2127 new_end_byte: 10,
2128 start_position: Point { row: 0, column: 6 },
2129 old_end_position: Point { row: 0, column: 11 },
2130 new_end_position: Point { row: 0, column: 10 }
2131 },
2132 InputEdit {
2133 start_byte: 12,
2134 old_end_byte: 17,
2135 new_end_byte: 12,
2136 start_position: Point { row: 0, column: 12 },
2137 old_end_position: Point { row: 1, column: 4 },
2138 new_end_position: Point { row: 0, column: 12 }
2141 );
2143 // Testing with the official example from tree-sitter
2144 let mut doc = Rope::from("fn test() {}");
2145 let transaction =
2146 Transaction::change(&doc, vec![(8, 8, Some("a: u32".into()))].into_iter());
2147 let edits = generate_edits(&doc, transaction.changes());
2148 transaction.apply(&mut doc);
2150 assert_eq!(doc, "fn test(a: u32) {}");
2151 assert_eq!(
2152 edits,
2153 &[InputEdit {
2154 start_byte: 8,
2155 old_end_byte: 8,
2156 new_end_byte: 14,
2157 start_position: Point { row: 0, column: 8 },
2158 old_end_position: Point { row: 0, column: 8 },
2159 new_end_position: Point { row: 0, column: 14 }
2160 }]
2161 );
2164 #[test]
2165 fn test_load_runtime_file() {
2166 // Test to make sure we can load some data from the runtime directory.
2167 let contents = load_runtime_file("rust", "indents.scm").unwrap();
2168 assert!(!contents.is_empty());
2170 let results = load_runtime_file("rust", "does-not-exist");
2171 assert!(results.is_err());