3 chars::char_is_line_ending,
6 transaction::{ChangeSet, Operation},
7 Rope, RopeSlice, Tendril,
10use ahash::RandomState;
11use arc_swap::{ArcSwap, Guard};
12use bitflags::bitflags;
13use hashbrown::raw::RawTable;
14use slotmap::{DefaultKey as LayerId, HopSlotMap};
19 collections::{HashMap, VecDeque},
22 mem::{replace, transmute},
28use once_cell::sync::{Lazy, OnceCell};
29use serde::{Deserialize, Serialize};
31use helix_loader::grammar::{get_language, load_runtime_file};
33fn deserialize_regex<'de, D>(deserializer: D) -> Result<Option<Regex>, D::Error>
35 D: serde::Deserializer<'de>,
37 Option::<String>::deserialize(deserializer)?
38 .map(|buf| Regex::new(&buf).map_err(serde::de::Error::custom))
42fn deserialize_lsp_config<'de, D>(deserializer: D) -> Result<Option<serde_json::Value>, D::Error>
44 D: serde::Deserializer<'de>,
46 Option::<toml::Value>::deserialize(deserializer)?
47 .map(|toml| toml.try_into().map_err(serde::de::Error::custom))
51pub fn deserialize_auto_pairs<'de, D>(deserializer: D) -> Result<Option<AutoPairs>, D::Error>
53 D: serde::Deserializer<'de>,
55 Ok(Option::<AutoPairConfig>::deserialize(deserializer)?.and_then(AutoPairConfig::into))
58fn default_timeout() -> u64 {
62#[derive(Debug, Serialize, Deserialize)]
63pub struct Configuration {
64 pub language: Vec<LanguageConfiguration>,
67impl Default for Configuration {
68 fn default() -> Self {
69 crate::config::default_syntax_loader()
73// largely based on tree-sitter/cli/src/loader.rs
74#[derive(Debug, Serialize, Deserialize)]
75#[serde(rename_all = "kebab-case", deny_unknown_fields)]
76pub struct LanguageConfiguration {
77 #[serde(rename = "name")]
78 pub language_id: String, // c-sharp, rust
79 pub scope: String, // source.rust
80 pub file_types: Vec<FileType>, // filename extension or ends_with? <Gemfile, rb, etc>
82 pub shebangs: Vec<String>, // interpreter(s) associated with language
83 pub roots: Vec<String>, // these indicate project roots <.git, Cargo.toml>
84 pub comment_token: Option<String>,
85 pub max_line_length: Option<usize>,
87 #[serde(default, skip_serializing, deserialize_with = "deserialize_lsp_config")]
88 pub config: Option<serde_json::Value>,
91 pub auto_format: bool,
93 #[serde(skip_serializing_if = "Option::is_none")]
94 pub formatter: Option<FormatterConfiguration>,
97 pub diagnostic_severity: Severity,
99 pub grammar: Option<String>, // tree-sitter grammar name, defaults to language_id
102 #[serde(default, skip_serializing, deserialize_with = "deserialize_regex")]
103 pub injection_regex: Option<Regex>,
107 pub(crate) highlight_config: OnceCell<Option<Arc<HighlightConfiguration>>>,
108 // tags_config OnceCell<> https://github.com/tree-sitter/tree-sitter/pull/583
109 #[serde(skip_serializing_if = "Option::is_none")]
110 pub language_server: Option<LanguageServerConfiguration>,
111 #[serde(skip_serializing_if = "Option::is_none")]
112 pub indent: Option<IndentationConfiguration>,
115 pub(crate) indent_query: OnceCell<Option<Query>>,
117 pub(crate) textobject_query: OnceCell<Option<TextObjectQuery>>,
118 #[serde(skip_serializing_if = "Option::is_none")]
119 pub debugger: Option<DebugAdapterConfig>,
121 /// Automatic insertion of pairs to parentheses, brackets,
122 /// etc. Defaults to true. Optionally, this can be a list of 2-tuples
123 /// to specify a list of characters to pair. This overrides the
125 #[serde(default, skip_serializing, deserialize_with = "deserialize_auto_pairs")]
126 pub auto_pairs: Option<AutoPairs>,
128 pub rulers: Option<Vec<u16>>, // if set, override editor's rulers
131#[derive(Debug, PartialEq, Eq, Hash)]
133 /// The extension of the file, either the `Path::extension` or the full
134 /// filename if the file does not have an extension.
136 /// The suffix of a file. This is compared to a given file's absolute
137 /// path, so it can be used to detect files based on their directories.
141impl Serialize for FileType {
142 fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
144 S: serde::Serializer,
146 use serde::ser::SerializeMap;
149 FileType::Extension(extension) => serializer.serialize_str(extension),
150 FileType::Suffix(suffix) => {
151 let mut map = serializer.serialize_map(Some(1))?;
152 map.serialize_entry("suffix", &suffix.replace(std::path::MAIN_SEPARATOR, "/"))?;
159impl<'de> Deserialize<'de> for FileType {
160 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
162 D: serde::de::Deserializer<'de>,
164 struct FileTypeVisitor;
166 impl<'de> serde::de::Visitor<'de> for FileTypeVisitor {
167 type Value = FileType;
169 fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
170 formatter.write_str("string or table")
173 fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
177 Ok(FileType::Extension(value.to_string()))
180 fn visit_map<M>(self, mut map: M) -> Result<Self::Value, M::Error>
182 M: serde::de::MapAccess<'de>,
184 match map.next_entry::<String, String>()? {
185 Some((key, suffix)) if key == "suffix" => Ok(FileType::Suffix(
186 suffix.replace('/', &std::path::MAIN_SEPARATOR.to_string()),
188 Some((key, _value)) => Err(serde::de::Error::custom(format!(
189 "unknown key in `file-types` list: {}",
192 None => Err(serde::de::Error::custom(
193 "expected a `suffix` key in the `file-types` entry",
199 deserializer.deserialize_any(FileTypeVisitor)
203#[derive(Debug, Serialize, Deserialize)]
204#[serde(rename_all = "kebab-case")]
205pub struct LanguageServerConfiguration {
208 #[serde(skip_serializing_if = "Vec::is_empty")]
209 pub args: Vec<String>,
210 #[serde(default, skip_serializing_if = "HashMap::is_empty")]
211 pub environment: HashMap<String, String>,
212 #[serde(default = "default_timeout")]
214 pub language_id: Option<String>,
217#[derive(Debug, Clone, Serialize, Deserialize)]
218#[serde(rename_all = "kebab-case")]
219pub struct FormatterConfiguration {
222 #[serde(skip_serializing_if = "Vec::is_empty")]
223 pub args: Vec<String>,
226#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]
227#[serde(rename_all = "kebab-case")]
228pub struct AdvancedCompletion {
229 pub name: Option<String>,
230 pub completion: Option<String>,
231 pub default: Option<String>,
234#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]
235#[serde(rename_all = "kebab-case", untagged)]
236pub enum DebugConfigCompletion {
238 Advanced(AdvancedCompletion),
241#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]
243pub enum DebugArgumentValue {
249#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]
250#[serde(rename_all = "kebab-case")]
251pub struct DebugTemplate {
254 pub completion: Vec<DebugConfigCompletion>,
255 pub args: HashMap<String, DebugArgumentValue>,
258#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]
259#[serde(rename_all = "kebab-case")]
260pub struct DebugAdapterConfig {
262 pub transport: String,
266 pub args: Vec<String>,
267 pub port_arg: Option<String>,
268 pub templates: Vec<DebugTemplate>,
270 pub quirks: DebuggerQuirks,
273// Different workarounds for adapters' differences
274#[derive(Debug, Default, PartialEq, Eq, Clone, Serialize, Deserialize)]
275pub struct DebuggerQuirks {
277 pub absolute_paths: bool,
280#[derive(Debug, Serialize, Deserialize)]
281#[serde(rename_all = "kebab-case")]
282pub struct IndentationConfiguration {
283 pub tab_width: usize,
287/// Configuration for auto pairs
288#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
289#[serde(rename_all = "kebab-case", deny_unknown_fields, untagged)]
290pub enum AutoPairConfig {
291 /// Enables or disables auto pairing. False means disabled. True means to use the default pairs.
294 /// The mappings of pairs.
295 Pairs(HashMap<char, char>),
298impl Default for AutoPairConfig {
299 fn default() -> Self {
300 AutoPairConfig::Enable(true)
304impl From<&AutoPairConfig> for Option<AutoPairs> {
305 fn from(auto_pair_config: &AutoPairConfig) -> Self {
306 match auto_pair_config {
307 AutoPairConfig::Enable(false) => None,
308 AutoPairConfig::Enable(true) => Some(AutoPairs::default()),
309 AutoPairConfig::Pairs(pairs) => Some(AutoPairs::new(pairs.iter())),
314impl From<AutoPairConfig> for Option<AutoPairs> {
315 fn from(auto_pairs_config: AutoPairConfig) -> Self {
316 (&auto_pairs_config).into()
320impl FromStr for AutoPairConfig {
321 type Err = std::str::ParseBoolError;
323 // only do bool parsing for runtime setting
324 fn from_str(s: &str) -> Result<Self, Self::Err> {
325 let enable: bool = s.parse()?;
326 Ok(AutoPairConfig::Enable(enable))
331pub struct TextObjectQuery {
336pub enum CapturedNode<'a> {
338 /// Guaranteed to be not empty
339 Grouped(Vec<Node<'a>>),
342impl<'a> CapturedNode<'a> {
343 pub fn start_byte(&self) -> usize {
345 Self::Single(n) => n.start_byte(),
346 Self::Grouped(ns) => ns[0].start_byte(),
350 pub fn end_byte(&self) -> usize {
352 Self::Single(n) => n.end_byte(),
353 Self::Grouped(ns) => ns.last().unwrap().end_byte(),
357 pub fn byte_range(&self) -> std::ops::Range<usize> {
358 self.start_byte()..self.end_byte()
362/// The maximum number of in-progress matches a TS cursor can consider at once.
363/// This is set to a constant in order to avoid performance problems for medium to large files. Set with `set_match_limit`.
364/// Using such a limit means that we lose valid captures, so there is fundamentally a tradeoff here.
367/// Old tree sitter versions used a limit of 32 by default until this limit was removed in version `0.19.5` (must now be set manually).
368/// However, this causes performance issues for medium to large files.
369/// In helix, this problem caused treesitter motions to take multiple seconds to complete in medium-sized rust files (3k loc).
372/// Neovim also encountered this problem and reintroduced this limit after it was removed upstream
373/// (see <https://github.com/neovim/neovim/issues/14897> and <https://github.com/neovim/neovim/pull/14915>).
374/// The number used here is fundamentally a tradeoff between breaking some obscure edge cases and performance.
377/// Neovim chose 64 for this value somewhat arbitrarily (<https://github.com/neovim/neovim/pull/18397>).
378/// 64 is too low for some languages though. In particular, it breaks some highlighting for record fields in Erlang record definitions.
379/// This number can be increased if new syntax highlight breakages are found, as long as the performance penalty is not too high.
380const TREE_SITTER_MATCH_LIMIT: u32 = 256;
382impl TextObjectQuery {
383 /// Run the query on the given node and return sub nodes which match given
384 /// capture ("function.inside", "class.around", etc).
386 /// Captures may contain multiple nodes by using quantifiers (+, *, etc),
387 /// and support for this is partial and could use improvement.
390 /// (comment)+ @capture
399 pub fn capture_nodes<'a>(
403 slice: RopeSlice<'a>,
404 cursor: &'a mut QueryCursor,
405 ) -> Option<impl Iterator<Item = CapturedNode<'a>>> {
406 self.capture_nodes_any(&[capture_name], node, slice, cursor)
409 /// Find the first capture that exists out of all given `capture_names`
410 /// and return sub nodes that match this capture.
411 pub fn capture_nodes_any<'a>(
413 capture_names: &[&str],
415 slice: RopeSlice<'a>,
416 cursor: &'a mut QueryCursor,
417 ) -> Option<impl Iterator<Item = CapturedNode<'a>>> {
418 let capture_idx = capture_names
420 .find_map(|cap| self.query.capture_index_for_name(cap))?;
422 cursor.set_match_limit(TREE_SITTER_MATCH_LIMIT);
425 .captures(&self.query, node, RopeProvider(slice))
426 .filter_map(move |(mat, _)| {
427 let nodes: Vec<_> = mat
430 .filter_map(|cap| (cap.index == capture_idx).then_some(cap.node))
434 Some(CapturedNode::Grouped(nodes))
436 nodes.into_iter().map(CapturedNode::Single).next()
444pub fn read_query(language: &str, filename: &str) -> String {
445 static INHERITS_REGEX: Lazy<Regex> =
446 Lazy::new(|| Regex::new(r";+\s*inherits\s*:?\s*([a-z_,()-]+)\s*").unwrap());
448 let query = load_runtime_file(language, filename).unwrap_or_default();
450 // replaces all "; inherits <language>(,<language>)*" with the queries of the given language(s)
452 .replace_all(&query, |captures: ®ex::Captures| {
455 .map(|language| format!("\n{}\n", read_query(language, filename)))
461impl LanguageConfiguration {
462 fn initialize_highlight(&self, scopes: &[String]) -> Option<Arc<HighlightConfiguration>> {
463 let highlights_query = read_query(&self.language_id, "highlights.scm");
464 // always highlight syntax errors
465 // highlights_query += "\n(ERROR) @error";
467 let injections_query = read_query(&self.language_id, "injections.scm");
468 let locals_query = read_query(&self.language_id, "locals.scm");
470 if highlights_query.is_empty() {
473 let language = get_language(self.grammar.as_deref().unwrap_or(&self.language_id))
476 "Failed to load tree-sitter parser for language {:?}: {}",
482 let config = HighlightConfiguration::new(
488 .map_err(|err| log::error!("Could not parse queries for language {:?}. Are your grammars out of sync? Try running 'hx --grammar fetch' and 'hx --grammar build'. This query could not be parsed: {:?}", self.language_id, err))
491 config.configure(scopes);
492 Some(Arc::new(config))
496 pub fn reconfigure(&self, scopes: &[String]) {
497 if let Some(Some(config)) = self.highlight_config.get() {
498 config.configure(scopes);
502 pub fn highlight_config(&self, scopes: &[String]) -> Option<Arc<HighlightConfiguration>> {
503 self.highlight_config
504 .get_or_init(|| self.initialize_highlight(scopes))
508 pub fn is_highlight_initialized(&self) -> bool {
509 self.highlight_config.get().is_some()
512 pub fn indent_query(&self) -> Option<&Query> {
514 .get_or_init(|| self.load_query("indents.scm"))
518 pub fn textobject_query(&self) -> Option<&TextObjectQuery> {
519 self.textobject_query
521 self.load_query("textobjects.scm")
522 .map(|query| TextObjectQuery { query })
527 pub fn scope(&self) -> &str {
531 fn load_query(&self, kind: &str) -> Option<Query> {
532 let query_text = read_query(&self.language_id, kind);
533 if query_text.is_empty() {
536 let lang = self.highlight_config.get()?.as_ref()?.language;
537 Query::new(lang, &query_text)
540 "Failed to parse {} queries for {}: {}",
550// Expose loader as Lazy<> global since it's always static?
555 language_configs: Vec<Arc<LanguageConfiguration>>,
556 language_config_ids_by_extension: HashMap<String, usize>, // Vec<usize>
557 language_config_ids_by_suffix: HashMap<String, usize>,
558 language_config_ids_by_shebang: HashMap<String, usize>,
560 scopes: ArcSwap<Vec<String>>,
564 pub fn new(config: Configuration) -> Self {
565 let mut loader = Self {
566 language_configs: Vec::new(),
567 language_config_ids_by_extension: HashMap::new(),
568 language_config_ids_by_suffix: HashMap::new(),
569 language_config_ids_by_shebang: HashMap::new(),
570 scopes: ArcSwap::from_pointee(Vec::new()),
573 for config in config.language {
575 let language_id = loader.language_configs.len();
577 for file_type in &config.file_types {
578 // entry().or_insert(Vec::new).push(language_id);
580 FileType::Extension(extension) => loader
581 .language_config_ids_by_extension
582 .insert(extension.clone(), language_id),
583 FileType::Suffix(suffix) => loader
584 .language_config_ids_by_suffix
585 .insert(suffix.clone(), language_id),
588 for shebang in &config.shebangs {
590 .language_config_ids_by_shebang
591 .insert(shebang.clone(), language_id);
594 loader.language_configs.push(Arc::new(config));
600 pub fn language_config_for_file_name(&self, path: &Path) -> Option<Arc<LanguageConfiguration>> {
601 // Find all the language configurations that match this file name
602 // or a suffix of the file name.
603 let configuration_id = path
605 .and_then(|n| n.to_str())
606 .and_then(|file_name| self.language_config_ids_by_extension.get(file_name))
609 .and_then(|extension| extension.to_str())
610 .and_then(|extension| self.language_config_ids_by_extension.get(extension))
613 self.language_config_ids_by_suffix
615 .find_map(|(file_type, id)| {
616 if path.to_str()?.ends_with(file_type) {
624 configuration_id.and_then(|&id| self.language_configs.get(id).cloned())
626 // TODO: content_regex handling conflict resolution
629 pub fn language_config_for_shebang(&self, source: &Rope) -> Option<Arc<LanguageConfiguration>> {
630 let line = Cow::from(source.line(0));
631 static SHEBANG_REGEX: Lazy<Regex> = Lazy::new(|| {
632 Regex::new(r"^#!\s*(?:\S*[/\\](?:env\s+(?:\-\S+\s+)*)?)?([^\s\.\d]+)").unwrap()
634 let configuration_id = SHEBANG_REGEX
636 .and_then(|cap| self.language_config_ids_by_shebang.get(&cap[1]));
638 configuration_id.and_then(|&id| self.language_configs.get(id).cloned())
641 pub fn language_config_for_scope(&self, scope: &str) -> Option<Arc<LanguageConfiguration>> {
642 self.language_configs
644 .find(|config| config.scope == scope)
648 pub fn language_config_for_language_id(&self, id: &str) -> Option<Arc<LanguageConfiguration>> {
649 self.language_configs
651 .find(|config| config.language_id == id)
655 pub fn language_configuration_for_injection_string(
658 ) -> Option<Arc<LanguageConfiguration>> {
659 let mut best_match_length = 0;
660 let mut best_match_position = None;
661 for (i, configuration) in self.language_configs.iter().enumerate() {
662 if let Some(injection_regex) = &configuration.injection_regex {
663 if let Some(mat) = injection_regex.find(string) {
664 let length = mat.end() - mat.start();
665 if length > best_match_length {
666 best_match_position = Some(i);
667 best_match_length = length;
673 if let Some(i) = best_match_position {
674 let configuration = &self.language_configs[i];
675 return Some(configuration.clone());
680 pub fn language_configs(&self) -> impl Iterator<Item = &Arc<LanguageConfiguration>> {
681 self.language_configs.iter()
684 pub fn set_scopes(&self, scopes: Vec<String>) {
685 self.scopes.store(Arc::new(scopes));
687 // Reconfigure existing grammars
691 .filter(|cfg| cfg.is_highlight_initialized())
693 config.reconfigure(&self.scopes());
697 pub fn scopes(&self) -> Guard<Arc<Vec<String>>> {
703 parser: tree_sitter::Parser,
704 pub cursors: Vec<QueryCursor>,
707// could also just use a pool, or a single instance?
709 pub static PARSER: RefCell<TsParser> = RefCell::new(TsParser {
710 parser: Parser::new(),
717 layers: HopSlotMap<LayerId, LanguageLayer>,
722fn byte_range_to_str(range: std::ops::Range<usize>, source: RopeSlice) -> Cow<str> {
723 Cow::from(source.byte_slice(range))
727 pub fn new(source: &Rope, config: Arc<HighlightConfiguration>, loader: Arc<Loader>) -> Self {
728 let root_layer = LanguageLayer {
732 flags: LayerUpdateFlags::empty(),
735 end_byte: usize::MAX,
736 start_point: Point::new(0, 0),
737 end_point: Point::new(usize::MAX, usize::MAX),
741 // track scope_descriptor: a Vec of scopes for item in tree
743 let mut layers = HopSlotMap::default();
744 let root = layers.insert(root_layer);
746 let mut syntax = Self {
753 .update(source, source, &ChangeSet::new(source))
763 changeset: &ChangeSet,
764 ) -> Result<(), Error> {
765 let mut queue = VecDeque::new();
766 queue.push_back(self.root);
768 let scopes = self.loader.scopes.load();
769 let injection_callback = |language: &str| {
771 .language_configuration_for_injection_string(language)
772 .and_then(|language_config| language_config.highlight_config(&scopes))
775 // Convert the changeset into tree sitter edits.
776 let edits = generate_edits(old_source, changeset);
778 // This table allows inverse indexing of `layers`.
779 // That is by hashing a `Layer` you can find
780 // the `LayerId` of an existing equivalent `Layer` in `layers`.
782 // It is used to determine if a new layer exists for an injection
783 // or if an existing layer needs to be updated.
784 let mut layers_table = RawTable::with_capacity(self.layers.len());
785 let layers_hasher = RandomState::new();
786 // Use the edits to update all layers markers
787 fn point_add(a: Point, b: Point) -> Point {
789 Point::new(a.row.saturating_add(b.row), b.column)
791 Point::new(0, a.column.saturating_add(b.column))
794 fn point_sub(a: Point, b: Point) -> Point {
796 Point::new(a.row.saturating_sub(b.row), a.column)
798 Point::new(0, a.column.saturating_sub(b.column))
802 for (layer_id, layer) in self.layers.iter_mut() {
803 // The root layer always covers the whole range (0..usize::MAX)
804 if layer.depth == 0 {
805 layer.flags = LayerUpdateFlags::MODIFIED;
809 if !edits.is_empty() {
810 for range in &mut layer.ranges {
811 // Roughly based on https://github.com/tree-sitter/tree-sitter/blob/ddeaa0c7f534268b35b4f6cb39b52df082754413/lib/src/subtree.c#L691-L720
812 for edit in edits.iter().rev() {
813 let is_pure_insertion = edit.old_end_byte == edit.start_byte;
815 // if edit is after range, skip
816 if edit.start_byte > range.end_byte {
817 // TODO: || (is_noop && edit.start_byte == range.end_byte)
821 // if edit is before range, shift entire range by len
822 if edit.old_end_byte < range.start_byte {
824 edit.new_end_byte + (range.start_byte - edit.old_end_byte);
825 range.start_point = point_add(
826 edit.new_end_position,
827 point_sub(range.start_point, edit.old_end_position),
830 range.end_byte = edit
832 .saturating_add(range.end_byte - edit.old_end_byte);
833 range.end_point = point_add(
834 edit.new_end_position,
835 point_sub(range.end_point, edit.old_end_position),
838 layer.flags |= LayerUpdateFlags::MOVED;
840 // if the edit starts in the space before and extends into the range
841 else if edit.start_byte < range.start_byte {
842 range.start_byte = edit.new_end_byte;
843 range.start_point = edit.new_end_position;
845 range.end_byte = range
847 .saturating_sub(edit.old_end_byte)
848 .saturating_add(edit.new_end_byte);
849 range.end_point = point_add(
850 edit.new_end_position,
851 point_sub(range.end_point, edit.old_end_position),
853 layer.flags = LayerUpdateFlags::MODIFIED;
855 // If the edit is an insertion at the start of the tree, shift
856 else if edit.start_byte == range.start_byte && is_pure_insertion {
857 range.start_byte = edit.new_end_byte;
858 range.start_point = edit.new_end_position;
859 layer.flags |= LayerUpdateFlags::MOVED;
861 range.end_byte = range
863 .saturating_sub(edit.old_end_byte)
864 .saturating_add(edit.new_end_byte);
865 range.end_point = point_add(
866 edit.new_end_position,
867 point_sub(range.end_point, edit.old_end_position),
869 layer.flags = LayerUpdateFlags::MODIFIED;
875 let hash = layers_hasher.hash_one(layer);
876 // Safety: insert_no_grow is unsafe because it assumes that the table
877 // has enough capacity to hold additional elements.
878 // This is always the case as we reserved enough capacity above.
879 unsafe { layers_table.insert_no_grow(hash, layer_id) };
882 PARSER.with(|ts_parser| {
883 let ts_parser = &mut ts_parser.borrow_mut();
884 let mut cursor = ts_parser.cursors.pop().unwrap_or_else(QueryCursor::new);
885 // TODO: might need to set cursor range
886 cursor.set_byte_range(0..usize::MAX);
887 cursor.set_match_limit(TREE_SITTER_MATCH_LIMIT);
889 let source_slice = source.slice(..);
891 while let Some(layer_id) = queue.pop_front() {
892 let layer = &mut self.layers[layer_id];
894 // Mark the layer as touched
895 layer.flags |= LayerUpdateFlags::TOUCHED;
897 // If a tree already exists, notify it of changes.
898 if let Some(tree) = &mut layer.tree {
901 .intersects(LayerUpdateFlags::MODIFIED | LayerUpdateFlags::MOVED)
903 for edit in edits.iter().rev() {
904 // Apply the edits in reverse.
905 // If we applied them in order then edit 1 would disrupt the positioning of edit 2.
910 if layer.flags.contains(LayerUpdateFlags::MODIFIED) {
911 // Re-parse the tree.
912 layer.parse(&mut ts_parser.parser, source)?;
915 // always parse if this layer has never been parsed before
916 layer.parse(&mut ts_parser.parser, source)?;
919 // Switch to an immutable borrow.
920 let layer = &self.layers[layer_id];
922 // Process injections.
923 let matches = cursor.matches(
924 &layer.config.injections_query,
925 layer.tree().root_node(),
926 RopeProvider(source_slice),
928 let mut injections = Vec::new();
930 let (language_name, content_node, included_children) = injection_for_match(
932 &layer.config.injections_query,
937 // Explicitly remove this match so that none of its other captures will remain
938 // in the stream of captures.
941 // If a language is found with the given name, then add a new language layer
942 // to the highlighted document.
943 if let (Some(language_name), Some(content_node)) = (language_name, content_node)
945 if let Some(config) = (injection_callback)(&language_name) {
947 intersect_ranges(&layer.ranges, &[content_node], included_children);
949 if !ranges.is_empty() {
950 injections.push((config, ranges));
956 // Process combined injections.
957 if let Some(combined_injections_query) = &layer.config.combined_injections_query {
958 let mut injections_by_pattern_index =
960 (None, Vec::new(), IncludedChildren::default());
961 combined_injections_query.pattern_count()
963 let matches = cursor.matches(
964 combined_injections_query,
965 layer.tree().root_node(),
966 RopeProvider(source_slice),
969 let entry = &mut injections_by_pattern_index[mat.pattern_index];
970 let (language_name, content_node, included_children) = injection_for_match(
972 combined_injections_query,
976 if language_name.is_some() {
977 entry.0 = language_name;
979 if let Some(content_node) = content_node {
980 entry.1.push(content_node);
982 entry.2 = included_children;
984 for (lang_name, content_nodes, included_children) in injections_by_pattern_index
986 if let (Some(lang_name), false) = (lang_name, content_nodes.is_empty()) {
987 if let Some(config) = (injection_callback)(&lang_name) {
988 let ranges = intersect_ranges(
993 if !ranges.is_empty() {
994 injections.push((config, ranges));
1001 let depth = layer.depth + 1;
1002 // TODO: can't inline this since matches borrows self.layers
1003 for (config, ranges) in injections {
1004 let new_layer = LanguageLayer {
1009 flags: LayerUpdateFlags::empty(),
1012 // Find an identical existing layer
1013 let layer = layers_table
1014 .get(layers_hasher.hash_one(&new_layer), |&it| {
1015 self.layers[it] == new_layer
1019 // ...or insert a new one.
1020 let layer_id = layer.unwrap_or_else(|| self.layers.insert(new_layer));
1022 queue.push_back(layer_id);
1025 // TODO: pre-process local scopes at this time, rather than highlight?
1026 // would solve problems with locals not working across boundaries
1029 // Return the cursor back in the pool.
1030 ts_parser.cursors.push(cursor);
1032 // Reset all `LayerUpdateFlags` and remove all untouched layers
1033 self.layers.retain(|_, layer| {
1034 replace(&mut layer.flags, LayerUpdateFlags::empty())
1035 .contains(LayerUpdateFlags::TOUCHED)
1042 pub fn tree(&self) -> &Tree {
1043 self.layers[self.root].tree()
1046 /// Iterate over the highlighted regions for a given slice of source code.
1047 pub fn highlight_iter<'a>(
1049 source: RopeSlice<'a>,
1050 range: Option<std::ops::Range<usize>>,
1051 cancellation_flag: Option<&'a AtomicUsize>,
1052 ) -> impl Iterator<Item = Result<HighlightEvent, Error>> + 'a {
1053 let mut layers = self
1056 .filter_map(|(_, layer)| {
1057 // TODO: if range doesn't overlap layer range, skip it
1059 // Reuse a cursor from the pool if available.
1060 let mut cursor = PARSER.with(|ts_parser| {
1061 let highlighter = &mut ts_parser.borrow_mut();
1062 highlighter.cursors.pop().unwrap_or_else(QueryCursor::new)
1065 // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which
1066 // prevents them from being moved. But both of these values are really just
1067 // pointers, so it's actually ok to move them.
1069 unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) };
1071 // if reusing cursors & no range this resets to whole range
1072 cursor_ref.set_byte_range(range.clone().unwrap_or(0..usize::MAX));
1073 cursor_ref.set_match_limit(TREE_SITTER_MATCH_LIMIT);
1075 let mut captures = cursor_ref
1077 &layer.config.query,
1078 layer.tree().root_node(),
1079 RopeProvider(source),
1083 // If there's no captures, skip the layer
1086 Some(HighlightIterLayer {
1087 highlight_end_stack: Vec::new(),
1088 scope_stack: vec![LocalScope {
1090 range: 0..usize::MAX,
1091 local_defs: Vec::new(),
1095 captures: RefCell::new(captures),
1096 config: layer.config.as_ref(), // TODO: just reuse `layer`
1097 depth: layer.depth, // TODO: just reuse `layer`
1100 .collect::<Vec<_>>();
1102 layers.sort_unstable_by_key(|layer| layer.sort_key());
1104 let mut result = HighlightIter {
1106 byte_offset: range.map_or(0, |r| r.start),
1111 last_highlight_range: None,
1113 result.sort_layers();
1118 // comment_strings_for_pos
1122 // suggested_indent_for_line_at_buffer_row
1123 // suggested_indent_for_buffer_row
1124 // indent_level_for_line
1130 /// Flags that track the status of a layer
1131 /// in the `Sytaxn::update` function
1132 struct LayerUpdateFlags : u32{
1133 const MODIFIED = 0b001;
1134 const MOVED = 0b010;
1135 const TOUCHED = 0b100;
1140pub struct LanguageLayer {
1143 pub config: Arc<HighlightConfiguration>,
1144 pub(crate) tree: Option<Tree>,
1145 pub ranges: Vec<Range>,
1147 flags: LayerUpdateFlags,
1150/// This PartialEq implementation only checks if that
1151/// two layers are theoretically identical (meaning they highlight the same text range with the same language).
1152/// It does not check whether the layers have the same internal treesitter
1154impl PartialEq for LanguageLayer {
1155 fn eq(&self, other: &Self) -> bool {
1156 self.depth == other.depth
1157 && self.config.language == other.config.language
1158 && self.ranges == other.ranges
1162/// Hash implementation belongs to PartialEq implementation above.
1163/// See its documentation for details.
1164impl Hash for LanguageLayer {
1165 fn hash<H: Hasher>(&self, state: &mut H) {
1166 self.depth.hash(state);
1167 // The transmute is necessary here because tree_sitter::Language does not derive Hash at the moment.
1168 // However it does use #[repr] transparent so the transmute here is safe
1169 // as `Language` (which `Grammar` is an alias for) is just a newtype wrapper around a (thin) pointer.
1170 // This is also compatible with the PartialEq implementation of language
1171 // as that is just a pointer comparison.
1172 let language: *const () = unsafe { transmute(self.config.language) };
1173 language.hash(state);
1174 self.ranges.hash(state);
1179 pub fn tree(&self) -> &Tree {
1181 self.tree.as_ref().unwrap()
1184 fn parse(&mut self, parser: &mut Parser, source: &Rope) -> Result<(), Error> {
1186 .set_included_ranges(&self.ranges)
1187 .map_err(|_| Error::InvalidRanges)?;
1190 .set_language(self.config.language)
1191 .map_err(|_| Error::InvalidLanguage)?;
1193 // unsafe { syntax.parser.set_cancellation_flag(cancellation_flag) };
1197 if byte <= source.len_bytes() {
1198 let (chunk, start_byte, _, _) = source.chunk_at_byte(byte);
1199 chunk[byte - start_byte..].as_bytes()
1207 .ok_or(Error::Cancelled)?;
1208 // unsafe { ts_parser.parser.set_cancellation_flag(None) };
1209 self.tree = Some(tree);
1214pub(crate) fn generate_edits(
1216 changeset: &ChangeSet,
1217) -> Vec<tree_sitter::InputEdit> {
1219 let mut old_pos = 0;
1221 let mut edits = Vec::new();
1223 if changeset.changes.is_empty() {
1227 let mut iter = changeset.changes.iter().peekable();
1229 // TODO; this is a lot easier with Change instead of Operation.
1231 fn point_at_pos(text: &Rope, pos: usize) -> (usize, Point) {
1232 let byte = text.char_to_byte(pos); // <- attempted to index past end
1233 let line = text.char_to_line(pos);
1234 let line_start_byte = text.line_to_byte(line);
1235 let col = byte - line_start_byte;
1237 (byte, Point::new(line, col))
1240 fn traverse(point: Point, text: &Tendril) -> Point {
1246 // TODO: there should be a better way here.
1247 let mut chars = text.chars().peekable();
1248 while let Some(ch) = chars.next() {
1249 if char_is_line_ending(ch) && !(ch == '\r' && chars.peek() == Some(&'\n')) {
1256 Point { row, column }
1259 while let Some(change) = iter.next() {
1260 let len = match change {
1261 Delete(i) | Retain(i) => *i,
1264 let mut old_end = old_pos + len;
1269 let (start_byte, start_position) = point_at_pos(old_text, old_pos);
1270 let (old_end_byte, old_end_position) = point_at_pos(old_text, old_end);
1273 edits.push(tree_sitter::InputEdit {
1274 start_byte, // old_pos to byte
1275 old_end_byte, // old_end to byte
1276 new_end_byte: start_byte, // old_pos to byte
1277 start_position, // old pos to coords
1278 old_end_position, // old_end to coords
1279 new_end_position: start_position, // old pos to coords
1283 let (start_byte, start_position) = point_at_pos(old_text, old_pos);
1285 // a subsequent delete means a replace, consume it
1286 if let Some(Delete(len)) = iter.peek() {
1287 old_end = old_pos + len;
1288 let (old_end_byte, old_end_position) = point_at_pos(old_text, old_end);
1293 edits.push(tree_sitter::InputEdit {
1294 start_byte, // old_pos to byte
1295 old_end_byte, // old_end to byte
1296 new_end_byte: start_byte + s.len(), // old_pos to byte + s.len()
1297 start_position, // old pos to coords
1298 old_end_position, // old_end to coords
1299 new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)
1303 edits.push(tree_sitter::InputEdit {
1304 start_byte, // old_pos to byte
1305 old_end_byte: start_byte, // same
1306 new_end_byte: start_byte + s.len(), // old_pos + s.len()
1307 start_position, // old pos to coords
1308 old_end_position: start_position, // same
1309 new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)
1319use std::sync::atomic::{AtomicUsize, Ordering};
1320use std::{iter, mem, ops, str, usize};
1322 Language as Grammar, Node, Parser, Point, Query, QueryCaptures, QueryCursor, QueryError,
1323 QueryMatch, Range, TextProvider, Tree, TreeCursor,
1326const CANCELLATION_CHECK_INTERVAL: usize = 100;
1328/// Indicates which highlight should be applied to a region of source code.
1329#[derive(Copy, Clone, Debug, PartialEq, Eq)]
1330pub struct Highlight(pub usize);
1332/// Represents the reason why syntax highlighting failed.
1333#[derive(Debug, PartialEq, Eq)]
1341/// Represents a single step in rendering a syntax-highlighted document.
1342#[derive(Copy, Clone, Debug)]
1343pub enum HighlightEvent {
1344 Source { start: usize, end: usize },
1345 HighlightStart(Highlight),
1349/// Contains the data needed to highlight code written in a particular language.
1351/// This struct is immutable and can be shared between threads.
1353pub struct HighlightConfiguration {
1354 pub language: Grammar,
1356 injections_query: Query,
1357 combined_injections_query: Option<Query>,
1358 highlights_pattern_index: usize,
1359 highlight_indices: ArcSwap<Vec<Option<Highlight>>>,
1360 non_local_variable_patterns: Vec<bool>,
1361 injection_content_capture_index: Option<u32>,
1362 injection_language_capture_index: Option<u32>,
1363 local_scope_capture_index: Option<u32>,
1364 local_def_capture_index: Option<u32>,
1365 local_def_value_capture_index: Option<u32>,
1366 local_ref_capture_index: Option<u32>,
1370struct LocalDef<'a> {
1372 value_range: ops::Range<usize>,
1373 highlight: Option<Highlight>,
1377struct LocalScope<'a> {
1379 range: ops::Range<usize>,
1380 local_defs: Vec<LocalDef<'a>>,
1384struct HighlightIter<'a> {
1385 source: RopeSlice<'a>,
1387 cancellation_flag: Option<&'a AtomicUsize>,
1388 layers: Vec<HighlightIterLayer<'a>>,
1390 next_event: Option<HighlightEvent>,
1391 last_highlight_range: Option<(usize, usize, u32)>,
1394// Adapter to convert rope chunks to bytes
1395pub struct ChunksBytes<'a> {
1396 chunks: ropey::iter::Chunks<'a>,
1398impl<'a> Iterator for ChunksBytes<'a> {
1399 type Item = &'a [u8];
1400 fn next(&mut self) -> Option<Self::Item> {
1401 self.chunks.next().map(str::as_bytes)
1405pub struct RopeProvider<'a>(pub RopeSlice<'a>);
1406impl<'a> TextProvider<'a> for RopeProvider<'a> {
1407 type I = ChunksBytes<'a>;
1409 fn text(&mut self, node: Node) -> Self::I {
1410 let fragment = self.0.byte_slice(node.start_byte()..node.end_byte());
1412 chunks: fragment.chunks(),
1417struct HighlightIterLayer<'a> {
1418 _tree: Option<Tree>,
1419 cursor: QueryCursor,
1420 captures: RefCell<iter::Peekable<QueryCaptures<'a, 'a, RopeProvider<'a>>>>,
1421 config: &'a HighlightConfiguration,
1422 highlight_end_stack: Vec<usize>,
1423 scope_stack: Vec<LocalScope<'a>>,
1427impl<'a> fmt::Debug for HighlightIterLayer<'a> {
1428 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1429 f.debug_struct("HighlightIterLayer").finish()
1433impl HighlightConfiguration {
1434 /// Creates a `HighlightConfiguration` for a given `Grammar` and set of highlighting
1439 /// * `language` - The Tree-sitter `Grammar` that should be used for parsing.
1440 /// * `highlights_query` - A string containing tree patterns for syntax highlighting. This
1441 /// should be non-empty, otherwise no syntax highlights will be added.
1442 /// * `injections_query` - A string containing tree patterns for injecting other languages
1443 /// into the document. This can be empty if no injections are desired.
1444 /// * `locals_query` - A string containing tree patterns for tracking local variable
1445 /// definitions and references. This can be empty if local variable tracking is not needed.
1447 /// Returns a `HighlightConfiguration` that can then be used with the `highlight` method.
1450 highlights_query: &str,
1451 injection_query: &str,
1453 ) -> Result<Self, QueryError> {
1454 // Concatenate the query strings, keeping track of the start offset of each section.
1455 let mut query_source = String::new();
1456 query_source.push_str(locals_query);
1457 let highlights_query_offset = query_source.len();
1458 query_source.push_str(highlights_query);
1460 // Construct a single query by concatenating the three query strings, but record the
1461 // range of pattern indices that belong to each individual string.
1462 let query = Query::new(language, &query_source)?;
1463 let mut highlights_pattern_index = 0;
1464 for i in 0..(query.pattern_count()) {
1465 let pattern_offset = query.start_byte_for_pattern(i);
1466 if pattern_offset < highlights_query_offset {
1467 highlights_pattern_index += 1;
1471 let mut injections_query = Query::new(language, injection_query)?;
1473 // Construct a separate query just for dealing with the 'combined injections'.
1474 // Disable the combined injection patterns in the main query.
1475 let mut combined_injections_query = Query::new(language, injection_query)?;
1476 let mut has_combined_queries = false;
1477 for pattern_index in 0..injections_query.pattern_count() {
1478 let settings = injections_query.property_settings(pattern_index);
1479 if settings.iter().any(|s| &*s.key == "injection.combined") {
1480 has_combined_queries = true;
1481 injections_query.disable_pattern(pattern_index);
1483 combined_injections_query.disable_pattern(pattern_index);
1486 let combined_injections_query = if has_combined_queries {
1487 Some(combined_injections_query)
1492 // Find all of the highlighting patterns that are disabled for nodes that
1493 // have been identified as local variables.
1494 let non_local_variable_patterns = (0..query.pattern_count())
1497 .property_predicates(i)
1499 .any(|(prop, positive)| !*positive && prop.key.as_ref() == "local")
1503 // Store the numeric ids for all of the special captures.
1504 let mut injection_content_capture_index = None;
1505 let mut injection_language_capture_index = None;
1506 let mut local_def_capture_index = None;
1507 let mut local_def_value_capture_index = None;
1508 let mut local_ref_capture_index = None;
1509 let mut local_scope_capture_index = None;
1510 for (i, name) in query.capture_names().iter().enumerate() {
1511 let i = Some(i as u32);
1512 match name.as_str() {
1513 "local.definition" => local_def_capture_index = i,
1514 "local.definition-value" => local_def_value_capture_index = i,
1515 "local.reference" => local_ref_capture_index = i,
1516 "local.scope" => local_scope_capture_index = i,
1521 for (i, name) in injections_query.capture_names().iter().enumerate() {
1522 let i = Some(i as u32);
1523 match name.as_str() {
1524 "injection.content" => injection_content_capture_index = i,
1525 "injection.language" => injection_language_capture_index = i,
1530 let highlight_indices = ArcSwap::from_pointee(vec![None; query.capture_names().len()]);
1535 combined_injections_query,
1536 highlights_pattern_index,
1538 non_local_variable_patterns,
1539 injection_content_capture_index,
1540 injection_language_capture_index,
1541 local_scope_capture_index,
1542 local_def_capture_index,
1543 local_def_value_capture_index,
1544 local_ref_capture_index,
1548 /// Get a slice containing all of the highlight names used in the configuration.
1549 pub fn names(&self) -> &[String] {
1550 self.query.capture_names()
1553 /// Set the list of recognized highlight names.
1555 /// Tree-sitter syntax-highlighting queries specify highlights in the form of dot-separated
1556 /// highlight names like `punctuation.bracket` and `function.method.builtin`. Consumers of
1557 /// these queries can choose to recognize highlights with different levels of specificity.
1558 /// For example, the string `function.builtin` will match against `function.builtin.constructor`
1559 /// but will not match `function.method.builtin` and `function.method`.
1561 /// When highlighting, results are returned as `Highlight` values, which contain the index
1562 /// of the matched highlight this list of highlight names.
1563 pub fn configure(&self, recognized_names: &[String]) {
1564 let mut capture_parts = Vec::new();
1565 let indices: Vec<_> = self
1569 .map(move |capture_name| {
1570 capture_parts.clear();
1571 capture_parts.extend(capture_name.split('.'));
1573 let mut best_index = None;
1574 let mut best_match_len = 0;
1575 for (i, recognized_name) in recognized_names.iter().enumerate() {
1576 let recognized_name = recognized_name;
1578 let mut matches = true;
1579 for (i, part) in recognized_name.split('.').enumerate() {
1580 match capture_parts.get(i) {
1581 Some(capture_part) if *capture_part == part => len += 1,
1588 if matches && len > best_match_len {
1589 best_index = Some(i);
1590 best_match_len = len;
1593 best_index.map(Highlight)
1597 self.highlight_indices.store(Arc::new(indices));
1601impl<'a> HighlightIterLayer<'a> {
1602 // First, sort scope boundaries by their byte offset in the document. At a
1603 // given position, emit scope endings before scope beginnings. Finally, emit
1604 // scope boundaries from deeper layers first.
1605 fn sort_key(&self) -> Option<(usize, bool, isize)> {
1606 let depth = -(self.depth as isize);
1607 let next_start = self
1611 .map(|(m, i)| m.captures[*i].node.start_byte());
1612 let next_end = self.highlight_end_stack.last().cloned();
1613 match (next_start, next_end) {
1614 (Some(start), Some(end)) => {
1616 Some((start, true, depth))
1618 Some((end, false, depth))
1621 (Some(i), None) => Some((i, true, depth)),
1622 (None, Some(j)) => Some((j, false, depth)),
1629enum IncludedChildren {
1635impl Default for IncludedChildren {
1636 fn default() -> Self {
1641// Compute the ranges that should be included when parsing an injection.
1642// This takes into account three things:
1643// * `parent_ranges` - The ranges must all fall within the *current* layer's ranges.
1644// * `nodes` - Every injection takes place within a set of nodes. The injection ranges
1645// are the ranges of those nodes.
1646// * `includes_children` - For some injections, the content nodes' children should be
1647// excluded from the nested document, so that only the content nodes' *own* content
1648// is reparsed. For other injections, the content nodes' entire ranges should be
1649// reparsed, including the ranges of their children.
1651 parent_ranges: &[Range],
1653 included_children: IncludedChildren,
1655 let mut cursor = nodes[0].walk();
1656 let mut result = Vec::new();
1657 let mut parent_range_iter = parent_ranges.iter();
1658 let mut parent_range = parent_range_iter
1660 .expect("Layers should only be constructed with non-empty ranges vectors");
1661 for node in nodes.iter() {
1662 let mut preceding_range = Range {
1664 start_point: Point::new(0, 0),
1665 end_byte: node.start_byte(),
1666 end_point: node.start_position(),
1668 let following_range = Range {
1669 start_byte: node.end_byte(),
1670 start_point: node.end_position(),
1671 end_byte: usize::MAX,
1672 end_point: Point::new(usize::MAX, usize::MAX),
1675 for excluded_range in node
1676 .children(&mut cursor)
1677 .filter_map(|child| match included_children {
1678 IncludedChildren::None => Some(child.range()),
1679 IncludedChildren::All => None,
1680 IncludedChildren::Unnamed => {
1681 if child.is_named() {
1688 .chain([following_range].iter().cloned())
1690 let mut range = Range {
1691 start_byte: preceding_range.end_byte,
1692 start_point: preceding_range.end_point,
1693 end_byte: excluded_range.start_byte,
1694 end_point: excluded_range.start_point,
1696 preceding_range = excluded_range;
1698 if range.end_byte < parent_range.start_byte {
1702 while parent_range.start_byte <= range.end_byte {
1703 if parent_range.end_byte > range.start_byte {
1704 if range.start_byte < parent_range.start_byte {
1705 range.start_byte = parent_range.start_byte;
1706 range.start_point = parent_range.start_point;
1709 if parent_range.end_byte < range.end_byte {
1710 if range.start_byte < parent_range.end_byte {
1712 start_byte: range.start_byte,
1713 start_point: range.start_point,
1714 end_byte: parent_range.end_byte,
1715 end_point: parent_range.end_point,
1718 range.start_byte = parent_range.end_byte;
1719 range.start_point = parent_range.end_point;
1721 if range.start_byte < range.end_byte {
1728 if let Some(next_range) = parent_range_iter.next() {
1729 parent_range = next_range;
1739impl<'a> HighlightIter<'a> {
1743 event: Option<HighlightEvent>,
1744 ) -> Option<Result<HighlightEvent, Error>> {
1746 if self.byte_offset < offset {
1747 result = Some(Ok(HighlightEvent::Source {
1748 start: self.byte_offset,
1751 self.byte_offset = offset;
1752 self.next_event = event;
1754 result = event.map(Ok);
1760 fn sort_layers(&mut self) {
1761 while !self.layers.is_empty() {
1762 if let Some(sort_key) = self.layers[0].sort_key() {
1764 while i + 1 < self.layers.len() {
1765 if let Some(next_offset) = self.layers[i + 1].sort_key() {
1766 if next_offset < sort_key {
1771 let layer = self.layers.remove(i + 1);
1772 PARSER.with(|ts_parser| {
1773 let highlighter = &mut ts_parser.borrow_mut();
1774 highlighter.cursors.push(layer.cursor);
1780 self.layers[0..(i + 1)].rotate_left(1);
1784 let layer = self.layers.remove(0);
1785 PARSER.with(|ts_parser| {
1786 let highlighter = &mut ts_parser.borrow_mut();
1787 highlighter.cursors.push(layer.cursor);
1794impl<'a> Iterator for HighlightIter<'a> {
1795 type Item = Result<HighlightEvent, Error>;
1797 fn next(&mut self) -> Option<Self::Item> {
1799 // If we've already determined the next highlight boundary, just return it.
1800 if let Some(e) = self.next_event.take() {
1804 // Periodically check for cancellation, returning `Cancelled` error if the
1805 // cancellation flag was flipped.
1806 if let Some(cancellation_flag) = self.cancellation_flag {
1807 self.iter_count += 1;
1808 if self.iter_count >= CANCELLATION_CHECK_INTERVAL {
1809 self.iter_count = 0;
1810 if cancellation_flag.load(Ordering::Relaxed) != 0 {
1811 return Some(Err(Error::Cancelled));
1816 // If none of the layers have any more highlight boundaries, terminate.
1817 if self.layers.is_empty() {
1818 let len = self.source.len_bytes();
1819 return if self.byte_offset < len {
1820 let result = Some(Ok(HighlightEvent::Source {
1821 start: self.byte_offset,
1824 self.byte_offset = len;
1831 // Get the next capture from whichever layer has the earliest highlight boundary.
1833 let layer = &mut self.layers[0];
1834 let captures = layer.captures.get_mut();
1835 if let Some((next_match, capture_index)) = captures.peek() {
1836 let next_capture = next_match.captures[*capture_index];
1837 range = next_capture.node.byte_range();
1839 // If any previous highlight ends before this node starts, then before
1840 // processing this capture, emit the source code up until the end of the
1841 // previous highlight, and an end event for that highlight.
1842 if let Some(end_byte) = layer.highlight_end_stack.last().cloned() {
1843 if end_byte <= range.start {
1844 layer.highlight_end_stack.pop();
1845 return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd));
1849 // If there are no more captures, then emit any remaining highlight end events.
1850 // And if there are none of those, then just advance to the end of the document.
1851 else if let Some(end_byte) = layer.highlight_end_stack.last().cloned() {
1852 layer.highlight_end_stack.pop();
1853 return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd));
1855 return self.emit_event(self.source.len_bytes(), None);
1858 let (mut match_, capture_index) = captures.next().unwrap();
1859 let mut capture = match_.captures[capture_index];
1861 // Remove from the local scope stack any local scopes that have already ended.
1862 while range.start > layer.scope_stack.last().unwrap().range.end {
1863 layer.scope_stack.pop();
1866 // If this capture is for tracking local variables, then process the
1867 // local variable info.
1868 let mut reference_highlight = None;
1869 let mut definition_highlight = None;
1870 while match_.pattern_index < layer.config.highlights_pattern_index {
1871 // If the node represents a local scope, push a new local scope onto
1873 if Some(capture.index) == layer.config.local_scope_capture_index {
1874 definition_highlight = None;
1875 let mut scope = LocalScope {
1877 range: range.clone(),
1878 local_defs: Vec::new(),
1880 for prop in layer.config.query.property_settings(match_.pattern_index) {
1881 if let "local.scope-inherits" = prop.key.as_ref() {
1883 prop.value.as_ref().map_or(true, |r| r.as_ref() == "true");
1886 layer.scope_stack.push(scope);
1888 // If the node represents a definition, add a new definition to the
1889 // local scope at the top of the scope stack.
1890 else if Some(capture.index) == layer.config.local_def_capture_index {
1891 reference_highlight = None;
1892 let scope = layer.scope_stack.last_mut().unwrap();
1894 let mut value_range = 0..0;
1895 for capture in match_.captures {
1896 if Some(capture.index) == layer.config.local_def_value_capture_index {
1897 value_range = capture.node.byte_range();
1901 let name = byte_range_to_str(range.clone(), self.source);
1902 scope.local_defs.push(LocalDef {
1907 definition_highlight = scope.local_defs.last_mut().map(|s| &mut s.highlight);
1909 // If the node represents a reference, then try to find the corresponding
1910 // definition in the scope stack.
1911 else if Some(capture.index) == layer.config.local_ref_capture_index
1912 && definition_highlight.is_none()
1914 definition_highlight = None;
1915 let name = byte_range_to_str(range.clone(), self.source);
1916 for scope in layer.scope_stack.iter().rev() {
1917 if let Some(highlight) = scope.local_defs.iter().rev().find_map(|def| {
1918 if def.name == name && range.start >= def.value_range.end {
1924 reference_highlight = highlight;
1927 if !scope.inherits {
1933 // Continue processing any additional matches for the same node.
1934 if let Some((next_match, next_capture_index)) = captures.peek() {
1935 let next_capture = next_match.captures[*next_capture_index];
1936 if next_capture.node == capture.node {
1937 capture = next_capture;
1938 match_ = captures.next().unwrap().0;
1947 // Otherwise, this capture must represent a highlight.
1948 // If this exact range has already been highlighted by an earlier pattern, or by
1949 // a different layer, then skip over this one.
1950 if let Some((last_start, last_end, last_depth)) = self.last_highlight_range {
1951 if range.start == last_start && range.end == last_end && layer.depth < last_depth {
1957 // If the current node was found to be a local variable, then skip over any
1958 // highlighting patterns that are disabled for local variables.
1959 if definition_highlight.is_some() || reference_highlight.is_some() {
1960 while layer.config.non_local_variable_patterns[match_.pattern_index] {
1961 if let Some((next_match, next_capture_index)) = captures.peek() {
1962 let next_capture = next_match.captures[*next_capture_index];
1963 if next_capture.node == capture.node {
1964 capture = next_capture;
1965 match_ = captures.next().unwrap().0;
1975 // Once a highlighting pattern is found for the current node, skip over
1976 // any later highlighting patterns that also match this node. Captures
1977 // for a given node are ordered by pattern index, so these subsequent
1978 // captures are guaranteed to be for highlighting, not injections or
1980 while let Some((next_match, next_capture_index)) = captures.peek() {
1981 let next_capture = next_match.captures[*next_capture_index];
1982 if next_capture.node == capture.node {
1989 let current_highlight = layer.config.highlight_indices.load()[capture.index as usize];
1991 // If this node represents a local definition, then store the current
1992 // highlight value on the local scope entry representing this node.
1993 if let Some(definition_highlight) = definition_highlight {
1994 *definition_highlight = current_highlight;
1997 // Emit a scope start event and push the node's end position to the stack.
1998 if let Some(highlight) = reference_highlight.or(current_highlight) {
1999 self.last_highlight_range = Some((range.start, range.end, layer.depth));
2000 layer.highlight_end_stack.push(range.end);
2002 .emit_event(range.start, Some(HighlightEvent::HighlightStart(highlight)));
2010fn injection_for_match<'a>(
2011 config: &HighlightConfiguration,
2013 query_match: &QueryMatch<'a, 'a>,
2014 source: RopeSlice<'a>,
2015) -> (Option<Cow<'a, str>>, Option<Node<'a>>, IncludedChildren) {
2016 let content_capture_index = config.injection_content_capture_index;
2017 let language_capture_index = config.injection_language_capture_index;
2019 let mut language_name = None;
2020 let mut content_node = None;
2021 for capture in query_match.captures {
2022 let index = Some(capture.index);
2023 if index == language_capture_index {
2024 let name = byte_range_to_str(capture.node.byte_range(), source);
2025 language_name = Some(name);
2026 } else if index == content_capture_index {
2027 content_node = Some(capture.node);
2031 let mut included_children = IncludedChildren::default();
2032 for prop in query.property_settings(query_match.pattern_index) {
2033 match prop.key.as_ref() {
2034 // In addition to specifying the language name via the text of a
2035 // captured node, it can also be hard-coded via a `#set!` predicate
2036 // that sets the injection.language key.
2037 "injection.language" => {
2038 if language_name.is_none() {
2039 language_name = prop.value.as_ref().map(|s| s.as_ref().into())
2043 // By default, injections do not include the *children* of an
2044 // `injection.content` node - only the ranges that belong to the
2045 // node itself. This can be changed using a `#set!` predicate that
2046 // sets the `injection.include-children` key.
2047 "injection.include-children" => included_children = IncludedChildren::All,
2049 // Some queries might only exclude named children but include unnamed
2050 // children in their `injection.content` node. This can be enabled using
2051 // a `#set!` predicate that sets the `injection.include-unnamed-children` key.
2052 "injection.include-unnamed-children" => included_children = IncludedChildren::Unnamed,
2057 (language_name, content_node, included_children)
2060pub struct Merge<I> {
2062 spans: Box<dyn Iterator<Item = (usize, std::ops::Range<usize>)>>,
2064 next_event: Option<HighlightEvent>,
2065 next_span: Option<(usize, std::ops::Range<usize>)>,
2067 queue: Vec<HighlightEvent>,
2070/// Merge a list of spans into the highlight event stream.
2071pub fn merge<I: Iterator<Item = HighlightEvent>>(
2073 spans: Vec<(usize, std::ops::Range<usize>)>,
2075 let spans = Box::new(spans.into_iter());
2076 let mut merge = Merge {
2083 merge.next_event = merge.iter.next();
2084 merge.next_span = merge.spans.next();
2088impl<I: Iterator<Item = HighlightEvent>> Iterator for Merge<I> {
2089 type Item = HighlightEvent;
2090 fn next(&mut self) -> Option<Self::Item> {
2091 use HighlightEvent::*;
2092 if let Some(event) = self.queue.pop() {
2097 match (self.next_event, &self.next_span) {
2098 // this happens when range is partially or fully offscreen
2099 (Some(Source { start, .. }), Some((span, range))) if start > range.start => {
2100 if start > range.end {
2101 self.next_span = self.spans.next();
2103 self.next_span = Some((*span, start..range.end));
2110 match (self.next_event, &self.next_span) {
2111 (Some(HighlightStart(i)), _) => {
2112 self.next_event = self.iter.next();
2113 Some(HighlightStart(i))
2115 (Some(HighlightEnd), _) => {
2116 self.next_event = self.iter.next();
2119 (Some(Source { start, end }), Some((_, range))) if start < range.start => {
2120 let intersect = range.start.min(end);
2121 let event = Source {
2126 if end == intersect {
2127 // the event is complete
2128 self.next_event = self.iter.next();
2130 // subslice the event
2131 self.next_event = Some(Source {
2139 (Some(Source { start, end }), Some((span, range))) if start == range.start => {
2140 let intersect = range.end.min(end);
2141 let event = HighlightStart(Highlight(*span));
2143 // enqueue in reverse order
2144 self.queue.push(HighlightEnd);
2145 self.queue.push(Source {
2150 if end == intersect {
2151 // the event is complete
2152 self.next_event = self.iter.next();
2154 // subslice the event
2155 self.next_event = Some(Source {
2161 if intersect == range.end {
2162 self.next_span = self.spans.next();
2164 self.next_span = Some((*span, intersect..range.end));
2169 (Some(event), None) => {
2170 self.next_event = self.iter.next();
2173 // Can happen if cursor at EOF and/or diagnostic reaches past the end.
2174 // We need to actually emit events for the cursor-at-EOF situation,
2175 // even though the range is past the end of the text. This needs to be
2176 // handled appropriately by the drawing code by not assuming that
2177 // all `Source` events point to valid indices in the rope.
2178 (None, Some((span, range))) => {
2179 let event = HighlightStart(Highlight(*span));
2180 self.queue.push(HighlightEnd);
2181 self.queue.push(Source {
2185 self.next_span = self.spans.next();
2188 (None, None) => None,
2189 e => unreachable!("{:?}", e),
2194fn node_is_visible(node: &Node) -> bool {
2195 node.is_missing() || (node.is_named() && node.language().node_kind_is_visible(node.kind_id()))
2198pub fn pretty_print_tree<W: fmt::Write>(fmt: &mut W, node: Node) -> fmt::Result {
2199 if node.child_count() == 0 {
2200 if node_is_visible(&node) {
2201 write!(fmt, "({})", node.kind())
2203 write!(fmt, "\"{}\"", node.kind())
2206 pretty_print_tree_impl(fmt, &mut node.walk(), 0)
2210fn pretty_print_tree_impl<W: fmt::Write>(
2212 cursor: &mut TreeCursor,
2215 let node = cursor.node();
2216 let visible = node_is_visible(&node);
2219 let indentation_columns = depth * 2;
2220 write!(fmt, "{:indentation_columns$}", "")?;
2222 if let Some(field_name) = cursor.field_name() {
2223 write!(fmt, "{}: ", field_name)?;
2226 write!(fmt, "({}", node.kind())?;
2230 if cursor.goto_first_child() {
2232 if node_is_visible(&cursor.node()) {
2233 fmt.write_char('\n')?;
2236 pretty_print_tree_impl(fmt, cursor, depth + 1)?;
2238 if !cursor.goto_next_sibling() {
2243 let moved = cursor.goto_parent();
2244 // The parent of the first child must exist, and must be `node`.
2245 debug_assert!(moved);
2246 debug_assert!(cursor.node() == node);
2250 fmt.write_char(')')?;
2259 use crate::{Rope, Transaction};
2262 fn test_textobject_queries() {
2264 (line_comment)+ @quantified_nodes
2265 ((line_comment)+) @quantified_nodes_grouped
2266 ((line_comment) (line_comment)) @multiple_nodes_grouped
2268 let source = Rope::from_str(
2275 let loader = Loader::new(Configuration { language: vec![] });
2276 let language = get_language("rust").unwrap();
2278 let query = Query::new(language, query_str).unwrap();
2279 let textobject = TextObjectQuery { query };
2280 let mut cursor = QueryCursor::new();
2282 let config = HighlightConfiguration::new(language, "", "", "").unwrap();
2283 let syntax = Syntax::new(&source, Arc::new(config), Arc::new(loader));
2285 let root = syntax.tree().root_node();
2286 let mut test = |capture, range| {
2287 let matches: Vec<_> = textobject
2288 .capture_nodes(capture, root, source.slice(..), &mut cursor)
2293 matches[0].byte_range(),
2295 "@{} expected {:?}",
2301 test("quantified_nodes", 1..36);
2302 // NOTE: Enable after implementing proper node group capturing
2303 // test("quantified_nodes_grouped", 1..36);
2304 // test("multiple_nodes_grouped", 1..36);
2309 let highlight_names: Vec<String> = [
2318 "punctuation.bracket",
2319 "punctuation.delimiter",
2327 "variable.parameter",
2334 let loader = Loader::new(Configuration { language: vec![] });
2336 let language = get_language("rust").unwrap();
2337 let config = HighlightConfiguration::new(
2339 &std::fs::read_to_string("../runtime/grammars/sources/rust/queries/highlights.scm")
2341 &std::fs::read_to_string("../runtime/grammars/sources/rust/queries/injections.scm")
2346 config.configure(&highlight_names);
2348 let source = Rope::from_str(
2354 let syntax = Syntax::new(&source, Arc::new(config), Arc::new(loader));
2355 let tree = syntax.tree();
2356 let root = tree.root_node();
2357 assert_eq!(root.kind(), "source_file");
2363 "(struct_item name: (type_identifier) body: (field_declaration_list)) ",
2364 "(function_item name: (identifier) parameters: (parameters) body: (block)))"
2368 let struct_node = root.child(0).unwrap();
2369 assert_eq!(struct_node.kind(), "struct_item");
2373 fn test_input_edits() {
2374 use tree_sitter::InputEdit;
2376 let doc = Rope::from("hello world!\ntest 123");
2377 let transaction = Transaction::change(
2379 vec![(6, 11, Some("test".into())), (12, 17, None)].into_iter(),
2381 let edits = generate_edits(&doc, transaction.changes());
2382 // transaction.apply(&mut state);
2391 start_position: Point { row: 0, column: 6 },
2392 old_end_position: Point { row: 0, column: 11 },
2393 new_end_position: Point { row: 0, column: 10 }
2399 start_position: Point { row: 0, column: 12 },
2400 old_end_position: Point { row: 1, column: 4 },
2401 new_end_position: Point { row: 0, column: 12 }
2406 // Testing with the official example from tree-sitter
2407 let mut doc = Rope::from("fn test() {}");
2409 Transaction::change(&doc, vec![(8, 8, Some("a: u32".into()))].into_iter());
2410 let edits = generate_edits(&doc, transaction.changes());
2411 transaction.apply(&mut doc);
2413 assert_eq!(doc, "fn test(a: u32) {}");
2420 start_position: Point { row: 0, column: 8 },
2421 old_end_position: Point { row: 0, column: 8 },
2422 new_end_position: Point { row: 0, column: 14 }
2428 fn assert_pretty_print(
2429 language_name: &str,
2435 let source = Rope::from_str(source);
2437 let loader = Loader::new(Configuration { language: vec![] });
2438 let language = get_language(language_name).unwrap();
2440 let config = HighlightConfiguration::new(language, "", "", "").unwrap();
2441 let syntax = Syntax::new(&source, Arc::new(config), Arc::new(loader));
2446 .descendant_for_byte_range(start, end)
2449 let mut output = String::new();
2450 pretty_print_tree(&mut output, root).unwrap();
2452 assert_eq!(expected, output);
2456 fn test_pretty_print() {
2457 let source = r#"/// Hello"#;
2458 assert_pretty_print("rust", source, "(line_comment)", 0, source.len());
2460 // A large tree should be indented with fields:
2461 let source = r#"fn main() {
2462 println!("Hello, World!");
2464 assert_pretty_print(
2469 " name: (identifier)\n",
2470 " parameters: (parameters)\n",
2472 " (expression_statement\n",
2473 " (macro_invocation\n",
2474 " macro: (identifier)\n",
2476 " (string_literal))))))",
2482 // Selecting a token should print just that token:
2483 let source = r#"fn main() {}"#;
2484 assert_pretty_print("rust", source, r#""fn""#, 0, 1);
2486 // Error nodes are printed as errors:
2487 let source = r#"}{"#;
2488 assert_pretty_print("rust", source, "(ERROR)", 0, source.len());
2490 // Fields broken under unnamed nodes are determined correctly.
2491 // In the following source, `object` belongs to the `singleton_method`
2492 // rule but `name` and `body` belong to an unnamed helper `_method_rest`.
2493 // This can cause a bug with a pretty-printing implementation that
2494 // uses `Node::field_name_for_child` to determine field names but is
2495 // fixed when using `TreeCursor::field_name`.
2496 let source = "def self.method_name
2499 assert_pretty_print(
2503 "(singleton_method\n",
2504 " object: (self)\n",
2505 " name: (identifier)\n",
2506 " body: (body_statement\n",
2515 fn test_load_runtime_file() {
2516 // Test to make sure we can load some data from the runtime directory.
2517 let contents = load_runtime_file("rust", "indents.scm").unwrap();
2518 assert!(!contents.is_empty());
2520 let results = load_runtime_file("rust", "does-not-exist");
2521 assert!(results.is_err());
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list