@@ -0,0 +1,993 @@
1+pub struct LanguageMode {
2+ parser: Parser,
3+}
4+
5+impl LanguageMode {
6+ // buffer, grammar, config, grammars, sync_timeout?
7+ pub fn new() -> Self {
8+ unimplemented!()
9+ // make a new root layer
10+ // track markers of injections
11+ //
12+ // track scope_descriptor: a Vec of scopes for item in tree
13+ //
14+ // fetch grammar for parser based on language string
15+ // update root layer
16+ }
17+
18+ // fn buffer_changed -> call layer.update(range, new_text) on root layer and then all marker layers
19+
20+ // call this on transaction.apply() -> buffer_changed(changes)
21+ //
22+ // fn parse(language, old_tree, ranges)
23+ //
24+ // fn tree() -> Tree
25+ //
26+ // <!--update_for_injection(grammar)-->
27+
28+ // Highlighting
29+ // fn highlight_iter() -> iterates over all the scopes
30+ // on_tokenize
31+ // on_change_highlighting
32+
33+ // Commenting
34+ // comment_strings_for_pos
35+ // is_commented
36+
37+ // Indentation
38+ // suggested_indent_for_line_at_buffer_row
39+ // suggested_indent_for_buffer_row
40+ // indent_level_for_line
41+
42+ // TODO: Folding
43+
44+ // Syntax APIs
45+ // get_syntax_node_containing_range ->
46+ // ...
47+ // get_syntax_node_at_pos
48+ // buffer_range_for_scope_at_pos
49+}
50+
51+pub struct LanguageLayer {
52+ // mode
53+// grammar
54+// depth
55+// tree: Tree,
56+}
57+
58+impl LanguageLayer {
59+ // fn highlight_iter() -> same as Mode but for this layer. Mode composits these
60+ // fn buffer_changed
61+ // fn update(range)
62+ // fn update_injections()
63+}
64+
65+// -- refactored from tree-sitter-highlight to be able to retain state
66+// TODO: add seek() to iter
67+
68+use std::sync::atomic::{AtomicUsize, Ordering};
69+use std::{iter, mem, ops, str, usize};
70+use tree_sitter::{
71+ Language, Node, Parser, Point, Query, QueryCaptures, QueryCursor, QueryError, QueryMatch,
72+ Range, Tree,
73+};
74+
75+const CANCELLATION_CHECK_INTERVAL: usize = 100;
76+const BUFFER_HTML_RESERVE_CAPACITY: usize = 10 * 1024;
77+const BUFFER_LINES_RESERVE_CAPACITY: usize = 1000;
78+
79+/// Indicates which highlight should be applied to a region of source code.
80+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
81+pub struct Highlight(pub usize);
82+
83+/// Represents the reason why syntax highlighting failed.
84+#[derive(Debug, PartialEq, Eq)]
85+pub enum Error {
86+ Cancelled,
87+ InvalidLanguage,
88+ Unknown,
89+}
90+
91+/// Represents a single step in rendering a syntax-highlighted document.
92+#[derive(Copy, Clone, Debug)]
93+pub enum HighlightEvent {
94+ Source { start: usize, end: usize },
95+ HighlightStart(Highlight),
96+ HighlightEnd,
97+}
98+
99+/// Contains the data neeeded to higlight code written in a particular language.
100+///
101+/// This struct is immutable and can be shared between threads.
102+pub struct HighlightConfiguration {
103+ pub language: Language,
104+ pub query: Query,
105+ combined_injections_query: Option<Query>,
106+ locals_pattern_index: usize,
107+ highlights_pattern_index: usize,
108+ highlight_indices: Vec<Option<Highlight>>,
109+ non_local_variable_patterns: Vec<bool>,
110+ injection_content_capture_index: Option<u32>,
111+ injection_language_capture_index: Option<u32>,
112+ local_scope_capture_index: Option<u32>,
113+ local_def_capture_index: Option<u32>,
114+ local_def_value_capture_index: Option<u32>,
115+ local_ref_capture_index: Option<u32>,
116+}
117+
118+/// Performs syntax highlighting, recognizing a given list of highlight names.
119+///
120+/// For the best performance `Highlighter` values should be reused between
121+/// syntax highlighting calls. A separate highlighter is needed for each thread that
122+/// is performing highlighting.
123+pub struct Highlighter {
124+ parser: Parser,
125+ cursors: Vec<QueryCursor>,
126+}
127+
128+#[derive(Debug)]
129+struct LocalDef<'a> {
130+ name: &'a str,
131+ value_range: ops::Range<usize>,
132+ highlight: Option<Highlight>,
133+}
134+
135+#[derive(Debug)]
136+struct LocalScope<'a> {
137+ inherits: bool,
138+ range: ops::Range<usize>,
139+ local_defs: Vec<LocalDef<'a>>,
140+}
141+
142+struct HighlightIter<'a, F>
143+where
144+ F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
145+{
146+ source: &'a [u8],
147+ byte_offset: usize,
148+ highlighter: &'a mut Highlighter,
149+ injection_callback: F,
150+ cancellation_flag: Option<&'a AtomicUsize>,
151+ layers: Vec<HighlightIterLayer<'a>>,
152+ iter_count: usize,
153+ next_event: Option<HighlightEvent>,
154+ last_highlight_range: Option<(usize, usize, usize)>,
155+}
156+
157+struct HighlightIterLayer<'a> {
158+ _tree: Tree,
159+ cursor: QueryCursor,
160+ captures: iter::Peekable<QueryCaptures<'a, &'a [u8]>>,
161+ config: &'a HighlightConfiguration,
162+ highlight_end_stack: Vec<usize>,
163+ scope_stack: Vec<LocalScope<'a>>,
164+ ranges: Vec<Range>,
165+ depth: usize,
166+}
167+
168+impl Highlighter {
169+ pub fn new() -> Self {
170+ Highlighter {
171+ parser: Parser::new(),
172+ cursors: Vec::new(),
173+ }
174+ }
175+
176+ pub fn parser(&mut self) -> &mut Parser {
177+ &mut self.parser
178+ }
179+
180+ /// Iterate over the highlighted regions for a given slice of source code.
181+ pub fn highlight<'a>(
182+ &'a mut self,
183+ config: &'a HighlightConfiguration,
184+ source: &'a [u8],
185+ cancellation_flag: Option<&'a AtomicUsize>,
186+ mut injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
187+ ) -> Result<impl Iterator<Item = Result<HighlightEvent, Error>> + 'a, Error> {
188+ let layers = HighlightIterLayer::new(
189+ source,
190+ self,
191+ cancellation_flag,
192+ &mut injection_callback,
193+ config,
194+ 0,
195+ vec![Range {
196+ start_byte: 0,
197+ end_byte: usize::MAX,
198+ start_point: Point::new(0, 0),
199+ end_point: Point::new(usize::MAX, usize::MAX),
200+ }],
201+ )?;
202+ assert_ne!(layers.len(), 0);
203+ let mut result = HighlightIter {
204+ source,
205+ byte_offset: 0,
206+ injection_callback,
207+ cancellation_flag,
208+ highlighter: self,
209+ iter_count: 0,
210+ layers: layers,
211+ next_event: None,
212+ last_highlight_range: None,
213+ };
214+ result.sort_layers();
215+ Ok(result)
216+ }
217+}
218+
219+impl HighlightConfiguration {
220+ /// Creates a `HighlightConfiguration` for a given `Language` and set of highlighting
221+ /// queries.
222+ ///
223+ /// # Parameters
224+ ///
225+ /// * `language` - The Tree-sitter `Language` that should be used for parsing.
226+ /// * `highlights_query` - A string containing tree patterns for syntax highlighting. This
227+ /// should be non-empty, otherwise no syntax highlights will be added.
228+ /// * `injections_query` - A string containing tree patterns for injecting other languages
229+ /// into the document. This can be empty if no injections are desired.
230+ /// * `locals_query` - A string containing tree patterns for tracking local variable
231+ /// definitions and references. This can be empty if local variable tracking is not needed.
232+ ///
233+ /// Returns a `HighlightConfiguration` that can then be used with the `highlight` method.
234+ pub fn new(
235+ language: Language,
236+ highlights_query: &str,
237+ injection_query: &str,
238+ locals_query: &str,
239+ ) -> Result<Self, QueryError> {
240+ // Concatenate the query strings, keeping track of the start offset of each section.
241+ let mut query_source = String::new();
242+ query_source.push_str(injection_query);
243+ let locals_query_offset = query_source.len();
244+ query_source.push_str(locals_query);
245+ let highlights_query_offset = query_source.len();
246+ query_source.push_str(highlights_query);
247+
248+ // Construct a single query by concatenating the three query strings, but record the
249+ // range of pattern indices that belong to each individual string.
250+ let mut query = Query::new(language, &query_source)?;
251+ let mut locals_pattern_index = 0;
252+ let mut highlights_pattern_index = 0;
253+ for i in 0..(query.pattern_count()) {
254+ let pattern_offset = query.start_byte_for_pattern(i);
255+ if pattern_offset < highlights_query_offset {
256+ if pattern_offset < highlights_query_offset {
257+ highlights_pattern_index += 1;
258+ }
259+ if pattern_offset < locals_query_offset {
260+ locals_pattern_index += 1;
261+ }
262+ }
263+ }
264+
265+ // Construct a separate query just for dealing with the 'combined injections'.
266+ // Disable the combined injection patterns in the main query.
267+ let mut combined_injections_query = Query::new(language, injection_query)?;
268+ let mut has_combined_queries = false;
269+ for pattern_index in 0..locals_pattern_index {
270+ let settings = query.property_settings(pattern_index);
271+ if settings.iter().any(|s| &*s.key == "injection.combined") {
272+ has_combined_queries = true;
273+ query.disable_pattern(pattern_index);
274+ } else {
275+ combined_injections_query.disable_pattern(pattern_index);
276+ }
277+ }
278+ let combined_injections_query = if has_combined_queries {
279+ Some(combined_injections_query)
280+ } else {
281+ None
282+ };
283+
284+ // Find all of the highlighting patterns that are disabled for nodes that
285+ // have been identified as local variables.
286+ let non_local_variable_patterns = (0..query.pattern_count())
287+ .map(|i| {
288+ query
289+ .property_predicates(i)
290+ .iter()
291+ .any(|(prop, positive)| !*positive && prop.key.as_ref() == "local")
292+ })
293+ .collect();
294+
295+ // Store the numeric ids for all of the special captures.
296+ let mut injection_content_capture_index = None;
297+ let mut injection_language_capture_index = None;
298+ let mut local_def_capture_index = None;
299+ let mut local_def_value_capture_index = None;
300+ let mut local_ref_capture_index = None;
301+ let mut local_scope_capture_index = None;
302+ for (i, name) in query.capture_names().iter().enumerate() {
303+ let i = Some(i as u32);
304+ match name.as_str() {
305+ "injection.content" => injection_content_capture_index = i,
306+ "injection.language" => injection_language_capture_index = i,
307+ "local.definition" => local_def_capture_index = i,
308+ "local.definition-value" => local_def_value_capture_index = i,
309+ "local.reference" => local_ref_capture_index = i,
310+ "local.scope" => local_scope_capture_index = i,
311+ _ => {}
312+ }
313+ }
314+
315+ let highlight_indices = vec![None; query.capture_names().len()];
316+ Ok(HighlightConfiguration {
317+ language,
318+ query,
319+ combined_injections_query,
320+ locals_pattern_index,
321+ highlights_pattern_index,
322+ highlight_indices,
323+ non_local_variable_patterns,
324+ injection_content_capture_index,
325+ injection_language_capture_index,
326+ local_def_capture_index,
327+ local_def_value_capture_index,
328+ local_ref_capture_index,
329+ local_scope_capture_index,
330+ })
331+ }
332+
333+ /// Get a slice containing all of the highlight names used in the configuration.
334+ pub fn names(&self) -> &[String] {
335+ self.query.capture_names()
336+ }
337+
338+ /// Set the list of recognized highlight names.
339+ ///
340+ /// Tree-sitter syntax-highlighting queries specify highlights in the form of dot-separated
341+ /// highlight names like `punctuation.bracket` and `function.method.builtin`. Consumers of
342+ /// these queries can choose to recognize highlights with different levels of specificity.
343+ /// For example, the string `function.builtin` will match against `function.method.builtin`
344+ /// and `function.builtin.constructor`, but will not match `function.method`.
345+ ///
346+ /// When highlighting, results are returned as `Highlight` values, which contain the index
347+ /// of the matched highlight this list of highlight names.
348+ pub fn configure(&mut self, recognized_names: &[String]) {
349+ let mut capture_parts = Vec::new();
350+ self.highlight_indices.clear();
351+ self.highlight_indices
352+ .extend(self.query.capture_names().iter().map(move |capture_name| {
353+ capture_parts.clear();
354+ capture_parts.extend(capture_name.split('.'));
355+
356+ let mut best_index = None;
357+ let mut best_match_len = 0;
358+ for (i, recognized_name) in recognized_names.iter().enumerate() {
359+ let mut len = 0;
360+ let mut matches = true;
361+ for part in recognized_name.split('.') {
362+ len += 1;
363+ if !capture_parts.contains(&part) {
364+ matches = false;
365+ break;
366+ }
367+ }
368+ if matches && len > best_match_len {
369+ best_index = Some(i);
370+ best_match_len = len;
371+ }
372+ }
373+ best_index.map(Highlight)
374+ }));
375+ }
376+}
377+
378+impl<'a> HighlightIterLayer<'a> {
379+ /// Create a new 'layer' of highlighting for this document.
380+ ///
381+ /// In the even that the new layer contains "combined injections" (injections where multiple
382+ /// disjoint ranges are parsed as one syntax tree), these will be eagerly processed and
383+ /// added to the returned vector.
384+ fn new<F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a>(
385+ source: &'a [u8],
386+ highlighter: &mut Highlighter,
387+ cancellation_flag: Option<&'a AtomicUsize>,
388+ injection_callback: &mut F,
389+ mut config: &'a HighlightConfiguration,
390+ mut depth: usize,
391+ mut ranges: Vec<Range>,
392+ ) -> Result<Vec<Self>, Error> {
393+ let mut result = Vec::with_capacity(1);
394+ let mut queue = Vec::new();
395+ loop {
396+ if highlighter.parser.set_included_ranges(&ranges).is_ok() {
397+ highlighter
398+ .parser
399+ .set_language(config.language)
400+ .map_err(|_| Error::InvalidLanguage)?;
401+
402+ unsafe { highlighter.parser.set_cancellation_flag(cancellation_flag) };
403+ let tree = highlighter
404+ .parser
405+ .parse(source, None)
406+ .ok_or(Error::Cancelled)?;
407+ unsafe { highlighter.parser.set_cancellation_flag(None) };
408+ let mut cursor = highlighter.cursors.pop().unwrap_or(QueryCursor::new());
409+
410+ // Process combined injections.
411+ if let Some(combined_injections_query) = &config.combined_injections_query {
412+ let mut injections_by_pattern_index =
413+ vec![(None, Vec::new(), false); combined_injections_query.pattern_count()];
414+ let matches =
415+ cursor.matches(combined_injections_query, tree.root_node(), |n: Node| {
416+ &source[n.byte_range()]
417+ });
418+ for mat in matches {
419+ let entry = &mut injections_by_pattern_index[mat.pattern_index];
420+ let (language_name, content_node, include_children) =
421+ injection_for_match(config, combined_injections_query, &mat, source);
422+ if language_name.is_some() {
423+ entry.0 = language_name;
424+ }
425+ if let Some(content_node) = content_node {
426+ entry.1.push(content_node);
427+ }
428+ entry.2 = include_children;
429+ }
430+ for (lang_name, content_nodes, includes_children) in injections_by_pattern_index
431+ {
432+ if let (Some(lang_name), false) = (lang_name, content_nodes.is_empty()) {
433+ if let Some(next_config) = (injection_callback)(lang_name) {
434+ let ranges = Self::intersect_ranges(
435+ &ranges,
436+ &content_nodes,
437+ includes_children,
438+ );
439+ if !ranges.is_empty() {
440+ queue.push((next_config, depth + 1, ranges));
441+ }
442+ }
443+ }
444+ }
445+ }
446+
447+ // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which
448+ // prevents them from being moved. But both of these values are really just
449+ // pointers, so it's actually ok to move them.
450+ let tree_ref = unsafe { mem::transmute::<_, &'static Tree>(&tree) };
451+ let cursor_ref =
452+ unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) };
453+ let captures = cursor_ref
454+ .captures(&config.query, tree_ref.root_node(), move |n: Node| {
455+ &source[n.byte_range()]
456+ })
457+ .peekable();
458+
459+ result.push(HighlightIterLayer {
460+ highlight_end_stack: Vec::new(),
461+ scope_stack: vec![LocalScope {
462+ inherits: false,
463+ range: 0..usize::MAX,
464+ local_defs: Vec::new(),
465+ }],
466+ cursor,
467+ depth,
468+ _tree: tree,
469+ captures,
470+ config,
471+ ranges,
472+ });
473+ }
474+
475+ if queue.is_empty() {
476+ break;
477+ } else {
478+ let (next_config, next_depth, next_ranges) = queue.remove(0);
479+ config = next_config;
480+ depth = next_depth;
481+ ranges = next_ranges;
482+ }
483+ }
484+
485+ Ok(result)
486+ }
487+
488+ // Compute the ranges that should be included when parsing an injection.
489+ // This takes into account three things:
490+ // * `parent_ranges` - The ranges must all fall within the *current* layer's ranges.
491+ // * `nodes` - Every injection takes place within a set of nodes. The injection ranges
492+ // are the ranges of those nodes.
493+ // * `includes_children` - For some injections, the content nodes' children should be
494+ // excluded from the nested document, so that only the content nodes' *own* content
495+ // is reparsed. For other injections, the content nodes' entire ranges should be
496+ // reparsed, including the ranges of their children.
497+ fn intersect_ranges(
498+ parent_ranges: &[Range],
499+ nodes: &[Node],
500+ includes_children: bool,
501+ ) -> Vec<Range> {
502+ let mut cursor = nodes[0].walk();
503+ let mut result = Vec::new();
504+ let mut parent_range_iter = parent_ranges.iter();
505+ let mut parent_range = parent_range_iter
506+ .next()
507+ .expect("Layers should only be constructed with non-empty ranges vectors");
508+ for node in nodes.iter() {
509+ let mut preceding_range = Range {
510+ start_byte: 0,
511+ start_point: Point::new(0, 0),
512+ end_byte: node.start_byte(),
513+ end_point: node.start_position(),
514+ };
515+ let following_range = Range {
516+ start_byte: node.end_byte(),
517+ start_point: node.end_position(),
518+ end_byte: usize::MAX,
519+ end_point: Point::new(usize::MAX, usize::MAX),
520+ };
521+
522+ for excluded_range in node
523+ .children(&mut cursor)
524+ .filter_map(|child| {
525+ if includes_children {
526+ None
527+ } else {
528+ Some(child.range())
529+ }
530+ })
531+ .chain([following_range].iter().cloned())
532+ {
533+ let mut range = Range {
534+ start_byte: preceding_range.end_byte,
535+ start_point: preceding_range.end_point,
536+ end_byte: excluded_range.start_byte,
537+ end_point: excluded_range.start_point,
538+ };
539+ preceding_range = excluded_range;
540+
541+ if range.end_byte < parent_range.start_byte {
542+ continue;
543+ }
544+
545+ while parent_range.start_byte <= range.end_byte {
546+ if parent_range.end_byte > range.start_byte {
547+ if range.start_byte < parent_range.start_byte {
548+ range.start_byte = parent_range.start_byte;
549+ range.start_point = parent_range.start_point;
550+ }
551+
552+ if parent_range.end_byte < range.end_byte {
553+ if range.start_byte < parent_range.end_byte {
554+ result.push(Range {
555+ start_byte: range.start_byte,
556+ start_point: range.start_point,
557+ end_byte: parent_range.end_byte,
558+ end_point: parent_range.end_point,
559+ });
560+ }
561+ range.start_byte = parent_range.end_byte;
562+ range.start_point = parent_range.end_point;
563+ } else {
564+ if range.start_byte < range.end_byte {
565+ result.push(range);
566+ }
567+ break;
568+ }
569+ }
570+
571+ if let Some(next_range) = parent_range_iter.next() {
572+ parent_range = next_range;
573+ } else {
574+ return result;
575+ }
576+ }
577+ }
578+ }
579+ result
580+ }
581+
582+ // First, sort scope boundaries by their byte offset in the document. At a
583+ // given position, emit scope endings before scope beginnings. Finally, emit
584+ // scope boundaries from deeper layers first.
585+ fn sort_key(&mut self) -> Option<(usize, bool, isize)> {
586+ let depth = -(self.depth as isize);
587+ let next_start = self
588+ .captures
589+ .peek()
590+ .map(|(m, i)| m.captures[*i].node.start_byte());
591+ let next_end = self.highlight_end_stack.last().cloned();
592+ match (next_start, next_end) {
593+ (Some(start), Some(end)) => {
594+ if start < end {
595+ Some((start, true, depth))
596+ } else {
597+ Some((end, false, depth))
598+ }
599+ }
600+ (Some(i), None) => Some((i, true, depth)),
601+ (None, Some(j)) => Some((j, false, depth)),
602+ _ => None,
603+ }
604+ }
605+}
606+
607+impl<'a, F> HighlightIter<'a, F>
608+where
609+ F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
610+{
611+ fn emit_event(
612+ &mut self,
613+ offset: usize,
614+ event: Option<HighlightEvent>,
615+ ) -> Option<Result<HighlightEvent, Error>> {
616+ let result;
617+ if self.byte_offset < offset {
618+ result = Some(Ok(HighlightEvent::Source {
619+ start: self.byte_offset,
620+ end: offset,
621+ }));
622+ self.byte_offset = offset;
623+ self.next_event = event;
624+ } else {
625+ result = event.map(Ok);
626+ }
627+ self.sort_layers();
628+ result
629+ }
630+
631+ fn sort_layers(&mut self) {
632+ while !self.layers.is_empty() {
633+ if let Some(sort_key) = self.layers[0].sort_key() {
634+ let mut i = 0;
635+ while i + 1 < self.layers.len() {
636+ if let Some(next_offset) = self.layers[i + 1].sort_key() {
637+ if next_offset < sort_key {
638+ i += 1;
639+ continue;
640+ }
641+ }
642+ break;
643+ }
644+ if i > 0 {
645+ &self.layers[0..(i + 1)].rotate_left(1);
646+ }
647+ break;
648+ } else {
649+ let layer = self.layers.remove(0);
650+ self.highlighter.cursors.push(layer.cursor);
651+ }
652+ }
653+ }
654+
655+ fn insert_layer(&mut self, mut layer: HighlightIterLayer<'a>) {
656+ if let Some(sort_key) = layer.sort_key() {
657+ let mut i = 1;
658+ while i < self.layers.len() {
659+ if let Some(sort_key_i) = self.layers[i].sort_key() {
660+ if sort_key_i > sort_key {
661+ self.layers.insert(i, layer);
662+ return;
663+ }
664+ i += 1;
665+ } else {
666+ self.layers.remove(i);
667+ }
668+ }
669+ self.layers.push(layer);
670+ }
671+ }
672+}
673+
674+impl<'a, F> Iterator for HighlightIter<'a, F>
675+where
676+ F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
677+{
678+ type Item = Result<HighlightEvent, Error>;
679+
680+ fn next(&mut self) -> Option<Self::Item> {
681+ 'main: loop {
682+ // If we've already determined the next highlight boundary, just return it.
683+ if let Some(e) = self.next_event.take() {
684+ return Some(Ok(e));
685+ }
686+
687+ // Periodically check for cancellation, returning `Cancelled` error if the
688+ // cancellation flag was flipped.
689+ if let Some(cancellation_flag) = self.cancellation_flag {
690+ self.iter_count += 1;
691+ if self.iter_count >= CANCELLATION_CHECK_INTERVAL {
692+ self.iter_count = 0;
693+ if cancellation_flag.load(Ordering::Relaxed) != 0 {
694+ return Some(Err(Error::Cancelled));
695+ }
696+ }
697+ }
698+
699+ // If none of the layers have any more highlight boundaries, terminate.
700+ if self.layers.is_empty() {
701+ return if self.byte_offset < self.source.len() {
702+ let result = Some(Ok(HighlightEvent::Source {
703+ start: self.byte_offset,
704+ end: self.source.len(),
705+ }));
706+ self.byte_offset = self.source.len();
707+ result
708+ } else {
709+ None
710+ };
711+ }
712+
713+ // Get the next capture from whichever layer has the earliest highlight boundary.
714+ let range;
715+ let layer = &mut self.layers[0];
716+ if let Some((next_match, capture_index)) = layer.captures.peek() {
717+ let next_capture = next_match.captures[*capture_index];
718+ range = next_capture.node.byte_range();
719+
720+ // If any previous highlight ends before this node starts, then before
721+ // processing this capture, emit the source code up until the end of the
722+ // previous highlight, and an end event for that highlight.
723+ if let Some(end_byte) = layer.highlight_end_stack.last().cloned() {
724+ if end_byte <= range.start {
725+ layer.highlight_end_stack.pop();
726+ return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd));
727+ }
728+ }
729+ }
730+ // If there are no more captures, then emit any remaining highlight end events.
731+ // And if there are none of those, then just advance to the end of the document.
732+ else if let Some(end_byte) = layer.highlight_end_stack.last().cloned() {
733+ layer.highlight_end_stack.pop();
734+ return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd));
735+ } else {
736+ return self.emit_event(self.source.len(), None);
737+ };
738+
739+ let (mut match_, capture_index) = layer.captures.next().unwrap();
740+ let mut capture = match_.captures[capture_index];
741+
742+ // If this capture represents an injection, then process the injection.
743+ if match_.pattern_index < layer.config.locals_pattern_index {
744+ let (language_name, content_node, include_children) =
745+ injection_for_match(&layer.config, &layer.config.query, &match_, &self.source);
746+
747+ // Explicitly remove this match so that none of its other captures will remain
748+ // in the stream of captures.
749+ match_.remove();
750+
751+ // If a language is found with the given name, then add a new language layer
752+ // to the highlighted document.
753+ if let (Some(language_name), Some(content_node)) = (language_name, content_node) {
754+ if let Some(config) = (self.injection_callback)(language_name) {
755+ let ranges = HighlightIterLayer::intersect_ranges(
756+ &self.layers[0].ranges,
757+ &[content_node],
758+ include_children,
759+ );
760+ if !ranges.is_empty() {
761+ match HighlightIterLayer::new(
762+ self.source,
763+ self.highlighter,
764+ self.cancellation_flag,
765+ &mut self.injection_callback,
766+ config,
767+ self.layers[0].depth + 1,
768+ ranges,
769+ ) {
770+ Ok(layers) => {
771+ for layer in layers {
772+ self.insert_layer(layer);
773+ }
774+ }
775+ Err(e) => return Some(Err(e)),
776+ }
777+ }
778+ }
779+ }
780+
781+ self.sort_layers();
782+ continue 'main;
783+ }
784+
785+ // Remove from the local scope stack any local scopes that have already ended.
786+ while range.start > layer.scope_stack.last().unwrap().range.end {
787+ layer.scope_stack.pop();
788+ }
789+
790+ // If this capture is for tracking local variables, then process the
791+ // local variable info.
792+ let mut reference_highlight = None;
793+ let mut definition_highlight = None;
794+ while match_.pattern_index < layer.config.highlights_pattern_index {
795+ // If the node represents a local scope, push a new local scope onto
796+ // the scope stack.
797+ if Some(capture.index) == layer.config.local_scope_capture_index {
798+ definition_highlight = None;
799+ let mut scope = LocalScope {
800+ inherits: true,
801+ range: range.clone(),
802+ local_defs: Vec::new(),
803+ };
804+ for prop in layer.config.query.property_settings(match_.pattern_index) {
805+ match prop.key.as_ref() {
806+ "local.scope-inherits" => {
807+ scope.inherits =
808+ prop.value.as_ref().map_or(true, |r| r.as_ref() == "true");
809+ }
810+ _ => {}
811+ }
812+ }
813+ layer.scope_stack.push(scope);
814+ }
815+ // If the node represents a definition, add a new definition to the
816+ // local scope at the top of the scope stack.
817+ else if Some(capture.index) == layer.config.local_def_capture_index {
818+ reference_highlight = None;
819+ definition_highlight = None;
820+ let scope = layer.scope_stack.last_mut().unwrap();
821+
822+ let mut value_range = 0..0;
823+ for capture in match_.captures {
824+ if Some(capture.index) == layer.config.local_def_value_capture_index {
825+ value_range = capture.node.byte_range();
826+ }
827+ }
828+
829+ if let Ok(name) = str::from_utf8(&self.source[range.clone()]) {
830+ scope.local_defs.push(LocalDef {
831+ name,
832+ value_range,
833+ highlight: None,
834+ });
835+ definition_highlight =
836+ scope.local_defs.last_mut().map(|s| &mut s.highlight);
837+ }
838+ }
839+ // If the node represents a reference, then try to find the corresponding
840+ // definition in the scope stack.
841+ else if Some(capture.index) == layer.config.local_ref_capture_index {
842+ if definition_highlight.is_none() {
843+ definition_highlight = None;
844+ if let Ok(name) = str::from_utf8(&self.source[range.clone()]) {
845+ for scope in layer.scope_stack.iter().rev() {
846+ if let Some(highlight) =
847+ scope.local_defs.iter().rev().find_map(|def| {
848+ if def.name == name && range.start >= def.value_range.end {
849+ Some(def.highlight)
850+ } else {
851+ None
852+ }
853+ })
854+ {
855+ reference_highlight = highlight;
856+ break;
857+ }
858+ if !scope.inherits {
859+ break;
860+ }
861+ }
862+ }
863+ }
864+ }
865+
866+ // Continue processing any additional matches for the same node.
867+ if let Some((next_match, next_capture_index)) = layer.captures.peek() {
868+ let next_capture = next_match.captures[*next_capture_index];
869+ if next_capture.node == capture.node {
870+ capture = next_capture;
871+ match_ = layer.captures.next().unwrap().0;
872+ continue;
873+ }
874+ }
875+
876+ self.sort_layers();
877+ continue 'main;
878+ }
879+
880+ // Otherwise, this capture must represent a highlight.
881+ // If this exact range has already been highlighted by an earlier pattern, or by
882+ // a different layer, then skip over this one.
883+ if let Some((last_start, last_end, last_depth)) = self.last_highlight_range {
884+ if range.start == last_start && range.end == last_end && layer.depth < last_depth {
885+ self.sort_layers();
886+ continue 'main;
887+ }
888+ }
889+
890+ // If the current node was found to be a local variable, then skip over any
891+ // highlighting patterns that are disabled for local variables.
892+ if definition_highlight.is_some() || reference_highlight.is_some() {
893+ while layer.config.non_local_variable_patterns[match_.pattern_index] {
894+ if let Some((next_match, next_capture_index)) = layer.captures.peek() {
895+ let next_capture = next_match.captures[*next_capture_index];
896+ if next_capture.node == capture.node {
897+ capture = next_capture;
898+ match_ = layer.captures.next().unwrap().0;
899+ continue;
900+ }
901+ }
902+
903+ self.sort_layers();
904+ continue 'main;
905+ }
906+ }
907+
908+ // Once a highlighting pattern is found for the current node, skip over
909+ // any later highlighting patterns that also match this node. Captures
910+ // for a given node are ordered by pattern index, so these subsequent
911+ // captures are guaranteed to be for highlighting, not injections or
912+ // local variables.
913+ while let Some((next_match, next_capture_index)) = layer.captures.peek() {
914+ let next_capture = next_match.captures[*next_capture_index];
915+ if next_capture.node == capture.node {
916+ layer.captures.next();
917+ } else {
918+ break;
919+ }
920+ }
921+
922+ let current_highlight = layer.config.highlight_indices[capture.index as usize];
923+
924+ // If this node represents a local definition, then store the current
925+ // highlight value on the local scope entry representing this node.
926+ if let Some(definition_highlight) = definition_highlight {
927+ *definition_highlight = current_highlight;
928+ }
929+
930+ // Emit a scope start event and push the node's end position to the stack.
931+ if let Some(highlight) = reference_highlight.or(current_highlight) {
932+ self.last_highlight_range = Some((range.start, range.end, layer.depth));
933+ layer.highlight_end_stack.push(range.end);
934+ return self
935+ .emit_event(range.start, Some(HighlightEvent::HighlightStart(highlight)));
936+ }
937+
938+ self.sort_layers();
939+ }
940+ }
941+}
942+
943+fn injection_for_match<'a>(
944+ config: &HighlightConfiguration,
945+ query: &'a Query,
946+ query_match: &QueryMatch<'a>,
947+ source: &'a [u8],
948+) -> (Option<&'a str>, Option<Node<'a>>, bool) {
949+ let content_capture_index = config.injection_content_capture_index;
950+ let language_capture_index = config.injection_language_capture_index;
951+
952+ let mut language_name = None;
953+ let mut content_node = None;
954+ for capture in query_match.captures {
955+ let index = Some(capture.index);
956+ if index == language_capture_index {
957+ language_name = capture.node.utf8_text(source).ok();
958+ } else if index == content_capture_index {
959+ content_node = Some(capture.node);
960+ }
961+ }
962+
963+ let mut include_children = false;
964+ for prop in query.property_settings(query_match.pattern_index) {
965+ match prop.key.as_ref() {
966+ // In addition to specifying the language name via the text of a
967+ // captured node, it can also be hard-coded via a `#set!` predicate
968+ // that sets the injection.language key.
969+ "injection.language" => {
970+ if language_name.is_none() {
971+ language_name = prop.value.as_ref().map(|s| s.as_ref())
972+ }
973+ }
974+
975+ // By default, injections do not include the *children* of an
976+ // `injection.content` node - only the ranges that belong to the
977+ // node itself. This can be changed using a `#set!` predicate that
978+ // sets the `injection.include-children` key.
979+ "injection.include-children" => include_children = true,
980+ _ => {}
981+ }
982+ }
983+
984+ (language_name, content_node, include_children)
985+}
986+
987+fn shrink_and_clear<T>(vec: &mut Vec<T>, capacity: usize) {
988+ if vec.len() > capacity {
989+ vec.truncate(capacity);
990+ vec.shrink_to_fit();
991+ }
992+ vec.clear();
993+}