@@ -2,6 +2,7 @@ use crate::{Change, Rope, RopeSlice, Transaction};
22pub use helix_syntax::Lang;
33pub use helix_syntax::{get_language, get_language_name};
44
5+use std::borrow::Cow;
56use std::cell::RefCell;
67use std::collections::HashMap;
78use std::path::{Path, PathBuf};
@@ -177,6 +178,22 @@ pub struct Syntax {
177178 pub(crate) root_layer: LanguageLayer,
178179}
179180
181+fn byte_range_to_str<'a>(range: std::ops::Range<usize>, source: RopeSlice<'a>) -> Cow<'a, str> {
182+ let start_char = source.byte_to_char(range.start);
183+ let end_char = source.byte_to_char(range.end);
184+ Cow::from(source.slice(start_char..end_char))
185+}
186+
187+fn node_to_bytes<'a>(node: Node, source: RopeSlice<'a>) -> Cow<'a, [u8]> {
188+ let start_char = source.byte_to_char(node.start_byte());
189+ let end_char = source.byte_to_char(node.end_byte());
190+ let fragment = source.slice(start_char..end_char);
191+ match fragment.as_str() {
192+ Some(fragment) => Cow::Borrowed(fragment.as_bytes()),
193+ None => Cow::Owned(String::from(fragment).into_bytes()),
194+ }
195+}
196+
180197impl Syntax {
181198 // buffer, grammar, config, grammars, sync_timeout?
182199 pub fn new(
@@ -248,7 +265,7 @@ impl Syntax {
248265 /// Iterate over the highlighted regions for a given slice of source code.
249266 pub fn highlight_iter<'a>(
250267 &self,
251− source: &'a [u8],
268+ source: RopeSlice<'a>,
252269 range: Option<std::ops::Range<usize>>,
253270 cancellation_flag: Option<&'a AtomicUsize>,
254271 mut injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
@@ -271,7 +288,8 @@ impl Syntax {
271288
272289 let captures = cursor_ref
273290 .captures(query_ref, tree_ref.root_node(), move |n: Node| {
274− &source[n.byte_range()]
291+ // &source[n.byte_range()]
292+ node_to_bytes(n, source)
275293 })
276294 .peekable();
277295
@@ -390,7 +408,8 @@ impl LanguageLayer {
390408 // vec![(None, Vec::new(), false); combined_injections_query.pattern_count()];
391409 // let matches =
392410 // cursor.matches(combined_injections_query, tree.root_node(), |n: Node| {
393− // &source[n.byte_range()]
411+ // // &source[n.byte_range()]
412+ // node_to_bytes(n, source)
394413 // });
395414 // for mat in matches {
396415 // let entry = &mut injections_by_pattern_index[mat.pattern_index];
@@ -653,7 +672,7 @@ pub struct HighlightConfiguration {
653672
654673#[derive(Debug)]
655674struct LocalDef<'a> {
656− name: &'a str,
675+ name: Cow<'a, str>,
657676 value_range: ops::Range<usize>,
658677 highlight: Option<Highlight>,
659678}
@@ -669,7 +688,7 @@ struct HighlightIter<'a, F>
669688where
670689 F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
671690{
672− source: &'a [u8],
691+ source: RopeSlice<'a>,
673692 byte_offset: usize,
674693 injection_callback: F,
675694 cancellation_flag: Option<&'a AtomicUsize>,
@@ -682,7 +701,7 @@ where
682701struct HighlightIterLayer<'a> {
683702 _tree: Option<Tree>,
684703 cursor: QueryCursor,
685− captures: iter::Peekable<QueryCaptures<'a, &'a [u8]>>,
704+ captures: iter::Peekable<QueryCaptures<'a, Cow<'a, [u8]>>>,
686705 config: &'a HighlightConfiguration,
687706 highlight_end_stack: Vec<usize>,
688707 scope_stack: Vec<LocalScope<'a>>,
@@ -856,7 +875,7 @@ impl<'a> HighlightIterLayer<'a> {
856875 /// disjoint ranges are parsed as one syntax tree), these will be eagerly processed and
857876 /// added to the returned vector.
858877 fn new<F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a>(
859− source: &'a [u8],
878+ source: RopeSlice<'a>,
860879 cancellation_flag: Option<&'a AtomicUsize>,
861880 injection_callback: &mut F,
862881 mut config: &'a HighlightConfiguration,
@@ -880,7 +899,18 @@ impl<'a> HighlightIterLayer<'a> {
880899 unsafe { highlighter.parser.set_cancellation_flag(cancellation_flag) };
881900 let tree = highlighter
882901 .parser
883− .parse(source, None)
902+ .parse_with(
903+ &mut |byte, _| {
904+ if byte <= source.len_bytes() {
905+ let (chunk, start_byte, _, _) = source.chunk_at_byte(byte);
906+ chunk[byte - start_byte..].as_bytes()
907+ } else {
908+ // out of range
909+ &[]
910+ }
911+ },
912+ None,
913+ )
884914 .ok_or(Error::Cancelled)?;
885915 unsafe { highlighter.parser.set_cancellation_flag(None) };
886916 let mut cursor = highlighter.cursors.pop().unwrap_or_else(QueryCursor::new);
@@ -895,7 +925,10 @@ impl<'a> HighlightIterLayer<'a> {
895925 let matches = cursor.matches(
896926 combined_injections_query,
897927 tree.root_node(),
898− |n: Node| &source[n.byte_range()],
928+ |n: Node| {
929+ // &source[n.byte_range()]
930+ node_to_bytes(n, source)
931+ },
899932 );
900933 for mat in matches {
901934 let entry = &mut injections_by_pattern_index[mat.pattern_index];
@@ -919,7 +952,7 @@ impl<'a> HighlightIterLayer<'a> {
919952 {
920953 if let (Some(lang_name), false) = (lang_name, content_nodes.is_empty())
921954 {
922− if let Some(next_config) = (injection_callback)(lang_name) {
955+ if let Some(next_config) = (injection_callback)(&lang_name) {
923956 let ranges = Self::intersect_ranges(
924957 &ranges,
925958 &content_nodes,
@@ -943,7 +976,8 @@ impl<'a> HighlightIterLayer<'a> {
943976 unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) };
944977 let captures = cursor_ref
945978 .captures(&config.query, tree_ref.root_node(), move |n: Node| {
946− &source[n.byte_range()]
979+ // &source[n.byte_range()]
980+ node_to_bytes(n, source)
947981 })
948982 .peekable();
949983
@@ -1195,12 +1229,13 @@ where
11951229
11961230 // If none of the layers have any more highlight boundaries, terminate.
11971231 if self.layers.is_empty() {
1198− return if self.byte_offset < self.source.len() {
1232+ let len = self.source.len_bytes();
1233+ return if self.byte_offset < len {
11991234 let result = Some(Ok(HighlightEvent::Source {
12001235 start: self.byte_offset,
1201− end: self.source.len(),
1236+ end: len,
12021237 }));
1203− self.byte_offset = self.source.len();
1238+ self.byte_offset = len;
12041239 result
12051240 } else {
12061241 None
@@ -1240,7 +1275,7 @@ where
12401275 // If this capture represents an injection, then process the injection.
12411276 if match_.pattern_index < layer.config.locals_pattern_index {
12421277 let (language_name, content_node, include_children) =
1243− injection_for_match(&layer.config, &layer.config.query, &match_, &self.source);
1278+ injection_for_match(&layer.config, &layer.config.query, &match_, self.source);
12441279
12451280 // Explicitly remove this match so that none of its other captures will remain
12461281 // in the stream of captures.
@@ -1249,7 +1284,7 @@ where
12491284 // If a language is found with the given name, then add a new language layer
12501285 // to the highlighted document.
12511286 if let (Some(language_name), Some(content_node)) = (language_name, content_node) {
1252− if let Some(config) = (self.injection_callback)(language_name) {
1287+ if let Some(config) = (self.injection_callback)(&language_name) {
12531288 let ranges = HighlightIterLayer::intersect_ranges(
12541289 &self.layers[0].ranges,
12551290 &[content_node],
@@ -1320,15 +1355,13 @@ where
13201355 }
13211356 }
13221357
1323− if let Ok(name) = str::from_utf8(&self.source[range.clone()]) {
1324− scope.local_defs.push(LocalDef {
1325− name,
1326− value_range,
1327− highlight: None,
1328− });
1329− definition_highlight =
1330− scope.local_defs.last_mut().map(|s| &mut s.highlight);
1331− }
1358+ let name = byte_range_to_str(range.clone(), self.source);
1359+ scope.local_defs.push(LocalDef {
1360+ name,
1361+ value_range,
1362+ highlight: None,
1363+ });
1364+ definition_highlight = scope.local_defs.last_mut().map(|s| &mut s.highlight);
13321365 }
13331366 // If the node represents a reference, then try to find the corresponding
13341367 // definition in the scope stack.
@@ -1336,21 +1369,20 @@ where
13361369 && definition_highlight.is_none()
13371370 {
13381371 definition_highlight = None;
1339− if let Ok(name) = str::from_utf8(&self.source[range.clone()]) {
1340− for scope in layer.scope_stack.iter().rev() {
1341− if let Some(highlight) = scope.local_defs.iter().rev().find_map(|def| {
1342− if def.name == name && range.start >= def.value_range.end {
1343− Some(def.highlight)
1344− } else {
1345− None
1346− }
1347− }) {
1348− reference_highlight = highlight;
1349− break;
1350− }
1351− if !scope.inherits {
1352− break;
1372+ let name = byte_range_to_str(range.clone(), self.source);
1373+ for scope in layer.scope_stack.iter().rev() {
1374+ if let Some(highlight) = scope.local_defs.iter().rev().find_map(|def| {
1375+ if def.name == name && range.start >= def.value_range.end {
1376+ Some(def.highlight)
1377+ } else {
1378+ None
13531379 }
1380+ }) {
1381+ reference_highlight = highlight;
1382+ break;
1383+ }
1384+ if !scope.inherits {
1385+ break;
13541386 }
13551387 }
13561388 }
@@ -1436,8 +1468,8 @@ fn injection_for_match<'a>(
14361468 config: &HighlightConfiguration,
14371469 query: &'a Query,
14381470 query_match: &QueryMatch<'a>,
1439− source: &'a [u8],
1440−) -> (Option<&'a str>, Option<Node<'a>>, bool) {
1471+ source: RopeSlice<'a>,
1472+) -> (Option<Cow<'a, str>>, Option<Node<'a>>, bool) {
14411473 let content_capture_index = config.injection_content_capture_index;
14421474 let language_capture_index = config.injection_language_capture_index;
14431475
@@ -1446,7 +1478,8 @@ fn injection_for_match<'a>(
14461478 for capture in query_match.captures {
14471479 let index = Some(capture.index);
14481480 if index == language_capture_index {
1449− language_name = capture.node.utf8_text(source).ok();
1481+ let name = byte_range_to_str(capture.node.byte_range(), source);
1482+ language_name = Some(name);
14501483 } else if index == content_capture_index {
14511484 content_node = Some(capture.node);
14521485 }
@@ -1460,7 +1493,7 @@ fn injection_for_match<'a>(
14601493 // that sets the injection.language key.
14611494 "injection.language" => {
14621495 if language_name.is_none() {
1463− language_name = prop.value.as_ref().map(|s| s.as_ref())
1496+ language_name = prop.value.as_ref().map(|s| s.as_ref().into())
14641497 }
14651498 }
14661499