on master help sign in
inject language based on file extension & shebang (#3970)

* inject language based on file extension Nodes can now be captured with "injection.filename". If this capture contains a valid file extension known to Helix, then the content will be highlighted as that language. * inject language by shebang Nodes can now be captured with "injection.shebang". If this capture contains a valid shebang line known to Helix, then the content will be highlighted as the language the shebang calls for. * add documentation for language injection * nix: fix highlights The `@` is now highlighted properly on either side of the function arg. Also, extending the phases with `buildPhase = prev.buildPhase + ''''` is now highlighted properly. Fix highlighting of `''$` style escapes (requires tree-sitter-nix bump) Fix `inherit` highlighting. * simplify injection_for_match Split out injection pair logic into its own method to make the overall flow easier to follow. Also transform the top-level function into a method on a HighlightConfiguration. * markdown: add shebang injection query

commit9c6c63a2be30252a6207f4aebb5e0f76f746b4c8 authorTimothy DeHerrera <tim@nrdxp.dev> dateFri 28 Apr 2023, 03:21 (4 years ago) parent204d1eb
8 files changed+216−84
book/src/SUMMARY.md+1 book/src/guides/injection.md+57 helix-core/src/syntax.rs+130−77 helix-term/src/ui/markdown.rs+4−2 languages.toml+1−1 runtime/queries/markdown/injections.scm+4 runtime/queries/nix/highlights.scm+4−1 runtime/queries/nix/injections.scm+15−3
book/src/SUMMARY.md+1
@@ -16,3 +16,4 @@
1616 - [Adding languages](./guides/adding_languages.md)
1717 - [Adding textobject queries](./guides/textobject.md)
1818 - [Adding indent queries](./guides/indent.md)
19+ - [Adding injection queries](./guides/injection.md)
book/src/guides/injection.mdnew file+57
@@ -0,0 +1,57 @@
1+# Adding Injection Queries
2+
3+Writing language injection queries allows one to highlight a specific node as a different language.
4+In addition to the [standard](upstream-docs) language injection options used by tree-sitter, there
5+are a few Helix specific extensions that allow for more control.
6+
7+And example of a simple query that would highlight all strings as bash in Nix:
8+```scm
9+((string_expression (string_fragment) @injection.content)
10+ (#set! injection.language "bash"))
11+```
12+
13+## Capture Types
14+
15+- `@injection.language` (standard):
16+The captured node may contain the language name used to highlight the node captured by
17+`@injection.content`.
18+
19+- `@injection.content` (standard):
20+Marks the content to be highlighted as the language captured with `@injection.language` _et al_.
21+
22+- `@injection.filename` (extension):
23+The captured node may contain a filename with a file-extension known to Helix,
24+highlighting `@injection.content` as that language. This uses the language extensions defined in
25+both the default languages.toml distributed with Helix, as well as user defined languages.
26+
27+- `@injection.shebang` (extension):
28+The captured node may contain a shebang used to choose a language to highlight as. This also uses
29+the shebangs defined in the default and user `languages.toml`.
30+
31+## Settings
32+
33+- `injection.combined` (standard):
34+Indicates that all the matching nodes in the tree should have their content parsed as one
35+nested document.
36+
37+- `injection.language` (standard):
38+Forces the captured content to be highlighted as the given language
39+
40+- `injection.include-children` (standard):
41+Indicates that the content node’s entire text should be re-parsed, including the text of its child
42+nodes. By default, child nodes’ text will be excluded from the injected document.
43+
44+- `injection.include-unnamed-children` (extension):
45+Same as `injection.include-children` but only for unnamed child nodes.
46+
47+## Predicates
48+
49+- `#eq?` (standard):
50+The first argument (a capture) must be equal to the second argument
51+(a capture or a string).
52+
53+- `#match?` (standard):
54+The first argument (a capture) must match the regex given in the
55+second argument (a string).
56+
57+[upstream-docs]: http://tree-sitter.github.io/tree-sitter/syntax-highlighting#language-injection
helix-core/src/syntax.rs+130−77
@@ -662,9 +662,8 @@ impl Loader {
662662
663663 pub fn language_config_for_shebang(&self, source: &Rope) -> Option<Arc<LanguageConfiguration>> {
664664 let line = Cow::from(source.line(0));
665− static SHEBANG_REGEX: Lazy<Regex> = Lazy::new(|| {
666− Regex::new(r"^#!\s*(?:\S*[/\\](?:env\s+(?:\-\S+\s+)*)?)?([^\s\.\d]+)").unwrap()
667− });
665+ static SHEBANG_REGEX: Lazy<Regex> =
666+ Lazy::new(|| Regex::new(&["^", SHEBANG].concat()).unwrap());
668667 let configuration_id = SHEBANG_REGEX
669668 .captures(&line)
670669 .and_then(|cap| self.language_config_ids_by_shebang.get(&cap[1]));
@@ -686,15 +685,14 @@ impl Loader {
686685 .cloned()
687686 }
688687
689− pub fn language_configuration_for_injection_string(
690− &self,
691− string: &str,
692− ) -> Option<Arc<LanguageConfiguration>> {
688+ /// Unlike language_config_for_language_id, which only returns Some for an exact id, this
689+ /// function will perform a regex match on the given string to find the closest language match.
690+ pub fn language_config_for_name(&self, name: &str) -> Option<Arc<LanguageConfiguration>> {
693691 let mut best_match_length = 0;
694692 let mut best_match_position = None;
695693 for (i, configuration) in self.language_configs.iter().enumerate() {
696694 if let Some(injection_regex) = &configuration.injection_regex {
697− if let Some(mat) = injection_regex.find(string) {
695+ if let Some(mat) = injection_regex.find(name) {
698696 let length = mat.end() - mat.start();
699697 if length > best_match_length {
700698 best_match_position = Some(i);
@@ -704,11 +702,20 @@ impl Loader {
704702 }
705703 }
706704
707− if let Some(i) = best_match_position {
708− let configuration = &self.language_configs[i];
709− return Some(configuration.clone());
705+ best_match_position.map(|i| self.language_configs[i].clone())
706+ }
707+
708+ pub fn language_configuration_for_injection_string(
709+ &self,
710+ capture: &InjectionLanguageMarker,
711+ ) -> Option<Arc<LanguageConfiguration>> {
712+ match capture {
713+ InjectionLanguageMarker::Name(string) => self.language_config_for_name(string),
714+ InjectionLanguageMarker::Filename(file) => self.language_config_for_file_name(file),
715+ InjectionLanguageMarker::Shebang(shebang) => {
716+ self.language_config_for_language_id(shebang)
717+ }
710718 }
711− None
712719 }
713720
714721 pub fn language_configs(&self) -> impl Iterator<Item = &Arc<LanguageConfiguration>> {
@@ -800,7 +807,7 @@ impl Syntax {
800807 queue.push_back(self.root);
801808
802809 let scopes = self.loader.scopes.load();
803− let injection_callback = |language: &str| {
810+ let injection_callback = |language: &InjectionLanguageMarker| {
804811 self.loader
805812 .language_configuration_for_injection_string(language)
806813 .and_then(|language_config| language_config.highlight_config(&scopes))
@@ -961,12 +968,9 @@ impl Syntax {
961968 );
962969 let mut injections = Vec::new();
963970 for mat in matches {
964− let (language_name, content_node, included_children) = injection_for_match(
965− &layer.config,
966− &layer.config.injections_query,
967− &mat,
968− source_slice,
969− );
971+ let (injection_capture, content_node, included_children) = layer
972+ .config
973+ .injection_for_match(&layer.config.injections_query, &mat, source_slice);
970974
971975 // Explicitly remove this match so that none of its other captures will remain
972976 // in the stream of captures.
@@ -974,9 +978,10 @@ impl Syntax {
974978
975979 // If a language is found with the given name, then add a new language layer
976980 // to the highlighted document.
977− if let (Some(language_name), Some(content_node)) = (language_name, content_node)
981+ if let (Some(injection_capture), Some(content_node)) =
982+ (injection_capture, content_node)
978983 {
979− if let Some(config) = (injection_callback)(&language_name) {
984+ if let Some(config) = (injection_callback)(&injection_capture) {
980985 let ranges =
981986 intersect_ranges(&layer.ranges, &[content_node], included_children);
982987
@@ -1001,14 +1006,11 @@ impl Syntax {
10011006 );
10021007 for mat in matches {
10031008 let entry = &mut injections_by_pattern_index[mat.pattern_index];
1004− let (language_name, content_node, included_children) = injection_for_match(
1005− &layer.config,
1006− combined_injections_query,
1007− &mat,
1008− source_slice,
1009− );
1010− if language_name.is_some() {
1011− entry.0 = language_name;
1009+ let (injection_capture, content_node, included_children) = layer
1010+ .config
1011+ .injection_for_match(combined_injections_query, &mat, source_slice);
1012+ if injection_capture.is_some() {
1013+ entry.0 = injection_capture;
10121014 }
10131015 if let Some(content_node) = content_node {
10141016 entry.1.push(content_node);
@@ -1395,6 +1397,8 @@ pub struct HighlightConfiguration {
13951397 non_local_variable_patterns: Vec<bool>,
13961398 injection_content_capture_index: Option<u32>,
13971399 injection_language_capture_index: Option<u32>,
1400+ injection_filename_capture_index: Option<u32>,
1401+ injection_shebang_capture_index: Option<u32>,
13981402 local_scope_capture_index: Option<u32>,
13991403 local_def_capture_index: Option<u32>,
14001404 local_def_value_capture_index: Option<u32>,
@@ -1538,6 +1542,8 @@ impl HighlightConfiguration {
15381542 // Store the numeric ids for all of the special captures.
15391543 let mut injection_content_capture_index = None;
15401544 let mut injection_language_capture_index = None;
1545+ let mut injection_filename_capture_index = None;
1546+ let mut injection_shebang_capture_index = None;
15411547 let mut local_def_capture_index = None;
15421548 let mut local_def_value_capture_index = None;
15431549 let mut local_ref_capture_index = None;
@@ -1558,6 +1564,8 @@ impl HighlightConfiguration {
15581564 match name.as_str() {
15591565 "injection.content" => injection_content_capture_index = i,
15601566 "injection.language" => injection_language_capture_index = i,
1567+ "injection.filename" => injection_filename_capture_index = i,
1568+ "injection.shebang" => injection_shebang_capture_index = i,
15611569 _ => {}
15621570 }
15631571 }
@@ -1573,6 +1581,8 @@ impl HighlightConfiguration {
15731581 non_local_variable_patterns,
15741582 injection_content_capture_index,
15751583 injection_language_capture_index,
1584+ injection_filename_capture_index,
1585+ injection_shebang_capture_index,
15761586 local_scope_capture_index,
15771587 local_def_capture_index,
15781588 local_def_value_capture_index,
@@ -1631,6 +1641,90 @@ impl HighlightConfiguration {
16311641
16321642 self.highlight_indices.store(Arc::new(indices));
16331643 }
1644+
1645+ fn injection_pair<'a>(
1646+ &self,
1647+ query_match: &QueryMatch<'a, 'a>,
1648+ source: RopeSlice<'a>,
1649+ ) -> (Option<InjectionLanguageMarker<'a>>, Option<Node<'a>>) {
1650+ let mut injection_capture = None;
1651+ let mut content_node = None;
1652+
1653+ for capture in query_match.captures {
1654+ let index = Some(capture.index);
1655+ if index == self.injection_language_capture_index {
1656+ let name = byte_range_to_str(capture.node.byte_range(), source);
1657+ injection_capture = Some(InjectionLanguageMarker::Name(name));
1658+ } else if index == self.injection_filename_capture_index {
1659+ let name = byte_range_to_str(capture.node.byte_range(), source);
1660+ let path = Path::new(name.as_ref()).to_path_buf();
1661+ injection_capture = Some(InjectionLanguageMarker::Filename(path.into()));
1662+ } else if index == self.injection_shebang_capture_index {
1663+ let node_slice = source.byte_slice(capture.node.byte_range());
1664+
1665+ // some languages allow space and newlines before the actual string content
1666+ // so a shebang could be on either the first or second line
1667+ let lines = if let Ok(end) = node_slice.try_line_to_byte(2) {
1668+ node_slice.byte_slice(..end)
1669+ } else {
1670+ node_slice
1671+ };
1672+
1673+ static SHEBANG_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(SHEBANG).unwrap());
1674+
1675+ injection_capture = SHEBANG_REGEX
1676+ .captures(&Cow::from(lines))
1677+ .map(|cap| InjectionLanguageMarker::Shebang(cap[1].to_owned()))
1678+ } else if index == self.injection_content_capture_index {
1679+ content_node = Some(capture.node);
1680+ }
1681+ }
1682+ (injection_capture, content_node)
1683+ }
1684+
1685+ fn injection_for_match<'a>(
1686+ &self,
1687+ query: &'a Query,
1688+ query_match: &QueryMatch<'a, 'a>,
1689+ source: RopeSlice<'a>,
1690+ ) -> (
1691+ Option<InjectionLanguageMarker<'a>>,
1692+ Option<Node<'a>>,
1693+ IncludedChildren,
1694+ ) {
1695+ let (mut injection_capture, content_node) = self.injection_pair(query_match, source);
1696+
1697+ let mut included_children = IncludedChildren::default();
1698+ for prop in query.property_settings(query_match.pattern_index) {
1699+ match prop.key.as_ref() {
1700+ // In addition to specifying the language name via the text of a
1701+ // captured node, it can also be hard-coded via a `#set!` predicate
1702+ // that sets the injection.language key.
1703+ "injection.language" if injection_capture.is_none() => {
1704+ injection_capture = prop
1705+ .value
1706+ .as_ref()
1707+ .map(|s| InjectionLanguageMarker::Name(s.as_ref().into()));
1708+ }
1709+
1710+ // By default, injections do not include the *children* of an
1711+ // `injection.content` node - only the ranges that belong to the
1712+ // node itself. This can be changed using a `#set!` predicate that
1713+ // sets the `injection.include-children` key.
1714+ "injection.include-children" => included_children = IncludedChildren::All,
1715+
1716+ // Some queries might only exclude named children but include unnamed
1717+ // children in their `injection.content` node. This can be enabled using
1718+ // a `#set!` predicate that sets the `injection.include-unnamed-children` key.
1719+ "injection.include-unnamed-children" => {
1720+ included_children = IncludedChildren::Unnamed
1721+ }
1722+ _ => {}
1723+ }
1724+ }
1725+
1726+ (injection_capture, content_node, included_children)
1727+ }
16341728}
16351729
16361730impl<'a> HighlightIterLayer<'a> {
@@ -2042,56 +2136,15 @@ impl<'a> Iterator for HighlightIter<'a> {
20422136 }
20432137}
20442138
2045−fn injection_for_match<'a>(
2046− config: &HighlightConfiguration,
2047− query: &'a Query,
2048− query_match: &QueryMatch<'a, 'a>,
2049− source: RopeSlice<'a>,
2050−) -> (Option<Cow<'a, str>>, Option<Node<'a>>, IncludedChildren) {
2051− let content_capture_index = config.injection_content_capture_index;
2052− let language_capture_index = config.injection_language_capture_index;
2053−
2054− let mut language_name = None;
2055− let mut content_node = None;
2056− for capture in query_match.captures {
2057− let index = Some(capture.index);
2058− if index == language_capture_index {
2059− let name = byte_range_to_str(capture.node.byte_range(), source);
2060− language_name = Some(name);
2061− } else if index == content_capture_index {
2062− content_node = Some(capture.node);
2063− }
2064− }
2065−
2066− let mut included_children = IncludedChildren::default();
2067− for prop in query.property_settings(query_match.pattern_index) {
2068− match prop.key.as_ref() {
2069− // In addition to specifying the language name via the text of a
2070− // captured node, it can also be hard-coded via a `#set!` predicate
2071− // that sets the injection.language key.
2072− "injection.language" => {
2073− if language_name.is_none() {
2074− language_name = prop.value.as_ref().map(|s| s.as_ref().into())
2075− }
2076− }
2077−
2078− // By default, injections do not include the *children* of an
2079− // `injection.content` node - only the ranges that belong to the
2080− // node itself. This can be changed using a `#set!` predicate that
2081− // sets the `injection.include-children` key.
2082− "injection.include-children" => included_children = IncludedChildren::All,
2083−
2084− // Some queries might only exclude named children but include unnamed
2085− // children in their `injection.content` node. This can be enabled using
2086− // a `#set!` predicate that sets the `injection.include-unnamed-children` key.
2087− "injection.include-unnamed-children" => included_children = IncludedChildren::Unnamed,
2088− _ => {}
2089− }
2090− }
2091−
2092− (language_name, content_node, included_children)
2139+#[derive(Debug, Clone)]
2140+pub enum InjectionLanguageMarker<'a> {
2141+ Name(Cow<'a, str>),
2142+ Filename(Cow<'a, Path>),
2143+ Shebang(String),
20932144}
20942145
2146+const SHEBANG: &str = r"#!\s*(?:\S*[/\\](?:env\s+(?:\-\S+\s+)*)?)?([^\s\.\d]+)";
2147+
20952148pub struct Merge<I> {
20962149 iter: I,
20972150 spans: Box<dyn Iterator<Item = (usize, std::ops::Range<usize>)>>,
helix-term/src/ui/markdown.rs+4−2
@@ -9,7 +9,7 @@ use std::sync::Arc;
99use pulldown_cmark::{CodeBlockKind, Event, HeadingLevel, Options, Parser, Tag};
1010
1111use helix_core::{
12− syntax::{self, HighlightEvent, Syntax},
12+ syntax::{self, HighlightEvent, InjectionLanguageMarker, Syntax},
1313 Rope,
1414};
1515use helix_view::{
@@ -47,7 +47,9 @@ pub fn highlighted_code_block<'a>(
4747
4848 let rope = Rope::from(text.as_ref());
4949 let syntax = config_loader
50− .language_configuration_for_injection_string(language)
50+ .language_configuration_for_injection_string(&InjectionLanguageMarker::Name(
51+ language.into(),
52+ ))
5153 .and_then(|config| config.highlight_config(theme.scopes()))
5254 .map(|config| Syntax::new(&rope, config, Arc::clone(&config_loader)));
5355
languages.toml+1−1
@@ -591,7 +591,7 @@ indent = { tab-width = 2, unit = " " }
591591
592592[[grammar]]
593593name = "nix"
594−source = { git = "https://github.com/nix-community/tree-sitter-nix", rev = "6b71a810c0acd49b980c50fc79092561f7cee307" }
594+source = { git = "https://github.com/nix-community/tree-sitter-nix", rev = "1b69cf1fa92366eefbe6863c184e5d2ece5f187d" }
595595
596596[[language]]
597597name = "ruby"
runtime/queries/markdown/injections.scm+4
@@ -1,5 +1,9 @@
11; From nvim-treesitter/nvim-treesitter
22
3+(fenced_code_block
4+ (code_fence_content) @injection.shebang @injection.content
5+ (#set! injection.include-unnamed-children))
6+
37(fenced_code_block
48 (info_string
59 (language) @injection.language)
runtime/queries/nix/highlights.scm+4−1
@@ -47,8 +47,10 @@
4747(float_expression) @constant.numeric.float
4848
4949(escape_sequence) @constant.character.escape
50+(dollar_escape) @constant.character.escape
5051
5152(function_expression
53+ "@"? @punctuation.delimiter
5254 universal: (identifier) @variable.parameter
5355 "@"? @punctuation.delimiter
5456)
@@ -82,7 +84,8 @@
8284(binding
8385 attrpath: (attrpath attr: (identifier)) @variable.other.member)
8486
85−(inherit_from attrs: (inherited_attrs attr: (identifier) @variable))
87+(inherit_from attrs: (inherited_attrs attr: (identifier) @variable.other.member))
88+(inherited_attrs attr: (identifier) @variable)
8689
8790(has_attr_expression
8891 expression: (_)
runtime/queries/nix/injections.scm+15−3
@@ -10,9 +10,11 @@
1010; such as those of stdenv.mkDerivation.
1111((binding
1212 attrpath: (attrpath (identifier) @_path)
13− expression: (indented_string_expression
14− (string_fragment) @injection.content))
15− (#match? @_path "(^\\w*Phase|(pre|post)\\w*|(.*\\.)?\\w*([sS]cript|[hH]ook)|(.*\\.)?startup)$")
13+ expression: [
14+ (indented_string_expression (string_fragment) @injection.content)
15+ (binary_expression (indented_string_expression (string_fragment) @injection.content))
16+ ])
17+ (#match? @_path "(^\\w*Phase|command|(pre|post)\\w*|(.*\\.)?\\w*([sS]cript|[hH]ook)|(.*\\.)?startup)$")
1618 (#set! injection.language "bash")
1719 (#set! injection.combined))
1820
@@ -150,3 +152,13 @@
150152; (#match? @_func "(^|\\.)writeFSharp(Bin)?$")
151153; (#set! injection.language "f-sharp")
152154; (#set! injection.combined))
155+
156+((apply_expression
157+ function: (apply_expression function: (_) @_func
158+ argument: (string_expression (string_fragment) @injection.filename))
159+ argument: (indented_string_expression (string_fragment) @injection.content))
160+ (#match? @_func "(^|\\.)write(Text|Script(Bin)?)$")
161+ (#set! injection.combined))
162+
163+((indented_string_expression (string_fragment) @injection.shebang @injection.content)
164+ (#set! injection.combined))