9c6c63a · helix · git.iko.soy

git.iko.soy/helix

on master help sign in

inject language based on file extension & shebang (#3970)

9c6c63aTimothy DeHerrera4 years ago

book/src/SUMMARY.md+1

compare with master

unifiedsplit

@@ -16,3 +16,4 @@

16 - [Adding languages](./guides/adding_languages.md)

17 - [Adding textobject queries](./guides/textobject.md)

18 - [Adding indent queries](./guides/indent.md)

19+ - [Adding injection queries](./guides/injection.md)

book/src/guides/injection.mdnew file+57

@@ -0,0 +1,57 @@

1+# Adding Injection Queries

3+Writing language injection queries allows one to highlight a specific node as a different language.

4+In addition to the [standard](upstream-docs) language injection options used by tree-sitter, there

5+are a few Helix specific extensions that allow for more control.

7+And example of a simple query that would highlight all strings as bash in Nix:

8+```scm

9+((string_expression (string_fragment) @injection.content)

10+ (#set! injection.language "bash"))

11+```

12+

13+## Capture Types

14+

15+- `@injection.language` (standard):

16+The captured node may contain the language name used to highlight the node captured by

17+`@injection.content`.

18+

19+- `@injection.content` (standard):

20+Marks the content to be highlighted as the language captured with `@injection.language` _et al_.

21+

22+- `@injection.filename` (extension):

23+The captured node may contain a filename with a file-extension known to Helix,

24+highlighting `@injection.content` as that language. This uses the language extensions defined in

25+both the default languages.toml distributed with Helix, as well as user defined languages.

26+

27+- `@injection.shebang` (extension):

28+The captured node may contain a shebang used to choose a language to highlight as. This also uses

29+the shebangs defined in the default and user `languages.toml`.

30+

31+## Settings

32+

33+- `injection.combined` (standard):

34+Indicates that all the matching nodes in the tree should have their content parsed as one

35+nested document.

36+

37+- `injection.language` (standard):

38+Forces the captured content to be highlighted as the given language

39+

40+- `injection.include-children` (standard):

41+Indicates that the content node’s entire text should be re-parsed, including the text of its child

42+nodes. By default, child nodes’ text will be excluded from the injected document.

43+

44+- `injection.include-unnamed-children` (extension):

45+Same as `injection.include-children` but only for unnamed child nodes.

46+

47+## Predicates

48+

49+- `#eq?` (standard):

50+The first argument (a capture) must be equal to the second argument

51+(a capture or a string).

52+

53+- `#match?` (standard):

54+The first argument (a capture) must match the regex given in the

55+second argument (a string).

56+

57+[upstream-docs]: http://tree-sitter.github.io/tree-sitter/syntax-highlighting#language-injection

helix-core/src/syntax.rs+130−77

@@ -662,9 +662,8 @@ impl Loader {

662

663 pub fn language_config_for_shebang(&self, source: &Rope) -> Option<Arc<LanguageConfiguration>> {

664 let line = Cow::from(source.line(0));

665− static SHEBANG_REGEX: Lazy<Regex> = Lazy::new(|| {

665+ static SHEBANG_REGEX: Lazy<Regex> =

666− Regex::new(r"^#!\s*(?:\S*[/\\](?:env\s+(?:\-\S+\s+)*)?)?([^\s\.\d]+)").unwrap()

666+ Lazy::new(|| Regex::new(&["^", SHEBANG].concat()).unwrap());

667− });

668 let configuration_id = SHEBANG_REGEX

667 let configuration_id = SHEBANG_REGEX

669 .captures(&line)

668 .captures(&line)

670 .and_then(|cap| self.language_config_ids_by_shebang.get(&cap[1]));

669 .and_then(|cap| self.language_config_ids_by_shebang.get(&cap[1]));

@@ -686,15 +685,14 @@ impl Loader {

686 .cloned()

685 .cloned()

687 }

686 }

688

687

689− pub fn language_configuration_for_injection_string(

688+ /// Unlike language_config_for_language_id, which only returns Some for an exact id, this

690− &self,

689+ /// function will perform a regex match on the given string to find the closest language match.

691− string: &str,

692− ) -> Option<Arc<LanguageConfiguration>> {

690+ pub fn language_config_for_name(&self, name: &str) -> Option<Arc<LanguageConfiguration>> {

693 let mut best_match_length = 0;

691 let mut best_match_length = 0;

694 let mut best_match_position = None;

692 let mut best_match_position = None;

695 for (i, configuration) in self.language_configs.iter().enumerate() {

693 for (i, configuration) in self.language_configs.iter().enumerate() {

696 if let Some(injection_regex) = &configuration.injection_regex {

694 if let Some(injection_regex) = &configuration.injection_regex {

697− if let Some(mat) = injection_regex.find(string) {

695+ if let Some(mat) = injection_regex.find(name) {

698 let length = mat.end() - mat.start();

696 let length = mat.end() - mat.start();

699 if length > best_match_length {

697 if length > best_match_length {

700 best_match_position = Some(i);

698 best_match_position = Some(i);

@@ -704,11 +702,20 @@ impl Loader {

704 }

702 }

705 }

703 }

706

704

707− if let Some(i) = best_match_position {

705+ best_match_position.map(|i| self.language_configs[i].clone())

708− let configuration = &self.language_configs[i];

706+ }

709− return Some(configuration.clone());

707+

708+ pub fn language_configuration_for_injection_string(

709+ &self,

710+ capture: &InjectionLanguageMarker,

711+ ) -> Option<Arc<LanguageConfiguration>> {

712+ match capture {

713+ InjectionLanguageMarker::Name(string) => self.language_config_for_name(string),

714+ InjectionLanguageMarker::Filename(file) => self.language_config_for_file_name(file),

715+ InjectionLanguageMarker::Shebang(shebang) => {

716+ self.language_config_for_language_id(shebang)

717+ }

710 }

718 }

711− None

712 }

719 }

713

720

714 pub fn language_configs(&self) -> impl Iterator<Item = &Arc<LanguageConfiguration>> {

721 pub fn language_configs(&self) -> impl Iterator<Item = &Arc<LanguageConfiguration>> {

@@ -800,7 +807,7 @@ impl Syntax {

800 queue.push_back(self.root);

807 queue.push_back(self.root);

801

808

802 let scopes = self.loader.scopes.load();

809 let scopes = self.loader.scopes.load();

803− let injection_callback = |language: &str| {

810+ let injection_callback = |language: &InjectionLanguageMarker| {

804 self.loader

811 self.loader

805 .language_configuration_for_injection_string(language)

812 .language_configuration_for_injection_string(language)

806 .and_then(|language_config| language_config.highlight_config(&scopes))

813 .and_then(|language_config| language_config.highlight_config(&scopes))

@@ -961,12 +968,9 @@ impl Syntax {

961 );

968 );

962 let mut injections = Vec::new();

969 let mut injections = Vec::new();

963 for mat in matches {

970 for mat in matches {

964− let (language_name, content_node, included_children) = injection_for_match(

971+ let (injection_capture, content_node, included_children) = layer

965− &layer.config,

972+ .config

966− &layer.config.injections_query,

973+ .injection_for_match(&layer.config.injections_query, &mat, source_slice);

967− &mat,

968− source_slice,

969− );

970

974

971 // Explicitly remove this match so that none of its other captures will remain

975 // Explicitly remove this match so that none of its other captures will remain

972 // in the stream of captures.

976 // in the stream of captures.

@@ -974,9 +978,10 @@ impl Syntax {

974

978

975 // If a language is found with the given name, then add a new language layer

979 // If a language is found with the given name, then add a new language layer

976 // to the highlighted document.

980 // to the highlighted document.

977− if let (Some(language_name), Some(content_node)) = (language_name, content_node)

981+ if let (Some(injection_capture), Some(content_node)) =

982+ (injection_capture, content_node)

978 {

983 {

979− if let Some(config) = (injection_callback)(&language_name) {

984+ if let Some(config) = (injection_callback)(&injection_capture) {

980 let ranges =

985 let ranges =

981 intersect_ranges(&layer.ranges, &[content_node], included_children);

986 intersect_ranges(&layer.ranges, &[content_node], included_children);

982

987

@@ -1001,14 +1006,11 @@ impl Syntax {

1001 );

1006 );

1002 for mat in matches {

1007 for mat in matches {

1003 let entry = &mut injections_by_pattern_index[mat.pattern_index];

1008 let entry = &mut injections_by_pattern_index[mat.pattern_index];

1004− let (language_name, content_node, included_children) = injection_for_match(

1009+ let (injection_capture, content_node, included_children) = layer

1005− &layer.config,

1010+ .config

1006− combined_injections_query,

1011+ .injection_for_match(combined_injections_query, &mat, source_slice);

1007− &mat,

1012+ if injection_capture.is_some() {

1008− source_slice,

1013+ entry.0 = injection_capture;

1009− );

1010− if language_name.is_some() {

1011− entry.0 = language_name;

1012 }

1014 }

1013 if let Some(content_node) = content_node {

1015 if let Some(content_node) = content_node {

1014 entry.1.push(content_node);

1016 entry.1.push(content_node);

@@ -1395,6 +1397,8 @@ pub struct HighlightConfiguration {

1395 non_local_variable_patterns: Vec<bool>,

1397 non_local_variable_patterns: Vec<bool>,

1396 injection_content_capture_index: Option<u32>,

1398 injection_content_capture_index: Option<u32>,

1397 injection_language_capture_index: Option<u32>,

1399 injection_language_capture_index: Option<u32>,

1400+ injection_filename_capture_index: Option<u32>,

1401+ injection_shebang_capture_index: Option<u32>,

1398 local_scope_capture_index: Option<u32>,

1402 local_scope_capture_index: Option<u32>,

1399 local_def_capture_index: Option<u32>,

1403 local_def_capture_index: Option<u32>,

1400 local_def_value_capture_index: Option<u32>,

1404 local_def_value_capture_index: Option<u32>,

@@ -1538,6 +1542,8 @@ impl HighlightConfiguration {

1538 // Store the numeric ids for all of the special captures.

1542 // Store the numeric ids for all of the special captures.

1539 let mut injection_content_capture_index = None;

1543 let mut injection_content_capture_index = None;

1540 let mut injection_language_capture_index = None;

1544 let mut injection_language_capture_index = None;

1545+ let mut injection_filename_capture_index = None;

1546+ let mut injection_shebang_capture_index = None;

1541 let mut local_def_capture_index = None;

1547 let mut local_def_capture_index = None;

1542 let mut local_def_value_capture_index = None;

1548 let mut local_def_value_capture_index = None;

1543 let mut local_ref_capture_index = None;

1549 let mut local_ref_capture_index = None;

@@ -1558,6 +1564,8 @@ impl HighlightConfiguration {

1558 match name.as_str() {

1564 match name.as_str() {

1559 "injection.content" => injection_content_capture_index = i,

1565 "injection.content" => injection_content_capture_index = i,

1560 "injection.language" => injection_language_capture_index = i,

1566 "injection.language" => injection_language_capture_index = i,

1567+ "injection.filename" => injection_filename_capture_index = i,

1568+ "injection.shebang" => injection_shebang_capture_index = i,

1561 _ => {}

1569 _ => {}

1562 }

1570 }

1563 }

1571 }

@@ -1573,6 +1581,8 @@ impl HighlightConfiguration {

1573 non_local_variable_patterns,

1581 non_local_variable_patterns,

1574 injection_content_capture_index,

1582 injection_content_capture_index,

1575 injection_language_capture_index,

1583 injection_language_capture_index,

1584+ injection_filename_capture_index,

1585+ injection_shebang_capture_index,

1576 local_scope_capture_index,

1586 local_scope_capture_index,

1577 local_def_capture_index,

1587 local_def_capture_index,

1578 local_def_value_capture_index,

1588 local_def_value_capture_index,

@@ -1631,6 +1641,90 @@ impl HighlightConfiguration {

1631

1641

1632 self.highlight_indices.store(Arc::new(indices));

1642 self.highlight_indices.store(Arc::new(indices));

1633 }

1643 }

1644+

1645+ fn injection_pair<'a>(

1646+ &self,

1647+ query_match: &QueryMatch<'a, 'a>,

1648+ source: RopeSlice<'a>,

1649+ ) -> (Option<InjectionLanguageMarker<'a>>, Option<Node<'a>>) {

1650+ let mut injection_capture = None;

1651+ let mut content_node = None;

1652+

1653+ for capture in query_match.captures {

1654+ let index = Some(capture.index);

1655+ if index == self.injection_language_capture_index {

1656+ let name = byte_range_to_str(capture.node.byte_range(), source);

1657+ injection_capture = Some(InjectionLanguageMarker::Name(name));

1658+ } else if index == self.injection_filename_capture_index {

1659+ let name = byte_range_to_str(capture.node.byte_range(), source);

1660+ let path = Path::new(name.as_ref()).to_path_buf();

1661+ injection_capture = Some(InjectionLanguageMarker::Filename(path.into()));

1662+ } else if index == self.injection_shebang_capture_index {

1663+ let node_slice = source.byte_slice(capture.node.byte_range());

1664+

1665+ // some languages allow space and newlines before the actual string content

1666+ // so a shebang could be on either the first or second line

1667+ let lines = if let Ok(end) = node_slice.try_line_to_byte(2) {

1668+ node_slice.byte_slice(..end)

1669+ } else {

1670+ node_slice

1671+ };

1672+

1673+ static SHEBANG_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(SHEBANG).unwrap());

1674+

1675+ injection_capture = SHEBANG_REGEX

1676+ .captures(&Cow::from(lines))

1677+ .map(|cap| InjectionLanguageMarker::Shebang(cap[1].to_owned()))

1678+ } else if index == self.injection_content_capture_index {

1679+ content_node = Some(capture.node);

1680+ }

1681+ }

1682+ (injection_capture, content_node)

1683+ }

1684+

1685+ fn injection_for_match<'a>(

1686+ &self,

1687+ query: &'a Query,

1688+ query_match: &QueryMatch<'a, 'a>,

1689+ source: RopeSlice<'a>,

1690+ ) -> (

1691+ Option<InjectionLanguageMarker<'a>>,

1692+ Option<Node<'a>>,

1693+ IncludedChildren,

1694+ ) {

1695+ let (mut injection_capture, content_node) = self.injection_pair(query_match, source);

1696+

1697+ let mut included_children = IncludedChildren::default();

1698+ for prop in query.property_settings(query_match.pattern_index) {

1699+ match prop.key.as_ref() {

1700+ // In addition to specifying the language name via the text of a

1701+ // captured node, it can also be hard-coded via a `#set!` predicate

1702+ // that sets the injection.language key.

1703+ "injection.language" if injection_capture.is_none() => {

1704+ injection_capture = prop

1705+ .value

1706+ .as_ref()

1707+ .map(|s| InjectionLanguageMarker::Name(s.as_ref().into()));

1708+ }

1709+

1710+ // By default, injections do not include the *children* of an

1711+ // `injection.content` node - only the ranges that belong to the

1712+ // node itself. This can be changed using a `#set!` predicate that

1713+ // sets the `injection.include-children` key.

1714+ "injection.include-children" => included_children = IncludedChildren::All,

1715+

1716+ // Some queries might only exclude named children but include unnamed

1717+ // children in their `injection.content` node. This can be enabled using

1718+ // a `#set!` predicate that sets the `injection.include-unnamed-children` key.

1719+ "injection.include-unnamed-children" => {

1720+ included_children = IncludedChildren::Unnamed

1721+ }

1722+ _ => {}

1723+ }

1724+ }

1725+

1726+ (injection_capture, content_node, included_children)

1727+ }

1634}

1728}

1635

1729

1636impl<'a> HighlightIterLayer<'a> {

1730impl<'a> HighlightIterLayer<'a> {

@@ -2042,56 +2136,15 @@ impl<'a> Iterator for HighlightIter<'a> {

2042 }

2136 }

2043}

2137}

2044

2138

2045−fn injection_for_match<'a>(

2139+#[derive(Debug, Clone)]

2046− config: &HighlightConfiguration,

2140+pub enum InjectionLanguageMarker<'a> {

2047− query: &'a Query,

2141+ Name(Cow<'a, str>),

2048− query_match: &QueryMatch<'a, 'a>,

2142+ Filename(Cow<'a, Path>),

2049− source: RopeSlice<'a>,

2143+ Shebang(String),

2050−) -> (Option<Cow<'a, str>>, Option<Node<'a>>, IncludedChildren) {

2051− let content_capture_index = config.injection_content_capture_index;

2052− let language_capture_index = config.injection_language_capture_index;

2053−

2054− let mut language_name = None;

2055− let mut content_node = None;

2056− for capture in query_match.captures {

2057− let index = Some(capture.index);

2058− if index == language_capture_index {

2059− let name = byte_range_to_str(capture.node.byte_range(), source);

2060− language_name = Some(name);

2061− } else if index == content_capture_index {

2062− content_node = Some(capture.node);

2063− }

2064− }

2065−

2066− let mut included_children = IncludedChildren::default();

2067− for prop in query.property_settings(query_match.pattern_index) {

2068− match prop.key.as_ref() {

2069− // In addition to specifying the language name via the text of a

2070− // captured node, it can also be hard-coded via a `#set!` predicate

2071− // that sets the injection.language key.

2072− "injection.language" => {

2073− if language_name.is_none() {

2074− language_name = prop.value.as_ref().map(|s| s.as_ref().into())

2075− }

2076− }

2077−

2078− // By default, injections do not include the *children* of an

2079− // `injection.content` node - only the ranges that belong to the

2080− // node itself. This can be changed using a `#set!` predicate that

2081− // sets the `injection.include-children` key.

2082− "injection.include-children" => included_children = IncludedChildren::All,

2083−

2084− // Some queries might only exclude named children but include unnamed

2085− // children in their `injection.content` node. This can be enabled using

2086− // a `#set!` predicate that sets the `injection.include-unnamed-children` key.

2087− "injection.include-unnamed-children" => included_children = IncludedChildren::Unnamed,

2088− _ => {}

2089− }

2090− }

2091−

2092− (language_name, content_node, included_children)

2093}

2144}

2094

2145

2146+const SHEBANG: &str = r"#!\s*(?:\S*[/\\](?:env\s+(?:\-\S+\s+)*)?)?([^\s\.\d]+)";

2147+

2095pub struct Merge<I> {

2148pub struct Merge<I> {

2096 iter: I,

2149 iter: I,

2097 spans: Box<dyn Iterator<Item = (usize, std::ops::Range<usize>)>>,

2150 spans: Box<dyn Iterator<Item = (usize, std::ops::Range<usize>)>>,

helix-term/src/ui/markdown.rs+4−2

@@ -9,7 +9,7 @@ use std::sync::Arc;

9use pulldown_cmark::{CodeBlockKind, Event, HeadingLevel, Options, Parser, Tag};

11use helix_core::{

12− syntax::{self, HighlightEvent, Syntax},

12+ syntax::{self, HighlightEvent, InjectionLanguageMarker, Syntax},

13 Rope,

14};

15use helix_view::{

@@ -47,7 +47,9 @@ pub fn highlighted_code_block<'a>(

48 let rope = Rope::from(text.as_ref());

49 let syntax = config_loader

50− .language_configuration_for_injection_string(language)

50+ .language_configuration_for_injection_string(&InjectionLanguageMarker::Name(

51+ language.into(),

52+ ))

51 .and_then(|config| config.highlight_config(theme.scopes()))

53 .and_then(|config| config.highlight_config(theme.scopes()))

52 .map(|config| Syntax::new(&rope, config, Arc::clone(&config_loader)));

54 .map(|config| Syntax::new(&rope, config, Arc::clone(&config_loader)));

languages.toml+1−1

@@ -591,7 +591,7 @@ indent = { tab-width = 2, unit = " " }

591

592[[grammar]]

593name = "nix"

594−source = { git = "https://github.com/nix-community/tree-sitter-nix", rev = "6b71a810c0acd49b980c50fc79092561f7cee307" }

594+source = { git = "https://github.com/nix-community/tree-sitter-nix", rev = "1b69cf1fa92366eefbe6863c184e5d2ece5f187d" }

595

596[[language]]

597name = "ruby"

runtime/queries/markdown/injections.scm+4

@@ -1,5 +1,9 @@

1; From nvim-treesitter/nvim-treesitter

3+(fenced_code_block

4+ (code_fence_content) @injection.shebang @injection.content

5+ (#set! injection.include-unnamed-children))

3(fenced_code_block

7(fenced_code_block

4 (info_string

8 (info_string

5 (language) @injection.language)

9 (language) @injection.language)

runtime/queries/nix/highlights.scm+4−1

@@ -47,8 +47,10 @@

47(float_expression) @constant.numeric.float

49(escape_sequence) @constant.character.escape

50+(dollar_escape) @constant.character.escape

51(function_expression

52(function_expression

53+ "@"? @punctuation.delimiter

52 universal: (identifier) @variable.parameter

54 universal: (identifier) @variable.parameter

53 "@"? @punctuation.delimiter

55 "@"? @punctuation.delimiter

54)

56)

@@ -82,7 +84,8 @@

82(binding

84(binding

83 attrpath: (attrpath attr: (identifier)) @variable.other.member)

85 attrpath: (attrpath attr: (identifier)) @variable.other.member)

85−(inherit_from attrs: (inherited_attrs attr: (identifier) @variable))

87+(inherit_from attrs: (inherited_attrs attr: (identifier) @variable.other.member))

88+(inherited_attrs attr: (identifier) @variable)

87(has_attr_expression

90(has_attr_expression

88 expression: (_)

91 expression: (_)

runtime/queries/nix/injections.scm+15−3

@@ -10,9 +10,11 @@

10; such as those of stdenv.mkDerivation.

11((binding

12 attrpath: (attrpath (identifier) @_path)

13+ expression: [

13− expression: (indented_string_expression

14+ (indented_string_expression (string_fragment) @injection.content)

14− (string_fragment) @injection.content))

15+ (binary_expression (indented_string_expression (string_fragment) @injection.content))

16+ ])

16 (#set! injection.language "bash")

18 (#set! injection.language "bash")

17 (#set! injection.combined))

19 (#set! injection.combined))

@@ -150,3 +152,13 @@

150; (#match? @_func "(^|\\.)writeFSharp(Bin)?$")

152; (#match? @_func "(^|\\.)writeFSharp(Bin)?$")

151; (#set! injection.language "f-sharp")

153; (#set! injection.language "f-sharp")

152; (#set! injection.combined))

154; (#set! injection.combined))

155+

156+((apply_expression

157+ function: (apply_expression function: (_) @_func

158+ argument: (string_expression (string_fragment) @injection.filename))

159+ argument: (indented_string_expression (string_fragment) @injection.content))

160+ (#match? @_func "(^|\\.)write(Text|Script(Bin)?)$")

161+ (#set! injection.combined))

162+

163+((indented_string_expression (string_fragment) @injection.shebang @injection.content)

164+ (#set! injection.combined))