enso-org/enso
1
1
Fork 0
mirror of https://github.com/enso-org/enso.git synced 2024-11-23 16:18:23 +03:00
Code Issues Projects Releases Wiki Activity

fix span bug in doc comments (#3808)

Browse Source
This commit is contained in:
Kaz Wesley 2022-10-18 13:37:36 -07:00 committed by GitHub
parent 17f73988e8
commit feb8eb4f83
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 189 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
engine/runtime/src/test/java/org/enso/compiler/EnsoCompilerTest.java
View File

@ -407,6 +407,31 @@ public class EnsoCompilerTest {
""");
}
@Test
@Ignore // Documented TypeSignatures within a type body are not handled yet in TreeToIr.
public void testTestGroup() throws Exception {
parseTest("""
type Test
## Creates a new test group, describing properties of the object
described by `self`.
Arguments:
- name: The name of the test group.
- behaviors: An action containing a set of specs for the group.
- pending: A reason for why the test is pending, or `Nothing` when it is not
pending.
> Example
Adding a test group.
from Standard.Test import Test, Test_Suite
example_group = Test_Suite.run <|
Test.group "Number" <| Nothing
group : Text -> Any -> (Text | Nothing) -> Nothing
""");
}
@Test
public void testReverseListType() throws Exception {
parseTest("""

32
lib/rust/parser/debug/tests/parse.rs
View File

@ -1019,13 +1019,13 @@ fn case_expression() {
let code = [
"case a of",
" Some -> x",
" Int ->",
" Int -> x",
];
#[rustfmt::skip]
let expected = block![
(CaseOf (Ident a) #(
(((Ident Some) "->" (Ident x)))
(((Ident Int) "->" ()))))
((() (Ident Some) "->" (Ident x)))
((() (Ident Int) "->" (Ident x)))))
];
test(&code.join("\n"), expected);
@ -1037,7 +1037,7 @@ fn case_expression() {
#[rustfmt::skip]
let expected = block![
(CaseOf (Ident a) #(
(((App (App (Ident Vector_2d) (Ident x)) (Ident y)) "->" (Ident x)))))];
((() (App (App (Ident Vector_2d) (Ident x)) (Ident y)) "->" (Ident x)))))];
test(&code.join("\n"), expected);
#[rustfmt::skip]
@ -1049,8 +1049,8 @@ fn case_expression() {
#[rustfmt::skip]
let expected = block![
(CaseOf (Ident self) #(
(((Ident Vector_2d) "->" (Ident x)))
(((Wildcard -1) "->" (Ident x)))))];
((() (Ident Vector_2d) "->" (Ident x)))
((() (Wildcard -1) "->" (Ident x)))))];
test(&code.join("\n"), expected);
#[rustfmt::skip]
@ -1062,8 +1062,8 @@ fn case_expression() {
#[rustfmt::skip]
let expected = block![
(CaseOf (Ident foo) #(
(((TypeAnnotated (Ident v) ":" (Ident My_Type)) "->" (Ident x)))
(((TypeAnnotated (Ident v) ":"
((() (TypeAnnotated (Ident v) ":" (Ident My_Type)) "->" (Ident x)))
((() (TypeAnnotated (Ident v) ":"
(Group (App (App (Ident My_Type) (Wildcard -1)) (Wildcard -1))))
"->" (Ident x)))))];
test(&code.join("\n"), expected);
@ -1077,18 +1077,18 @@ fn case_by_type() {
}
}
test_case!("f:A->B -> x",
((TypeAnnotated (Ident f) ":" (OprApp (Ident A) (Ok "->") (Ident B))) "->" (Ident x)));
(() (TypeAnnotated (Ident f) ":" (OprApp (Ident A) (Ok "->") (Ident B))) "->" (Ident x)));
test_case!("f : A->B -> x",
((TypeAnnotated (Ident f) ":" (OprApp (Ident A) (Ok "->") (Ident B))) "->" (Ident x)));
(() (TypeAnnotated (Ident f) ":" (OprApp (Ident A) (Ok "->") (Ident B))) "->" (Ident x)));
test_case!("v : A -> x->x",
((TypeAnnotated (Ident v) ":" (Ident A)) "->" (OprApp (Ident x) (Ok "->") (Ident x))));
(() (TypeAnnotated (Ident v) ":" (Ident A)) "->" (OprApp (Ident x) (Ok "->") (Ident x))));
test_case!("v : A -> x -> x",
((TypeAnnotated (Ident v) ":" (Ident A)) "->" (OprApp (Ident x) (Ok "->") (Ident x))));
(() (TypeAnnotated (Ident v) ":" (Ident A)) "->" (OprApp (Ident x) (Ok "->") (Ident x))));
test_case!("v:A->x->x",
((TypeAnnotated (Ident v) ":" (Ident A)) "->" (OprApp (Ident x) (Ok "->") (Ident x))));
test_case!("v:A->x", ((TypeAnnotated (Ident v) ":" (Ident A)) "->" (Ident x)));
(() (TypeAnnotated (Ident v) ":" (Ident A)) "->" (OprApp (Ident x) (Ok "->") (Ident x))));
test_case!("v:A->x", (() (TypeAnnotated (Ident v) ":" (Ident A)) "->" (Ident x)));
test_case!("v : A -> _ + x",
((TypeAnnotated (Ident v) ":" (Ident A)) "->"
(() (TypeAnnotated (Ident v) ":" (Ident A)) "->"
(TemplateFunction 1 (OprApp (Wildcard 0) (Ok "+") (Ident x)))));
}
@ -1101,7 +1101,7 @@ fn pattern_match_auto_scope() {
];
#[rustfmt::skip]
let expected = block![
(CaseOf (Ident self) #((((App (Ident Vector_2d) (AutoScope)) "->" (Ident x)))))];
(CaseOf (Ident self) #(((() (App (Ident Vector_2d) (AutoScope)) "->" (Ident x)))))];
test(&code.join("\n"), expected);
}

119
lib/rust/parser/src/lexer.rs
View File

@ -108,7 +108,9 @@ pub enum State {
/// Reading a multi-line text literal.
MultilineText {
/// Indentation level of the quote symbol introducing the block.
indent: VisibleOffset,
quote_indent: VisibleOffset,
/// Indentation level of the first line of the block.
initial_indent: Option<VisibleOffset>,
},
}
@ -168,6 +170,14 @@ impl<'s> Lexer<'s> {
self.run_and_get_offset(|this| this.spaces());
}
/// Consume spaces after parsing a [`Token`] and update the internal spacing info. Doesn't
/// consume more than the specified [`VisibleOffset`] of spaces.
#[inline(always)]
fn spaces_after_lexeme_with_limit(&mut self, limit: VisibleOffset) {
(self.last_spaces_visible_offset, self.last_spaces_offset) =
self.run_and_get_offset(|this| this.spaces_with_limit(limit));
}
/// Run the provided function. If it consumed any chars, return the [`Token`] containing the
/// provided function output. Returns [`None`] otherwise.
#[inline(always)]
@ -342,6 +352,19 @@ impl<'s> Lexer<'s> {
}
total_visible_offset
}
/// Consume visible space characters and return their visible offset.
#[inline(always)]
fn spaces_with_limit(&mut self, limit: VisibleOffset) -> VisibleOffset {
let mut total_visible_offset = VisibleOffset(0);
while let Some(visible_offset) = self.space() {
total_visible_offset += visible_offset;
if total_visible_offset >= limit {
break;
}
}
total_visible_offset
}
}
@ -835,29 +858,32 @@ impl<'s> Lexer<'s> {
fn multiline_text(
&mut self,
open_quote_start: (Bytes, Offset<'s>),
indent: VisibleOffset,
quote_indent: VisibleOffset,
text_type: TextType,
) {
let open_quote_end = self.mark();
let token =
self.make_token(open_quote_start, open_quote_end.clone(), token::Variant::text_start());
self.output.push(token);
let mut initial_indent = None;
if text_type.expects_initial_newline() && let Some(newline) = self.line_break() {
self.output.push(newline.with_variant(token::Variant::text_initial_newline()));
if self.last_spaces_visible_offset > quote_indent {
initial_indent = self.last_spaces_visible_offset.into();
}
}
let text_start = self.mark();
self.text_content(
Some(text_start),
None,
text_type.is_interpolated(),
State::MultilineText { indent },
Some(indent),
State::MultilineText { quote_indent, initial_indent },
);
}
fn inline_quote(&mut self, quote_char: char, text_type: TextType) {
let is_interpolated = text_type.is_interpolated();
self.text_content(None, quote_char.into(), is_interpolated, State::InlineText, None);
self.text_content(None, quote_char.into(), is_interpolated, State::InlineText);
}
fn end_splice(&mut self, state: State) {
@ -869,8 +895,8 @@ impl<'s> Lexer<'s> {
self.output.push(token);
match state {
State::InlineText => self.inline_quote('\'', TextType::Interpolated),
State::MultilineText { indent } => {
self.text_content(None, None, true, State::MultilineText { indent }, Some(indent));
State::MultilineText { .. } => {
self.text_content(None, None, true, state);
}
}
}
@ -880,24 +906,27 @@ impl<'s> Lexer<'s> {
start: Option<(Bytes, Offset<'s>)>,
closing_char: Option<char>,
interpolate: bool,
state: State,
multiline: Option<VisibleOffset>,
mut state: State,
) -> TextEndedAt {
let mut text_start = start.unwrap_or_else(|| self.mark());
let is_multiline = matches!(state, State::MultilineText { .. });
while let Some(char) = self.current_char {
if closing_char == Some(char) || (multiline.is_none() && is_newline_char(char)) {
if closing_char == Some(char) || (!is_multiline && is_newline_char(char)) {
break;
}
let before_newline = self.mark();
let mut newline = self.take_1('\r');
newline = newline || self.take_1('\n');
if newline {
let indent = multiline.unwrap();
if newline && let State::MultilineText { quote_indent, initial_indent } = &mut state {
let text_end = self.mark();
self.spaces_after_lexeme();
if let Some(indent) = *initial_indent {
self.spaces_after_lexeme_with_limit(indent);
} else {
self.spaces_after_lexeme();
}
if let Some(char) = self.current_char && !is_newline_char(char) {
let block_indent = self.last_spaces_visible_offset;
if block_indent <= indent {
if block_indent <= *quote_indent {
let token = self.make_token(
text_start,
before_newline.clone(),
@ -913,13 +942,16 @@ impl<'s> Lexer<'s> {
self.output.push(token);
return TextEndedAt::End;
}
if initial_indent.is_none() {
*initial_indent = block_indent.into();
}
};
let token =
self.make_token(text_start, text_end.clone(), token::Variant::text_section());
if !(token.code.is_empty() && token.left_offset.code.is_empty()) {
self.make_token(text_start.clone(), text_end.clone(), token::Variant::text_section());
if !token.code.is_empty() {
self.output.push(token);
text_start = self.mark();
}
text_start = self.mark();
continue;
}
if interpolate && char == '\\' {
@ -942,13 +974,15 @@ impl<'s> Lexer<'s> {
continue;
}
if interpolate && char == '`' {
let splice_quote_start = self.mark();
let mut splice_quote_start = self.mark();
let token = self.make_token(
text_start,
text_start.clone(),
splice_quote_start.clone(),
token::Variant::text_section(),
);
if !(token.code.is_empty() && token.left_offset.code.is_empty()) {
if token.code.is_empty() {
splice_quote_start = text_start;
} else {
self.output.push(token);
}
self.take_next();
@ -1078,7 +1112,7 @@ enum TextEndedAt {
End,
}
#[derive(PartialEq, Eq, Copy, Clone)]
#[derive(PartialEq, Eq, Copy, Clone, Debug)]
enum TextType {
Raw,
Interpolated,
@ -1095,49 +1129,6 @@ impl TextType {
}
}
/// Move whitespace characters from the end of `left` to the beginning of `right` until the visible
/// length of `left` is not longer than `target`.
#[allow(unsafe_code)]
pub fn untrim(target: VisibleOffset, left: &mut Offset, right: &mut Code) {
let mut utf8 = 0;
let mut utf16 = 0;
let mut trimmed = VisibleOffset(0);
for c in left.code.repr.chars().rev() {
if left.visible - trimmed <= target {
break;
}
utf8 += c.len_utf8();
utf16 += c.len_utf16();
trimmed += space_char_visible_size(c).unwrap();
}
if utf8 == 0 && utf16 == 0 {
return;
}
left.visible = left.visible - trimmed;
left.code.utf16 -= utf16;
let len = left.code.repr.len() - utf8;
unsafe {
match right.repr {
Cow::Borrowed(s) if s.as_ptr() == left.code.repr.as_ptr().add(left.code.repr.len()) => {
let p = s.as_ptr().sub(utf8);
let len = s.len() + utf8;
right.repr = Cow::Borrowed(str::from_utf8_unchecked(slice::from_raw_parts(p, len)));
}
_ => {
let mut s = String::with_capacity(len + right.repr.len());
s += &left.code.repr[len..];
right.utf16 += utf16;
s += &right.repr;
right.repr = Cow::Owned(s);
}
}
}
left.code.repr = match &left.code.repr {
Cow::Borrowed(s) => Cow::Borrowed(&s[..len]),
Cow::Owned(s) => Cow::Owned(s[..len].to_string()),
};
}
// ================

1
lib/rust/parser/src/lib.rs
View File

@ -85,6 +85,7 @@
#![feature(specialization)]
#![feature(if_let_guard)]
#![feature(box_patterns)]
#![feature(option_get_or_insert_default)]
// === Standard Linter Configuration ===
#![deny(non_ascii_idents)]
#![warn(unsafe_code)]

103
lib/rust/parser/src/macros/built_in.rs
View File

@ -418,7 +418,7 @@ pub fn case<'s>() -> Definition<'s> {
crate::macro_definition! {("case", everything(), "of", everything()) case_body}
}
fn case_body<'s>(segments: NonEmptyVec<MatchedSegment<'s>>) -> syntax::Tree<'s> {
fn case_body(segments: NonEmptyVec<MatchedSegment>) -> syntax::Tree {
use operator::resolve_operator_precedence_if_non_empty;
use syntax::tree::*;
let (of, mut rest) = segments.pop();
@ -427,51 +427,50 @@ fn case_body<'s>(segments: NonEmptyVec<MatchedSegment<'s>>) -> syntax::Tree<'s>
let expression = case.result.tokens();
let expression = resolve_operator_precedence_if_non_empty(expression);
let of_ = into_ident(of.header);
let mut case_lines: Vec<CaseLine> = vec![];
let mut case_builder = CaseBuilder::default();
let finish_line = |case: Option<Case<'s>>, case_lines: &mut Vec<CaseLine<'s>>| {
if let Some(case) = case {
if case_lines.is_empty() {
case_lines.push(default());
}
case_lines.last_mut().unwrap().case = Some(case);
}
};
for item in of.result.tokens() {
if let syntax::Item::Block(items) = item {
for item in items {
if let syntax::Item::Token(syntax::Token {
left_offset,
code,
variant: syntax::token::Variant::Newline(_),
}) = item
{
finish_line(case_builder.finish(), &mut case_lines);
let newline = syntax::token::newline(left_offset, code).into();
case_lines.push(CaseLine { newline, ..default() });
continue;
}
case_builder.push(item);
}
continue;
items.into_iter().for_each(|item| case_builder.push(item));
} else {
case_builder.push(item);
}
case_builder.push(item);
}
finish_line(case_builder.finish(), &mut case_lines);
Tree::case_of(case_, expression, of_, case_lines)
let (case_lines, any_invalid) = case_builder.finish();
let tree = Tree::case_of(case_, expression, of_, case_lines);
if any_invalid {
return tree.with_error("Invalid case expression.");
}
tree
}
#[derive(Default)]
struct CaseBuilder<'s> {
tokens: Vec<syntax::Item<'s>>,
resolver: operator::Precedence<'s>,
pattern: Option<syntax::Tree<'s>>,
arrow: Option<syntax::token::Operator<'s>>,
spaces: bool,
// Case components
documentation: Option<syntax::tree::DocComment<'s>>,
pattern: Option<syntax::Tree<'s>>,
arrow: Option<syntax::token::Operator<'s>>,
// Within-case state
spaces: bool,
tokens: Vec<syntax::Item<'s>>,
resolver: operator::Precedence<'s>,
// Output
case_lines: Vec<syntax::tree::CaseLine<'s>>,
any_invalid: bool,
}
impl<'s> CaseBuilder<'s> {
fn push(&mut self, token: syntax::Item<'s>) {
if let syntax::Item::Token(syntax::Token {
left_offset,
code,
variant: syntax::token::Variant::Newline(_),
}) = token
{
self.finish_line();
let newline = syntax::token::newline(left_offset, code).into();
self.case_lines.push(syntax::tree::CaseLine { newline, ..default() });
return;
}
if self.arrow.is_none() &&
let syntax::Item::Token(syntax::Token { left_offset, code, variant: syntax::token::Variant::Operator(op) }) = &token
&& op.properties.is_arrow()
@ -487,7 +486,7 @@ impl<'s> CaseBuilder<'s> {
self.tokens.push(token);
}
fn finish(&mut self) -> Option<syntax::tree::Case<'s>> {
fn finish_line(&mut self) {
if self.arrow.is_none() && !self.spaces {
for (i, token) in self.tokens.iter().enumerate() {
if let syntax::Item::Token(syntax::Token { left_offset, code, variant: syntax::token::Variant::Operator(op) }) = &token
@ -500,10 +499,27 @@ impl<'s> CaseBuilder<'s> {
}
}
}
self.spaces = false;
self.resolver.extend(self.tokens.drain(..));
let pattern = self.pattern.take();
let arrow = self.arrow.take();
let expression = match self.resolver.finish() {
Some(syntax::Tree {
span,
variant:
box syntax::tree::Variant::Documented(syntax::tree::Documented {
mut documentation,
expression: None,
}),
}) if self.documentation.is_none() => {
documentation.open.left_offset += span.left_offset;
if self.case_lines.is_empty() {
self.case_lines.push(default());
}
let mut case = self.case_lines.last_mut().unwrap().case.get_or_insert_default();
case.documentation = documentation.into();
return;
}
Some(syntax::Tree {
span,
variant:
@ -518,9 +534,22 @@ impl<'s> CaseBuilder<'s> {
e => e,
};
if pattern.is_none() && arrow.is_none() && expression.is_none() {
return None;
return;
}
Some(syntax::tree::Case { pattern, arrow, expression })
self.any_invalid =
self.any_invalid || pattern.is_none() || arrow.is_none() || expression.is_none();
if self.case_lines.is_empty() {
self.case_lines.push(default());
}
let mut case = &mut self.case_lines.last_mut().unwrap().case.get_or_insert_default();
case.pattern = pattern;
case.arrow = arrow;
case.expression = expression;
}
fn finish(mut self) -> (Vec<syntax::tree::CaseLine<'s>>, bool) {
self.finish_line();
(self.case_lines, self.any_invalid)
}
}
@ -636,7 +665,7 @@ fn splice_body(segments: NonEmptyVec<MatchedSegment>) -> syntax::Tree {
let expression = segment.result.tokens();
let expression = operator::resolve_operator_precedence_if_non_empty(expression);
let splice = syntax::tree::TextElement::Splice { open, expression, close };
syntax::Tree::text_literal(default(), default(), vec![splice], default(), default(), default())
syntax::Tree::text_literal(default(), default(), vec![splice], default(), default())
}
fn into_open_symbol(token: syntax::token::Token) -> syntax::token::OpenSymbol {

6
lib/rust/parser/src/source/code.rs
View File

@ -123,6 +123,12 @@ impl Length {
pub fn is_zero(&self) -> bool {
self.utf8 == 0
}
/// Return the length in UTF-8 code units (bytes).
#[inline(always)]
pub fn utf8_bytes(&self) -> usize {
self.utf8
}
}
impl Add for Length {

70
lib/rust/parser/src/syntax/tree.rs
View File

@ -141,9 +141,6 @@ macro_rules! with_ast_definition { ($f:ident ($($args:tt)*)) => { $f! { $($args)
#[serde(skip)]
#[reflect(skip)]
pub closed: bool,
#[serde(skip)]
#[reflect(skip)]
pub trim: VisibleOffset,
},
/// A simple application, like `print "hello"`.
App {
@ -449,7 +446,10 @@ pub struct TypeConstructorDef<'s> {
impl<'s> span::Builder<'s> for TypeConstructorDef<'s> {
fn add_to_span(&mut self, span: Span<'s>) -> Span<'s> {
span.add(&mut self.constructor).add(&mut self.arguments).add(&mut self.block)
span.add(&mut self.documentation)
.add(&mut self.constructor)
.add(&mut self.arguments)
.add(&mut self.block)
}
}
@ -517,10 +517,6 @@ pub struct DocComment<'s> {
pub open: token::TextStart<'s>,
/// The documentation text.
pub elements: Vec<TextElement<'s>>,
/// The minimum indent among all lines of the text content.
#[serde(skip)]
#[reflect(skip)]
pub trim: VisibleOffset,
/// Empty lines between the comment and the item.
pub newlines: Vec<token::Newline<'s>>,
}
@ -651,21 +647,22 @@ impl<'s> span::Builder<'s> for CaseLine<'s> {
/// A case-expression in a case-of expression.
#[derive(Clone, Debug, Default, Eq, PartialEq, Visitor, Serialize, Reflect, Deserialize)]
pub struct Case<'s> {
/// Documentation, if present.
pub documentation: Option<DocComment<'s>>,
/// The pattern being matched. It is an error for this to be absent.
pub pattern: Option<Tree<'s>>,
pub pattern: Option<Tree<'s>>,
/// Token.
pub arrow: Option<token::Operator<'s>>,
pub arrow: Option<token::Operator<'s>>,
/// The expression associated with the pattern. It is an error for this to be empty.
pub expression: Option<Tree<'s>>,
pub expression: Option<Tree<'s>>,
}
impl<'s> Case<'s> {
/// Return a mutable reference to the `left_offset` of this object (which will actually belong
/// to one of the object's children, if it has any).
pub fn left_offset_mut(&mut self) -> Option<&mut Offset<'s>> {
self.pattern
.as_mut()
.map(|p| &mut p.span.left_offset)
None.or_else(|| self.documentation.as_mut().map(|t| &mut t.open.left_offset))
.or_else(|| self.pattern.as_mut().map(|t| &mut t.span.left_offset))
.or_else(|| self.arrow.as_mut().map(|t| &mut t.left_offset))
.or_else(|| self.expression.as_mut().map(|e| &mut e.span.left_offset))
}
@ -768,14 +765,12 @@ pub fn apply<'s>(mut func: Tree<'s>, mut arg: Tree<'s>) -> Tree<'s> {
match (&mut *func.variant, &mut *arg.variant) {
(Variant::TextLiteral(lhs), Variant::TextLiteral(rhs)) if !lhs.closed => {
join_text_literals(lhs, rhs, mem::take(&mut arg.span));
if lhs.open.is_some() && lhs.closed {
trim_text(lhs.trim, &mut lhs.elements);
}
if let TextLiteral { open: Some(open), newline: None, elements, closed: true, close: None, trim } = lhs && open.code.starts_with('#') {
if let TextLiteral { open: Some(open), newline: None, elements, closed: true, close: None } = lhs
&& open.code.starts_with('#') {
let mut open = open.clone();
open.left_offset += func.span.left_offset;
let elements = mem::take(elements);
let doc = DocComment { open, elements, trim: *trim, newlines: default() };
let doc = DocComment { open, elements, newlines: default() };
return Tree::documented(doc, default());
}
func
@ -848,9 +843,6 @@ fn join_text_literals<'s>(
rhs: &mut TextLiteral<'s>,
rhs_span: Span<'s>,
) {
if rhs.trim != VisibleOffset(0) && (lhs.trim == VisibleOffset(0) || rhs.trim < lhs.trim) {
lhs.trim = rhs.trim;
}
match rhs.elements.first_mut() {
Some(TextElement::Section { text }) => text.left_offset += rhs_span.left_offset,
Some(TextElement::Escape { token }) => token.left_offset += rhs_span.left_offset,
@ -865,26 +857,6 @@ fn join_text_literals<'s>(
lhs.closed = rhs.closed;
}
fn trim_text(trim: VisibleOffset, elements: &mut Vec<TextElement>) {
let mut remaining = elements.len();
let mut carried_offset = Offset::default();
elements.retain_mut(|e| {
remaining -= 1;
let (offset, code) = match e {
TextElement::Section { text } => (&mut text.left_offset, &mut text.code),
TextElement::Escape { token } => (&mut token.left_offset, &mut token.code),
TextElement::Splice { open, .. } => (&mut open.left_offset, &mut open.code),
};
*offset += mem::take(&mut carried_offset);
crate::lexer::untrim(trim, offset, code);
if remaining != 0 && code.is_empty() {
carried_offset = mem::take(offset);
return false;
}
true
});
}
/// Join two nodes with an operator, in a way appropriate for their types.
///
/// For most operands this will simply construct an `OprApp`; however, a non-operator block (i.e. an
@ -968,24 +940,22 @@ impl<'s> From<Token<'s>> for Tree<'s> {
token::Variant::NumberBase(base) =>
Tree::number(Some(token.with_variant(base)), None, None),
token::Variant::TextStart(open) =>
Tree::text_literal(Some(token.with_variant(open)), default(), default(), default(), default(), default()),
Tree::text_literal(Some(token.with_variant(open)), default(), default(), default(), default()),
token::Variant::TextSection(section) => {
let trim = token.left_offset.visible;
let section = TextElement::Section { text: token.with_variant(section) };
Tree::text_literal(default(), default(), vec![section], default(), default(), trim)
Tree::text_literal(default(), default(), vec![section], default(), default())
}
token::Variant::TextEscape(escape) => {
let trim = token.left_offset.visible;
let token = token.with_variant(escape);
let section = TextElement::Escape { token };
Tree::text_literal(default(), default(), vec![section], default(), default(), trim)
Tree::text_literal(default(), default(), vec![section], default(), default())
}
token::Variant::TextEnd(_) if token.code.is_empty() =>
Tree::text_literal(default(), default(), default(), default(), true, default()),
Tree::text_literal(default(), default(), default(), default(), true),
token::Variant::TextEnd(close) =>
Tree::text_literal(default(), default(), default(), Some(token.with_variant(close)), true, default()),
Tree::text_literal(default(), default(), default(), Some(token.with_variant(close)), true),
token::Variant::TextInitialNewline(_) =>
Tree::text_literal(default(), Some(token::newline(token.left_offset, token.code)), default(), default(), default(), default()),
Tree::text_literal(default(), Some(token::newline(token.left_offset, token.code)), default(), default(), default()),
token::Variant::Wildcard(wildcard) => Tree::wildcard(token.with_variant(wildcard), default()),
token::Variant::AutoScope(t) => Tree::auto_scope(token.with_variant(t)),
token::Variant::OpenSymbol(s) =>