Fix approximately a bajillion fmt and parsing bugs

(discovered by fuzzing)

There's more to come, but this seems like a good batch for now.
This commit is contained in:
Joshua Warner 2023-01-11 19:44:29 -08:00
parent 8f62eeaf7e
commit 0b8e68f70d
No known key found for this signature in database
GPG Key ID: 89AD497003F93FDD
68 changed files with 1011 additions and 229 deletions

View File

@ -12,7 +12,7 @@ use roc_region::all::Loc;
/// Does an AST node need parens around it?
///
/// Usually not, but there are two cases where it may be required
/// Usually not, but there are a few cases where it may be required
///
/// 1. In a function type, function types are in parens
///
@ -25,11 +25,19 @@ use roc_region::all::Loc;
/// Just (Just a)
/// List (List a)
/// reverse (reverse l)
///
/// 3. In a chain of binary operators, things like nested defs require parens.
///
/// a + (
/// x = 3
/// x + 1
/// )
#[derive(PartialEq, Eq, Clone, Copy, Debug)]
pub enum Parens {
NotNeeded,
InFunctionType,
InApply,
InOperator,
}
/// In an AST node, do we show newlines around it
@ -263,7 +271,6 @@ impl<'a> Formattable for TypeAnnotation<'a> {
}
Apply(pkg, name, arguments) => {
buf.indent(indent);
// NOTE apply is never multiline
let write_parens = parens == Parens::InApply && !arguments.is_empty();
if write_parens {
@ -277,11 +284,38 @@ impl<'a> Formattable for TypeAnnotation<'a> {
buf.push_str(name);
for argument in *arguments {
buf.spaces(1);
argument
.value
.format_with_options(buf, Parens::InApply, Newlines::No, indent);
let needs_indent = except_last(arguments).any(|a| a.is_multiline())
|| arguments
.last()
.map(|a| {
a.is_multiline()
&& (!a.extract_spaces().before.is_empty()
|| !is_outdentable(&a.value))
})
.unwrap_or_default();
let arg_indent = if needs_indent {
indent + INDENT
} else {
indent
};
for arg in arguments.iter() {
if needs_indent {
let arg = arg.extract_spaces();
fmt_spaces(buf, arg.before.iter(), arg_indent);
buf.ensure_ends_with_newline();
arg.item.format_with_options(
buf,
Parens::InApply,
Newlines::Yes,
arg_indent,
);
fmt_spaces(buf, arg.after.iter(), arg_indent);
} else {
buf.spaces(1);
arg.format_with_options(buf, Parens::InApply, Newlines::No, arg_indent);
}
}
if write_parens {
@ -372,6 +406,13 @@ impl<'a> Formattable for TypeAnnotation<'a> {
}
}
fn is_outdentable(ann: &TypeAnnotation) -> bool {
matches!(
ann.extract_spaces().item,
TypeAnnotation::Tuple { .. } | TypeAnnotation::Record { .. }
)
}
/// Fields are subtly different on the type and term level:
///
/// > type: { x : Int, y : Bool }
@ -691,3 +732,11 @@ impl<'a> Formattable for HasAbilities<'a> {
}
}
}
pub fn except_last<T>(items: &[T]) -> impl Iterator<Item = &T> {
if items.is_empty() {
items.iter()
} else {
items[..items.len() - 1].iter()
}
}

View File

@ -77,6 +77,7 @@ impl<'a> Formattable for TypeDef<'a> {
for var in *vars {
buf.spaces(1);
fmt_pattern(buf, &var.value, indent, Parens::NotNeeded);
buf.indent(indent);
}
buf.push_str(" :");
@ -95,6 +96,7 @@ impl<'a> Formattable for TypeDef<'a> {
for var in *vars {
buf.spaces(1);
fmt_pattern(buf, &var.value, indent, Parens::NotNeeded);
buf.indent(indent);
}
buf.push_str(" :=");
@ -136,6 +138,7 @@ impl<'a> Formattable for TypeDef<'a> {
for var in *vars {
buf.spaces(1);
fmt_pattern(buf, &var.value, indent, Parens::NotNeeded);
buf.indent(indent);
}
buf.push_str(" has");
@ -191,6 +194,7 @@ impl<'a> Formattable for ValueDef<'a> {
match self {
Annotation(loc_pattern, loc_annotation) => {
loc_pattern.format(buf, indent);
buf.indent(indent);
if loc_annotation.is_multiline() {
buf.push_str(" :");
@ -390,6 +394,7 @@ pub fn fmt_body<'a, 'buf>(
indent: u16,
) {
pattern.format_with_options(buf, Parens::InApply, Newlines::No, indent);
buf.indent(indent);
buf.push_str(" =");
if body.is_multiline() {
@ -415,7 +420,7 @@ pub fn fmt_body<'a, 'buf>(
);
}
}
Expr::BinOps(_, _) => {
Expr::Defs(..) | Expr::BinOps(_, _) => {
// Binop chains always get a newline. Otherwise you can have things like:
//
// something = foo

View File

@ -1,4 +1,4 @@
use crate::annotation::{Formattable, Newlines, Parens};
use crate::annotation::{except_last, Formattable, Newlines, Parens};
use crate::collection::{fmt_collection, Braces};
use crate::def::fmt_defs;
use crate::pattern::fmt_pattern;
@ -142,7 +142,7 @@ impl<'a> Formattable for Expr<'a> {
}
ParensAround(sub_expr) => {
if parens == Parens::NotNeeded && !sub_expr_requests_parens(sub_expr) {
sub_expr.format_with_options(buf, Parens::NotNeeded, Newlines::Yes, indent);
sub_expr.format_with_options(buf, Parens::NotNeeded, newlines, indent);
} else {
let should_add_newlines = match sub_expr {
Expr::Closure(..)
@ -202,86 +202,111 @@ impl<'a> Formattable for Expr<'a> {
buf.push_str("crash");
}
Apply(loc_expr, loc_args, _) => {
// Sadly this assertion fails in practice. The fact that the parser produces code like this is going to
// confuse the formatter, because it depends on being able to "see" spaces that logically come before the inner
// expr in several places - which is necessarily the case when the `loc_expr` of the apply itself has
// SpaceBefore.
//
// TODO: enforce in the type system that spaces must be pushed to the "outside".
// In other words, Expr::Apply should look something like the following, and there shouldn't be Expr::SpaceBefore and ::SpaceAfter.
//
// ```
// Apply(&'a SpaceAfter<Loc<Expr<'a>>>, &'a [&'a SpaceBefore<Loc<Expr<'a>>>], CalledVia),
// ```
//
// assert!(loc_expr.extract_spaces().before.is_empty(), "{:#?}", self);
buf.indent(indent);
if apply_needs_parens && !loc_args.is_empty() {
buf.push('(');
}
// should_reflow_outdentable, aka should we transform this:
//
// ```
// foo bar
// [
// 1,
// 2,
// ]
// ```
//
// Into this:
//
// ```
// foo bar [
// 1,
// 2,
// ]
// ```
let should_reflow_outdentable = loc_expr.extract_spaces().after.is_empty()
&& except_last(loc_args).all(|a| !a.is_multiline())
&& loc_args
.last()
.map(|a| {
a.extract_spaces().item.is_multiline()
&& matches!(
a.value.extract_spaces().item,
Expr::Tuple(_) | Expr::List(_) | Expr::Record(_)
)
&& a.extract_spaces().before == [CommentOrNewline::Newline]
})
.unwrap_or_default();
let needs_indent = !should_reflow_outdentable
&& (!loc_expr.extract_spaces().after.is_empty()
|| except_last(loc_args).any(|a| a.is_multiline())
|| loc_args
.last()
.map(|a| {
a.is_multiline()
&& (!a.extract_spaces().before.is_empty()
|| !is_outdentable(&a.value))
})
.unwrap_or_default());
let arg_indent = if needs_indent {
indent + INDENT
} else {
indent
};
loc_expr.format_with_options(buf, Parens::InApply, Newlines::Yes, indent);
let multiline_args = loc_args.iter().any(|loc_arg| loc_arg.is_multiline());
for loc_arg in loc_args.iter() {
if should_reflow_outdentable {
buf.spaces(1);
let mut found_multiline_expr = false;
let mut iter = loc_args.iter().peekable();
// Ignore any comments+newlines before/after.
// We checked above that there's only a single newline before the last arg,
// which we're intentionally ignoring.
while let Some(loc_arg) = iter.next() {
if iter.peek().is_none() {
found_multiline_expr = match loc_arg.value {
SpaceBefore(sub_expr, spaces) => match sub_expr {
Record { .. } | List { .. } => {
let is_only_newlines = spaces.iter().all(|s| s.is_newline());
is_only_newlines
&& !found_multiline_expr
&& sub_expr.is_multiline()
}
_ => false,
},
Record { .. } | List { .. } | Closure { .. } => {
!found_multiline_expr && loc_arg.is_multiline()
}
_ => false,
}
let arg = loc_arg.extract_spaces();
arg.item.format_with_options(
buf,
Parens::InApply,
Newlines::Yes,
arg_indent,
);
} else if needs_indent {
let arg = loc_arg.extract_spaces();
fmt_spaces(buf, arg.before.iter(), arg_indent);
buf.ensure_ends_with_newline();
arg.item.format_with_options(
buf,
Parens::InApply,
Newlines::Yes,
arg_indent,
);
fmt_spaces(buf, arg.after.iter(), arg_indent);
} else {
found_multiline_expr = loc_arg.is_multiline();
}
}
let should_outdent_last_arg = found_multiline_expr;
if multiline_args && !should_outdent_last_arg {
let arg_indent = indent + INDENT;
for loc_arg in loc_args.iter() {
buf.newline();
loc_arg.format_with_options(buf, Parens::InApply, Newlines::No, arg_indent);
}
} else if multiline_args && should_outdent_last_arg {
let mut iter = loc_args.iter().peekable();
while let Some(loc_arg) = iter.next() {
buf.spaces(1);
if iter.peek().is_none() {
match loc_arg.value {
SpaceBefore(sub_expr, _) => {
sub_expr.format_with_options(
buf,
Parens::InApply,
Newlines::Yes,
indent,
);
}
_ => {
loc_arg.format_with_options(
buf,
Parens::InApply,
Newlines::Yes,
indent,
);
}
}
} else {
loc_arg.format_with_options(
buf,
Parens::InApply,
Newlines::Yes,
indent,
);
}
}
} else {
for loc_arg in loc_args.iter() {
buf.spaces(1);
loc_arg.format_with_options(buf, Parens::InApply, Newlines::Yes, indent);
loc_arg.format_with_options(
buf,
Parens::InApply,
Newlines::Yes,
arg_indent,
);
}
}
@ -337,24 +362,28 @@ impl<'a> Formattable for Expr<'a> {
fmt_backpassing(buf, loc_patterns, loc_body, loc_ret, indent);
}
Defs(defs, ret) => {
// It should theoretically be impossible to *parse* an empty defs list.
// (Canonicalization can remove defs later, but that hasn't happened yet!)
debug_assert!(!defs.is_empty());
{
let indent = if parens == Parens::InOperator {
buf.indent(indent);
buf.push('(');
buf.newline();
indent + INDENT
} else {
indent
};
fmt_defs(buf, defs, indent);
// It should theoretically be impossible to *parse* an empty defs list.
// (Canonicalization can remove defs later, but that hasn't happened yet!)
debug_assert!(!defs.is_empty());
match &ret.value {
SpaceBefore(sub_expr, spaces) => {
let empty_line_before_return = empty_line_before_expr(&ret.value);
let has_inline_comment = has_line_comment_before(&ret.value);
fmt_defs(buf, defs, indent);
if has_inline_comment {
match &ret.value {
SpaceBefore(sub_expr, spaces) => {
buf.spaces(1);
format_spaces(buf, spaces, newlines, indent);
fmt_spaces(buf, spaces.iter(), indent);
if !empty_line_before_return {
buf.newline();
}
buf.indent(indent);
sub_expr.format_with_options(
buf,
@ -362,17 +391,21 @@ impl<'a> Formattable for Expr<'a> {
Newlines::Yes,
indent,
);
} else {
}
_ => {
buf.ensure_ends_with_newline();
buf.indent(indent);
// Even if there were no defs, which theoretically should never happen,
// still print the return value.
ret.format_with_options(buf, Parens::NotNeeded, Newlines::Yes, indent);
}
}
_ => {
buf.ensure_ends_with_newline();
buf.indent(indent);
// Even if there were no defs, which theoretically should never happen,
// still print the return value.
ret.format_with_options(buf, Parens::NotNeeded, Newlines::Yes, indent);
}
}
if parens == Parens::InOperator {
buf.ensure_ends_with_newline();
buf.indent(indent);
buf.push(')');
}
}
Expect(condition, continuation) => {
@ -387,7 +420,7 @@ impl<'a> Formattable for Expr<'a> {
When(loc_condition, branches) => fmt_when(buf, loc_condition, branches, indent),
Tuple(items) => fmt_collection(buf, indent, Braces::Round, *items, Newlines::No),
List(items) => fmt_collection(buf, indent, Braces::Square, *items, Newlines::No),
BinOps(lefts, right) => fmt_binops(buf, lefts, right, false, parens, indent),
BinOps(lefts, right) => fmt_binops(buf, lefts, right, false, indent),
UnaryOp(sub_expr, unary_op) => {
buf.indent(indent);
match &unary_op.value {
@ -461,6 +494,13 @@ pub(crate) fn format_sq_literal(buf: &mut Buf, s: &str) {
buf.push('\'');
}
fn is_outdentable(expr: &Expr) -> bool {
matches!(
expr.extract_spaces().item,
Expr::Tuple(_) | Expr::List(_) | Expr::Record(_) | Expr::Closure(..)
)
}
fn starts_with_newline(expr: &Expr) -> bool {
use roc_parse::ast::Expr::*;
@ -599,7 +639,6 @@ fn fmt_binops<'a, 'buf>(
lefts: &'a [(Loc<Expr<'a>>, Loc<BinOp>)],
loc_right_side: &'a Loc<Expr<'a>>,
part_of_multi_line_binops: bool,
apply_needs_parens: Parens,
indent: u16,
) {
let is_multiline = part_of_multi_line_binops
@ -609,7 +648,7 @@ fn fmt_binops<'a, 'buf>(
for (loc_left_side, loc_binop) in lefts {
let binop = loc_binop.value;
loc_left_side.format_with_options(buf, apply_needs_parens, Newlines::No, indent);
loc_left_side.format_with_options(buf, Parens::InOperator, Newlines::No, indent);
if is_multiline {
buf.ensure_ends_with_newline();
@ -623,7 +662,7 @@ fn fmt_binops<'a, 'buf>(
buf.spaces(1);
}
loc_right_side.format_with_options(buf, apply_needs_parens, Newlines::Yes, indent);
loc_right_side.format_with_options(buf, Parens::InOperator, Newlines::Yes, indent);
}
fn format_spaces<'a, 'buf>(
@ -642,44 +681,6 @@ fn format_spaces<'a, 'buf>(
}
}
fn has_line_comment_before<'a>(expr: &'a Expr<'a>) -> bool {
use roc_parse::ast::Expr::*;
match expr {
SpaceBefore(_, spaces) => {
matches!(spaces.iter().next(), Some(CommentOrNewline::LineComment(_)))
}
_ => false,
}
}
fn empty_line_before_expr<'a>(expr: &'a Expr<'a>) -> bool {
use roc_parse::ast::Expr::*;
match expr {
SpaceBefore(_, spaces) => {
let mut has_at_least_one_newline = false;
for comment_or_newline in spaces.iter() {
match comment_or_newline {
CommentOrNewline::Newline => {
if has_at_least_one_newline {
return true;
} else {
has_at_least_one_newline = true;
}
}
CommentOrNewline::LineComment(_) | CommentOrNewline::DocComment(_) => {}
}
}
false
}
_ => false,
}
}
fn is_when_patterns_multiline(when_branch: &WhenBranch) -> bool {
let patterns = when_branch.patterns;
let (first_pattern, rest) = patterns.split_first().unwrap();
@ -1204,19 +1205,16 @@ fn fmt_backpassing<'a, 'buf>(
indent
};
let pattern_needs_parens = loc_patterns
.iter()
.any(|p| pattern_needs_parens_when_backpassing(&p.value));
if pattern_needs_parens {
buf.indent(indent);
buf.push('(');
}
let mut it = loc_patterns.iter().peekable();
while let Some(loc_pattern) = it.next() {
loc_pattern.format(buf, indent);
let needs_parens = if pattern_needs_parens_when_backpassing(&loc_pattern.value) {
Parens::InApply
} else {
Parens::NotNeeded
};
loc_pattern.format_with_options(buf, needs_parens, Newlines::No, indent);
if it.peek().is_some() {
if arguments_are_multiline {
@ -1229,10 +1227,6 @@ fn fmt_backpassing<'a, 'buf>(
}
}
if pattern_needs_parens {
buf.push(')');
}
if arguments_are_multiline {
buf.newline();
buf.indent(indent);
@ -1479,6 +1473,8 @@ fn sub_expr_requests_parens(expr: &Expr<'_>) -> bool {
})
}
Expr::If(_, _) => true,
Expr::SpaceBefore(e, _) => sub_expr_requests_parens(e),
Expr::SpaceAfter(e, _) => sub_expr_requests_parens(e),
_ => false,
}
}

View File

@ -72,12 +72,11 @@ fn fmt_spaces_max_consecutive_newlines<'a, 'buf, I>(
// Only ever print two newlines back to back.
// (Two newlines renders as one blank line.)
let mut consecutive_newlines = 0;
let mut encountered_comment = false;
for space in spaces {
match space {
Newline => {
if !encountered_comment && (consecutive_newlines < max_consecutive_newlines) {
if consecutive_newlines < max_consecutive_newlines {
buf.newline();
// Don't bother incrementing it if we're already over the limit.
@ -90,14 +89,14 @@ fn fmt_spaces_max_consecutive_newlines<'a, 'buf, I>(
fmt_comment(buf, comment);
buf.newline();
encountered_comment = true;
consecutive_newlines = 1;
}
DocComment(docs) => {
buf.indent(indent);
fmt_docs(buf, docs);
buf.newline();
encountered_comment = true;
consecutive_newlines = 1;
}
}
}

View File

@ -1,7 +1,6 @@
use std::fmt::Debug;
use crate::header::{AppHeader, HostedHeader, InterfaceHeader, PackageHeader, PlatformHeader};
use crate::ident::Ident;
use crate::parser::ESingleQuote;
use bumpalo::collections::{String, Vec};
use bumpalo::Bump;
@ -807,51 +806,6 @@ pub enum Base {
}
impl<'a> Pattern<'a> {
pub fn from_ident(arena: &'a Bump, ident: Ident<'a>) -> Pattern<'a> {
match ident {
Ident::Tag(string) => Pattern::Tag(string),
Ident::OpaqueRef(string) => Pattern::OpaqueRef(string),
Ident::Access { module_name, parts } => {
if parts.len() == 1 {
// This is valid iff there is no module.
let ident = parts.iter().next().unwrap();
if module_name.is_empty() {
Pattern::Identifier(ident)
} else {
Pattern::QualifiedIdentifier { module_name, ident }
}
} else {
// This is definitely malformed.
let mut buf =
String::with_capacity_in(module_name.len() + (2 * parts.len()), arena);
let mut any_parts_printed = if module_name.is_empty() {
false
} else {
buf.push_str(module_name);
true
};
for part in parts.iter() {
if any_parts_printed {
buf.push('.');
} else {
any_parts_printed = true;
}
buf.push_str(part);
}
Pattern::Malformed(buf.into_bump_str())
}
}
Ident::RecordAccessorFunction(string) => Pattern::Malformed(string),
Ident::TupleAccessorFunction(string) => Pattern::Malformed(string),
Ident::Malformed(string, _problem) => Pattern::Malformed(string),
}
}
/// Check that patterns are equivalent, meaning they have the same shape, but may have
/// different locations/whitespace
pub fn equivalent(&self, other: &Self) -> bool {

View File

@ -378,6 +378,10 @@ where
}
}
fn begins_with_crlf(bytes: &[u8]) -> bool {
bytes.len() >= 2 && bytes[0] == b'\r' && bytes[1] == b'\n'
}
pub fn spaces<'a, E>() -> impl Parser<'a, &'a [CommentOrNewline<'a>], E>
where
E: 'a + SpaceProblem,
@ -399,6 +403,7 @@ where
let is_doc_comment = state.bytes().first() == Some(&b'#')
&& (state.bytes().get(1) == Some(&b' ')
|| state.bytes().get(1) == Some(&b'\n')
|| begins_with_crlf(&state.bytes()[1..])
|| state.bytes().get(1) == None);
if is_doc_comment {
@ -422,7 +427,10 @@ where
newlines.push(comment);
state.advance_mut(len);
if state.bytes().first() == Some(&b'\n') {
if begins_with_crlf(state.bytes()) {
state.advance_mut(1);
state = state.advance_newline();
} else if state.bytes().first() == Some(&b'\n') {
state = state.advance_newline();
}

View File

@ -4,7 +4,7 @@ use crate::ast::{
};
use crate::blankspace::{
space0_after_e, space0_around_e_no_after_indent_check, space0_around_ee, space0_before_e,
space0_e, spaces, spaces_around, spaces_before,
space0_before_optional_after, space0_e, spaces, spaces_around, spaces_before,
};
use crate::ident::{integer_ident, lowercase_ident, parse_ident, Accessor, Ident};
use crate::keyword;
@ -42,7 +42,7 @@ pub fn test_parse_expr<'a>(
state: State<'a>,
) -> Result<Loc<Expr<'a>>, EExpr<'a>> {
let parser = skip_second!(
space0_before_e(loc_expr(true), EExpr::IndentStart,),
space0_before_optional_after(loc_expr(true), EExpr::IndentStart, EExpr::IndentEnd),
expr_end()
);
@ -255,7 +255,10 @@ fn loc_possibly_negative_or_negated_term<'a>(
// this will parse negative numbers, which the unary negate thing up top doesn't (for now)
loc!(specialize(EExpr::Number, number_literal_help())),
loc!(map_with_arena!(
and!(loc!(word1(b'!', EExpr::Start)), loc_term(options)),
and!(
loc!(word1(b'!', EExpr::Start)),
space0_before_e(loc_term(options), EExpr::IndentStart)
),
|arena: &'a Bump, (loc_op, loc_expr): (Loc<_>, _)| {
Expr::UnaryOp(arena.alloc(loc_expr), Loc::at(loc_op.region, UnaryOp::Not))
}
@ -668,7 +671,7 @@ pub fn parse_single_def<'a>(
let (_, ann_type, state) = parser.parse(arena, state, min_indent)?;
let region = Region::span_across(&loc_pattern.region, &ann_type.region);
match &loc_pattern.value {
match &loc_pattern.value.extract_spaces().item {
Pattern::Apply(
Loc {
value: Pattern::Tag(name),
@ -740,7 +743,7 @@ pub fn parse_single_def<'a>(
opaque_signature_with_space_before().parse(arena, state, min_indent + 1)?;
let region = Region::span_across(&loc_pattern.region, &signature.region);
match &loc_pattern.value {
match &loc_pattern.value.extract_spaces().item {
Pattern::Apply(
Loc {
value: Pattern::Tag(name),
@ -1890,7 +1893,7 @@ fn expr_to_pattern_help<'a>(arena: &'a Bump, expr: &Expr<'a>) -> Result<Pattern<
Expr::Str(string) => Ok(Pattern::StrLiteral(*string)),
Expr::SingleQuote(string) => Ok(Pattern::SingleQuote(string)),
Expr::MalformedIdent(string, _problem) => Ok(Pattern::Malformed(string)),
Expr::MalformedIdent(string, problem) => Ok(Pattern::MalformedIdent(string, *problem)),
}
}

View File

@ -290,6 +290,10 @@ fn chomp_integer_part(buffer: &[u8]) -> Result<&str, Progress> {
)
}
fn is_plausible_ident_continue(ch: char) -> bool {
ch == '_' || is_alnum(ch)
}
#[inline(always)]
fn chomp_part<F, G>(leading_is_good: F, rest_is_good: G, buffer: &[u8]) -> Result<&str, Progress>
where
@ -317,6 +321,15 @@ where
}
}
if let Ok((next, _width)) = char::from_utf8_slice_start(&buffer[chomped..]) {
// This would mean we have e.g.:
// * identifier followed by a _
// * an integer followed by an alphabetic char
if is_plausible_ident_continue(next) {
return Err(NoProgress);
}
}
if chomped == 0 {
Err(NoProgress)
} else {

View File

@ -535,6 +535,9 @@ pub enum EPattern<'a> {
IndentStart(Position),
IndentEnd(Position),
AsIndentStart(Position),
RecordAccessorFunction(Position),
TupleAccessorFunction(Position),
}
#[derive(Debug, Clone, PartialEq, Eq)]

View File

@ -406,13 +406,13 @@ fn loc_ident_pattern_help<'a>(
))
}
}
Ident::RecordAccessorFunction(string) | Ident::TupleAccessorFunction(string) => Ok((
Ident::RecordAccessorFunction(_string) => Err((
MadeProgress,
Loc {
region: loc_ident.region,
value: Pattern::Malformed(string),
},
state,
EPattern::RecordAccessorFunction(loc_ident.region.start()),
)),
Ident::TupleAccessorFunction(_string) => Err((
MadeProgress,
EPattern::TupleAccessorFunction(loc_ident.region.start()),
)),
Ident::Malformed(malformed, problem) => {
debug_assert!(!malformed.is_empty());

View File

@ -326,8 +326,12 @@ pub fn parse_str_like_literal<'a>() -> impl Parser<'a, StrLikeLiteral<'a>, EStri
if state.bytes().starts_with(b"\"\"\"") {
// ending the string; don't use the last newline
segments
.push(StrSegment::Plaintext(utf8(state.clone(), without_newline)?));
if !without_newline.is_empty() {
segments.push(StrSegment::Plaintext(utf8(
state.clone(),
without_newline,
)?));
}
} else {
segments
.push(StrSegment::Plaintext(utf8(state.clone(), with_newline)?));

View File

@ -19,9 +19,9 @@ mod test_parse {
use bumpalo::collections::vec::Vec;
use bumpalo::{self, Bump};
use roc_parse::ast::Expr::{self, *};
use roc_parse::ast::StrLiteral::*;
use roc_parse::ast::StrSegment::*;
use roc_parse::ast::{self, EscapedChar};
use roc_parse::ast::{CommentOrNewline, StrLiteral::*};
use roc_parse::module::module_defs;
use roc_parse::parser::{Parser, SyntaxError};
use roc_parse::state::State;
@ -322,6 +322,22 @@ mod test_parse {
assert_eq!(std::mem::size_of::<roc_parse::ast::Expr>(), 40);
}
#[test]
fn parse_two_line_comment_with_crlf() {
let src = "# foo\r\n# bar\r\n42";
assert_parses_to(
src,
Expr::SpaceBefore(
&Expr::Num("42"),
&[
CommentOrNewline::LineComment(" foo"),
// We used to have a bug where there was an extra CommentOrNewline::Newline between these.
CommentOrNewline::LineComment(" bar"),
],
),
);
}
// PARSE ERROR
// TODO this should be parse error, but isn't!

View File

@ -6,6 +6,9 @@ license = "UPL-1.0"
edition = "2021"
description = "Tests for the parse + fmt crates."
[features]
"parse_debug_trace" = ["roc_parse/parse_debug_trace"]
[dependencies]
roc_collections = { path = "../collections" }
roc_region = { path = "../region" }

View File

@ -219,7 +219,7 @@ impl<'a> Input<'a> {
* * * AST after formatting:\n{:#?}\n\n",
self.as_str(),
output.as_ref().as_str(),
ast_normalized,
actual,
reparsed_ast_normalized
);
}
@ -229,7 +229,10 @@ impl<'a> Input<'a> {
let reformatted = reparsed_ast.format();
if output != reformatted {
eprintln!("Formatting bug; formatting is not stable.\nOriginal code:\n{}\n\nFormatted code:\n{}\n\n", self.as_str(), output.as_ref().as_str());
eprintln!("Formatting bug; formatting is not stable.\nOriginal code:\n{}\n\nFormatted code:\n{}\n\nAST:\n{:#?}\n\n",
self.as_str(),
output.as_ref().as_str(),
actual);
eprintln!("Reformatting the formatted code changed it again, as follows:\n\n");
assert_multiline_str_eq!(output.as_ref().as_str(), reformatted.as_ref().as_str());

View File

@ -1 +1 @@
Expr(BadExprEnd(@20), @0)
Expr(BadExprEnd(@22), @0)

View File

@ -0,0 +1,25 @@
Apply(
@0-1 SpaceAfter(
Var {
module_name: "",
ident: "f",
},
[
Newline,
],
),
[
@2-4 Num(
"-5",
),
@5-6 SpaceBefore(
Num(
"2",
),
[
Newline,
],
),
],
Space,
)

View File

@ -0,0 +1,42 @@
Defs(
Defs {
tags: [
Index(0),
],
regions: [
@0-3,
],
space_before: [
Slice(start = 0, length = 0),
],
space_after: [
Slice(start = 0, length = 0),
],
spaces: [],
type_defs: [
Alias {
header: TypeHeader {
name: @0-1 "F",
vars: [],
},
ann: @2-3 BoundVariable(
"e",
),
},
],
value_defs: [],
},
@7-8 SpaceBefore(
Var {
module_name: "",
ident: "q",
},
[
LineComment(
"",
),
Newline,
Newline,
],
),
)

View File

@ -0,0 +1,13 @@
ParensAround(
SpaceAfter(
Var {
module_name: "",
ident: "i",
},
[
LineComment(
"abc",
),
],
),
)

View File

@ -0,0 +1,54 @@
Defs(
Defs {
tags: [
Index(0),
],
regions: [
@0-7,
],
space_before: [
Slice(start = 0, length = 0),
],
space_after: [
Slice(start = 0, length = 0),
],
spaces: [],
type_defs: [
Alias {
header: TypeHeader {
name: @0-1 "Z",
vars: [
@3-4 SpaceAfter(
SpaceBefore(
Identifier(
"h",
),
[
LineComment(
"",
),
],
),
[
Newline,
],
),
],
},
ann: @6-7 BoundVariable(
"a",
),
},
],
value_defs: [],
},
@8-9 SpaceBefore(
Var {
module_name: "",
ident: "j",
},
[
Newline,
],
),
)

View File

@ -0,0 +1,43 @@
Defs(
Defs {
tags: [
Index(2147483648),
],
regions: [
@0-5,
],
space_before: [
Slice(start = 0, length = 0),
],
space_after: [
Slice(start = 0, length = 0),
],
spaces: [],
type_defs: [],
value_defs: [
Annotation(
@0-1 SpaceAfter(
Identifier(
"w",
),
[
LineComment(
"",
),
],
),
@4-5 BoundVariable(
"n",
),
),
],
},
@6-7 SpaceBefore(
Tag(
"Q",
),
[
Newline,
],
),
)

View File

@ -0,0 +1,44 @@
Defs(
Defs {
tags: [
Index(2147483648),
],
regions: [
@0-5,
],
space_before: [
Slice(start = 0, length = 0),
],
space_after: [
Slice(start = 0, length = 0),
],
spaces: [],
type_defs: [],
value_defs: [
Body(
@0-1 SpaceAfter(
Identifier(
"t",
),
[
LineComment(
"",
),
],
),
@4-5 Num(
"3",
),
),
],
},
@6-7 SpaceBefore(
Var {
module_name: "",
ident: "e",
},
[
Newline,
],
),
)

View File

@ -16,8 +16,11 @@ Defs(
type_defs: [],
value_defs: [
Body(
@0-7 Malformed(
@0-7 MalformedIdent(
"my_list",
Underscore(
@3,
),
),
@10-58 List(
Collection {

View File

@ -16,8 +16,11 @@ Defs(
type_defs: [],
value_defs: [
Body(
@0-7 Malformed(
@0-7 MalformedIdent(
"my_list",
Underscore(
@3,
),
),
@10-26 List(
[

View File

@ -16,8 +16,11 @@ Defs(
type_defs: [],
value_defs: [
Body(
@0-7 Malformed(
@0-7 MalformedIdent(
"my_list",
Underscore(
@3,
),
),
@10-27 List(
Collection {

View File

@ -0,0 +1,29 @@
Backpassing(
[
@0-3 Apply(
@0-1 Tag(
"F",
),
[
@2-3 NumLiteral(
"1",
),
],
),
@5-6 Identifier(
"r",
),
],
@10-11 Var {
module_name: "",
ident: "a",
},
@12-13 SpaceBefore(
Tag(
"W",
),
[
Newline,
],
),
)

View File

@ -0,0 +1,23 @@
Apply(
@0-1 Var {
module_name: "",
ident: "e",
},
[
@1-10 Str(
Block(
[
[
Plaintext(
"\"",
),
EscapedChar(
DoubleQuote,
),
],
],
),
),
],
Space,
)

View File

@ -0,0 +1,8 @@
UnaryOp(
@1-7 Str(
Block(
[],
),
),
@0-1 Not,
)

View File

@ -0,0 +1,58 @@
Defs(
Defs {
tags: [
Index(2147483648),
],
regions: [
@0-9,
],
space_before: [
Slice(start = 0, length = 0),
],
space_after: [
Slice(start = 0, length = 0),
],
spaces: [],
type_defs: [],
value_defs: [
Body(
@0-1 Identifier(
"a",
),
@2-9 Apply(
@2-3 SpaceAfter(
Tag(
"A",
),
[
Newline,
],
),
[
@5-7 UnaryOp(
@6-7 Var {
module_name: "",
ident: "g",
},
@5-6 Negate,
),
@8-9 Var {
module_name: "",
ident: "a",
},
],
Space,
),
),
],
},
@10-11 SpaceBefore(
Var {
module_name: "",
ident: "a",
},
[
Newline,
],
),
)

View File

@ -0,0 +1,3 @@
a=A
-g a
a

View File

@ -0,0 +1,64 @@
Defs(
Defs {
tags: [
Index(2147483648),
],
regions: [
@0-7,
],
space_before: [
Slice(start = 0, length = 0),
],
space_after: [
Slice(start = 0, length = 0),
],
spaces: [],
type_defs: [],
value_defs: [
Body(
@0-1 Identifier(
"x",
),
@2-7 Defs(
Defs {
tags: [
Index(2147483648),
],
regions: [
@2-5,
],
space_before: [
Slice(start = 0, length = 0),
],
space_after: [
Slice(start = 0, length = 0),
],
spaces: [],
type_defs: [],
value_defs: [
Annotation(
@2-3 Identifier(
"a",
),
@4-5 BoundVariable(
"n",
),
),
],
},
@6-7 Num(
"4",
),
),
),
],
},
@8-9 SpaceBefore(
Underscore(
"",
),
[
Newline,
],
),
)

View File

@ -0,0 +1,10 @@
ParensAround(
SpaceBefore(
Tag(
"A",
),
[
Newline,
],
),
)

View File

@ -0,0 +1,49 @@
BinOps(
[
(
@0-1 SpaceAfter(
Num(
"7",
),
[
Newline,
],
),
@2-4 Equals,
),
],
@5-11 ParensAround(
Defs(
Defs {
tags: [
Index(0),
],
regions: [
@5-8,
],
space_before: [
Slice(start = 0, length = 0),
],
space_after: [
Slice(start = 0, length = 0),
],
spaces: [],
type_defs: [
Alias {
header: TypeHeader {
name: @5-6 "Q",
vars: [],
},
ann: @7-8 BoundVariable(
"c",
),
},
],
value_defs: [],
},
@9-11 Num(
"42",
),
),
),
)

View File

@ -0,0 +1,51 @@
Defs(
Defs {
tags: [
Index(0),
],
regions: [
@0-6,
],
space_before: [
Slice(start = 0, length = 0),
],
space_after: [
Slice(start = 0, length = 0),
],
spaces: [],
type_defs: [
Alias {
header: TypeHeader {
name: @0-1 "A",
vars: [],
},
ann: @2-6 Apply(
"",
"A",
[
@5-6 SpaceBefore(
Apply(
"",
"A",
[],
),
[
Newline,
],
),
],
),
},
],
value_defs: [],
},
@7-8 SpaceBefore(
Var {
module_name: "",
ident: "p",
},
[
Newline,
],
),
)

View File

@ -0,0 +1,42 @@
Defs(
Defs {
tags: [
Index(0),
],
regions: [
@0-3,
],
space_before: [
Slice(start = 0, length = 0),
],
space_after: [
Slice(start = 0, length = 0),
],
spaces: [],
type_defs: [
Alias {
header: TypeHeader {
name: @0-1 "J",
vars: [],
},
ann: @2-3 Apply(
"",
"R",
[],
),
},
],
value_defs: [],
},
@5-8 SpaceBefore(
MalformedIdent(
"n_p",
Underscore(
@7,
),
),
[
Newline,
],
),
)

View File

@ -0,0 +1,3 @@
a : F
F : h
abc

View File

@ -0,0 +1,55 @@
Defs(
Defs {
tags: [
Index(2147483648),
Index(0),
],
regions: [
@0-3,
@4-8,
],
space_before: [
Slice(start = 0, length = 0),
Slice(start = 0, length = 1),
],
space_after: [
Slice(start = 0, length = 0),
Slice(start = 1, length = 0),
],
spaces: [
Newline,
],
type_defs: [
Alias {
header: TypeHeader {
name: @4-5 "F",
vars: [],
},
ann: @7-8 BoundVariable(
"h",
),
},
],
value_defs: [
Annotation(
@0-1 Identifier(
"a",
),
@2-3 Apply(
"",
"F",
[],
),
),
],
},
@9-12 SpaceBefore(
Var {
module_name: "",
ident: "abc",
},
[
Newline,
],
),
)

View File

@ -0,0 +1,4 @@
a:F
F
:h
abc

View File

@ -213,7 +213,6 @@ mod test_fmt {
indoc!(
r#"
x = 0 # comment
x
"#
),
@ -1410,6 +1409,7 @@ mod test_fmt {
r#"
f = \x ->
# 1st
# 2nd
x
@ -1693,6 +1693,7 @@ mod test_fmt {
),
);
// TODO: do we want to override the user's intent like this?
expr_formats_to(
indoc!(
r#"
@ -1821,6 +1822,7 @@ mod test_fmt {
),
);
// TODO: do we want to override the user's intent like this?
expr_formats_to(
indoc!(
r#"

View File

@ -252,9 +252,15 @@ mod test_snapshots {
pass/basic_tuple.expr,
pass/basic_var.expr,
pass/bound_variable.expr,
pass/call_with_newlines.expr,
pass/closure_with_underscores.expr,
pass/comment_after_annotation.expr,
pass/comment_after_def.moduledefs,
pass/comment_after_expr_in_parens.expr,
pass/comment_after_op.expr,
pass/comment_after_tag_in_def.expr,
pass/comment_before_colon_def.expr,
pass/comment_before_equals_def.expr,
pass/comment_before_op.expr,
pass/comment_inside_empty_list.expr,
pass/comment_with_non_ascii.expr,
@ -303,26 +309,34 @@ mod test_snapshots {
pass/mixed_docs.expr,
pass/module_def_newline.moduledefs,
pass/multi_backpassing.expr,
pass/multi_backpassing_with_apply.expr,
pass/multi_char_string.expr,
pass/multiline_string.expr,
pass/multiline_string_in_apply.expr,
pass/multiline_tuple_with_comments.expr,
pass/multiline_type_signature.expr,
pass/multiline_type_signature_with_comment.expr,
pass/multiple_fields.expr,
pass/multiple_operators.expr,
pass/neg_inf_float.expr,
pass/negate_multiline_string.expr,
pass/negative_float.expr,
pass/negative_in_apply_def.expr,
pass/negative_int.expr,
pass/nested_def_annotation.moduledefs,
pass/nested_def_without_newline.expr,
pass/nested_if.expr,
pass/nested_module.header,
pass/newline_after_equals.expr, // Regression test for https://github.com/roc-lang/roc/issues/51
pass/newline_after_mul.expr,
pass/newline_after_paren.expr,
pass/newline_after_sub.expr,
pass/newline_and_spaces_before_less_than.expr,
pass/newline_before_add.expr,
pass/newline_before_operator_with_defs.expr,
pass/newline_before_sub.expr,
pass/newline_in_packages.full,
pass/newline_in_type_alias_application.expr,
pass/newline_in_type_def.expr,
pass/newline_inside_empty_list.expr,
pass/newline_singleton_list.expr,
@ -407,7 +421,9 @@ mod test_snapshots {
pass/unary_not.expr,
pass/unary_not_with_parens.expr,
pass/underscore_backpassing.expr,
pass/underscore_expr_in_def.expr,
pass/underscore_in_assignment_pattern.expr,
pass/value_def_confusion.expr,
pass/var_else.expr,
pass/var_if.expr,
pass/var_is.expr,