enso-org/enso
1
1
Fork 0
mirror of https://github.com/enso-org/enso.git synced 2024-11-22 22:10:15 +03:00
Code Issues Projects Releases Wiki Activity

Parser: Full constructor syntax for type definitions; Field syntax; Complex operator sections; Template functions; Text improvements; Operator methods; eliminate Unsupported; better ArgumentDefinitions (#3716)

Browse Source 
I believe all parse failures remaining after these changes are because the new parser is intentionally stricter about some things. I'll be reviewing those failures and opening a bug to change the library/tests code.

Implements:
- https://www.pivotaltracker.com/story/show/182941610: full type def syntax
- https://www.pivotaltracker.com/story/show/182497490: field syntax
- https://www.pivotaltracker.com/story/show/182497395: complex operator sections
- https://www.pivotaltracker.com/story/show/182497236: template functions
- `codeRepr` without leading whitespace
- text literals: interpret escape sequences in lexer
- the multiline text-literal left-trim algorithm
- type operator-methods
- the `<=` operator is no longer treated as a modifier
- https://www.pivotaltracker.com/story/show/183315038: eliminate Unsupported
- use ArgumentDefinition for type constructor arguments
- more detailed ArgumentDefinition type
This commit is contained in:
Kaz Wesley 2022-10-04 21:45:31 -07:00 committed by GitHub
parent c460b7c4a4
commit 44a031f9f0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 2025 additions and 1316 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/CODEOWNERS vendored
View File

@ -11,6 +11,7 @@ Cargo.toml
/lib/rust/metamodel/ @kazcw @wdanilo @jaroslavtulach
/lib/rust/parser/ @kazcw @wdanilo @jaroslavtulach
/lib/rust/profiler/ @kazcw @MichaelMauderer @wdanilo
/lib/rust/reflect/ @MichaelMauderer @mwu-tow @farmaazon @wdanilo @kazcw @wdanilo @jaroslavtulach
/integration-test/ @MichaelMauderer @wdanilo @farmaazon @kazcw
/tools/build-performance/ @kazcw @mwu-tow @wdanilo

2
distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso
View File

@ -863,7 +863,7 @@ type Column
size = ['length', self.length]
name = ['name', self.name]
max_data = 100
data = ['data', self.to_vector.take (First max_data)])
data = ['data', self.to_vector.take (First max_data)]
Json.from_pairs [size, name, data] . to_text
## ALIAS Sum Columns

2
distribution/lib/Standard/Test/0.0.0-dev/src/Main.enso
View File

@ -605,7 +605,7 @@ type Verbs
start_with : Text -> Text -> Assertion
start_with self subject argument =
if subject.starts_with argument then Success else
fail (subject.to_text + " did not start with " + argument.to_text))
fail (subject.to_text + " did not start with " + argument.to_text)
## PRIVATE

5
engine/runtime/src/main/java/org/enso/compiler/EnsoCompiler.java
View File

@ -4,7 +4,6 @@ import com.oracle.truffle.api.source.Source;
import org.enso.compiler.core.IR;
import org.enso.syntax2.Parser;
import org.enso.syntax2.Tree;
import org.enso.syntax2.UnsupportedSyntaxException;
public final class EnsoCompiler implements AutoCloseable {
private final Parser parser;
@ -18,12 +17,12 @@ public final class EnsoCompiler implements AutoCloseable {
this.parser.close();
}
IR.Module compile(Source src) throws UnsupportedSyntaxException {
IR.Module compile(Source src) {
var tree = parse(src);
return generateIR(tree);
}
Tree parse(Source src) throws UnsupportedSyntaxException {
Tree parse(Source src) {
return parser.parse(src.getCharacters());
}

487
engine/runtime/src/main/java/org/enso/compiler/TreeToIr.java
View File

@ -51,6 +51,7 @@ import org.enso.syntax2.Tree;
import scala.Option;
import scala.collection.immutable.List;
import scala.jdk.javaapi.CollectionConverters;
final class TreeToIr {
static final TreeToIr MODULE = new TreeToIr();
@ -305,7 +306,7 @@ final class TreeToIr {
getIdentifiedLocation(fn),
meta(), diag()
);
var args = translateArgumentsDefs(fn.getArgs());
var args = translateArgumentsDefinition(fn.getArgs());
var body = translateExpression(fn.getBody(), false);
yield new IR$Module$Scope$Definition$Method$Binding(
@ -386,22 +387,8 @@ final class TreeToIr {
};
}
private List<IR.DefinitionArgument> translateArgumentsDefs(java.util.List<ArgumentDefinition> args) {
ArrayList<Tree> params = new ArrayList<>();
for (var d : args) {
params.add(d.getPattern());
}
return translateArgumentsDefinition(params);
}
private List<IR.DefinitionArgument> translateArgumentsDefinition(java.util.List<Tree> params) {
return translateArgumentsDefinition(params, nil());
}
private List<IR.DefinitionArgument> translateArgumentsDefinition(java.util.List<Tree> params, List<IR.DefinitionArgument> args) {
for (var p : params) {
var m = translateArgumentDefinition(p, false);
args = cons(m, args);
}
return args.reverse();
private List<IR.DefinitionArgument> translateArgumentsDefinition(java.util.List<ArgumentDefinition> args) {
return CollectionConverters.asScala(args.stream().map(p -> translateArgumentDefinition(p)).iterator()).toList();
}
/** Translates the body of a type expression.
@ -410,14 +397,8 @@ final class TreeToIr {
* @return the [[IR]] representation of `body`
*/
private scala.collection.immutable.List<IR> translateTypeBody(java.util.List<Line> block, boolean found) {
List<IR> result = nil();
for (var line : block) {
var expr = translateTypeBodyExpression(line);
if (expr != null) {
result = cons(expr, result);
}
}
return result.reverse();
var ir = block.stream().map(line -> translateTypeBodyExpression(line)).filter(line -> line != null).iterator();
return CollectionConverters.asScala(ir).toList();
}
/** Translates any expression that can be found in the body of a type
@ -587,23 +568,76 @@ final class TreeToIr {
);
yield prefix;
}
case "->" -> {
if (insideTypeSignature) {
List<IR.Expression> args = nil();
Tree treeBody = null;
Tree.OprApp head = app;
while (head != null) {
// FIXME: Handle parameterized types here.
var literal = buildName(head.getLhs());
args = cons(literal, args);
treeBody = head.getRhs();
head = switch (treeBody) {
case Tree.OprApp a -> {
var opr = a.getOpr().getRight();
if (opr != null && "->".equals(opr.codeRepr()))
yield a;
yield null;
}
default -> null;
};
}
args = args.reverse();
var body = translateExpression(treeBody, false);
yield new IR$Type$Function(args, body, Option.empty(), meta(), diag());
} else {
// Old-style lambdas; this syntax will be eliminated after the parser transition is complete.
var arg = app.getLhs();
var isSuspended = false;
if (arg instanceof Tree.UnaryOprApp susApp && "~".equals(susApp.getOpr().codeRepr())) {
arg = susApp.getRhs();
isSuspended = true;
}
IR.Name name = switch (arg) {
case Tree.Wildcard wild -> new IR$Name$Blank(getIdentifiedLocation(wild.getToken()), meta(), diag());
case Tree.Ident id -> {
IR.Expression identifier = translateIdent(id, false);
yield switch (identifier) {
case IR.Name name_ -> name_;
default -> throw new UnhandledEntity(identifier, "translateExpression");
};
}
default -> throw new UnhandledEntity(arg, "translateExpressiontranslateArgumentDefinition");
};
var arg_ = new IR$DefinitionArgument$Specified(
name,
Option.empty(),
Option.empty(),
isSuspended,
getIdentifiedLocation(arg),
meta(),
diag()
);
List<IR.DefinitionArgument> args = cons(arg_, nil());
var body = translateExpression(app.getRhs(), false);
yield new IR$Function$Lambda(args, body, getIdentifiedLocation(tree), true, meta(), diag());
}
}
default -> {
if (op.getProperties() != null) {
var lhs = translateCallArgument(app.getLhs(), insideTypeSignature);
var rhs = translateCallArgument(app.getRhs(), insideTypeSignature);
yield new IR$Application$Operator$Binary(
lhs,
new IR$Name$Literal(
op.codeRepr(), true,
getIdentifiedLocation(app),
meta(), diag()
),
rhs,
var lhs = translateCallArgument(app.getLhs(), insideTypeSignature);
var rhs = translateCallArgument(app.getRhs(), insideTypeSignature);
yield new IR$Application$Operator$Binary(
lhs,
new IR$Name$Literal(
op.codeRepr(), true,
getIdentifiedLocation(app),
meta(), diag()
);
}
throw new UnhandledEntity(tree, op.codeRepr());
),
rhs,
getIdentifiedLocation(app),
meta(), diag()
);
}
};
}
@ -701,33 +735,6 @@ final class TreeToIr {
var t = fullTxt.substring(1, fullTxt.length() - 1);
yield new IR$Literal$Text(t, getIdentifiedLocation(txt), meta(), diag());
}
case Tree.Arrow arrow when insideTypeSignature -> {
List<IR.Expression> args = nil();
Tree treeBody = null;
{
Tree.Arrow head = arrow;
while (head != null) {
for (var a : head.getArguments()) {
var literal = buildName(a);
args = cons(literal, args);
}
treeBody = head.getBody();
head = switch (treeBody) {
case Tree.Arrow a -> a;
default -> null;
};
}
args = args.reverse();
}
var body = translateExpression(treeBody, isMethod);
yield new IR$Type$Function(args, body, Option.empty(), meta(), diag());
}
case Tree.Arrow arrow -> {
var arguments = translateArgumentsDefinition(arrow.getArguments());
var body = translateExpression(arrow.getBody(), isMethod);
yield new IR$Function$Lambda(arguments, body, getIdentifiedLocation(arrow), true, meta(), diag());
}
default -> throw new UnhandledEntity(tree, "translateExpression");
};
/*
@ -1061,132 +1068,37 @@ final class TreeToIr {
/** Translates an argument definition from [[AST]] into [[IR]].
*
* @param arg the argument to translate
* @param isSuspended `true` if the argument is suspended, otherwise `false`
* @return the [[IR]] representation of `arg`
* @tailrec
*/
IR.DefinitionArgument translateArgumentDefinition(Tree arg, boolean isSuspended) {
var core = maybeManyParensed(arg);
return switch (core) {
case null -> null;
case Tree.OprApp app when isOperator(":", app.getOpr()) -> {
yield switch (translateIdent(app.getLhs(), false)) {
case IR.Name name -> {
var type = translateQualifiedNameOrExpression(app.getRhs());
yield new IR$DefinitionArgument$Specified(
name,
Option.apply(type), Option.empty(),
false, getIdentifiedLocation(app), meta(), diag()
);
}
default -> throw new UnhandledEntity(app.getLhs(), "translateArgumentDefinition");
};
}
case Tree.OprApp withValue when isOperator("=", withValue.getOpr()) -> {
var defaultValue = translateExpression(withValue.getRhs(), false);
yield switch (withValue.getLhs()) {
case Tree.OprApp app when isOperator(":", app.getOpr()) -> {
yield switch (translateIdent(app.getLhs(), false)) {
case IR.Name name -> {
var type = translateQualifiedNameOrExpression(app.getRhs());
yield new IR$DefinitionArgument$Specified(
name,
Option.apply(type), Option.apply(defaultValue),
false, getIdentifiedLocation(app), meta(), diag()
);
}
default -> throw new UnhandledEntity(app.getLhs(), "translateArgumentDefinition");
};
}
case Tree.TypeAnnotated anno -> {
yield null;
}
default -> throw new UnhandledEntity(withValue.getLhs(), "translateArgumentDefinition");
};
}
case Tree.OprSectionBoundary bound -> {
yield translateArgumentDefinition(bound.getAst(), isSuspended);
}
case Tree.TypeAnnotated anno -> {
yield null;
}
/*
case AstView.AscribedArgument(name, ascType, mValue, isSuspended) =>
translateIdent(name) match {
case name: IR.Name =>
DefinitionArgument.Specified(
name,
Some(translateQualifiedNameOrExpression(ascType)),
mValue.map(translateExpression(_)),
isSuspended,
getIdentifiedLocation(arg)
)
case _ =>
throw new UnhandledEntity(arg, "translateArgumentDefinition")
}
case AstView.LazyAssignedArgumentDefinition(name, value) =>
translateIdent(name) match {
case name: IR.Name =>
DefinitionArgument.Specified(
name,
None,
Some(translateExpression(value)),
suspended = true,
getIdentifiedLocation(arg)
)
case _ =>
throw new UnhandledEntity(arg, "translateArgumentDefinition")
}
case AstView.LazyArgument(arg) =>
translateArgumentDefinition(arg, isSuspended = true)
*/
*
* @param def the argument to translate
* @return the [[IR]] representation of `arg`
*/
IR.DefinitionArgument translateArgumentDefinition(ArgumentDefinition def) {
Tree pattern = def.getPattern();
IR.Name name = switch (pattern) {
case Tree.Wildcard wild -> new IR$Name$Blank(getIdentifiedLocation(wild.getToken()), meta(), diag());
case Tree.Ident id -> {
IR.Expression identifier = translateIdent(id, false);
yield switch (identifier) {
case IR.Name name -> new IR$DefinitionArgument$Specified(
name,
Option.empty(),
Option.empty(),
isSuspended,
getIdentifiedLocation(arg),
meta(), diag()
);
default -> throw new UnhandledEntity(arg, "translateArgumentDefinition");
case IR.Name name_ -> name_;
// TODO: Other types of pattern. Needs IR support.
default -> throw new UnhandledEntity(pattern, "translateArgumentDefinition");
};
}
case Tree.UnaryOprApp opr when "~".equals(opr.getOpr().codeRepr()) -> {
yield translateArgumentDefinition(opr.getRhs(), true);
}
/*
case AstView.AssignedArgument(name, value) =>
translateIdent(name) match {
case name: IR.Name =>
DefinitionArgument.Specified(
name,
None,
Some(translateExpression(value)),
isSuspended,
getIdentifiedLocation(arg)
)
case _ =>
throw new UnhandledEntity(arg, "translateArgumentDefinition")
}
*/
case Tree.Wildcard wild -> {
var loc = getIdentifiedLocation(arg);
yield new IR$DefinitionArgument$Specified(
new IR$Name$Blank(loc, meta(), diag()),
Option.empty(),
Option.empty(),
isSuspended,
loc,
meta(), diag()
);
}
default -> throw new UnhandledEntity(core, "translateArgumentDefinition");
// TODO: Other types of pattern. Needs IR support.
default -> throw new UnhandledEntity(pattern, "translateArgumentDefinition");
};
boolean isSuspended = def.getSuspension() != null;
var ascribedType = Option.apply(def.getType()).map(ascription -> translateExpression(ascription.getType(), true));
var defaultValue = Option.apply(def.getDefault()).map(default_ -> translateExpression(default_.getExpression(), false));
return new IR$DefinitionArgument$Specified(
name,
ascribedType,
defaultValue,
isSuspended,
getIdentifiedLocation(def),
meta(),
diag()
);
}
private List<IR.CallArgument> translateCallArguments(List<Tree> args, List<IR.CallArgument> res, boolean insideTypeSignature) {
@ -1565,61 +1477,30 @@ final class TreeToIr {
* @return the [[IR]] representation of `imp`
*/
IR$Module$Scope$Import translateImport(Tree.Import imp) {
if (imp.getImport() != null) {
if (imp.getFrom() != null) {
var qualifiedName = buildQualifiedName(imp.getFrom().getBody());
var onlyNames = imp.getImport().getBody();
var isAll = isAll(onlyNames);
return new IR$Module$Scope$Import$Module(
qualifiedName, Option.empty(), isAll, Option.empty(),
Option.empty(), getIdentifiedLocation(imp), false,
meta(), diag()
);
} else if (imp.getPolyglot() != null) {
List<IR.Name> qualifiedName = buildNames(imp.getImport().getBody(), '.', true);
StringBuilder pkg = new StringBuilder();
String cls = extractPackageAndName(qualifiedName, pkg);
Option<String> rename = imp.getImportAs() == null ? Option.empty() :
Option.apply(buildName(imp.getImportAs().getBody()).name());
return new IR$Module$Scope$Import$Polyglot(
new IR$Module$Scope$Import$Polyglot$Java(pkg.toString(), cls),
rename, getIdentifiedLocation(imp),
meta(), diag()
);
} else {
var qualifiedName = buildQualifiedName(imp.getImport().getBody());
Option<IR$Name$Literal> rename = imp.getImportAs() == null ? Option.empty() :
Option.apply(buildName(imp.getImportAs().getBody()));
return new IR$Module$Scope$Import$Module(
qualifiedName, rename, false, Option.empty(),
Option.empty(), getIdentifiedLocation(imp), false,
meta(), diag()
);
}
Option<IR$Name$Literal> rename = Option.apply(imp.getAs()).map(as -> buildName(as.getBody()));
if (imp.getPolyglot() != null) {
List<IR.Name> qualifiedName = buildNames(imp.getImport().getBody(), '.', true);
StringBuilder pkg = new StringBuilder();
String cls = extractPackageAndName(qualifiedName, pkg);
return new IR$Module$Scope$Import$Polyglot(
new IR$Module$Scope$Import$Polyglot$Java(pkg.toString(), cls),
rename.map(name -> name.name()), getIdentifiedLocation(imp),
meta(), diag()
);
}
/*
imp match {
case AST.Import(path, rename, isAll, onlyNames, hiddenNames) =>
IR.Module.Scope.Import.Module(
IR.Name.Qualified(path.map(buildName(_)).toList, None),
rename.map(buildName(_)),
isAll,
onlyNames.map(_.map(buildName(_)).toList),
hiddenNames.map(_.map(buildName(_)).toList),
getIdentifiedLocation(imp)
)
case _ =>
IR.Error.Syntax(imp, IR.Error.Syntax.InvalidImport)
IR$Name$Qualified qualifiedName;
// TODO: onlyNames, hiddenNames
if (imp.getFrom() != null) {
qualifiedName = buildQualifiedName(imp.getFrom().getBody());
} else {
qualifiedName = buildQualifiedName(imp.getImport().getBody());
}
*/
return new IR$Error$Syntax(imp, IR$Error$Syntax$InvalidImport$.MODULE$, meta(), diag());
}
private boolean isAll(Tree onlyNames) {
return switch (onlyNames) {
case Tree.Ident id -> buildName(id).name().equals("all");
default -> false;
};
var isAll = imp.getAll() != null;
return new IR$Module$Scope$Import$Module(
qualifiedName, rename, isAll, Option.empty(),
Option.empty(), getIdentifiedLocation(imp), false,
meta(), diag()
);
}
@SuppressWarnings("unchecked")
@ -1645,55 +1526,31 @@ final class TreeToIr {
* @return the [[IR]] representation of `imp`
*/
IR$Module$Scope$Export$Module translateExport(Tree.Export exp) {
if (exp.getExport() != null) {
if (exp.getFrom() != null) {
var qualifiedName = buildQualifiedName(exp.getFrom().getBody());
var onlyBodies = exp.getExport().getBody();
var isAll = isAll(onlyBodies);
final Option<List<IR$Name$Literal>> onlyNames = isAll ? Option.empty() :
Option.apply((List<IR$Name$Literal>) (Object)buildNames(onlyBodies, ',', false));
Option<IR$Name$Literal> rename = Option.apply(exp.getAs()).map(as -> buildName(as.getBody()));
if (exp.getFrom() != null) {
var qualifiedName = buildQualifiedName(exp.getFrom().getBody());
var onlyBodies = exp.getExport().getBody();
var isAll = exp.getAll() != null;
final Option<List<IR$Name$Literal>> onlyNames = isAll ? Option.empty() :
Option.apply((List<IR$Name$Literal>) (Object)buildNames(onlyBodies, ',', false));
var hidingList = exp.getHiding() == null ? nil() : buildNames(exp.getHiding().getBody(), ',', false);
final Option<List<IR$Name$Literal>> hidingNames = hidingList.isEmpty() ? Option.empty() :
Option.apply((List<IR$Name$Literal>) (Object)hidingList);
var hidingList = exp.getHiding() == null ? nil() : buildNames(exp.getHiding().getBody(), ',', false);
final Option<List<IR$Name$Literal>> hidingNames = hidingList.isEmpty() ? Option.empty() :
Option.apply((List<IR$Name$Literal>) (Object)hidingList);
Option<IR$Name$Literal> rename;
if (exp.getFromAs() != null) {
rename = Option.apply(buildName(exp.getFromAs().getBody()));
} else {
rename = Option.empty();
}
return new IR$Module$Scope$Export$Module(
qualifiedName, rename,
true, onlyNames, hidingNames, getIdentifiedLocation(exp), false,
meta(), diag()
return new IR$Module$Scope$Export$Module(
qualifiedName, rename,
true, onlyNames, hidingNames, getIdentifiedLocation(exp), false,
meta(), diag()
);
} else {
var qualifiedName = buildQualifiedName(exp.getExport().getBody());
return new IR$Module$Scope$Export$Module(
qualifiedName, rename, false, Option.empty(),
Option.empty(), getIdentifiedLocation(exp), false,
meta(), diag()
);
} else {
var qualifiedName = buildQualifiedName(exp.getExport().getBody());
Option<IR$Name$Literal> rename = exp.getExportAs() == null ? Option.empty() :
Option.apply(buildName(exp.getExportAs().getBody()));
return new IR$Module$Scope$Export$Module(
qualifiedName, rename, false, Option.empty(),
Option.empty(), getIdentifiedLocation(exp), false,
meta(), diag()
);
}
}
/*
exp match {
case AST.Export(path, rename, isAll, onlyNames, hiddenNames) =>
IR.Module.Scope.Export.Module(
IR.Name.Qualified(path.map(buildName(_)).toList, None),
rename.map(buildName(_)),
isAll,
onlyNames.map(_.map(buildName(_)).toList),
hiddenNames.map(_.map(buildName(_)).toList),
getIdentifiedLocation(exp)
)
}
case _ -> */
throw new UnhandledEntity(exp, "translateExport");
}
/** Translates an arbitrary invalid expression from the [[AST]] representation
@ -1758,9 +1615,17 @@ final class TreeToIr {
}
*/
private IR$Name$Literal buildName(Token name) {
return new IR$Name$Literal(
name.codeRepr(),
false,
getIdentifiedLocation(name),
meta(), diag()
);
}
private IR$Name$Literal buildName(Tree ident) {
return switch (ident) {
case Tree.Ident id -> buildName(id, id.getToken());
case Tree.Ident id -> buildName(id.getToken());
default -> throw new UnhandledEntity(ident, "buildName");
};
}
@ -1777,16 +1642,52 @@ final class TreeToIr {
}
private Option<IdentifiedLocation> getIdentifiedLocation(Tree ast) {
if (ast == null) {
return Option.empty();
return Option.apply(ast).map(ast_ -> {
int begin = Math.toIntExact(ast_.getStartCode());
int end = Math.toIntExact(ast_.getEndCode());
return new IdentifiedLocation(new Location(begin, end), Option.empty());
});
}
private Option<IdentifiedLocation> getIdentifiedLocation(ArgumentDefinition ast) {
// Note: In the IR, `DefinitionArgument` is a type of AST node; in the parser, it is not an AST-level type, but a
// type used only in specific locations. As a result, the IR expects it to have a source-code location, but the
// parser doesn't have a span for it. Here we synthesize one. This will not be necessary if we refactor IR so that
// `ArgumentDefinition` is not an AST node, though that change would have effects throughout the compiler and may
// not be worthwhile if IR is expected to be replaced.
long begin;
if (ast.getOpen() != null) {
begin = ast.getOpen().getStartCode();
} else if (ast.getOpen2() != null) {
begin = ast.getOpen2().getStartCode();
} else if (ast.getSuspension() != null) {
begin = ast.getSuspension().getStartCode();
} else {
begin = ast.getPattern().getStartCode();
}
int begin, len;
{
begin = Math.toIntExact(ast.getStartCode());
int end = Math.toIntExact(ast.getEndCode());
len = end - begin;
int begin_ = Math.toIntExact(begin);
long end;
if (ast.getClose() != null) {
end = ast.getClose().getEndCode();
} else if (ast.getDefault() != null) {
end = ast.getDefault().getEquals().getEndCode();
} else if (ast.getClose2() != null) {
end = ast.getClose2().getEndCode();
} else if (ast.getType() != null) {
end = ast.getType().getOperator().getEndCode();
} else {
end = ast.getPattern().getEndCode();
}
return Option.apply(new IdentifiedLocation(new Location(begin, begin + len), Option.empty()));
int end_ = Math.toIntExact(end);
return Option.apply(new IdentifiedLocation(new Location(begin_, end_), Option.empty()));
}
private Option<IdentifiedLocation> getIdentifiedLocation(Token ast) {
return Option.apply(ast).map(ast_ -> {
int begin = Math.toIntExact(ast_.getStartCode());
int end = Math.toIntExact(ast_.getEndCode());
return new IdentifiedLocation(new Location(begin, end), Option.empty());
});
}
private MetadataStorage meta() {
return MetadataStorage.apply(nil());

20
engine/runtime/src/test/java/org/enso/compiler/EnsoCompilerTest.java
View File

@ -12,7 +12,6 @@ import org.enso.compiler.core.IR;
import org.enso.syntax.text.AST.ASTOf;
import org.enso.syntax.text.Parser;
import org.enso.syntax.text.Shape;
import org.enso.syntax2.UnsupportedSyntaxException;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@ -40,7 +39,6 @@ public class EnsoCompilerTest {
}
@Test
@Ignore
public void testCase() throws Exception {
parseTest("""
type Msg
@ -62,7 +60,6 @@ public class EnsoCompilerTest {
import project.IO
import Standard.Base as Enso_List
from Standard.Base import all hiding Number, Boolean
from Standard.Table as Column_Module import Column
polyglot java import java.lang.Float
polyglot java import java.net.URI as Java_URI
@ -94,7 +91,6 @@ public class EnsoCompilerTest {
}
@Test
@Ignore
public void testFactorial() throws Exception {
parseTest("""
fac n = if n == 1 then 1 else n * fac n-1
@ -168,7 +164,6 @@ public class EnsoCompilerTest {
}
@Test
@Ignore
public void testIfThenBlock() throws Exception {
parseTest("""
from_java_set java_set =
@ -198,17 +193,21 @@ public class EnsoCompilerTest {
}
@Test
@Ignore
public void testSignature3() throws Exception {
parseTest("val = 123 : Int");
}
@Test
@Ignore
public void testSignature4() throws Exception {
parseTest("val = foo (123 : Int)");
}
@Test
@Ignore
public void testSignature5() throws Exception {
parseTest("val : List Int -> Int");
}
@Test
public void testExport1() throws Exception {
parseTest("export prj.Data.Foo");
@ -229,11 +228,6 @@ public class EnsoCompilerTest {
parseTest("from prj.Data.Foo export all hiding Bar, Baz");
}
@Test
public void testExportFromAsExport() throws Exception {
parseTest("from prj.Data.Foo as Bar export Baz, Quux");
}
@Test
public void testTextLiteral() throws Exception {
parseTest("""
@ -261,7 +255,7 @@ public class EnsoCompilerTest {
}
@SuppressWarnings("unchecked")
private void parseTest(String code) throws UnsupportedSyntaxException, IOException {
private void parseTest(String code) throws IOException {
var src = Source.newBuilder("enso", code, "test-" + Integer.toHexString(code.hashCode()) + ".enso").build();
var ir = ensoCompiler.compile(src);
assertNotNull("IR was generated", ir);

3
lib/rust/metamodel/lexpr/src/lib.rs
View File

@ -190,7 +190,10 @@ impl<'g> ToSExpr<'g> {
fn primitive(&self, primitive: Primitive, data: &mut &[u8]) -> Value {
match primitive {
Primitive::U32 => Value::Number(read_u32(data).into()),
Primitive::I32 => Value::Number((read_u32(data) as i32).into()),
Primitive::Char => Value::Char(char::try_from(read_u32(data)).unwrap()),
Primitive::U64 => Value::Number(read_u64(data).into()),
Primitive::I64 => Value::Number((read_u64(data) as i64).into()),
Primitive::Bool => {
let value = read_u8(data);
let value = match value {

5
lib/rust/metamodel/src/java/from_meta.rs
View File

@ -75,8 +75,11 @@ impl FromMeta {
fn primitive(&self, ty: &meta::Primitive) -> Result<Primitive, Class> {
match ty {
meta::Primitive::Bool => Ok(Primitive::Bool),
meta::Primitive::U64 => Ok(Primitive::Long { unsigned: true }),
meta::Primitive::I32 => Ok(Primitive::Int { unsigned: false }),
meta::Primitive::I64 => Ok(Primitive::Long { unsigned: false }),
meta::Primitive::U32 => Ok(Primitive::Int { unsigned: true }),
meta::Primitive::U64 => Ok(Primitive::Long { unsigned: true }),
meta::Primitive::Char => Ok(Primitive::Int { unsigned: true }),
meta::Primitive::String => Err(Class::string()),
meta::Primitive::Option(t0_) => Err(Class::optional(self.meta_to_java[t0_])),
meta::Primitive::Sequence(t0_) => Err(Class::list(self.meta_to_java[t0_])),

5
lib/rust/metamodel/src/meta/graphviz.rs
View File

@ -40,8 +40,11 @@ pub fn graph(typegraph: &TypeGraph) -> Graph {
graph.edges.push((sname.clone(), sname2, EdgeType::Field));
},
Data::Primitive(Primitive::U32)
| Data::Primitive(Primitive::Bool)
| Data::Primitive(Primitive::U64)
| Data::Primitive(Primitive::I32)
| Data::Primitive(Primitive::I64)
| Data::Primitive(Primitive::Bool)
| Data::Primitive(Primitive::Char)
| Data::Primitive(Primitive::String) => {}
Data::Primitive(Primitive::Sequence(t0)) => graph.edges.push((
sname.clone(),

16
lib/rust/metamodel/src/meta/mod.rs
View File

@ -103,6 +103,12 @@ pub enum Primitive {
U32,
/// An unsigned 64-bit integer.
U64,
/// A signed 32-bit integer.
I32,
/// A signed 64-bit integer.
I64,
/// A unicode code point (in the range 0 to 0x10FFFF).
Char,
/// An UTF-8-encoded string.
String,
/// Zero or more values of a type.
@ -360,8 +366,11 @@ impl TypeGraph {
rewrite(t1);
}
Data::Primitive(Primitive::U32)
| Data::Primitive(Primitive::Bool)
| Data::Primitive(Primitive::U64)
| Data::Primitive(Primitive::I32)
| Data::Primitive(Primitive::I64)
| Data::Primitive(Primitive::Bool)
| Data::Primitive(Primitive::Char)
| Data::Primitive(Primitive::String) => {}
}
}
@ -404,8 +413,11 @@ impl TypeGraph {
to_visit.insert(*t1);
}
Data::Primitive(Primitive::U32)
| Data::Primitive(Primitive::Bool)
| Data::Primitive(Primitive::U64)
| Data::Primitive(Primitive::I32)
| Data::Primitive(Primitive::I64)
| Data::Primitive(Primitive::Bool)
| Data::Primitive(Primitive::Char)
| Data::Primitive(Primitive::String) => {}
}
}

7
lib/rust/metamodel/src/meta/serialization.rs
View File

@ -273,12 +273,15 @@ impl<'g> ProgramBuilder<'g> {
match primitive {
// Value doesn't matter, but this will be recognizable in the output, and will tend not
// to encode compatibly with other types.
Primitive::U32 => self.emit(Op::U32(1234567890)),
Primitive::U32 | Primitive::I32 => self.emit(Op::U32(1234567890)),
// Value 1 chosen to detect errors better: 0 encodes the same way as Option::None.
Primitive::Bool => self.emit(Op::U8(1)),
// Value doesn't matter, but this will be recognizable in the output, and will tend not
// to encode compatibly with other types.
Primitive::U64 => self.emit(Op::U64(1234567890123456789)),
Primitive::Char => self.emit(Op::U32(0x10FFFF)),
// Value doesn't matter, but this will be recognizable in the output, and will tend not
// to encode compatibly with other types.
Primitive::U64 | Primitive::I64 => self.emit(Op::U64(1234567890123456789)),
Primitive::String => self.emit(Op::U64("".len() as u64)),
Primitive::Sequence(_) if basecase => self.emit(Op::U64(0)),
Primitive::Sequence(t0) => {

15
lib/rust/metamodel/src/rust/mod.rs
View File

@ -122,6 +122,10 @@ pub enum Primitive {
Usize,
/// A `u32`.
U32,
/// An `i32`.
I32,
/// A `char`.
Char,
/// A `String`.
String,
/// A `Vec<_>`.
@ -197,7 +201,12 @@ pub trait ReferencedTypes {
impl ReferencedTypes for Primitive {
fn referenced_types(&self) -> Vec<LazyType> {
match self {
Primitive::Bool | Primitive::Usize | Primitive::String | Primitive::U32 => vec![],
Primitive::Bool
| Primitive::Usize
| Primitive::String
| Primitive::U32
| Primitive::I32
| Primitive::Char => vec![],
Primitive::Vec(ty) | Primitive::Option(ty) => vec![*ty],
Primitive::Result(ty0, ty1) => vec![*ty0, *ty1],
}
@ -293,10 +302,12 @@ impl TypeData {
impl Primitive {
/// Get information about the composition operator relating the types this type is composed of.
pub fn type_type(&self) -> TypeType {
match &self {
match self {
Primitive::Bool
| Primitive::Usize
| Primitive::U32
| Primitive::I32
| Primitive::Char
| Primitive::String
| Primitive::Vec(_) => TypeType::Product,
Primitive::Option(_) | Primitive::Result(_, _) => TypeType::Sum,

2
lib/rust/metamodel/src/rust/to_meta.rs
View File

@ -140,6 +140,8 @@ impl ToMeta {
let primitive = match primitive {
Primitive::Bool => meta::Primitive::Bool,
Primitive::U32 => meta::Primitive::U32,
Primitive::I32 => meta::Primitive::I32,
Primitive::Char => meta::Primitive::Char,
// In platform-independent formats, a `usize` is serialized as 64 bits.
Primitive::Usize => meta::Primitive::U64,
Primitive::String => meta::Primitive::String,

9
lib/rust/parser/generate-java/java/org/enso/syntax2/Message.java
View File

@ -8,7 +8,6 @@ final class Message {
private final CharSequence context;
private final int base;
private final long metadata;
private boolean encounteredUnsupportedSyntax;
private long position;
Message(java.nio.ByteBuffer bufferIn, CharSequence contextIn, long baseIn, long metadataIn) {
@ -63,14 +62,6 @@ final class Message {
return "Message[buffer=" + buffer.position() + "]";
}
boolean getEncounteredUnsupportedSyntax() {
return encounteredUnsupportedSyntax;
}
void markEncounteredUnsupportedSyntax() {
encounteredUnsupportedSyntax = true;
}
java.util.UUID getUuid(long nodeOffset, long nodeLength) {
long high = Parser.getUuidHigh(metadata, nodeOffset, nodeLength);
long low = Parser.getUuidLow(metadata, nodeOffset, nodeLength);

9
lib/rust/parser/generate-java/java/org/enso/syntax2/Parser.java
View File

@ -1,7 +1,6 @@
package org.enso.syntax2;
import org.enso.syntax2.Message;
import org.enso.syntax2.UnsupportedSyntaxException;
import java.io.File;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
@ -57,7 +56,7 @@ public final class Parser implements AutoCloseable {
return new Parser(state);
}
public Tree parse(CharSequence input) throws UnsupportedSyntaxException {
public Tree parse(CharSequence input) {
byte[] inputBytes = input.toString().getBytes(StandardCharsets.UTF_8);
ByteBuffer inputBuf = ByteBuffer.allocateDirect(inputBytes.length);
inputBuf.put(inputBytes);
@ -66,11 +65,7 @@ public final class Parser implements AutoCloseable {
var metadata = getMetadata(state);
serializedTree.order(ByteOrder.LITTLE_ENDIAN);
var message = new Message(serializedTree, input, base, metadata);
var result = Tree.deserialize(message);
if (message.getEncounteredUnsupportedSyntax()) {
throw new UnsupportedSyntaxException(result);
}
return result;
return Tree.deserialize(message);
}
@Override

14
lib/rust/parser/generate-java/java/org/enso/syntax2/UnsupportedSyntaxException.java
View File

@ -1,14 +0,0 @@
package org.enso.syntax2;
public final class UnsupportedSyntaxException extends Exception {
private final Tree tree;
UnsupportedSyntaxException(Tree treeIn) {
super("Tree contains unsupported syntax. Details are in an `Unsupported` node in the tree.");
tree = treeIn;
}
public Tree getTree() {
return tree;
}
}

6
lib/rust/parser/generate-java/src/main.rs
View File

@ -39,14 +39,12 @@ fn main() {
let ast = enso_parser::syntax::Tree::reflect();
let tree = enso_parser::syntax::Tree::reflect().id;
let token = enso_parser::syntax::Token::<enso_parser::syntax::token::Variant>::reflect().id;
let unsupported = enso_parser::syntax::tree::Unsupported::reflect().id;
let (graph, rust_to_meta) = rust::to_meta(ast);
let (graph, meta_to_java) = java::from_meta(&graph, enso_parser_generate_java::EITHER_TYPE);
let mut graph = java::transform::optional_to_null(graph);
let rust_to_java = |id| meta_to_java[&rust_to_meta[&id]];
let (tree, token, unsupported) =
(rust_to_java(tree), rust_to_java(token), rust_to_java(unsupported));
serialization::derive(&mut graph, tree, token, unsupported);
let (tree, token) = (rust_to_java(tree), rust_to_java(token));
serialization::derive(&mut graph, tree, token);
let graph = java::to_syntax(&graph, enso_parser_generate_java::PACKAGE);
let mut args = std::env::args();
args.next().unwrap();

19
lib/rust/parser/generate-java/src/serialization.rs
View File

@ -20,9 +20,9 @@ const TREE_BEGIN: &str = "fieldSpanLeftOffsetCodeReprBegin";
const TREE_LEN: &str = "fieldSpanLeftOffsetCodeReprLen";
/// Derive deserialization for all types in the typegraph.
pub fn derive(graph: &mut TypeGraph, tree: ClassId, token: ClassId, unsupported: ClassId) {
pub fn derive(graph: &mut TypeGraph, tree: ClassId, token: ClassId) {
let source = "source";
impl_deserialize(graph, tree, token, unsupported, source);
impl_deserialize(graph, tree, token, source);
graph[token].methods.push(impl_getter(CODE_GETTER));
graph[tree].methods.push(impl_getter(CODE_GETTER));
}
@ -30,13 +30,7 @@ pub fn derive(graph: &mut TypeGraph, tree: ClassId, token: ClassId, unsupported:
// === Deserialization Methods ===
fn impl_deserialize(
graph: &mut TypeGraph,
tree: ClassId,
token: ClassId,
unsupported: ClassId,
source: &str,
) {
fn impl_deserialize(graph: &mut TypeGraph, tree: ClassId, token: ClassId, source: &str) {
// Add UUIDs to types
let uuid = Class::builtin("java.util.UUID", vec![]);
let uuid = graph.classes.insert(uuid);
@ -123,11 +117,6 @@ fn impl_deserialize(
let class = &graph[id];
let mut deserialization =
bincode::DeserializerBuilder::new(id, crate::SERIALIZATION_SUPPORT, crate::EITHER_TYPE);
if id == unsupported {
deserialization.pre_hook(|bincode::HookInput { message }| {
format!("{message}.markEncounteredUnsupportedSyntax();\n")
});
}
if id == tree || class.parent == Some(tree) {
deserialization.materialize(tree_source, context_materializer());
deserialization.materialize(tree_id, uuid_materializer());
@ -179,6 +168,6 @@ fn end_code_token() -> impl for<'a> Fn(MaterializerInput<'a>) -> String + 'stati
fn impl_getter(name: &str) -> Method {
let mut method = syntax::Method::new(name, syntax::Type::named("String"));
method.body =
"return source.subSequence((int)startWhitespace, (int)endCode).toString();\n".to_string();
"return source.subSequence((int)startCode, (int)endCode).toString();\n".to_string();
Method::Raw(method)
}

398
lib/rust/parser/src/lexer.rs
View File

@ -376,8 +376,18 @@ fn is_octal_digit(t: char) -> bool {
/// Check whether the provided character is a hexadecimal digit.
#[inline(always)]
fn is_hexadecimal_digit(t: char) -> bool {
is_decimal_digit(t) || ('a'..='f').contains(&t) || ('A'..='F').contains(&t)
fn is_hexadecimal_digit(c: char) -> bool {
decode_hexadecimal_digit(c).is_some()
}
#[inline(always)]
fn decode_hexadecimal_digit(c: char) -> Option<u8> {
Some(match c {
'0'..='9' => c as u8 - b'0',
'a'..='f' => 10 + (c as u8 - b'a'),
'A'..='F' => 10 + (c as u8 - b'A'),
_ => return None,
})
}
impl<'s> Lexer<'s> {
@ -593,13 +603,7 @@ impl<'s> Lexer<'s> {
self.submit_token(token);
return;
}
let only_eq = token.code.chars().all(|t| t == '=');
let is_mod = token.code.ends_with('=') && !only_eq;
let tp = if is_mod {
token::Variant::modifier()
} else {
token::Variant::operator(analyze_operator(&token.code))
};
let tp = token::Variant::operator(analyze_operator(&token.code));
let token = token.with_variant(tp);
self.submit_token(token);
}
@ -623,24 +627,39 @@ fn analyze_operator(token: &str) -> token::OperatorProperties {
"~" =>
return operator
.with_unary_prefix_mode(token::Precedence::max())
.as_compile_time_operation(),
.as_compile_time_operation()
.as_suspension(),
"-" =>
return operator
.with_unary_prefix_mode(token::Precedence::max())
.with_binary_infix_precedence(14),
// "There are a few operators with the lowest precedence possible."
"=" => return operator.with_binary_infix_precedence(1).as_assignment(),
// - These 3 "consume everything to the right".
"=" =>
return operator
.with_binary_infix_precedence(1)
.as_right_associative()
.with_lhs_section_termination(operator::SectionTermination::Unwrap)
.as_assignment(),
":" =>
return operator
.with_binary_infix_precedence(2)
.with_binary_infix_precedence(1)
.as_right_associative()
.with_lhs_section_termination(operator::SectionTermination::Reify)
.as_compile_time_operation()
.as_type_annotation(),
"->" =>
return operator.with_binary_infix_precedence(3).as_compile_time_operation().as_arrow(),
return operator
.with_binary_infix_precedence(1)
.as_right_associative()
.with_lhs_section_termination(operator::SectionTermination::Unwrap)
.as_compile_time_operation()
.as_arrow(),
"|" | "\\\\" | "&" => return operator.with_binary_infix_precedence(4),
">>" | "<<" => return operator.with_binary_infix_precedence(5),
"|>" | "|>>" | "<|" | "<<|" => return operator.with_binary_infix_precedence(6),
// Other special operators.
"<=" | ">=" => return operator.with_binary_infix_precedence(14),
"==" => return operator.with_binary_infix_precedence(1),
"," =>
return operator
@ -672,22 +691,25 @@ fn analyze_operator(token: &str) -> token::OperatorProperties {
'<' | '>' => 14,
'+' | '-' => 15,
'*' | '/' | '%' => 16,
_ => return operator,
_ => 17,
};
operator.with_binary_infix_precedence(binary)
}
// ==============
// === Symbol ===
// ==============
// ===============
// === Symbols ===
// ===============
impl<'s> Lexer<'s> {
/// Parse a symbol.
fn symbol(&mut self) {
if let Some(token) = self.token(|this| this.take_1(&['(', ')', '{', '}', '[', ']'])) {
self.submit_token(token.with_variant(token::Variant::symbol()));
if let Some(token) = self.token(|this| this.take_1(&['(', '{', '['])) {
self.submit_token(token.with_variant(token::Variant::open_symbol()));
}
if let Some(token) = self.token(|this| this.take_1(&[')', '}', ']'])) {
self.submit_token(token.with_variant(token::Variant::close_symbol()));
}
}
}
@ -750,28 +772,17 @@ impl<'s> Lexer<'s> {
impl<'s> Lexer<'s> {
/// Read a text literal.
fn text(&mut self) {
if self.current_char == Some('`') {
match self.stack.last().copied() {
Some(State::InlineText) | Some(State::MultilineText { .. }) => {
let splice_quote_start = self.mark();
self.take_next();
let splice_quote_end = self.mark();
let token = self.make_token(
splice_quote_start,
splice_quote_end,
token::Variant::Symbol(token::variant::Symbol()),
);
self.output.push(token);
match self.stack.pop().unwrap() {
State::InlineText => self.inline_quote('\''),
State::MultilineText { indent } => self.text_lines(indent, true),
}
}
None => return,
}
}
let quote_char = match self.current_char {
Some(char @ ('"' | '\'')) => char,
Some('`') => {
if let Some(state) = self.stack.pop() {
self.end_splice(state);
} else {
let token = self.token(|this| this.take_next()).unwrap();
self.submit_token(token.with_variant(token::Variant::invalid()));
}
return;
}
_ => return,
};
let indent = self.last_spaces_visible_offset;
@ -790,15 +801,22 @@ impl<'s> Lexer<'s> {
self.take_next();
}
if multiline {
while self.current_char.is_some() {
let mut newline = self.take_1('\r');
newline = newline || self.take_1('\n');
if newline {
break;
}
}
let before_space = self.mark();
self.spaces_after_lexeme();
let text_start = self.mark();
let token = self.make_token(open_quote_start, text_start.clone(),
let token = self.make_token(open_quote_start, before_space,
token::Variant::TextStart(token::variant::TextStart()));
self.output.push(token);
let interpolate = quote_char == '\'';
if self.text_content(None, interpolate, State::MultilineText { indent }) {
return;
}
self.text_lines(indent, interpolate);
self.text_content(Some(text_start), None, interpolate, State::MultilineText { indent }, Some(indent));
return;
} else {
// Exactly two quote characters: Open and shut case.
let close_quote_end = self.mark();
@ -821,7 +839,7 @@ impl<'s> Lexer<'s> {
}
fn inline_quote(&mut self, quote_char: char) {
if self.text_content(Some(quote_char), quote_char == '\'', State::InlineText) {
if self.text_content(None, Some(quote_char), quote_char == '\'', State::InlineText, None) {
return;
}
if let Some(char) = self.current_char && char == quote_char {
@ -834,22 +852,86 @@ impl<'s> Lexer<'s> {
}
}
fn end_splice(&mut self, state: State) {
let splice_quote_start = self.mark();
self.take_next();
let splice_quote_end = self.mark();
let token = self.make_token(
splice_quote_start,
splice_quote_end,
token::Variant::CloseSymbol(token::variant::CloseSymbol()),
);
self.output.push(token);
match state {
State::InlineText => self.inline_quote('\''),
State::MultilineText { indent } => self.text_lines(indent, true),
}
}
fn text_content(
&mut self,
start: Option<(Bytes, Offset<'s>)>,
closing_char: Option<char>,
interpolate: bool,
state: State,
multiline: Option<VisibleOffset>,
) -> bool {
let mut text_start = self.mark();
let mut text_start = start.unwrap_or_else(|| self.mark());
while let Some(char) = self.current_char {
if is_newline_char(char) || closing_char == Some(char) {
if closing_char == Some(char) || (multiline.is_none() && is_newline_char(char)) {
break;
}
let before_newline = self.mark();
let mut newline = self.take_1('\r');
newline = newline || self.take_1('\n');
if newline {
let indent = multiline.unwrap();
let text_end = self.mark();
self.spaces_after_lexeme();
if let Some(char) = self.current_char {
if self.last_spaces_visible_offset <= indent && !is_newline_char(char) {
let token = self.make_token(
text_start,
before_newline.clone(),
token::Variant::TextSection(token::variant::TextSection()),
);
if !(token.code.is_empty() && token.left_offset.code.is_empty()) {
self.output.push(token);
}
let token = self.make_token(
before_newline,
text_end,
token::Variant::Newline(token::variant::Newline()),
);
self.output.push(token);
self.spaces_after_lexeme();
return false;
}
};
let token = self.make_token(
text_start,
text_end.clone(),
token::Variant::TextSection(token::variant::TextSection()),
);
if !(token.code.is_empty() && token.left_offset.code.is_empty()) {
self.output.push(token);
}
text_start = self.mark();
continue;
}
if char == '\\' {
let backslash_start = self.mark();
self.take_next();
if let Some(char) = self.current_char && (interpolate || closing_char == Some(char)) {
self.text_escape(char, interpolate, backslash_start, &mut text_start);
let token = self.make_token(
text_start,
backslash_start.clone(),
token::Variant::TextSection(token::variant::TextSection()),
);
if !token.code.is_empty() {
self.output.push(token);
}
text_start = self.text_escape(backslash_start, char);
continue;
}
}
@ -860,7 +942,7 @@ impl<'s> Lexer<'s> {
splice_quote_start.clone(),
token::Variant::TextSection(token::variant::TextSection()),
);
if !token.code.is_empty() {
if !(token.code.is_empty() && token.left_offset.code.is_empty()) {
self.output.push(token);
}
self.take_next();
@ -868,10 +950,9 @@ impl<'s> Lexer<'s> {
let token = self.make_token(
splice_quote_start,
splice_quote_end.clone(),
token::Variant::Symbol(token::variant::Symbol()),
token::Variant::OpenSymbol(token::variant::OpenSymbol()),
);
self.output.push(token);
self.spaces_after_lexeme();
self.stack.push(state);
return true;
}
@ -891,134 +972,80 @@ impl<'s> Lexer<'s> {
fn text_escape(
&mut self,
char: char,
interpolate: bool,
backslash_start: (Bytes, Offset<'s>),
text_start: &'_ mut (Bytes, Offset<'s>),
) {
let token = self.make_token(
text_start.clone(),
backslash_start.clone(),
token::Variant::TextSection(token::variant::TextSection()),
);
if !token.code.is_empty() {
self.output.push(token);
}
if interpolate && char == 'x' || char == 'u' || char == 'U' {
char: char,
) -> (Bytes, Offset<'s>) {
let leader = match char {
'x' => Some((2, false)),
'u' => Some((4, true)),
'U' => Some((8, false)),
_ => None,
};
if let Some((mut expect_len, accepts_delimiter)) = leader {
self.take_next();
let leader_end = self.mark();
let token = self.make_token(
backslash_start,
leader_end.clone(),
token::Variant::TextEscapeLeader(token::variant::TextEscapeLeader()),
);
self.output.push(token);
let (mut expect_len, accepts_delimiter) = match char {
'x' => (2, false),
'u' => (4, true),
'U' => (8, false),
_ => unreachable!(),
};
let delimited = accepts_delimiter && self.current_char == Some('{');
let mut sequence_start = leader_end.clone();
if delimited {
self.take_next();
sequence_start = self.mark();
let token = self.make_token(
leader_end,
sequence_start.clone(),
token::Variant::TextEscapeSequenceStart(
token::variant::TextEscapeSequenceStart(),
),
);
self.output.push(token);
expect_len = 6;
}
let mut value: u32 = 0;
for _ in 0..expect_len {
if let Some(c) = self.current_char && is_hexadecimal_digit(c) {
if let Some(c) = self.current_char && let Some(x) = decode_hexadecimal_digit(c) {
value = 16 * value + x as u32;
self.take_next();
} else {
break;
}
}
let sequence_end = self.mark();
let token = self.make_token(
sequence_start,
sequence_end.clone(),
token::Variant::TextEscapeHexDigits(token::variant::TextEscapeHexDigits()),
);
self.output.push(token);
let value = char::from_u32(value);
if delimited && self.current_char == Some('}') {
self.take_next();
let close_end = self.mark();
let token = self.make_token(
sequence_end,
close_end.clone(),
token::Variant::TextEscapeSequenceEnd(token::variant::TextEscapeSequenceEnd()),
);
self.output.push(token);
*text_start = close_end;
} else {
*text_start = sequence_end;
}
return;
let sequence_end = self.mark();
let token = self.make_token(
backslash_start,
sequence_end.clone(),
token::Variant::TextEscape(token::variant::TextEscape(value)),
);
self.output.push(token);
sequence_end
} else {
let value = match char {
'0' => Some('\0'),
'a' => Some('\x07'),
'b' => Some('\x08'),
'f' => Some('\x0C'),
'n' => Some('\x0A'),
'r' => Some('\x0D'),
't' => Some('\x09'),
'v' => Some('\x0B'),
'\\' => Some('\\'),
'"' => Some('"'),
'\'' => Some('\''),
'`' => Some('`'),
_ => None,
};
self.take_next();
let escape_end = self.mark();
let token = self.make_token(
backslash_start,
escape_end.clone(),
token::Variant::TextEscape(token::variant::TextEscape(value)),
);
self.output.push(token);
escape_end
}
let backslash_end = self.mark();
let token = self.make_token(
backslash_start,
backslash_end.clone(),
token::Variant::TextEscapeSymbol(token::variant::TextEscapeSymbol()),
);
self.output.push(token);
self.take_next();
let escaped_end = self.mark();
let token = self.make_token(
backslash_end,
escaped_end.clone(),
token::Variant::TextEscapeChar(token::variant::TextEscapeChar()),
);
self.output.push(token);
*text_start = escaped_end;
self.take_next();
}
/// Read the lines of a text literal.
fn text_lines(&mut self, indent: VisibleOffset, is_interpolated: bool) {
while self.current_char.is_some() {
let start = self.mark();
// Consume the newline and any spaces.
if !self.take_1('\n') && self.take_1('\r') {
self.take_1('\n');
}
let before_space = self.mark();
if start != before_space {
// Create a text section for the newline.
let newline = self.make_token(
start.clone(),
before_space.clone(),
token::Variant::TextSection(token::variant::TextSection()),
);
self.spaces_after_lexeme();
// Check indent, unless this is an empty line.
if let Some(char) = self.current_char {
if self.last_spaces_visible_offset <= indent && !is_newline_char(char) {
let token = self.make_token(
start,
before_space,
token::Variant::Newline(token::variant::Newline()),
);
self.output.push(token);
self.spaces_after_lexeme();
return;
}
};
self.output.push(newline);
}
// Output the line as a text section.
if self.text_content(None, is_interpolated, State::MultilineText { indent }) {
break;
}
}
self.text_content(
None,
None,
is_interpolated,
State::MultilineText { indent },
Some(indent),
);
}
fn mark(&mut self) -> (Bytes, Offset<'s>) {
@ -1045,6 +1072,49 @@ impl<'s> Lexer<'s> {
}
}
/// Move whitespace characters from the end of `left` to the beginning of `right` until the visible
/// length of `left` is not longer than `target`.
#[allow(unsafe_code)]
pub fn untrim(target: VisibleOffset, left: &mut Offset, right: &mut Code) {
let mut utf8 = 0;
let mut utf16 = 0;
let mut trimmed = VisibleOffset(0);
for c in left.code.repr.chars().rev() {
if left.visible - trimmed <= target {
break;
}
utf8 += c.len_utf8();
utf16 += c.len_utf16();
trimmed += space_char_visible_size(c).unwrap();
}
if utf8 == 0 && utf16 == 0 {
return;
}
left.visible = left.visible - trimmed;
left.code.utf16 -= utf16;
let len = left.code.repr.len() - utf8;
unsafe {
match right.repr {
Cow::Borrowed(s) if s.as_ptr() == left.code.repr.as_ptr().add(left.code.repr.len()) => {
let p = s.as_ptr().sub(utf8);
let len = s.len() + utf8;
right.repr = Cow::Borrowed(str::from_utf8_unchecked(slice::from_raw_parts(p, len)));
}
_ => {
let mut s = String::with_capacity(len + right.repr.len());
s += &left.code.repr[len..];
right.utf16 += utf16;
s += &right.repr;
right.repr = Cow::Owned(s);
}
}
}
left.code.repr = match &left.code.repr {
Cow::Borrowed(s) => Cow::Borrowed(&s[..len]),
Cow::Owned(s) => Cow::Owned(s[..len].to_string()),
};
}
// ================
@ -1168,6 +1238,14 @@ impl<'s> Lexer<'s> {
let block_end = self.marker_token(token::Variant::block_end());
self.submit_token(block_end);
}
if self.last_spaces_visible_offset != VisibleOffset(0) {
let left_offset_start = self.current_offset - self.last_spaces_offset;
let offset_code = self.input.slice(left_offset_start..self.current_offset);
let visible_offset = self.last_spaces_visible_offset;
let offset = Offset(visible_offset, offset_code);
let eof = token::variant::Variant::Newline(token::variant::Newline());
self.submit_token(Token(offset, "", eof));
}
let mut internal_error = self.internal_error.take();
if self.current_char != None {
let message = format!("Lexer did not consume all input. State: {self:?}");
@ -1581,9 +1659,11 @@ mod benches {
fn bench_idents(b: &mut Bencher) {
let reps = 1_000_000;
let str = "test ".repeat(reps);
// Trim the trailing space off.
let str = &str[..str.len() - 1];
b.iter(move || {
let lexer = Lexer::new(&str);
let lexer = Lexer::new(str);
assert_eq!(lexer.run().unwrap().len(), reps);
});
}

246
lib/rust/parser/src/lib.rs
View File

@ -83,6 +83,7 @@
#![feature(test)]
#![feature(specialization)]
#![feature(let_chains)]
#![feature(let_else)]
#![feature(if_let_guard)]
#![feature(box_patterns)]
// === Standard Linter Configuration ===
@ -105,6 +106,7 @@
use crate::prelude::*;
// ==============
// === Export ===
// ==============
@ -221,13 +223,7 @@ fn expression_to_statement(mut tree: syntax::Tree<'_>) -> syntax::Tree<'_> {
let variant = Box::new(Variant::TypeAnnotated(annotated));
return Tree::invalid(err, Tree { variant, span });
}
let tree_ = match &mut tree {
Tree { variant: box Variant::OprSectionBoundary(OprSectionBoundary { ast }), span } => {
left_offset += &span.left_offset;
ast
}
_ => &mut tree,
};
let tree_ = &mut tree;
let opr_app = match tree_ {
Tree { variant: box Variant::OprApp(opr_app), span } => {
left_offset += &span.left_offset;
@ -236,90 +232,206 @@ fn expression_to_statement(mut tree: syntax::Tree<'_>) -> syntax::Tree<'_> {
_ => return tree,
};
if let OprApp { lhs: Some(lhs), opr: Ok(opr), rhs } = opr_app && opr.properties.is_assignment() {
let (mut lhs_, args) = collect_arguments(lhs.clone());
let (mut leftmost, args) = collect_arguments(lhs.clone());
if let Some(rhs) = rhs {
if let Variant::Ident(ident) = &*lhs_.variant && ident.token.variant.is_type {
if let Variant::Ident(ident) = &*leftmost.variant && ident.token.variant.is_type {
// If the LHS is a type, this is a (destructuring) assignment.
let mut result = Tree::assignment(mem::take(lhs), mem::take(opr), mem::take(rhs));
left_offset += result.span.left_offset;
result.span.left_offset = left_offset;
let lhs = expression_to_pattern(mem::take(lhs));
let mut result = Tree::assignment(lhs, mem::take(opr), mem::take(rhs));
result.span.left_offset += left_offset;
return result;
}
if args.is_empty() && !is_body_block(rhs) {
// If the LHS has no arguments, and there is a RHS, and the RHS is not a body block,
// this is a variable assignment.
let mut result = Tree::assignment(lhs_, mem::take(opr), mem::take(rhs));
left_offset += result.span.left_offset;
result.span.left_offset = left_offset;
let mut result = Tree::assignment(leftmost, mem::take(opr), mem::take(rhs));
result.span.left_offset += left_offset;
return result;
}
}
if let Variant::Ident(Ident { token }) = &mut *lhs_.variant {
if let Variant::Ident(Ident { token }) = &mut *leftmost.variant {
// If this is not a variable assignment, and the leftmost leaf of the `App` tree is
// an identifier, this is a function definition.
let mut result = Tree::function(mem::take(token), args, mem::take(opr), mem::take(rhs));
left_offset += result.span.left_offset;
result.span.left_offset = left_offset;
result.span.left_offset += left_offset;
return result;
}
}
tree
}
fn collect_arguments(
mut lhs_: syntax::Tree,
) -> (syntax::Tree, Vec<syntax::tree::ArgumentDefinition>) {
fn expression_to_type(mut input: syntax::Tree<'_>) -> syntax::Tree<'_> {
use syntax::tree::*;
let mut args = vec![];
loop {
match lhs_.variant {
box Variant::App(App { func, arg }) => {
lhs_ = func.clone();
match &arg.variant {
box Variant::OprApp(OprApp { lhs: Some(lhs), opr: Ok(opr), rhs: Some(rhs) })
if opr.properties.is_assignment() => {
let equals = opr.clone();
let open = default();
let close = default();
let mut pattern = lhs.clone();
pattern.span.left_offset += arg.span.left_offset.clone();
let default = Some(ArgumentDefault { equals, expression: rhs.clone() });
args.push(ArgumentDefinition { open, pattern, default, close });
}
box Variant::Group(Group { open, body: Some(body), close }) if let box
Variant::OprApp(OprApp { lhs: Some(lhs), opr: Ok(opr), rhs: Some(rhs) }) = &body.variant
&& opr.properties.is_assignment() => {
let equals = opr.clone();
let mut open = open.clone();
open.left_offset += arg.span.left_offset.clone();
let open = Some(open);
let close = Some(close.clone());
let default = Some(ArgumentDefault { equals, expression: rhs.clone() });
let pattern = lhs.clone();
args.push(ArgumentDefinition { open, pattern, default, close });
}
_ => {
let open = default();
let close = default();
let default = default();
args.push(ArgumentDefinition { open, pattern: arg.clone(), default, close });
}
if let Variant::Wildcard(wildcard) = &mut *input.variant {
wildcard.de_bruijn_index = None;
return input;
}
let mut out = match input.variant {
box Variant::TemplateFunction(TemplateFunction { ast, .. }) => expression_to_type(ast),
box Variant::Group(Group { open, body: Some(body), close }) =>
Tree::group(open, Some(expression_to_type(body)), close),
box Variant::OprApp(OprApp { lhs, opr, rhs }) =>
Tree::opr_app(lhs.map(expression_to_type), opr, rhs.map(expression_to_type)),
box Variant::App(App { func, arg }) =>
Tree::app(expression_to_type(func), expression_to_type(arg)),
_ => return input,
};
out.span.left_offset += input.span.left_offset;
out
}
fn expression_to_pattern(mut input: syntax::Tree<'_>) -> syntax::Tree<'_> {
use syntax::tree::*;
if let Variant::Wildcard(wildcard) = &mut *input.variant {
wildcard.de_bruijn_index = None;
return input;
}
let mut out = match input.variant {
box Variant::TemplateFunction(TemplateFunction { ast, .. }) => expression_to_pattern(ast),
box Variant::Group(Group { open, body: Some(body), close }) =>
Tree::group(open, Some(expression_to_pattern(body)), close),
box Variant::App(App { func, arg }) =>
Tree::app(expression_to_pattern(func), expression_to_pattern(arg)),
box Variant::TypeAnnotated(TypeAnnotated { expression, operator, type_ }) =>
Tree::type_annotated(expression_to_pattern(expression), operator, type_),
_ => return input,
};
out.span.left_offset += input.span.left_offset;
out
}
fn collect_arguments(
mut tree: syntax::Tree,
) -> (syntax::Tree, Vec<syntax::tree::ArgumentDefinition>) {
if let box syntax::tree::Variant::OprApp(syntax::tree::OprApp {
lhs: None,
opr: Ok(opr),
rhs: Some(rhs),
}) = tree.variant
{
let syntax::token::Token { left_offset, code, .. } = opr;
let opr = syntax::token::ident(left_offset, code, false, 0, false, false);
let mut opr = Some(syntax::Tree::ident(opr));
let mut tree_ = rhs;
let mut left_offset = tree.span.left_offset;
syntax::tree::recurse_left_mut_while(&mut tree_, |tree| {
left_offset += mem::take(&mut tree.span.left_offset);
match &mut *tree.variant {
syntax::tree::Variant::App(syntax::tree::App { func, .. })
if !matches!(&*func.variant, syntax::tree::Variant::App(_)) =>
{
let mut func_ = func.clone();
func_.span.left_offset = mem::take(&mut left_offset);
*func = syntax::Tree::app(opr.take().unwrap(), func_);
false
}
_ => true,
}
box Variant::NamedApp(NamedApp { func, open, name, equals, arg, close }) => {
lhs_ = func.clone();
let open = open.clone();
let close = close.clone();
let equals = equals.clone();
let pattern = Tree::ident(name);
let default = Some(ArgumentDefault { equals, expression: arg.clone() });
args.push(ArgumentDefinition { open, pattern, default, close });
}
_ => break,
}
});
tree = tree_;
}
let mut args = vec![];
while let Some(arg) = parse_argument_application(&mut tree) {
args.push(arg);
}
args.reverse();
(lhs_, args)
(tree, args)
}
/// Try to parse the expression as an application of a function to an `ArgumentDefinition`. If it
/// matches, replace the expression with its LHS, and return the `ArgumentDefinition` node.
pub fn parse_argument_application<'s>(
expression: &'_ mut syntax::Tree<'s>,
) -> Option<syntax::tree::ArgumentDefinition<'s>> {
use syntax::tree::*;
match &mut expression.variant {
box Variant::App(App { func, arg }) => {
let arg = parse_argument_definition(arg.clone());
func.span.left_offset += mem::take(&mut expression.span.left_offset);
*expression = func.clone();
Some(arg)
}
box Variant::NamedApp(NamedApp { func, open, name, equals, arg, close }) => {
let open = mem::take(open);
let close = mem::take(close);
let equals = equals.clone();
let pattern = Tree::ident(name.clone());
let open2 = default();
let suspension = default();
let close2 = default();
let type_ = default();
let default = Some(ArgumentDefault { equals, expression: arg.clone() });
func.span.left_offset += mem::take(&mut expression.span.left_offset);
*expression = func.clone();
Some(ArgumentDefinition {
open,
open2,
pattern,
suspension,
default,
close2,
type_,
close,
})
}
_ => None,
}
}
/// Interpret the expression as an element of an argument definition sequence.
pub fn parse_argument_definition(mut pattern: syntax::Tree) -> syntax::tree::ArgumentDefinition {
use syntax::tree::*;
let mut open1 = default();
let mut close1 = default();
if let box Variant::Group(Group { mut open, body: Some(mut body), close }) = pattern.variant {
*(if let Some(open) = open.as_mut() {
&mut open.left_offset
} else {
&mut body.span.left_offset
}) += pattern.span.left_offset;
open1 = open;
close1 = close;
pattern = body;
}
let mut default_ = default();
if let Variant::OprApp(OprApp { lhs: Some(lhs), opr: Ok(opr), rhs: Some(rhs) }) = &*pattern.variant && opr.properties.is_assignment() {
let left_offset = pattern.span.left_offset;
default_ = Some(ArgumentDefault { equals: opr.clone(), expression: rhs.clone() });
pattern = lhs.clone();
pattern.span.left_offset += left_offset;
}
let mut open2 = default();
let mut close2 = default();
if let box Variant::Group(Group { mut open, body: Some(mut body), close }) = pattern.variant {
*(if let Some(open) = open.as_mut() {
&mut open.left_offset
} else {
&mut body.span.left_offset
}) += pattern.span.left_offset;
open2 = open;
close2 = close;
pattern = body;
}
let mut type__ = default();
if let box Variant::TypeAnnotated(TypeAnnotated { mut expression, operator, type_ }) =
pattern.variant
{
expression.span.left_offset += pattern.span.left_offset;
type__ = Some(ArgumentType { operator, type_ });
pattern = expression;
}
let mut suspension = default();
if let Variant::UnaryOprApp(UnaryOprApp { opr, rhs: Some(rhs) }) = &*pattern.variant && opr.properties.is_suspension() {
let mut opr = opr.clone();
opr.left_offset += pattern.span.left_offset;
suspension = Some(opr);
pattern = rhs.clone();
}
let pattern = expression_to_pattern(pattern);
let open = open1;
let close = close1;
let type_ = type__;
ArgumentDefinition { open, open2, pattern, suspension, default: default_, close2, type_, close }
}
/// Return whether the expression is a body block.

300
lib/rust/parser/src/macros/built_in.rs
View File

@ -32,17 +32,20 @@ fn register_import_macros(macros: &mut resolver::SegmentMap<'_>) {
use crate::macro_definition;
let defs = [
macro_definition! {("import", everything()) import_body},
macro_definition! {("import", everything(), "as", everything()) import_body},
macro_definition! {("import", everything(), "as", identifier()) import_body},
macro_definition! {("import", everything(), "hiding", everything()) import_body},
macro_definition! {("polyglot", everything(), "import", everything()) import_body},
macro_definition! {
("polyglot", everything(), "import", everything(), "as", everything()) import_body},
("polyglot", everything(), "import", everything(), "as", identifier()) import_body},
macro_definition! {
("polyglot", everything(), "import", everything(), "hiding", everything()) import_body},
macro_definition! {
("from", everything(), "import", everything(), "hiding", everything()) import_body},
macro_definition! {
("from", everything(), "as", everything(), "import", everything()) import_body},
("from", everything(), "import", nothing(), "all", nothing()) import_body},
macro_definition! {
("from", everything(), "import", nothing(), "all", nothing(), "hiding", everything())
import_body},
macro_definition! {("from", everything(), "import", everything()) import_body},
];
for def in defs {
@ -51,41 +54,48 @@ fn register_import_macros(macros: &mut resolver::SegmentMap<'_>) {
}
fn import_body(segments: NonEmptyVec<MatchedSegment>) -> syntax::Tree {
use operator::resolve_operator_precedence_if_non_empty;
let mut polyglot = None;
let mut from = None;
let mut from_as = None;
let mut import = None;
let mut import_as = None;
let mut all = None;
let mut as_ = None;
let mut hiding = None;
for segment in segments {
let header = segment.header;
let body = resolve_operator_precedence_if_non_empty(segment.result.tokens());
let body = operator::resolve_operator_precedence_if_non_empty(segment.result.tokens());
let field = match header.code.as_ref() {
"polyglot" => &mut polyglot,
"from" => &mut from,
"as" if import.is_none() => &mut from_as,
"import" => &mut import,
"as" => &mut import_as,
"all" => {
all = Some(into_ident(header));
continue;
}
"as" => &mut as_,
"hiding" => &mut hiding,
_ => unreachable!(),
};
*field = Some(syntax::tree::MultiSegmentAppSegment { header, body });
}
let import = import.unwrap();
syntax::Tree::import(polyglot, from, from_as, import, import_as, hiding)
syntax::Tree::import(polyglot, from, import, all, as_, hiding)
}
fn register_export_macros(macros: &mut resolver::SegmentMap<'_>) {
use crate::macro_definition;
let defs = [
macro_definition! {("export", everything()) export_body},
macro_definition! {("export", everything(), "as", everything()) export_body},
macro_definition! {("export", everything(), "as", identifier()) export_body},
macro_definition! {("from", everything(), "export", everything()) export_body},
macro_definition! {
("from", everything(), "export", nothing(), "all", nothing()) export_body},
macro_definition! {
("from", everything(), "export", everything(), "hiding", everything()) export_body},
macro_definition! {
("from", everything(), "as", everything(), "export", everything()) export_body},
("from", everything(), "export", nothing(), "all", nothing(), "hiding", everything())
export_body},
macro_definition! {
("from", everything(), "as", identifier(), "export", everything()) export_body},
];
for def in defs {
macros.register(def);
@ -93,27 +103,29 @@ fn register_export_macros(macros: &mut resolver::SegmentMap<'_>) {
}
fn export_body(segments: NonEmptyVec<MatchedSegment>) -> syntax::Tree {
use operator::resolve_operator_precedence_if_non_empty;
let mut from = None;
let mut from_as = None;
let mut export = None;
let mut export_as = None;
let mut all = None;
let mut as_ = None;
let mut hiding = None;
for segment in segments {
let header = segment.header;
let body = resolve_operator_precedence_if_non_empty(segment.result.tokens());
let body = operator::resolve_operator_precedence_if_non_empty(segment.result.tokens());
let field = match header.code.as_ref() {
"from" => &mut from,
"as" if export.is_none() => &mut from_as,
"export" => &mut export,
"as" => &mut export_as,
"all" => {
all = Some(into_ident(header));
continue;
}
"as" => &mut as_,
"hiding" => &mut hiding,
_ => unreachable!(),
};
*field = Some(syntax::tree::MultiSegmentAppSegment { header, body });
}
let export = export.unwrap();
syntax::Tree::export(from, from_as, export, export_as, hiding)
syntax::Tree::export(from, export, all, as_, hiding)
}
/// If-then-else macro definition.
@ -132,68 +144,55 @@ pub fn group<'s>() -> Definition<'s> {
}
fn group_body(segments: NonEmptyVec<MatchedSegment>) -> syntax::Tree {
use operator::resolve_operator_precedence_if_non_empty;
let (close, mut segments) = segments.pop();
let close = into_symbol(close.header);
let close = into_close_symbol(close.header);
let segment = segments.pop().unwrap();
let open = into_symbol(segment.header);
let open = into_open_symbol(segment.header);
let body = segment.result.tokens();
let body = resolve_operator_precedence_if_non_empty(body);
syntax::Tree::group(open, body, close)
let body = operator::resolve_operator_precedence_if_non_empty(body);
syntax::Tree::group(Some(open), body, Some(close))
}
/// New type definition macro definition.
pub fn type_def<'s>() -> Definition<'s> {
use pattern::*;
#[rustfmt::skip]
let pattern =
identifier() / "name" % "type name" >>
many(identifier() % "type parameter" / "param") % "type parameters" >>
block(
everything() / "statements"
) % "type definition body";
crate::macro_definition! {
("type", pattern)
type_def_body
}
/// Type definitions.
fn type_def<'s>() -> Definition<'s> {
crate::macro_definition! {("type", everything()) type_def_body}
}
fn type_def_body(matched_segments: NonEmptyVec<MatchedSegment>) -> syntax::Tree {
// FIXME: This implementation of parsing constructors works for correct inputs, but doesn't
// handle incorrect syntax ideally. Issue: #182745069
use syntax::tree::*;
let segment = matched_segments.pop().0;
let match_tree = segment.result.into_var_map();
let mut v = match_tree.view();
let name = v.query("name").map(|name| name[0].clone()).unwrap_or_default();
let name = operator::resolve_operator_precedence_if_non_empty(name);
let no_params = [];
let params = v.nested().query("param").unwrap_or(&no_params);
let params = params
.iter()
.map(|tokens| {
operator::resolve_operator_precedence_if_non_empty(tokens.iter().cloned()).unwrap()
})
.collect_vec();
let mut constructors = default();
let mut body = default();
if let Some(items) = v.query("statements") {
let items = items[0].iter().cloned();
let mut builder = TypeDefBodyBuilder::default();
for syntax::tree::block::Line { newline, expression } in syntax::tree::block::lines(items) {
let header = into_ident(segment.header);
let mut tokens = segment.result.tokens().into_iter();
let mut i = 0;
for (i_, item) in tokens.as_slice().iter().enumerate() {
if matches!(item, syntax::Item::Block(_)) {
break;
}
i = i_ + 1;
}
let Some(first_line) = operator::Precedence::new().resolve_non_section(tokens.by_ref().take(i)) else {
let placeholder = Tree::ident(syntax::token::ident("", "", false, 0, false, false));
return placeholder.with_error("Expected identifier after `type` keyword.");
};
let (name, params) = crate::collect_arguments(first_line);
let name = match name {
Tree { variant: box Variant::Ident(Ident { mut token }), span } => {
token.left_offset += span.left_offset;
token
}
other => {
return other.with_error("Expected identifier after `type` keyword.");
}
};
let mut builder = TypeDefBodyBuilder::default();
if let Some(syntax::Item::Block(block)) = tokens.next() {
for block::Line { newline, expression } in block::lines(block) {
builder.line(newline, expression);
}
let (constructors_, body_) = builder.finish();
constructors = constructors_;
body = body_;
}
match name {
Some(name) => syntax::Tree::type_def(segment.header, name, params, constructors, body),
None => {
let name = syntax::Tree::ident(syntax::token::ident("", "", false, 0, false, false));
let result = syntax::Tree::type_def(segment.header, name, params, constructors, body);
result.with_error("Expected identifier after `type` keyword.")
}
}
debug_assert_eq!(tokens.next(), None);
let (constructors, body) = builder.finish();
Tree::type_def(header, name, params, constructors, body)
}
#[derive(Default)]
@ -239,45 +238,61 @@ impl<'s> TypeDefBodyBuilder<'s> {
(self.constructors, self.body)
}
/// Interpret the given expression as an `TypeConstructorDef`, if its syntax is compatible.
/// Interpret the given expression as a `TypeConstructorDef`, if its syntax is compatible.
fn to_constructor_line(
expression: syntax::Tree<'_>,
) -> Result<syntax::tree::TypeConstructorDef<'_>, syntax::Tree<'_>> {
use syntax::tree::*;
if let Tree {
variant:
box Variant::ArgumentBlockApplication(ArgumentBlockApplication {
lhs: Some(Tree { variant: box Variant::Ident(ident), span: span_ }),
arguments,
}),
span,
} = expression
{
let mut constructor = ident.token;
let mut left_offset = span.left_offset;
left_offset += &span_.left_offset;
left_offset += constructor.left_offset;
constructor.left_offset = left_offset;
let block = arguments;
let arguments = default();
return Ok(TypeConstructorDef { constructor, arguments, block });
}
let mut arguments = vec![];
let mut lhs = &expression;
let mut left_offset = crate::source::span::Offset::default();
while let Tree { variant: box Variant::App(App { func, arg }), span } = lhs {
left_offset += &span.left_offset;
lhs = func;
arguments.push(arg.clone());
}
if let Tree { variant: box Variant::Ident(Ident { token }), span } = lhs {
let mut constructor = token.clone();
left_offset += &span.left_offset;
left_offset += constructor.left_offset;
constructor.left_offset = left_offset;
arguments.reverse();
let mut left_offset = crate::source::Offset::default();
let mut last_argument_default = default();
let lhs = match &expression {
Tree {
variant:
box Variant::OprApp(OprApp { lhs: Some(lhs), opr: Ok(opr), rhs: Some(rhs) }),
span,
} if opr.properties.is_assignment() => {
left_offset += span.left_offset.clone();
last_argument_default = Some((opr.clone(), rhs.clone()));
lhs
}
Tree {
variant:
box Variant::ArgumentBlockApplication(ArgumentBlockApplication {
lhs: Some(Tree { variant: box Variant::Ident(ident), span: span_ }),
arguments,
}),
span,
} => {
let mut constructor = ident.token.clone();
let mut left_offset = span.left_offset.clone();
left_offset += &span_.left_offset;
left_offset += constructor.left_offset;
constructor.left_offset = left_offset;
let block = arguments
.iter()
.cloned()
.map(|block::Line { newline, expression }| ArgumentDefinitionLine {
newline,
argument: expression.map(crate::parse_argument_definition),
})
.collect();
let arguments = default();
return Ok(TypeConstructorDef { constructor, arguments, block });
}
_ => &expression,
};
let (constructor, mut arguments) = crate::collect_arguments(lhs.clone());
if let Tree { variant: box Variant::Ident(Ident { token }), span } = constructor && token.is_type {
let mut constructor = token;
left_offset += span.left_offset;
constructor.left_offset += left_offset;
if let Some((equals, expression)) = last_argument_default
&& let Some(ArgumentDefinition { open: None, default, close: None, .. })
= arguments.last_mut() && default.is_none() {
*default = Some(ArgumentDefault { equals, expression });
}
let block = default();
return Ok(TypeConstructorDef { constructor, arguments, block });
return Ok(TypeConstructorDef{ constructor, arguments, block });
}
Err(expression)
}
@ -292,14 +307,13 @@ pub fn lambda<'s>() -> Definition<'s> {
}
fn lambda_body(segments: NonEmptyVec<MatchedSegment>) -> syntax::Tree {
use operator::resolve_operator_precedence_if_non_empty;
let (segment, _) = segments.pop();
let operator = segment.header;
let syntax::token::Token { left_offset, code, .. } = operator;
let properties = syntax::token::OperatorProperties::default();
let operator = syntax::token::operator(left_offset, code, properties);
let arrow = segment.result.tokens();
let arrow = resolve_operator_precedence_if_non_empty(arrow);
let arrow = operator::resolve_operator_precedence_if_non_empty(arrow);
syntax::Tree::lambda(operator, arrow)
}
@ -310,12 +324,7 @@ pub fn case<'s>() -> Definition<'s> {
fn case_body(segments: NonEmptyVec<MatchedSegment>) -> syntax::Tree {
use operator::resolve_operator_precedence_if_non_empty;
use syntax::token;
use syntax::tree::*;
let into_ident = |token| {
let token::Token { left_offset, code, .. } = token;
token::ident(left_offset, code, false, 0, false, false)
};
let (of, mut rest) = segments.pop();
let case = rest.pop().unwrap();
let case_ = into_ident(case.header);
@ -324,26 +333,29 @@ fn case_body(segments: NonEmptyVec<MatchedSegment>) -> syntax::Tree {
let of_ = into_ident(of.header);
let body = of.result.tokens();
let body = resolve_operator_precedence_if_non_empty(body);
let mut initial = None;
let mut lines = vec![];
let mut case_lines = vec![];
if let Some(body) = body {
match body.variant {
box Variant::ArgumentBlockApplication(ArgumentBlockApplication { lhs, arguments }) => {
initial = lhs;
lines = arguments;
let mut left_offset = body.span.left_offset;
if let Some(initial) = initial.as_mut() {
left_offset += mem::take(&mut initial.span.left_offset);
initial.span.left_offset = left_offset;
} else if let Some(first) = lines.first_mut() {
left_offset += mem::take(&mut first.newline.left_offset);
first.newline.left_offset = left_offset;
if let Some(lhs) = lhs {
case_lines.push(CaseLine { case: Some(lhs.into()), ..default() });
}
case_lines.extend(arguments.into_iter().map(
|block::Line { newline, expression }| CaseLine {
newline: newline.into(),
case: expression.map(Case::from),
},
));
if let Some(left_offset) =
case_lines.first_mut().and_then(CaseLine::left_offset_mut)
{
*left_offset += body.span.left_offset;
}
}
_ => initial = Some(body),
_ => case_lines.push(CaseLine { case: Some(body.into()), ..default() }),
}
}
Tree::case(case_, expression, of_, initial, lines)
Tree::case_of(case_, expression, of_, case_lines)
}
/// Array literal.
@ -367,32 +379,37 @@ fn tuple_body(segments: NonEmptyVec<MatchedSegment>) -> syntax::Tree {
}
struct GroupedSequence<'s> {
left: syntax::token::Symbol<'s>,
left: syntax::token::OpenSymbol<'s>,
first: Option<syntax::Tree<'s>>,
rest: Vec<syntax::tree::OperatorDelimitedTree<'s>>,
right: syntax::token::Symbol<'s>,
right: syntax::token::CloseSymbol<'s>,
}
fn grouped_sequence(segments: NonEmptyVec<MatchedSegment>) -> GroupedSequence {
use operator::resolve_operator_precedence_if_non_empty;
use syntax::tree::*;
let (right, mut rest) = segments.pop();
let right_ = into_symbol(right.header);
let right_ = into_close_symbol(right.header);
let left = rest.pop().unwrap();
let left_ = into_symbol(left.header);
let left_ = into_open_symbol(left.header);
let expression = left.result.tokens();
let expression = resolve_operator_precedence_if_non_empty(expression);
let expression = operator::resolve_operator_precedence_if_non_empty(expression);
let mut rest = vec![];
let mut lhs_ = &expression;
let mut left_offset = crate::source::span::Offset::default();
while let Some(Tree {
variant: box Variant::OprApp(OprApp { lhs, opr: Ok(opr), rhs: Some(rhs) }), ..
variant: box Variant::OprApp(OprApp { lhs, opr: Ok(opr), rhs: Some(rhs) }), span
}) = lhs_ && opr.properties.is_sequence() {
lhs_ = lhs;
let operator = opr.clone();
let body = rhs.clone();
rest.push(OperatorDelimitedTree { operator, body });
left_offset += span.left_offset.clone();
}
rest.reverse();
let mut first = lhs_.clone();
if let Some(first) = &mut first {
first.span.left_offset += left_offset;
}
let first = lhs_.clone();
GroupedSequence { left: left_, first, rest, right: right_ }
}
@ -401,18 +418,27 @@ fn splice<'s>() -> Definition<'s> {
}
fn splice_body(segments: NonEmptyVec<MatchedSegment>) -> syntax::Tree {
use operator::resolve_operator_precedence_if_non_empty;
let (close, mut segments) = segments.pop();
let close = into_symbol(close.header);
let close = into_close_symbol(close.header);
let segment = segments.pop().unwrap();
let open = into_symbol(segment.header);
let open = into_open_symbol(segment.header);
let expression = segment.result.tokens();
let expression = resolve_operator_precedence_if_non_empty(expression);
let expression = operator::resolve_operator_precedence_if_non_empty(expression);
let splice = syntax::tree::TextElement::Splice { open, expression, close };
syntax::Tree::text_literal(default(), vec![splice], default(), default())
}
fn into_symbol(token: syntax::token::Token) -> syntax::token::Symbol {
fn into_open_symbol(token: syntax::token::Token) -> syntax::token::OpenSymbol {
let syntax::token::Token { left_offset, code, .. } = token;
syntax::token::symbol(left_offset, code)
syntax::token::open_symbol(left_offset, code)
}
fn into_close_symbol(token: syntax::token::Token) -> syntax::token::CloseSymbol {
let syntax::token::Token { left_offset, code, .. } = token;
syntax::token::close_symbol(left_offset, code)
}
fn into_ident(token: syntax::token::Token) -> syntax::token::Ident {
let syntax::token::Token { left_offset, code, .. } = token;
syntax::token::ident(left_offset, code, false, 0, false, false)
}

26
lib/rust/parser/src/macros/resolver.rs
View File

@ -421,6 +421,9 @@ impl<'s> Resolver<'s> {
fn process_token(&mut self, root_macro_map: &SegmentMap<'s>, token: Token<'s>) -> Step<'s> {
let repr = &**token.code;
if !token.variant.can_start_macro_segment() {
return Step::NormalToken(token.into());
}
if let Some(subsegments) = self.current_macro.possible_next_segments.get(repr) {
trace!("Entering next segment of the current macro.");
let mut new_match_tree =
@ -434,7 +437,7 @@ impl<'s> Resolver<'s> {
trace!("Next token reserved by parent macro. Resolving current macro.");
self.replace_current_with_parent_macro(parent_macro);
Step::MacroStackPop(token.into())
} else if let Some(segments) = root_macro_map.get(repr) && token.variant.can_start_macro() {
} else if let Some(segments) = root_macro_map.get(repr) {
trace!("Starting a new nested macro resolution.");
let mut matched_macro_def = default();
let mut current_macro = PartiallyMatchedMacro {
@ -507,9 +510,24 @@ impl<'s> Resolver<'s> {
let out = (macro_def.body)(pattern_matched_segments);
(out, not_used_items_of_last_segment)
} else {
let message = format!("Macro was not matched with any known macro definition.\nResolved segments: {resolved_segments:?}");
let error = syntax::tree::Error::new(message);
(syntax::tree::Tree::invalid(error, default()), default())
let mut segments = resolved_segments.into_iter();
let (header0, mut segment) = segments.next().unwrap();
let mut items = VecDeque::new();
items.append(&mut segment);
for (header, mut segment) in segments {
items.push_back(syntax::Item::Token(header));
items.append(&mut segment);
}
let mut body = String::new();
for item in &items {
match item {
syntax::Item::Token(token) => body.push_str(&token.code.repr),
syntax::Item::Block(_block) => body.push_str("<block>"),
syntax::Item::Tree(tree) => body.push_str(&tree.code()),
}
}
let header0 = syntax::Tree::from(header0).with_error("Invalid macro invocation.");
(header0, items)
}
}

47
lib/rust/parser/src/main.rs
View File

@ -4,6 +4,7 @@
// === Features ===
#![allow(incomplete_features)]
#![feature(allocator_api)]
#![feature(exact_size_is_empty)]
#![feature(test)]
#![feature(specialization)]
#![feature(let_chains)]
@ -35,7 +36,47 @@ use enso_parser::prelude::*;
fn main() {
init_wasm();
let ast = enso_parser::Parser::new().run("foo = 23");
println!("\n\n==================\n\n");
println!("{:#?}", ast);
let args = std::env::args().skip(1);
if args.is_empty() {
use std::io::Read;
let mut input = String::new();
std::io::stdin().read_to_string(&mut input).unwrap();
check_file("<stdin>", input.as_str());
} else {
args.for_each(|path| check_file(&path, &std::fs::read_to_string(&path).unwrap()));
}
}
fn check_file(path: &str, mut code: &str) {
if let Some((_meta, code_)) = enso_parser::metadata::parse(code) {
code = code_;
}
let ast = enso_parser::Parser::new().run(code);
let errors = RefCell::new(vec![]);
ast.map(|tree| {
if let enso_parser::syntax::tree::Variant::Invalid(err) = &*tree.variant {
errors.borrow_mut().push((err.clone(), tree.span.clone()));
}
});
for (error, span) in &*errors.borrow() {
let whitespace = &span.left_offset.code.repr;
let start = whitespace.as_ptr() as usize + whitespace.len() - code.as_ptr() as usize;
let mut line = 1;
let mut char = 0;
for (i, c) in code.char_indices() {
if i >= start {
break;
}
if c == '\n' {
line += 1;
char = 0;
} else {
char += 1;
}
}
eprintln!("{path}:{line}:{char}: {}", &error.error.message);
}
for (parsed, original) in ast.code().lines().zip(code.lines()) {
assert_eq!(parsed, original, "Bug: dropped tokens, while parsing: {}", path);
}
}

60
lib/rust/parser/src/serialization.rs
View File

@ -64,6 +64,29 @@ where D: serde::Deserializer<'de> {
// ==============
// === Tokens ===
// ==============
pub(crate) fn serialize_optional_char<S>(c: &Option<char>, s: S) -> Result<S::Ok, S::Error>
where S: serde::Serializer {
let value = c.map(|c| c as u32).unwrap_or(0xFFFF_FFFF);
s.serialize_u32(value)
}
pub(crate) fn deserialize_optional_char<'c, 'de, D>(
deserializer: D,
) -> Result<Option<char>, D::Error>
where D: serde::Deserializer<'de> {
let value = deserializer.deserialize_u32(DeserializeU32)?;
Ok(match value {
0xFFFF_FFFF => None,
x => Some(char::try_from(x).unwrap()),
})
}
// =============
// === Error ===
// =============
@ -99,3 +122,40 @@ impl<'de> serde::de::Visitor<'de> for DeserializeU64 {
Ok(i)
}
}
struct DeserializeU32;
impl<'de> serde::de::Visitor<'de> for DeserializeU32 {
type Value = u32;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
write!(formatter, "An unsigned 32-bit integer.")
}
fn visit_u32<E>(self, i: u32) -> Result<Self::Value, E>
where E: serde::de::Error {
Ok(i)
}
}
// ========================================
// === General purpose value transforms ===
// ========================================
pub(crate) fn serialize_optional_int<S>(x: &Option<u32>, s: S) -> Result<S::Ok, S::Error>
where S: serde::Serializer {
s.serialize_u32(x.unwrap_or(0xFFFF_FFFF))
}
pub(crate) fn deserialize_optional_int<'c, 'de, D>(
deserializer: D,
) -> Result<Option<u32>, D::Error>
where D: serde::Deserializer<'de> {
let value = deserializer.deserialize_u32(DeserializeU32)?;
Ok(match value {
0xFFFF_FFFF => None,
x => Some(x),
})
}

91
lib/rust/parser/src/syntax/item.rs
View File

@ -44,92 +44,7 @@ impl<'s> Item<'s> {
/// Convert this item to a [`Tree`].
pub fn to_ast(self) -> Tree<'s> {
match self {
Item::Token(token) => match token.variant {
token::Variant::Ident(ident) => Tree::ident(token.with_variant(ident)),
token::Variant::Digits(number) =>
Tree::number(None, Some(token.with_variant(number)), None),
token::Variant::NumberBase(base) =>
Tree::number(Some(token.with_variant(base)), None, None),
token::Variant::TextStart(open) => Tree::text_literal(
Some(token.with_variant(open)),
default(),
default(),
default(),
),
token::Variant::TextSection(section) => {
let trim = token.left_offset.visible;
let section = tree::TextElement::Section { text: token.with_variant(section) };
Tree::text_literal(default(), vec![section], default(), trim)
}
token::Variant::TextEscapeSymbol(escape) => {
let trim = token.left_offset.visible;
let backslash = Some(token.with_variant(escape));
let section = tree::TextElement::EscapeChar { backslash, char: None };
Tree::text_literal(default(), vec![section], default(), trim)
}
token::Variant::TextEscapeChar(escape) => {
let trim = token.left_offset.visible;
let char = Some(token.with_variant(escape));
let section = tree::TextElement::EscapeChar { backslash: None, char };
Tree::text_literal(default(), vec![section], default(), trim)
}
token::Variant::TextEscapeLeader(leader) => {
let trim = token.left_offset.visible;
let leader = Some(token.with_variant(leader));
let section = tree::TextElement::EscapeSequence {
leader,
open: None,
digits: None,
close: None,
};
Tree::text_literal(default(), vec![section], default(), trim)
}
token::Variant::TextEscapeHexDigits(digits) => {
let digits = Some(token.with_variant(digits));
let section = tree::TextElement::EscapeSequence {
leader: None,
open: None,
digits,
close: None,
};
Tree::text_literal(default(), vec![section], default(), default())
}
token::Variant::TextEscapeSequenceStart(t) => {
let open = Some(token.with_variant(t));
let section = tree::TextElement::EscapeSequence {
leader: None,
open,
digits: None,
close: None,
};
Tree::text_literal(default(), vec![section], default(), default())
}
token::Variant::TextEscapeSequenceEnd(t) => {
let close = Some(token.with_variant(t));
let section = tree::TextElement::EscapeSequence {
leader: None,
open: None,
digits: None,
close,
};
Tree::text_literal(default(), vec![section], default(), default())
}
token::Variant::TextEnd(close) => Tree::text_literal(
default(),
default(),
Some(token.with_variant(close)),
default(),
),
token::Variant::Wildcard(wildcard) => Tree::wildcard(token.with_variant(wildcard)),
token::Variant::AutoScope(t) => Tree::auto_scope(token.with_variant(t)),
_ => {
let message = format!("to_ast: Item::Token({token:?})");
let value = Tree::ident(
token.with_variant(token::variant::Ident(false, 0, false, false)),
);
Tree::with_unsupported(value, message)
}
},
Item::Token(token) => token.into(),
Item::Tree(ast) => ast,
Item::Block(items) => build_block(items),
}
@ -202,7 +117,9 @@ macro_rules! generate_variant_checks {
pub enum $enum:ident {
$(
$(#$variant_meta:tt)*
$variant:ident $({ $(pub $field:ident : $field_ty:ty),* $(,)? })?
$variant:ident $({
$($(#$field_meta:tt)* pub $field:ident : $field_ty:ty),* $(,)?
})?
),* $(,)?
}
) => { paste!{

372
lib/rust/parser/src/syntax/operator.rs
View File

@ -12,6 +12,70 @@ use crate::syntax::token::Token;
// === Precedence ===
// ==================
/// Operator precedence resolver.
#[derive(Debug)]
pub struct Precedence<'s> {
nospace_builder: ExpressionBuilder<'s>,
builder: ExpressionBuilder<'s>,
}
impl<'s> Default for Precedence<'s> {
fn default() -> Self {
Self::new()
}
}
impl<'s> Precedence<'s> {
/// Return a new operator precedence resolver.
pub fn new() -> Self {
Self {
nospace_builder: ExpressionBuilder { nospace: true, ..default() },
builder: ExpressionBuilder { nospace: false, ..default() },
}
}
/// Resolve precedence in a context where the result cannot be an operator section or template
/// function.
pub fn resolve_non_section(
&mut self,
items: impl IntoIterator<Item = syntax::Item<'s>>,
) -> Option<syntax::Tree<'s>> {
self.resolve_(items).map(|op| op.value)
}
/// Resolve precedence.
pub fn resolve(
&mut self,
items: impl IntoIterator<Item = syntax::Item<'s>>,
) -> Option<syntax::Tree<'s>> {
self.resolve_(items).map(syntax::Tree::from)
}
fn resolve_(
&mut self,
items: impl IntoIterator<Item = syntax::Item<'s>>,
) -> Option<Operand<syntax::Tree<'s>>> {
for item in items {
if starts_new_no_space_group(&item) {
self.builder.extend_from(&mut self.nospace_builder);
}
match item {
syntax::Item::Token(Token {
variant: token::Variant::Operator(opr),
left_offset,
code,
}) => self.nospace_builder.operator(Token(left_offset, code, opr)),
syntax::Item::Token(token) =>
self.nospace_builder.operand(syntax::Tree::from(token).into()),
syntax::Item::Tree(tree) => self.nospace_builder.operand(tree.into()),
syntax::Item::Block(_) => self.nospace_builder.operand(item.to_ast().into()),
}
}
self.builder.extend_from(&mut self.nospace_builder);
self.builder.finish()
}
}
/// Annotate expressions that should use spacing, because otherwise they are misleading. For
/// example, `if cond then.x else.y` is parsed as `if cond then .x else .y`, which after expansion
/// translates to `if cond then (\t -> t.x) else (\t -> t.y)`. However, for some macros spacing is
@ -19,7 +83,8 @@ use crate::syntax::token::Token;
fn annotate_tokens_that_need_spacing(item: syntax::Item) -> syntax::Item {
use syntax::tree::Variant::*;
item.map_tree(|ast| match &*ast.variant {
MultiSegmentApp(data) if !data.segments.first().header.is_symbol() =>
MultiSegmentApp(data)
if !matches!(data.segments.first().header.variant, token::Variant::OpenSymbol(_)) =>
ast.with_error("This expression cannot be used in a non-spaced equation."),
_ => ast,
})
@ -39,58 +104,24 @@ pub fn resolve_operator_precedence(items: NonEmptyVec<syntax::Item<'_>>) -> synt
pub fn resolve_operator_precedence_if_non_empty<'s>(
items: impl IntoIterator<Item = syntax::Item<'s>>,
) -> Option<syntax::Tree<'s>> {
type Tokens<'s> = Vec<syntax::Item<'s>>;
let mut flattened: Tokens<'s> = default();
let mut no_space_group: Tokens<'s> = default();
let process_no_space_group = |flattened: &mut Tokens<'s>, no_space_group: &mut Tokens<'s>| {
let tokens = no_space_group.drain(..);
if tokens.len() < 2 {
flattened.extend(tokens);
} else {
let tokens = tokens.map(annotate_tokens_that_need_spacing);
let ast = resolve_operator_precedence_internal(tokens, true).unwrap();
flattened.push(ast.into());
}
};
// Returns `true` for an item if that item should not follow any other item in a no-space group
// (i.e. the item has "space" before it).
let starts_new_no_space_group = |item: &syntax::item::Item| {
if item.left_visible_offset().width_in_spaces != 0 {
return true;
}
if let syntax::item::Item::Block(_) = item {
return true;
}
false
};
for item in items {
if starts_new_no_space_group(&item) {
process_no_space_group(&mut flattened, &mut no_space_group);
}
no_space_group.push(item);
}
process_no_space_group(&mut flattened, &mut no_space_group);
resolve_operator_precedence_internal(flattened, false)
let mut precedence = Precedence::new();
precedence.resolve(items)
}
fn resolve_operator_precedence_internal<'s>(
items: impl IntoIterator<Item = syntax::Item<'s>>,
nospace: bool,
) -> Option<syntax::Tree<'s>> {
let mut expression = ExpressionBuilder { nospace, ..default() };
for item in items {
if let syntax::Item::Token(Token {
variant: token::Variant::Operator(opr),
left_offset,
code,
}) = item
{
expression.operator(Token(left_offset, code, opr));
} else {
expression.operand(item);
}
// Returns `true` for an item if that item should not follow any other item in a no-space group
// (i.e. the item has "space" before it).
fn starts_new_no_space_group(item: &syntax::item::Item) -> bool {
if item.left_visible_offset().width_in_spaces != 0 {
return true;
}
expression.finish()
if let syntax::item::Item::Block(_) = item {
return true;
}
if let syntax::item::Item::Token(Token { variant: token::Variant::Operator(opr), .. }) = item
&& opr.properties.is_sequence() {
return true;
}
false
}
@ -104,31 +135,22 @@ fn resolve_operator_precedence_internal<'s>(
///
/// [^1](https://en.wikipedia.org/wiki/Operator-precedence_parser)
/// [^2](https://en.wikipedia.org/wiki/Shunting_yard_algorithm)
#[derive(Default)]
#[derive(Default, Debug)]
struct ExpressionBuilder<'s> {
was_section_used: bool,
output: Vec<syntax::Item<'s>>,
operator_stack: Vec<Operator<'s>>,
prev_type: Option<ItemType>,
precedence_error: Option<String>,
nospace: bool,
output: Vec<Operand<syntax::Tree<'s>>>,
operator_stack: Vec<Operator<'s>>,
prev_type: Option<ItemType>,
nospace: bool,
}
impl<'s> ExpressionBuilder<'s> {
/// Extend the expression with an operand.
pub fn operand(&mut self, item: syntax::Item<'s>) {
let item = if self.prev_type == Some(ItemType::Ast) {
// Multiple non-operators next to each other.
let lhs = self.output.pop().unwrap();
let lhs = lhs.to_ast();
let rhs = item.to_ast();
syntax::tree::apply(lhs, rhs).into()
} else {
// Non-operator that follows previously consumed operator.
item
};
self.prev_type = Some(ItemType::Ast);
self.output.push(item);
pub fn operand(&mut self, mut operand: Operand<syntax::Tree<'s>>) {
if self.prev_type.replace(ItemType::Ast) == Some(ItemType::Ast) {
operand =
self.output.pop().unwrap().map(|lhs| syntax::tree::apply(lhs, operand.into()));
}
self.output.push(operand);
}
/// Extend the expression with an operator.
@ -151,15 +173,7 @@ impl<'s> ExpressionBuilder<'s> {
// Binary operator section (no LHS).
(_, Some(prec), _) => self.binary_operator(prec, assoc, opr),
// Failed to compute a role for the operator; this should not be possible.
(_, None, None) => {
// We don't know the correct precedence, so we can't structure the tree correctly.
// Pick some arbitrary value so we can at least produce *some* tree containing all
// the right subexpressions; we'll wrap the expression in an `Invalid` node.
const ARBITRARY_PRECEDENCE: token::Precedence = token::Precedence { value: 20 };
let error = || format!("Precedence of: {:?}", opr.code);
self.precedence_error.get_or_insert_with(error);
self.binary_operator(ARBITRARY_PRECEDENCE, assoc, opr);
}
(_, None, None) => unreachable!(),
}
}
@ -173,11 +187,21 @@ impl<'s> ExpressionBuilder<'s> {
) {
if self.prev_type == Some(ItemType::Opr)
&& let Some(prev_opr) = self.operator_stack.last_mut()
&& let Arity::Binary(oprs) = &mut prev_opr.opr {
oprs.push(opr);
return;
&& let Arity::Binary { tokens, .. } = &mut prev_opr.opr {
if tokens.len() == 1 && opr.properties.is_type_annotation() {
let prev = match self.operator_stack.pop().unwrap().opr {
Arity::Binary { tokens, .. } => tokens.into_iter().next().unwrap(),
_ => unreachable!(),
};
let tp = token::Variant::ident(false, 0, false, false);
let prev = Token(prev.left_offset, prev.code, tp);
self.output.push(Operand::from(syntax::Tree::from(prev)));
} else {
tokens.push(opr);
return;
}
}
self.push_operator(prec, assoc, Arity::Binary(vec![opr]));
self.push_operator(prec, assoc, Arity::binary(opr));
}
/// Add an operator to the stack; [`reduce`] the stack first, as appropriate for the specified
@ -193,10 +217,10 @@ impl<'s> ExpressionBuilder<'s> {
// If the previous item was also an operator, this must be a unary operator following a
// binary operator; we cannot reduce the stack because the unary operator must be
// evaluated before the binary operator, regardless of precedence.
let mut rhs = self.output.pop().map(|rhs| rhs.to_ast());
let mut rhs = self.output.pop();
self.reduce(precedence, &mut rhs);
if let Some(rhs) = rhs {
self.output.push(rhs.into());
self.output.push(rhs);
}
}
self.operator_stack.push(opr);
@ -206,20 +230,37 @@ impl<'s> ExpressionBuilder<'s> {
/// Given a starting value, replace it with the result of successively applying to it all
/// operators in the `operator_stack` that have precedence greater than or equal to the
/// specified value, consuming LHS values from the `output` stack as needed.
fn reduce(&mut self, prec: token::Precedence, rhs: &mut Option<syntax::Tree<'s>>) {
fn reduce(&mut self, prec: token::Precedence, rhs: &mut Option<Operand<syntax::Tree<'s>>>) {
while let Some(opr) = self.operator_stack.pop_if(|opr| {
opr.precedence > prec
|| (opr.precedence == prec && opr.associativity == token::Associativity::Left)
}) {
let rhs_ = rhs.take();
let ast = match opr.opr {
Arity::Unary(opr) => syntax::Tree::unary_opr_app(opr, rhs_),
Arity::Binary(opr) => {
let lhs = self.output.pop().map(|t| t.to_ast());
let can_form_section = opr.len() != 1 || opr[0].properties.can_form_section();
self.was_section_used = self.was_section_used
|| (can_form_section && (lhs.is_none() || rhs_.is_none()));
syntax::tree::apply_operator(lhs, opr, rhs_)
Arity::Unary(opr) =>
Operand::from(rhs_).map(|item| syntax::Tree::unary_opr_app(opr, item)),
Arity::Binary { tokens, lhs_section_termination } => {
let lhs = self.output.pop();
if let Some(lhs_termination) = lhs_section_termination {
let lhs = match lhs_termination {
SectionTermination::Reify => lhs.map(syntax::Tree::from),
SectionTermination::Unwrap => lhs.map(|op| op.value),
};
let rhs = rhs_.map(syntax::Tree::from);
let ast = syntax::tree::apply_operator(lhs, tokens, rhs, self.nospace);
Operand::from(ast)
} else {
let rhs = rhs_.map(syntax::Tree::from);
let mut elided = 0;
if tokens.len() != 1 || tokens[0].properties.can_form_section() {
elided += lhs.is_none() as u32 + rhs.is_none() as u32;
}
let mut operand = Operand::from(lhs).map(|lhs| {
syntax::tree::apply_operator(lhs, tokens, rhs, self.nospace)
});
operand.elided += elided;
operand
}
}
};
*rhs = Some(ast);
@ -227,35 +268,38 @@ impl<'s> ExpressionBuilder<'s> {
}
/// Return an expression constructed from the accumulated state. Will return `None` only if no
/// inputs were provided.
pub fn finish(mut self) -> Option<syntax::Tree<'s>> {
/// inputs were provided. `self` will be reset to its initial state.
pub fn finish(&mut self) -> Option<Operand<syntax::Tree<'s>>> {
use ItemType::*;
let mut item =
(self.prev_type == Some(Ast)).and_option_from(|| self.output.pop().map(|t| t.to_ast()));
self.reduce(token::Precedence::min(), &mut item);
if !self.output.is_empty() {
panic!(
"Internal error. Not all tokens were consumed while constructing the expression."
);
}
let out = if self.was_section_used {
// This can't fail: `was_section_used` won't be true unless we had at least one input,
// and if we have at least one input, we have output.
let out = item.unwrap();
Some(syntax::Tree::opr_section_boundary(out))
} else {
item
};
if let Some(error) = self.precedence_error {
return Some(syntax::Tree::with_unsupported(out.unwrap(), error));
}
let mut out = (self.prev_type == Some(Ast)).and_option_from(|| self.output.pop());
self.reduce(token::Precedence::min(), &mut out);
debug_assert!(self.operator_stack.is_empty());
debug_assert!(
self.output.is_empty(),
"Internal error. Not all tokens were consumed while constructing the expression."
);
self.prev_type = None;
out
}
pub fn extend_from(&mut self, child: &mut Self) {
if child.output.is_empty() && let Some(op) = child.operator_stack.pop() {
match op.opr {
Arity::Unary(un) => self.operator(un),
Arity::Binary { tokens, .. } => tokens.into_iter().for_each(|op| self.operator(op)),
};
child.prev_type = None;
return;
}
if let Some(o) = child.finish() {
self.operand(o);
}
}
}
/// Classify an item as an operator, or operand; this is used in [`resolve_operator_precedence`] to
/// merge consecutive nodes of the same type.
#[derive(PartialEq, Eq)]
#[derive(PartialEq, Eq, Debug)]
enum ItemType {
Ast,
Opr,
@ -273,5 +317,101 @@ struct Operator<'s> {
#[derive(Debug)]
enum Arity<'s> {
Unary(token::Operator<'s>),
Binary(Vec<token::Operator<'s>>),
Binary {
tokens: Vec<token::Operator<'s>>,
lhs_section_termination: Option<SectionTermination>,
},
}
impl<'s> Arity<'s> {
fn binary(tok: token::Operator<'s>) -> Self {
let lhs_section_termination = tok.properties.lhs_section_termination();
let tokens = vec![tok];
Self::Binary { tokens, lhs_section_termination }
}
}
// === Operand ===
/// Wraps a value, tracking the number of wildcards or elided operands within it.
#[derive(Default, Debug)]
struct Operand<T> {
value: T,
elided: u32,
wildcards: u32,
}
/// Transpose.
impl<T> From<Option<Operand<T>>> for Operand<Option<T>> {
fn from(operand: Option<Operand<T>>) -> Self {
match operand {
Some(Operand { value, elided, wildcards }) =>
Self { value: Some(value), elided, wildcards },
None => default(),
}
}
}
/// Unit. Creates an Operand from a node.
impl<'s> From<syntax::Tree<'s>> for Operand<syntax::Tree<'s>> {
fn from(mut value: syntax::Tree<'s>) -> Self {
let elided = 0;
let wildcards = if let syntax::Tree {
variant:
box syntax::tree::Variant::Wildcard(syntax::tree::Wildcard { de_bruijn_index, .. }),
..
} = &mut value
{
debug_assert_eq!(*de_bruijn_index, None);
*de_bruijn_index = Some(0);
1
} else {
0
};
Self { value, wildcards, elided }
}
}
/// Counit. Bakes any information about elided operands into the tree.
impl<'s> From<Operand<syntax::Tree<'s>>> for syntax::Tree<'s> {
fn from(operand: Operand<syntax::Tree<'s>>) -> Self {
let Operand { mut value, elided, wildcards } = operand;
if elided != 0 {
value = syntax::Tree::opr_section_boundary(elided, value);
}
if wildcards != 0 {
value = syntax::Tree::template_function(wildcards, value);
}
value
}
}
impl<T> Operand<T> {
/// Operate on the contained value without altering the elided-operand information.
fn map<U>(self, f: impl FnOnce(T) -> U) -> Operand<U> {
let Self { value, elided, wildcards } = self;
let value = f(value);
Operand { value, elided, wildcards }
}
}
// === SectionTermination ===
/// Operator-section/template-function termination behavior of an operator with regard to an
/// operand.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum SectionTermination {
/// If the operand is an operator-section/template-function, indicate it by wrapping it in a
/// suitable node.
Reify,
/// Discard any operator-section/template-function properties associated with the operand.
Unwrap,
}
impl Default for SectionTermination {
fn default() -> Self {
Self::Reify
}
}

109
lib/rust/parser/src/syntax/token.rs
View File

@ -49,7 +49,8 @@
//! ```text
//! pub enum Variant {
//! Newline (variant::Newline),
//! Symbol (variant::Symbol),
//! OpenSymbol (variant::OpenSymbol),
//! CloseSymbol (variant::CloseSymbol),
//! Wildcard (variant::Wildcard),
//! Ident (variant::Ident),
//! // ... many more
@ -81,7 +82,8 @@
//! ```text
//! pub enum VariantMarker {
//! Newline,
//! Symbol,
//! OpenSymbol,
//! CloseSymbol,
//! Wildcard,
//! Ident,
//! // ... many more
@ -251,7 +253,8 @@ macro_rules! with_token_definition { ($f:ident ($($args:tt)*)) => { $f! { $($arg
#[derive(Default)]
pub enum Variant {
Newline,
Symbol,
OpenSymbol,
CloseSymbol,
BlockStart,
BlockEnd,
Wildcard {
@ -261,14 +264,18 @@ macro_rules! with_token_definition { ($f:ident ($($args:tt)*)) => { $f! { $($arg
Ident {
pub is_free: bool,
pub lift_level: usize,
#[serde(skip)]
#[reflect(skip)]
pub is_type: bool,
#[serde(skip)]
#[reflect(skip)]
pub is_default: bool,
},
Operator {
#[serde(skip)]
#[reflect(skip)]
pub properties: OperatorProperties,
},
Modifier,
DocComment,
Digits {
pub base: Option<Base>
},
@ -276,21 +283,30 @@ macro_rules! with_token_definition { ($f:ident ($($args:tt)*)) => { $f! { $($arg
TextStart,
TextEnd,
TextSection,
TextEscapeSymbol,
TextEscapeChar,
TextEscapeLeader,
TextEscapeHexDigits,
TextEscapeSequenceStart,
TextEscapeSequenceEnd,
TextEscape {
#[serde(serialize_with = "crate::serialization::serialize_optional_char")]
#[serde(deserialize_with = "crate::serialization::deserialize_optional_char")]
#[reflect(as = "char")]
pub value: Option<char>,
},
Invalid,
}
}}}
impl Variant {
/// Return whether this token can introduce a macro invocation.
pub fn can_start_macro(&self) -> bool {
// Prevent macro interpretation of symbols that have been lexically contextualized as text
// escape control characters.
!matches!(self, Variant::TextEscapeSymbol(_) | Variant::TextEscapeSequenceStart(_))
/// Return whether this token can introduce a macro segment.
pub fn can_start_macro_segment(&self) -> bool {
!matches!(
self,
// Prevent macro interpretation of symbols that have been lexically contextualized as
// text escape control characters.
Variant::TextEscape(_)
| Variant::TextSection(_)
| Variant::TextStart(_)
| Variant::TextEnd(_)
// Prevent macro interpretation of lexically-inappropriate tokens.
| Variant::Invalid(_)
)
}
}
@ -304,50 +320,23 @@ impl Default for Variant {
// === Operator properties ===
/// Properties of an operator that are identified when lexing.
#[derive(
Debug,
Clone,
Copy,
PartialEq,
Eq,
Serialize,
Reflect,
Deserialize,
PartialOrd,
Ord,
Default
)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Default)]
pub struct OperatorProperties {
// Precedence
#[serde(skip)]
#[reflect(skip)]
binary_infix_precedence: Option<Precedence>,
#[serde(skip)]
#[reflect(skip)]
unary_prefix_precedence: Option<Precedence>,
// Operator section behavior
lhs_section_termination: Option<crate::syntax::operator::SectionTermination>,
// Special properties
#[serde(skip)]
#[reflect(skip)]
is_compile_time_operation: bool,
#[serde(skip)]
#[reflect(skip)]
is_right_associative: bool,
#[serde(skip)]
#[reflect(skip)]
can_be_decimal_operator: bool,
// Unique operators
#[serde(skip)]
#[reflect(skip)]
is_type_annotation: bool,
#[serde(skip)]
#[reflect(skip)]
is_assignment: bool,
#[serde(skip)]
#[reflect(skip)]
is_arrow: bool,
#[serde(skip)]
#[reflect(skip)]
is_sequence: bool,
is_suspension: bool,
}
impl OperatorProperties {
@ -376,6 +365,13 @@ impl OperatorProperties {
Self { is_right_associative: true, ..self }
}
/// Return a copy of this operator, modified to have the specified LHS operator-section/
/// template-function behavior.
pub fn with_lhs_section_termination<T>(self, lhs_section_termination: T) -> Self
where T: Into<Option<crate::syntax::operator::SectionTermination>> {
Self { lhs_section_termination: lhs_section_termination.into(), ..self }
}
/// Return a copy of this operator, modified to be flagged as a type annotation operator.
pub fn as_type_annotation(self) -> Self {
Self { is_type_annotation: true, ..self }
@ -396,6 +392,11 @@ impl OperatorProperties {
Self { is_sequence: true, ..self }
}
/// Return a copy of this operator, modified to be flagged as the execution-suspension operator.
pub fn as_suspension(self) -> Self {
Self { is_suspension: true, ..self }
}
/// Return a copy of this operator, modified to allow an interpretion as a decmial point.
pub fn with_decimal_interpretation(self) -> Self {
Self { can_be_decimal_operator: true, ..self }
@ -421,6 +422,11 @@ impl OperatorProperties {
self.is_type_annotation
}
/// Return the LHS operator-section/template-function behavior of this operator.
pub fn lhs_section_termination(&self) -> Option<crate::syntax::operator::SectionTermination> {
self.lhs_section_termination
}
/// Return whether this operator is the assignment operator.
pub fn is_assignment(&self) -> bool {
self.is_assignment
@ -436,6 +442,11 @@ impl OperatorProperties {
self.is_sequence
}
/// Return whether this operator is the execution-suspension operator.
pub fn is_suspension(&self) -> bool {
self.is_suspension
}
/// Return this operator's associativity.
pub fn associativity(&self) -> Associativity {
match self.is_right_associative {
@ -502,7 +513,9 @@ macro_rules! generate_token_aliases {
pub enum $enum:ident {
$(
$(#$variant_meta:tt)*
$variant:ident $({ $(pub $field:ident : $field_ty:ty),* $(,)? })?
$variant:ident $({
$($(#$field_meta:tt)* pub $field:ident : $field_ty:ty),* $(,)?
})?
),* $(,)?
}
) => { paste!{

501
lib/rust/parser/src/syntax/tree.rs
View File

@ -4,6 +4,7 @@ use crate::prelude::*;
use crate::source::*;
use crate::syntax::*;
use crate::expression_to_pattern;
use crate::span_builder;
use enso_parser_syntax_tree_visitor::Visitor;
@ -85,11 +86,6 @@ macro_rules! with_ast_definition { ($f:ident ($($args:tt)*)) => { $f! { $($args)
pub error: Error,
pub ast: Tree<'s>,
},
/// Indicates a subtree in which an unimplemented case was reached.
Unsupported {
pub error: String,
pub ast: Tree<'s>,
},
/// A sequence of lines introduced by a line ending in an operator.
BodyBlock {
/// The lines of the block.
@ -126,6 +122,10 @@ macro_rules! with_ast_definition { ($f:ident ($($args:tt)*)) => { $f! { $($args)
/// The wildcard marker, `_`.
Wildcard {
pub token: token::Wildcard<'s>,
#[serde(serialize_with = "crate::serialization::serialize_optional_int")]
#[serde(deserialize_with = "crate::serialization::deserialize_optional_int")]
#[reflect(as = "i32")]
pub de_bruijn_index: Option<u32>,
},
/// The auto-scoping marker, `...`.
AutoScope {
@ -135,6 +135,8 @@ macro_rules! with_ast_definition { ($f:ident ($($args:tt)*)) => { $f! { $($args)
pub open: Option<token::TextStart<'s>>,
pub elements: Vec<TextElement<'s>>,
pub close: Option<token::TextEnd<'s>>,
#[serde(skip)]
#[reflect(skip)]
pub trim: VisibleOffset,
},
/// A simple application, like `print "hello"`.
@ -145,11 +147,11 @@ macro_rules! with_ast_definition { ($f:ident ($($args:tt)*)) => { $f! { $($args)
/// An application using an argument name, like `summarize_transaction (price = 100)`.
NamedApp {
pub func: Tree<'s>,
pub open: Option<token::Symbol<'s>>,
pub open: Option<token::OpenSymbol<'s>>,
pub name: token::Ident<'s>,
pub equals: token::Operator<'s>,
pub arg: Tree<'s>,
pub close: Option<token::Symbol<'s>>,
pub close: Option<token::CloseSymbol<'s>>,
},
/// Application using the `default` keyword.
DefaultApp {
@ -176,7 +178,12 @@ macro_rules! with_ast_definition { ($f:ident ($($args:tt)*)) => { $f! { $($args)
/// ([`OprApp`] with left operand missing), and the [`OprSectionBoundary`] will be placed
/// around the whole `.sum 1` expression.
OprSectionBoundary {
pub ast: Tree<'s>,
pub arguments: u32,
pub ast: Tree<'s>,
},
TemplateFunction {
pub arguments: u32,
pub ast: Tree<'s>,
},
/// An application of a multi-segment function, such as `if ... then ... else ...`. Each
/// segment starts with a token and contains an expression. Some multi-segment functions can
@ -194,11 +201,11 @@ macro_rules! with_ast_definition { ($f:ident ($($args:tt)*)) => { $f! { $($args)
/// - First zero or more type constructors, and their subordinate blocks.
/// - Then a block of statements, which may define methods or type methods.
TypeDef {
pub keyword: Token<'s>,
pub name: Tree<'s>,
pub params: Vec<Tree<'s>>,
pub keyword: token::Ident<'s>,
pub name: token::Ident<'s>,
pub params: Vec<ArgumentDefinition<'s>>,
pub constructors: Vec<TypeConstructorLine<'s>>,
pub block: Vec<block::Line<'s>>,
pub block: Vec<block::Line<'s>>,
},
/// A variable assignment, like `foo = bar 23`.
Assignment {
@ -223,26 +230,28 @@ macro_rules! with_ast_definition { ($f:ident ($($args:tt)*)) => { $f! { $($args)
},
/// An import statement.
Import {
pub polyglot: Option<MultiSegmentAppSegment<'s>>,
pub from: Option<MultiSegmentAppSegment<'s>>,
pub from_as: Option<MultiSegmentAppSegment<'s>>,
pub import: MultiSegmentAppSegment<'s>,
pub import_as: Option<MultiSegmentAppSegment<'s>>,
pub hiding: Option<MultiSegmentAppSegment<'s>>,
pub polyglot: Option<MultiSegmentAppSegment<'s>>,
pub from: Option<MultiSegmentAppSegment<'s>>,
pub import: MultiSegmentAppSegment<'s>,
pub all: Option<token::Ident<'s>>,
#[reflect(rename = "as")]
pub as_: Option<MultiSegmentAppSegment<'s>>,
pub hiding: Option<MultiSegmentAppSegment<'s>>,
},
/// An export statement.
Export {
pub from: Option<MultiSegmentAppSegment<'s>>,
pub from_as: Option<MultiSegmentAppSegment<'s>>,
pub export: MultiSegmentAppSegment<'s>,
pub export_as: Option<MultiSegmentAppSegment<'s>>,
pub hiding: Option<MultiSegmentAppSegment<'s>>,
pub from: Option<MultiSegmentAppSegment<'s>>,
pub export: MultiSegmentAppSegment<'s>,
pub all: Option<token::Ident<'s>>,
#[reflect(rename = "as")]
pub as_: Option<MultiSegmentAppSegment<'s>>,
pub hiding: Option<MultiSegmentAppSegment<'s>>,
},
/// An expression grouped by matched parentheses.
Group {
pub open: token::Symbol<'s>,
pub open: Option<token::OpenSymbol<'s>>,
pub body: Option<Tree<'s>>,
pub close: token::Symbol<'s>,
pub close: Option<token::CloseSymbol<'s>>,
},
/// Statement declaring the type of a variable.
TypeSignature {
@ -264,19 +273,12 @@ macro_rules! with_ast_definition { ($f:ident ($($args:tt)*)) => { $f! { $($args)
#[reflect(rename = "type")]
pub type_: Tree<'s>,
},
/// A `case` pattern-matching expression.
Case {
/// A `case _ of` pattern-matching expression.
CaseOf {
pub case: token::Ident<'s>,
pub expression: Option<Tree<'s>>,
pub of: token::Ident<'s>,
pub initial: Option<Tree<'s>>,
pub body: Vec<block::Line<'s>>,
},
/// An expression mapping arguments to an expression with the `->` operator.
Arrow {
pub arguments: Vec<Tree<'s>>,
pub arrow: token::Operator<'s>,
pub body: Option<Tree<'s>>,
pub cases: Vec<CaseLine<'s>>,
},
/// A lambda expression.
Lambda {
@ -285,17 +287,17 @@ macro_rules! with_ast_definition { ($f:ident ($($args:tt)*)) => { $f! { $($args)
},
/// An array literal.
Array {
pub left: token::Symbol<'s>,
pub left: token::OpenSymbol<'s>,
pub first: Option<Tree<'s>>,
pub rest: Vec<OperatorDelimitedTree<'s>>,
pub right: token::Symbol<'s>,
pub right: token::CloseSymbol<'s>,
},
/// A tuple literal.
Tuple {
pub left: token::Symbol<'s>,
pub left: token::OpenSymbol<'s>,
pub first: Option<Tree<'s>>,
pub rest: Vec<OperatorDelimitedTree<'s>>,
pub right: token::Symbol<'s>,
pub right: token::CloseSymbol<'s>,
},
}
}};}
@ -358,12 +360,6 @@ impl<'s> Tree<'s> {
pub fn with_error(self, message: impl Into<Cow<'static, str>>) -> Self {
Tree::invalid(Error::new(message), self)
}
/// Constructor.
pub fn with_unsupported(self, message: String) -> Self {
eprintln!("Unsupported syntax: {}", &message);
Tree::unsupported(message, self)
}
}
impl<'s> span::Builder<'s> for Error {
@ -402,9 +398,9 @@ pub struct TypeConstructorDef<'s> {
/// The identifier naming the type constructor.
pub constructor: token::Ident<'s>,
/// The arguments the type constructor accepts, specified inline.
pub arguments: Vec<Tree<'s>>,
pub arguments: Vec<ArgumentDefinition<'s>>,
/// The arguments the type constructor accepts, specified on their own lines.
pub block: Vec<block::Line<'s>>,
pub block: Vec<ArgumentDefinitionLine<'s>>,
}
impl<'s> span::Builder<'s> for TypeConstructorDef<'s> {
@ -413,6 +409,21 @@ impl<'s> span::Builder<'s> for TypeConstructorDef<'s> {
}
}
/// An argument specification on its own line.
#[derive(Clone, Debug, Eq, PartialEq, Visitor, Serialize, Reflect, Deserialize)]
pub struct ArgumentDefinitionLine<'s> {
/// The token beginning the line.
pub newline: token::Newline<'s>,
/// The argument definition, unless this is an empty line.
pub argument: Option<ArgumentDefinition<'s>>,
}
impl<'s> span::Builder<'s> for ArgumentDefinitionLine<'s> {
fn add_to_span(&mut self, span: Span<'s>) -> Span<'s> {
span.add(&mut self.newline).add(&mut self.argument)
}
}
// === Text literals ===
@ -426,31 +437,18 @@ pub enum TextElement<'s> {
text: token::TextSection<'s>,
},
/// An escaped character.
EscapeChar {
/// The \ character.
backslash: Option<token::TextEscapeSymbol<'s>>,
/// The escaped character.
char: Option<token::TextEscapeChar<'s>>,
},
/// A unicode escape sequence.
EscapeSequence {
/// The backslash and format characters.
leader: Option<token::TextEscapeLeader<'s>>,
/// The opening delimiter, if present.
open: Option<token::TextEscapeSequenceStart<'s>>,
/// The hex digits.
digits: Option<token::TextEscapeHexDigits<'s>>,
/// The closing delimiter, if present.
close: Option<token::TextEscapeSequenceEnd<'s>>,
Escape {
/// The escape sequence.
token: token::TextEscape<'s>,
},
/// An interpolated section within a text literal.
Splice {
/// The opening ` character.
open: token::Symbol<'s>,
open: token::OpenSymbol<'s>,
/// The interpolated expression.
expression: Option<Tree<'s>>,
/// The closing ` character.
close: token::Symbol<'s>,
close: token::CloseSymbol<'s>,
},
}
@ -458,9 +456,7 @@ impl<'s> span::Builder<'s> for TextElement<'s> {
fn add_to_span(&mut self, span: Span<'s>) -> Span<'s> {
match self {
TextElement::Section { text } => text.add_to_span(span),
TextElement::EscapeChar { backslash, char } => span.add(backslash).add(char),
TextElement::EscapeSequence { leader, open, digits, close } =>
span.add(leader).add(open).add(digits).add(close),
TextElement::Escape { token } => span.add(token),
TextElement::Splice { open, expression, close } =>
span.add(open).add(expression).add(close),
}
@ -501,19 +497,35 @@ impl<'s> span::Builder<'s> for FractionalDigits<'s> {
/// A function argument definition.
#[derive(Clone, Debug, Eq, PartialEq, Visitor, Serialize, Reflect, Deserialize)]
pub struct ArgumentDefinition<'s> {
/// Closing parenthesis (only present when a default is provided).
pub open: Option<token::Symbol<'s>>,
/// Opening parenthesis (outer).
pub open: Option<token::OpenSymbol<'s>>,
/// Opening parenthesis (inner).
pub open2: Option<token::OpenSymbol<'s>>,
/// An optional execution-suspension unary operator (~).
pub suspension: Option<token::Operator<'s>>,
/// The pattern being bound to an argument.
pub pattern: Tree<'s>,
pub pattern: Tree<'s>,
/// An optional type ascribed to an argument.
#[reflect(rename = "type")]
pub type_: Option<ArgumentType<'s>>,
/// Closing parenthesis (inner).
pub close2: Option<token::CloseSymbol<'s>>,
/// An optional default value for an argument.
pub default: Option<ArgumentDefault<'s>>,
/// Closing parenthesis (only present when a default is provided).
pub close: Option<token::Symbol<'s>>,
pub default: Option<ArgumentDefault<'s>>,
/// Closing parenthesis (outer).
pub close: Option<token::CloseSymbol<'s>>,
}
impl<'s> span::Builder<'s> for ArgumentDefinition<'s> {
fn add_to_span(&mut self, span: Span<'s>) -> Span<'s> {
span.add(&mut self.pattern).add(&mut self.default)
span.add(&mut self.open)
.add(&mut self.open2)
.add(&mut self.suspension)
.add(&mut self.pattern)
.add(&mut self.type_)
.add(&mut self.close2)
.add(&mut self.default)
.add(&mut self.close)
}
}
@ -532,6 +544,95 @@ impl<'s> span::Builder<'s> for ArgumentDefault<'s> {
}
}
/// A type ascribed to an argument definition.
#[derive(Clone, Debug, Eq, PartialEq, Visitor, Serialize, Reflect, Deserialize)]
pub struct ArgumentType<'s> {
/// The `:` token.
pub operator: token::Operator<'s>,
/// The type.
#[reflect(rename = "type")]
pub type_: Tree<'s>,
}
impl<'s> span::Builder<'s> for ArgumentType<'s> {
fn add_to_span(&mut self, span: Span<'s>) -> Span<'s> {
span.add(&mut self.operator).add(&mut self.type_)
}
}
// === CaseOf ===
/// A that may contain a case-expression in a case-of expression.
#[derive(Clone, Debug, Default, Eq, PartialEq, Visitor, Serialize, Reflect, Deserialize)]
pub struct CaseLine<'s> {
/// The token beginning the line. This will always be present, unless the first case-expression
/// is on the same line as the initial case-of.
pub newline: Option<token::Newline<'s>>,
/// The case-expression, if present.
pub case: Option<Case<'s>>,
}
impl<'s> CaseLine<'s> {
/// Return a mutable reference to the `left_offset` of this object (which will actually belong
/// to one of the object's children, if it has any).
pub fn left_offset_mut(&mut self) -> Option<&mut Offset<'s>> {
self.newline
.as_mut()
.map(|t| &mut t.left_offset)
.or_else(|| self.case.as_mut().and_then(Case::left_offset_mut))
}
}
impl<'s> span::Builder<'s> for CaseLine<'s> {
fn add_to_span(&mut self, span: Span<'s>) -> Span<'s> {
span.add(&mut self.newline).add(&mut self.case)
}
}
/// A case-expression in a case-of expression.
#[derive(Clone, Debug, Default, Eq, PartialEq, Visitor, Serialize, Reflect, Deserialize)]
pub struct Case<'s> {
/// The pattern being matched. It is an error for this to be absent.
pub pattern: Option<Tree<'s>>,
/// Token.
pub arrow: Option<token::Operator<'s>>,
/// The expression associated with the pattern. It is an error for this to be empty.
pub expression: Option<Tree<'s>>,
}
impl<'s> Case<'s> {
/// Return a mutable reference to the `left_offset` of this object (which will actually belong
/// to one of the object's children, if it has any).
pub fn left_offset_mut(&mut self) -> Option<&mut Offset<'s>> {
self.pattern
.as_mut()
.map(|p| &mut p.span.left_offset)
.or_else(|| self.arrow.as_mut().map(|t| &mut t.left_offset))
.or_else(|| self.expression.as_mut().map(|e| &mut e.span.left_offset))
}
}
impl<'s> span::Builder<'s> for Case<'s> {
fn add_to_span(&mut self, span: Span<'s>) -> Span<'s> {
span.add(&mut self.pattern).add(&mut self.arrow).add(&mut self.expression)
}
}
impl<'s> From<Tree<'s>> for Case<'s> {
fn from(tree: Tree<'s>) -> Self {
match tree.variant {
box Variant::OprApp(OprApp { lhs, opr: Ok(opr), rhs }) if opr.properties.is_arrow() => {
let pattern = lhs.map(expression_to_pattern);
let mut case = Case { pattern, arrow: opr.into(), expression: rhs };
*case.left_offset_mut().unwrap() += tree.span.left_offset;
case
}
_ => Case { expression: tree.into(), ..default() },
}
}
}
// === OprApp ===
@ -621,46 +722,9 @@ impl<'s> span::Builder<'s> for OperatorDelimitedTree<'s> {
/// application has special semantics.
pub fn apply<'s>(mut func: Tree<'s>, mut arg: Tree<'s>) -> Tree<'s> {
match &mut *func.variant {
Variant::OprApp(app)
if let Ok(opr) = &app.opr && !opr.properties.can_form_section() && app.rhs.is_none() => {
app.rhs = Some(arg);
return func;
}
Variant::TextLiteral(lhs) if let Variant::TextLiteral(rhs) = &mut *arg.variant
&& lhs.close.is_none() && rhs.open.is_none() => {
if rhs.trim != VisibleOffset(0) && (lhs.trim == VisibleOffset(0) || rhs.trim < lhs.trim) {
lhs.trim = rhs.trim;
}
match rhs.elements.first_mut() {
Some(TextElement::Section { text }) => text.left_offset = arg.span.left_offset,
Some(TextElement::EscapeChar { char: char_, .. }) => {
if let Some(char__) = char_ {
char__.left_offset = arg.span.left_offset;
if let Some(TextElement::EscapeChar { char, .. }) = lhs.elements.last_mut()
&& char.is_none() {
*char = mem::take(char_);
return func;
}
}
}
Some(TextElement::EscapeSequence { open: open_, digits: digits_, close: close_, .. }) => {
if let Some(TextElement::EscapeSequence { open, digits, close, .. })
= lhs.elements.last_mut() {
if open.is_none() {
*open = open_.clone();
}
if digits.is_none() {
*digits = digits_.clone();
}
*close = close_.clone();
return func;
}
}
Some(TextElement::Splice { open, .. }) => open.left_offset = arg.span.left_offset,
None => (),
}
lhs.elements.append(&mut rhs.elements);
lhs.close = rhs.close.take();
Variant::TextLiteral(lhs) if lhs.close.is_none()
&& let Tree { variant: box Variant::TextLiteral(rhs), span } = arg => {
join_text_literals(lhs, rhs, span);
return func;
}
Variant::Number(func_ @ Number { base: _, integer: None, fractional_digits: None })
@ -675,10 +739,20 @@ pub fn apply<'s>(mut func: Tree<'s>, mut arg: Tree<'s>) -> Tree<'s> {
}
match &mut *arg.variant {
Variant::ArgumentBlockApplication(block) if block.lhs.is_none() => {
let func_left_offset = mem::take(&mut func.span.left_offset);
let arg_left_offset = mem::replace(&mut arg.span.left_offset, func_left_offset);
if let Some(first) = block.arguments.first_mut() {
first.newline.left_offset += arg_left_offset;
}
block.lhs = Some(func);
arg
}
Variant::OperatorBlockApplication(block) if block.lhs.is_none() => {
let func_left_offset = mem::take(&mut func.span.left_offset);
let arg_left_offset = mem::replace(&mut arg.span.left_offset, func_left_offset);
if let Some(first) = block.expressions.first_mut() {
first.newline.left_offset += arg_left_offset;
}
block.lhs = Some(func);
arg
}
@ -688,7 +762,7 @@ pub fn apply<'s>(mut func: Tree<'s>, mut arg: Tree<'s>) -> Tree<'s> {
lhs.left_offset += arg.span.left_offset.clone();
Tree::named_app(func, None, lhs, opr.clone(), rhs.clone(), None)
}
Variant::Group(Group { open, body: Some(body), close }) if let box
Variant::Group(Group { open: Some(open), body: Some(body), close: Some(close) }) if let box
Variant::OprApp(OprApp { lhs: Some(lhs), opr: Ok(opr), rhs: Some(rhs) }) = &body.variant
&& opr.properties.is_assignment() && let Variant::Ident(lhs) = &*lhs.variant => {
let mut open = open.clone();
@ -706,6 +780,44 @@ pub fn apply<'s>(mut func: Tree<'s>, mut arg: Tree<'s>) -> Tree<'s> {
}
}
fn join_text_literals<'s>(
lhs: &'_ mut TextLiteral<'s>,
mut rhs: TextLiteral<'s>,
rhs_span: Span<'s>,
) {
if rhs.trim != VisibleOffset(0) && (lhs.trim == VisibleOffset(0) || rhs.trim < lhs.trim) {
lhs.trim = rhs.trim;
}
match rhs.elements.first_mut() {
Some(TextElement::Section { text }) => text.left_offset += rhs_span.left_offset,
Some(TextElement::Escape { token }) => token.left_offset += rhs_span.left_offset,
Some(TextElement::Splice { open, .. }) => open.left_offset += rhs_span.left_offset,
None => (),
}
lhs.elements.append(&mut rhs.elements);
lhs.close = rhs.close.take();
if lhs.open.is_some() {
let trim = lhs.trim;
let mut remaining = lhs.elements.len();
let mut carried_offset = Offset::default();
lhs.elements.retain_mut(|e| {
remaining -= 1;
let (offset, code) = match e {
TextElement::Section { text } => (&mut text.left_offset, &mut text.code),
TextElement::Escape { token } => (&mut token.left_offset, &mut token.code),
TextElement::Splice { open, .. } => (&mut open.left_offset, &mut open.code),
};
*offset += mem::take(&mut carried_offset);
crate::lexer::untrim(trim, offset, code);
if remaining != 0 && code.is_empty() {
carried_offset = mem::take(offset);
return false;
}
true
});
}
}
/// Join two nodes with an operator, in a way appropriate for their types.
///
/// For most operands this will simply construct an `OprApp`; however, a non-operator block (i.e. an
@ -715,6 +827,7 @@ pub fn apply_operator<'s>(
mut lhs: Option<Tree<'s>>,
opr: Vec<token::Operator<'s>>,
mut rhs: Option<Tree<'s>>,
nospace: bool,
) -> Tree<'s> {
let opr = match opr.len() {
0 => return apply(lhs.unwrap(), rhs.unwrap()),
@ -722,46 +835,37 @@ pub fn apply_operator<'s>(
_ => Err(MultipleOperatorError { operators: NonEmptyVec::try_from(opr).unwrap() }),
};
if let Ok(opr_) = &opr && opr_.properties.is_type_annotation() {
let (lhs, rhs) = match (lhs, rhs) {
(Some(lhs), Some(rhs)) => (lhs, rhs),
return match (lhs, rhs) {
(Some(lhs), Some(rhs)) => {
let rhs = crate::expression_to_type(rhs);
Tree::type_annotated(lhs, opr.unwrap(), rhs)
},
(lhs, rhs) => {
let invalid = Tree::opr_app(lhs, opr, rhs);
let err = Error::new("`:` operator must be applied to two operands.");
return Tree::invalid(err, invalid);
Tree::invalid(err, invalid)
}
};
return Tree::type_annotated(lhs, opr.unwrap(), rhs);
}
if let Ok(opr) = &opr && opr.properties.is_arrow() {
let mut args = vec![];
if let Some(mut lhs) = lhs {
while let Tree { variant: box Variant::App(App { func, arg }), span } = lhs {
lhs = func;
let mut left_offset = span.left_offset;
left_offset += mem::take(&mut lhs.span.left_offset);
lhs.span.left_offset = left_offset;
args.push(arg);
}
args.push(lhs);
args.reverse();
}
return Tree::arrow(args, opr.clone(), rhs);
}
if let Ok(opr) = &opr && opr.properties.can_be_decimal_operator()
if nospace
&& let Ok(opr) = &opr && opr.properties.can_be_decimal_operator()
&& let Some(lhs) = lhs.as_mut()
&& let box Variant::Number(lhs) = &mut lhs.variant
&& lhs.fractional_digits.is_none()
&& let box Variant::Number(lhs_) = &mut lhs.variant
&& lhs_.fractional_digits.is_none()
&& let Some(rhs) = rhs.as_mut()
&& let box Variant::Number(Number { base: None, integer: Some(digits), fractional_digits: None }) = &mut rhs.variant
{
let integer = mem::take(&mut lhs.integer);
let dot = opr.clone();
let digits = digits.clone();
return Tree::number(None, integer, Some(FractionalDigits { dot, digits }));
lhs_.fractional_digits = Some(FractionalDigits { dot, digits });
return lhs.clone();
}
if let Some(rhs_) = rhs.as_mut() {
if let Variant::ArgumentBlockApplication(block) = &mut *rhs_.variant {
if block.lhs.is_none() {
if let Some(first) = block.arguments.first_mut() {
first.newline.left_offset += mem::take(&mut rhs_.span.left_offset);
}
let ArgumentBlockApplication { lhs: _, arguments } = block;
let arguments = mem::take(arguments);
let rhs_ = block::body_from_lines(arguments);
@ -772,6 +876,109 @@ pub fn apply_operator<'s>(
Tree::opr_app(lhs, opr, rhs)
}
impl<'s> From<Token<'s>> for Tree<'s> {
fn from(token: Token<'s>) -> Self {
match token.variant {
token::Variant::Ident(ident) => Tree::ident(token.with_variant(ident)),
token::Variant::Digits(number) =>
Tree::number(None, Some(token.with_variant(number)), None),
token::Variant::NumberBase(base) =>
Tree::number(Some(token.with_variant(base)), None, None),
token::Variant::TextStart(open) =>
Tree::text_literal(Some(token.with_variant(open)), default(), default(), default()),
token::Variant::TextSection(section) => {
let trim = token.left_offset.visible;
let section = TextElement::Section { text: token.with_variant(section) };
Tree::text_literal(default(), vec![section], default(), trim)
}
token::Variant::TextEscape(escape) => {
let trim = token.left_offset.visible;
let token = token.with_variant(escape);
let section = TextElement::Escape { token };
Tree::text_literal(default(), vec![section], default(), trim)
}
token::Variant::TextEnd(close) =>
Tree::text_literal(default(), default(), Some(token.with_variant(close)), default()),
token::Variant::Wildcard(wildcard) => Tree::wildcard(token.with_variant(wildcard), default()),
token::Variant::AutoScope(t) => Tree::auto_scope(token.with_variant(t)),
token::Variant::OpenSymbol(s) =>
Tree::group(Some(token.with_variant(s)), default(), default()).with_error("Unmatched delimiter"),
token::Variant::CloseSymbol(s) =>
Tree::group(default(), default(), Some(token.with_variant(s))).with_error("Unmatched delimiter"),
// These should be unreachable: They are handled when assembling items into blocks,
// before parsing proper.
token::Variant::Newline(_)
| token::Variant::BlockStart(_)
| token::Variant::BlockEnd(_)
// This should be unreachable: `resolve_operator_precedence` doesn't calls `to_ast` for
// operators.
| token::Variant::Operator(_)
// Map an error case in the lexer to an error in the AST.
| token::Variant::Invalid(_) => {
let message = format!("Unexpected token: {token:?}");
let ident = token::variant::Ident(false, 0, false, false);
let value = Tree::ident(token.with_variant(ident));
Tree::with_error(value, message)
}
}
}
}
// =============================
// === Traversing operations ===
// =============================
/// Recurse into the lexical LHS of the tree, applying a function to each node reached, until the
/// function returns `false`.
pub fn recurse_left_mut_while<'s>(
mut tree: &'_ mut Tree<'s>,
mut f: impl FnMut(&'_ mut Tree<'s>) -> bool,
) {
while f(tree) {
tree = match &mut *tree.variant {
// No LHS.
Variant::Invalid(_)
| Variant::BodyBlock(_)
| Variant::Ident(_)
| Variant::Number(_)
| Variant::Wildcard(_)
| Variant::AutoScope(_)
| Variant::TextLiteral(_)
| Variant::UnaryOprApp(_)
| Variant::MultiSegmentApp(_)
| Variant::TypeDef(_)
| Variant::Function(_)
| Variant::Import(_)
| Variant::Export(_)
| Variant::Group(_)
| Variant::CaseOf(_)
| Variant::TypeSignature(_)
| Variant::Lambda(_)
| Variant::Array(_)
| Variant::Tuple(_) => break,
// Optional LHS.
Variant::ArgumentBlockApplication(ArgumentBlockApplication { lhs, .. })
| Variant::OperatorBlockApplication(OperatorBlockApplication { lhs, .. })
| Variant::OprApp(OprApp { lhs, .. }) =>
if let Some(lhs) = lhs {
lhs
} else {
break;
},
// Unconditional LHS.
Variant::App(App { func: lhs, .. })
| Variant::NamedApp(NamedApp { func: lhs, .. })
| Variant::OprSectionBoundary(OprSectionBoundary { ast: lhs, .. })
| Variant::TemplateFunction(TemplateFunction { ast: lhs, .. })
| Variant::DefaultApp(DefaultApp { func: lhs, .. })
| Variant::Assignment(Assignment { pattern: lhs, .. })
| Variant::TypeAnnotated(TypeAnnotated { expression: lhs, .. }) => lhs,
}
}
}
// ================
@ -956,6 +1163,20 @@ define_visitor_no_mut!(Item, visit_item);
crate::with_token_definition!(define_visitor_for_tokens());
// === Primitives ===
impl<'s, 'a> TreeVisitable<'s, 'a> for u32 {}
impl<'s, 'a> TreeVisitableMut<'s, 'a> for u32 {}
impl<'a, 't, 's> SpanVisitable<'s, 'a> for u32 {}
impl<'a, 't, 's> SpanVisitableMut<'s, 'a> for u32 {}
impl<'a, 't, 's> ItemVisitable<'s, 'a> for u32 {}
impl<'s> span::Builder<'s> for u32 {
fn add_to_span(&mut self, span: Span<'s>) -> Span<'s> {
span
}
}
// === TreeVisitable special cases ===
impl<'s, 'a> TreeVisitable<'s, 'a> for Tree<'s> {

65
lib/rust/parser/src/syntax/tree/block.rs
View File

@ -75,23 +75,34 @@ pub struct OperatorBlockExpression<'s> {
/// Interpret the given expression as an `OperatorBlockExpression`, if it fits the correct pattern.
fn to_operator_block_expression(
expression_: Tree<'_>,
mut tree: Tree<'_>,
) -> Result<OperatorBlockExpression<'_>, Tree<'_>> {
let tree_ = match &*expression_.variant {
Variant::OprSectionBoundary(OprSectionBoundary { ast }) => ast,
_ => return Err(expression_),
};
if let Variant::OprApp(OprApp { lhs: None, opr, rhs: Some(expression) }) = &*tree_.variant {
if expression.span.left_offset.visible.width_in_spaces < 1 {
return Err(expression_);
let mut left_operator = None;
recurse_left_mut_while(&mut tree, |tree| {
if let Variant::OprApp(OprApp { lhs: None, opr, rhs }) = &mut *tree.variant
&& let Some(rhs_) = rhs
&& rhs_.span.left_offset.visible.width_in_spaces >= 1
{
left_operator = Some(opr.clone());
*tree = mem::take(rhs).unwrap();
}
let mut operator = opr.clone();
operator.first_operator_mut().left_offset = expression_.span.left_offset;
let expression = expression.clone();
Ok(OperatorBlockExpression { operator, expression })
} else {
Err(expression_)
true
});
let Some(mut operator) = left_operator else {
return Err(tree);
};
operator.first_operator_mut().left_offset += mem::take(&mut tree.span.left_offset);
if let Variant::OprSectionBoundary(OprSectionBoundary { arguments, .. }) = &mut *tree.variant {
*arguments -= 1;
}
let expression = match *tree.variant {
Variant::OprSectionBoundary(OprSectionBoundary { ast, arguments: 0 }) => {
operator.first_operator_mut().left_offset += tree.span.left_offset;
ast
}
_ => tree,
};
Ok(OperatorBlockExpression { operator, expression })
}
impl<'s> span::Builder<'s> for OperatorBlockExpression<'s> {
@ -268,22 +279,23 @@ where
let newline = default();
let line = default();
let finished = default();
Lines { items, newline, line, finished }
let precedence = default();
Lines { items, newline, line, finished, precedence }
}
/// An iterator of [`Line`]s.
#[derive(Debug)]
pub struct Lines<'s, I> {
items: I,
newline: token::Newline<'s>,
line: Vec<Item<'s>>,
finished: bool,
items: I,
newline: token::Newline<'s>,
line: Vec<Item<'s>>,
finished: bool,
precedence: operator::Precedence<'s>,
}
impl<'s, I> Lines<'s, I> {
fn parse_current_line(&mut self, newline: token::Newline<'s>) -> Line<'s> {
let line = self.line.drain(..);
let expression = operator::resolve_operator_precedence_if_non_empty(line);
let expression = self.precedence.resolve(self.line.drain(..));
Line { newline, expression }
}
}
@ -302,15 +314,16 @@ where I: Iterator<Item = Item<'s>>
Item::Token(Token { variant: token::Variant::Newline(_), left_offset, code }) => {
let token = token::newline(left_offset, code);
let newline = mem::replace(&mut self.newline, token);
if newline.code.is_empty() && self.line.is_empty() {
// The block started with a real newline; ignore the implicit newline.
continue;
// If the block started with a real newline, ignore the implicit newline.
if !(newline.code.is_empty()
&& newline.left_offset.code.is_empty()
&& self.line.is_empty())
{
return self.parse_current_line(newline).into();
}
return self.parse_current_line(newline).into();
}
_ => {
self.line.push(item);
continue;
}
}
}

495
lib/rust/parser/tests/parse.rs
View File

@ -1,5 +1,7 @@
//! Parse expressions and compare their results to expected values.
// === Features ===
#![feature(let_else)]
// === Non-Standard Linter Configuration ===
#![allow(clippy::option_map_unit_fn)]
#![allow(clippy::precedence)]
@ -52,34 +54,29 @@ fn application() {
}
#[test]
fn parentheses_simple() {
let expected = block![(Group "(" (App (Ident a) (Ident b)) ")")];
test("(a b)", expected);
}
#[test]
fn section_simple() {
let expected_lhs = block![(OprSectionBoundary (OprApp () (Ok "+") (Ident a)))];
test("+ a", expected_lhs);
let expected_rhs = block![(OprSectionBoundary (OprApp (Ident a) (Ok "+") ()))];
test("a +", expected_rhs);
}
#[test]
fn parentheses_nested() {
fn parentheses() {
test("(a b)", block![(Group (App (Ident a) (Ident b)))]);
test("x)", block![(App (Ident x) (Invalid))]);
test("(x", block![(App (Invalid) (Ident x))]);
#[rustfmt::skip]
let expected = block![
(Group
"("
(App (Group "(" (App (Ident a) (Ident b)) ")")
(Ident c))
")")];
(App (Group (App (Ident a) (Ident b)))
(Ident c)))];
test("((a b) c)", expected);
}
#[test]
fn section_simple() {
let expected_lhs = block![(OprSectionBoundary 1 (OprApp () (Ok "+") (Ident a)))];
test("+ a", expected_lhs);
let expected_rhs = block![(OprSectionBoundary 1 (OprApp (Ident a) (Ok "+") ()))];
test("a +", expected_rhs);
}
#[test]
fn comments() {
test("# a b c", block![() ()]);
test("# a b c", block![()()]);
}
@ -87,8 +84,17 @@ fn comments() {
#[test]
fn type_definition_no_body() {
test("type Bool", block![(TypeDef (Ident type) (Ident Bool) #() #() #())]);
test("type Option a", block![(TypeDef (Ident type) (Ident Option) #((Ident a)) #() #())]);
test("type Bool", block![(TypeDef type Bool #() #() #())]);
test("type Option a", block![(TypeDef type Option #((() (Ident a) () ())) #() #())]);
test("type Option (a)", block![
(TypeDef type Option #((() (Ident a) () ())) #() #())]);
test("type Foo (a : Int)", block![
(TypeDef type Foo #((() (Ident a) (":" (Ident Int)) ())) #() #())]);
test("type A a=0", block![
(TypeDef type A #((() (Ident a) () ("=" (Number () "0" ())))) #() #())]);
test("type Existing_Headers (column_names : Vector Text)", block![
(TypeDef type Existing_Headers #(
(() (Ident column_names) (":" (App (Ident Vector) (Ident Text))) ())) #() #())]);
}
#[test]
@ -103,9 +109,9 @@ fn type_constructors() {
];
#[rustfmt::skip]
let expected = block![
(TypeDef (Ident type) (Ident Geo) #()
#(((Circle #() #((Ident radius) (Ident x))))
((Rectangle #((Ident width) (Ident height)) #()))
(TypeDef type Geo #()
#(((Circle #() #(((() (Ident radius) () ())) ((() (Ident x) () ())))))
((Rectangle #((() (Ident width) () ()) (() (Ident height) () ())) #()))
((Point #() #())))
#())
];
@ -116,14 +122,32 @@ fn type_constructors() {
fn type_methods() {
let code = ["type Geo", " number =", " x", " area self = x + x"];
#[rustfmt::skip]
let expected = block![
(TypeDef (Ident type) (Ident Geo) #() #()
let expected = block![
(TypeDef type Geo #() #()
#((Function number #() "=" (BodyBlock #((Ident x))))
(Function area #((() (Ident self) () ())) "=" (OprApp (Ident x) (Ok "+") (Ident x)))))
];
test(&code.join("\n"), expected);
}
#[test]
fn type_operator_methods() {
#[rustfmt::skip]
let code = [
"type Foo",
" + : Foo -> Foo -> Foo",
" + self b = b",
];
#[rustfmt::skip]
let expected = block![
(TypeDef type Foo #() #()
#((TypeSignature #"+" ":"
(OprApp (Ident Foo) (Ok "->") (OprApp (Ident Foo) (Ok "->") (Ident Foo))))
(Function #"+" #((() (Ident self) () ()) (() (Ident b) () ()))
"=" (Ident b))))];
test(&code.join("\n"), expected);
}
#[test]
fn type_def_full() {
let code = [
@ -140,9 +164,11 @@ fn type_def_full() {
];
#[rustfmt::skip]
let expected = block![
(TypeDef (Ident type) (Ident Geo) #()
#(((Circle #() #((TypeAnnotated (Ident radius) ":" (Ident float)) (Ident x))))
((Rectangle #((Ident width) (Ident height)) #()))
(TypeDef type Geo #()
#(((Circle #() #(
((() (Ident radius) (":" (Ident float)) ()))
((() (Ident x) () ())))))
((Rectangle #((() (Ident width) () ()) (() (Ident height) () ())) #()))
((Point #() #()))
(()))
#((Function number #() "=" (BodyBlock #((Ident x))))
@ -151,6 +177,20 @@ fn type_def_full() {
test(&code.join("\n"), expected);
}
#[test]
fn type_def_defaults() {
let code = ["type Result error ok=Nothing", " Ok value:ok = Nothing"];
#[rustfmt::skip]
let expected = block![
(TypeDef type Result #((() (Ident error) () ())
(() (Ident ok) () ("=" (Ident Nothing))))
#(((Ok #((() (Ident value) (":" (Ident ok)) ("=" (Ident Nothing))))
#())))
#())
];
test(&code.join("\n"), expected);
}
#[test]
fn type_def_nested() {
#[rustfmt::skip]
@ -161,9 +201,9 @@ fn type_def_nested() {
];
#[rustfmt::skip]
let expected = block![
(TypeDef (Ident type) (Ident Foo) #() #()
#((TypeDef (Ident type) (Ident Bar) #() #() #())
(TypeDef (Ident type) (Ident Baz) #() #() #())))
(TypeDef type Foo #() #()
#((TypeDef type Bar #() #() #())
(TypeDef type Baz #() #() #())))
];
test(&code.join("\n"), expected);
}
@ -181,7 +221,7 @@ fn assignment_simple() {
#[test]
fn function_inline_simple_args() {
test("foo a = x", block![(Function foo #((() (Ident a) () ())) "=" (Ident x))]);
test(" foo a = x", block![(Function foo #((() (Ident a) () ())) "=" (Ident x))]);
#[rustfmt::skip]
test("foo a b = x",
block![(Function foo #((() (Ident a) () ()) (() (Ident b) () ())) "=" (Ident x))]);
@ -192,6 +232,7 @@ fn function_inline_simple_args() {
#((() (Ident a) () ()) (() (Ident b) () ()) (() (Ident c) () ()))
"=" (Ident x))],
);
test(" foo _ = x", block![(Function foo #((() (Wildcard -1) () ())) "=" (Ident x))]);
}
#[test]
@ -202,7 +243,8 @@ fn function_block_noargs() {
#[test]
fn function_block_simple_args() {
test("foo a =", block![(Function foo #((() (Ident a) () ())) "=" ())]);
test("foo a b =", block![(Function foo #((() (Ident a) () ()) (() (Ident b) () ())) "=" ())]);
test("foo a b =", block![(Function foo #((() (Ident a) () ())
(() (Ident b) () ())) "=" ())]);
#[rustfmt::skip]
test(
"foo a b c =", block![
@ -218,8 +260,8 @@ fn function_block_simple_args() {
#[test]
fn named_arguments() {
let cases = [
("f x=y", block![(NamedApp (Ident f) () x "=" (Ident y) ())]),
("f (x = y)", block![(NamedApp (Ident f) "(" x "=" (Ident y) ")")]),
("f x=y", block![(NamedApp (Ident f) x "=" (Ident y))]),
("f (x = y)", block![(NamedApp (Ident f) x "=" (Ident y))]),
];
cases.into_iter().for_each(|(code, expected)| test(code, expected));
}
@ -238,17 +280,17 @@ fn default_arguments() {
#[rustfmt::skip]
let cases = [
("f x=1 = x",
block![(Function f #((() (Ident x) ("=" (Number () "1" ())) ())) "=" (Ident x))]),
block![(Function f #((() (Ident x) () ("=" (Number () "1" ())))) "=" (Ident x))]),
("f (x = 1) = x",
block![(Function f #(("(" (Ident x) ("=" (Number () "1" ())) ")")) "=" (Ident x))]),
block![(Function f #((() (Ident x) () ("=" (Number () "1" ())))) "=" (Ident x))]),
// Pattern in LHS:
("f ~x=1 = x", block![
(Function f
#((() (UnaryOprApp "~" (Ident x)) ("=" (Number () "1" ())) ()))
#(("~" (Ident x) () ("=" (Number () "1" ()))))
"=" (Ident x))]),
("f (~x = 1) = x", block![
(Function f
#(("(" (UnaryOprApp "~" (Ident x)) ("=" (Number () "1" ())) ")"))
#(("~" (Ident x) () ("=" (Number () "1" ()))))
"=" (Ident x))]),
];
cases.into_iter().for_each(|(code, expected)| test(code, expected));
@ -279,7 +321,7 @@ fn code_block_body() {
#[rustfmt::skip]
let expect = block![
(Function main #() "=" (BodyBlock #(
(OprSectionBoundary (OprApp () (Ok "+") (Ident x)))
(OprSectionBoundary 1 (OprApp () (Ok "+") (Ident x)))
(App (Ident print) (Ident x)))))
];
test(&code.join("\n"), expect);
@ -298,6 +340,19 @@ fn code_block_operator() {
test(&code.join("\n"), expect);
}
#[test]
//#[ignore] // WIP
fn dot_operator_blocks() {
let code = ["rect1", " . width = 7", " . center", " + x"];
#[rustfmt::skip]
let expected = block![
(OperatorBlockApplication (Ident rect1)
#(((Ok ".") (OprApp (Ident width) (Ok "=") (Number () "7" ())))
((Ok ".") (OperatorBlockApplication (Ident center)
#(((Ok "+") (Ident x))) #()))) #())];
test(&code.join("\n"), expected);
}
#[test]
fn code_block_argument_list() {
#[rustfmt::skip]
@ -321,7 +376,7 @@ fn code_block_argument_list() {
let expect = block![
(Assignment (Ident value) "="
(ArgumentBlockApplication (Ident foo) #(
(OprSectionBoundary (OprApp () (Ok "+") (Ident x)))
(OprSectionBoundary 1 (OprApp () (Ok "+") (Ident x)))
(Ident bar))))
];
test(&code.join("\n"), expect);
@ -369,6 +424,28 @@ fn code_block_with_following_statement() {
test(&code.join("\n"), expected);
}
#[test]
fn operator_block_nested() {
let code = ["foo", " + bar", " - baz"];
#[rustfmt::skip]
let expected = block![
(OperatorBlockApplication (Ident foo)
#(((Ok "+") (OperatorBlockApplication (Ident bar) #(((Ok "-") (Ident baz))) #())))
#())];
test(&code.join("\n"), expected);
}
#[test]
fn operator_section_in_operator_block() {
let code = ["foo", " + bar +"];
#[rustfmt::skip]
let expected = block![
(OperatorBlockApplication (Ident foo)
#(((Ok "+") (OprSectionBoundary 1 (OprApp (Ident bar) (Ok "+") ()))))
#())];
test(&code.join("\n"), expected);
}
// === Binary Operators ===
@ -410,6 +487,60 @@ fn pipeline_operators() {
test("a |> f", block![(OprApp (Ident a) (Ok "|>") (Ident f))]);
}
#[test]
fn accessor_operator() {
// Test that the accessor operator `.` is treated like any other operator.
let cases = [
("Console.", block![(OprSectionBoundary 1 (OprApp (Ident Console) (Ok ".") ()))]),
(".", block![(OprSectionBoundary 2 (OprApp () (Ok ".") ()))]),
(".log", block![(OprSectionBoundary 1 (OprApp () (Ok ".") (Ident log)))]),
];
cases.into_iter().for_each(|(code, expected)| test(code, expected));
}
#[test]
fn operator_sections() {
#[rustfmt::skip]
test(".map (+2 * 3) *7", block![
(OprSectionBoundary 1
(App (App (OprApp () (Ok ".") (Ident map))
(Group
(OprSectionBoundary 1 (OprApp (OprApp () (Ok "+") (Number () "2" ()))
(Ok "*") (Number () "3" ())))))
(OprSectionBoundary 1 (OprApp () (Ok "*") (Number () "7" ())))))]);
#[rustfmt::skip]
test(".sum 1", block![
(OprSectionBoundary 1 (App (OprApp () (Ok ".") (Ident sum)) (Number () "1" ())))]);
#[rustfmt::skip]
test("+1 + x", block![
(OprSectionBoundary 1 (OprApp (OprApp () (Ok "+") (Number () "1" ()))
(Ok "+") (Ident x)))]);
#[rustfmt::skip]
test("increment = 1 +", block![
(Assignment (Ident increment) "="
(OprSectionBoundary 1 (OprApp (Number () "1" ()) (Ok "+") ())))]);
}
#[test]
fn template_functions() {
#[rustfmt::skip]
test("_.map (_+2 * 3) _*7", block![
(TemplateFunction 1
(App (App (OprApp (Wildcard 0) (Ok ".") (Ident map))
(Group
(TemplateFunction 1
(OprApp (OprApp (Wildcard 0) (Ok "+") (Number () "2" ()))
(Ok "*") (Number () "3" ())))))
(TemplateFunction 1 (OprApp (Wildcard 0) (Ok "*") (Number () "7" ())))))]);
#[rustfmt::skip]
test("_.sum 1", block![
(TemplateFunction 1 (App (OprApp (Wildcard 0) (Ok ".") (Ident sum)) (Number () "1" ())))]);
#[rustfmt::skip]
test("_+1 + x", block![
(TemplateFunction 1 (OprApp (OprApp (Wildcard 0) (Ok "+") (Number () "1" ()))
(Ok "+") (Ident x)))]);
}
// === Unary Operators ===
@ -417,7 +548,7 @@ fn pipeline_operators() {
fn unevaluated_argument() {
let code = ["main ~foo = x"];
let expected = block![
(Function main #((() (UnaryOprApp "~" (Ident foo)) () ())) "=" (Ident x))
(Function main #(("~" (Ident foo) () ())) "=" (Ident x))
];
test(&code.join("\n"), expected);
}
@ -464,11 +595,11 @@ fn minus_binary() {
fn minus_section() {
#[rustfmt::skip]
let cases = [
("- x", block![(OprSectionBoundary (OprApp () (Ok "-") (Ident x)))]),
("(- x)", block![(Group "(" (OprSectionBoundary (OprApp () (Ok "-") (Ident x))) ")")]),
("- x", block![(OprSectionBoundary 1 (OprApp () (Ok "-") (Ident x)))]),
("(- x)", block![(Group (OprSectionBoundary 1 (OprApp () (Ok "-") (Ident x))))]),
("- (x * x)", block![
(OprSectionBoundary (OprApp () (Ok "-")
(Group "(" (OprApp (Ident x) (Ok "*") (Ident x)) ")")))]),
(OprSectionBoundary 1 (OprApp () (Ok "-")
(Group (OprApp (Ident x) (Ok "*") (Ident x)))))]),
];
cases.into_iter().for_each(|(code, expected)| test(code, expected));
}
@ -479,9 +610,9 @@ fn minus_unary() {
let cases = [
("f -x", block![(App (Ident f) (UnaryOprApp "-" (Ident x)))]),
("-x", block![(UnaryOprApp "-" (Ident x))]),
("(-x)", block![(Group "(" (UnaryOprApp "-" (Ident x)) ")")]),
("(-x)", block![(Group (UnaryOprApp "-" (Ident x)))]),
("-(x * x)", block![
(UnaryOprApp "-" (Group "(" (OprApp (Ident x) (Ok "*") (Ident x)) ")"))]),
(UnaryOprApp "-" (Group (OprApp (Ident x) (Ok "*") (Ident x))))]),
("x=-x", block![(Assignment (Ident x) "=" (UnaryOprApp "-" (Ident x)))]),
("-x+x", block![(OprApp (UnaryOprApp "-" (Ident x)) (Ok "+") (Ident x))]),
("-x*x", block![(OprApp (UnaryOprApp "-" (Ident x)) (Ok "*") (Ident x))]),
@ -498,46 +629,39 @@ fn import() {
#[rustfmt::skip]
let cases = [
("import project.IO", block![
(Import () () () ((Ident import) (OprApp (Ident project) (Ok ".") (Ident IO))) () ())]),
(Import () () ((Ident import) (OprApp (Ident project) (Ok ".") (Ident IO))) () () ())]),
("import Standard.Base as Enso_List", block![
(Import () () ()
(Import () ()
((Ident import) (OprApp (Ident Standard) (Ok ".") (Ident Base)))
()
((Ident as) (Ident Enso_List))
())]),
("from Standard.Base import all", block![
(Import ()
((Ident from) (OprApp (Ident Standard) (Ok ".") (Ident Base)))
()
((Ident import) (Ident all))
() ())]),
((Ident import) ())
all () ())]),
("from Standard.Base import all hiding Number, Boolean", block![
(Import ()
((Ident from) (OprApp (Ident Standard) (Ok ".") (Ident Base)))
()
((Ident import) (Ident all))
((Ident import) ())
all
()
((Ident hiding) (OprApp (Ident Number) (Ok ",") (Ident Boolean))))]),
("from Standard.Table as Column_Module import Column", block![
(Import ()
((Ident from) (OprApp (Ident Standard) (Ok ".") (Ident Table)))
((Ident as) (Ident Column_Module))
((Ident import) (Ident Column))
() ())]),
("polyglot java import java.lang.Float", block![
(Import
((Ident polyglot) (Ident java))
()
()
((Ident import)
(OprApp (OprApp (Ident java) (Ok ".") (Ident lang)) (Ok ".") (Ident Float)))
() ())]),
() () ())]),
("polyglot java import java.net.URI as Java_URI", block![
(Import
((Ident polyglot) (Ident java))
()
()
((Ident import)
(OprApp (OprApp (Ident java) (Ok ".") (Ident net)) (Ok ".") (Ident URI)))
()
((Ident as) (Ident Java_URI))
())]),
];
@ -549,32 +673,24 @@ fn export() {
#[rustfmt::skip]
let cases = [
("export prj.Data.Foo", block![
(Export () ()
(Export ()
((Ident export)
(OprApp (OprApp (Ident prj) (Ok ".") (Ident Data)) (Ok ".") (Ident Foo)))
() ())]),
() () ())]),
("export Foo as Bar", block![
(Export () () ((Ident export) (Ident Foo)) ((Ident as) (Ident Bar)) ())]),
(Export () ((Ident export) (Ident Foo)) () ((Ident as) (Ident Bar)) ())]),
("from Foo export Bar, Baz", block![
(Export
((Ident from) (Ident Foo))
()
((Ident export) (OprApp (Ident Bar) (Ok ",") (Ident Baz)))
() ())]),
() () ())]),
("from Foo export all hiding Bar, Baz", block![
(Export
((Ident from) (Ident Foo))
()
((Ident export) (Ident all))
((Ident export) ())
all
()
((Ident hiding) (OprApp (Ident Bar) (Ok ",") (Ident Baz))))]),
("from Foo as Bar export Baz, Quux", block![
(Export
((Ident from) (Ident Foo))
((Ident as) (Ident Bar))
((Ident export) (OprApp (Ident Baz) (Ok ",") (Ident Quux)))
() ())
]),
];
cases.into_iter().for_each(|(code, expected)| test(code, expected));
}
@ -628,7 +744,11 @@ fn type_annotations() {
("val = foo (x : Int)", block![
(Assignment (Ident val) "="
(App (Ident foo)
(Group "(" (TypeAnnotated (Ident x) ":" (Ident Int)) ")")))]),
(Group (TypeAnnotated (Ident x) ":" (Ident Int)))))]),
("(x : My_Type _)", block![
(Group (TypeAnnotated (Ident x) ":" (App (Ident My_Type) (Wildcard -1))))]),
("x : List Int -> Int", block![
(TypeSignature x ":" (OprApp (App (Ident List) (Ident Int)) (Ok "->") (Ident Int)))]),
];
cases.into_iter().for_each(|(code, expected)| test(code, expected));
}
@ -641,102 +761,125 @@ fn inline_text_literals() {
#[rustfmt::skip]
let cases = [
(r#""I'm an inline raw text!""#, block![
(TextLiteral #((Section "I'm an inline raw text!")) 0)]),
(TextLiteral #((Section "I'm an inline raw text!")))]),
(r#"zero_length = """#, block![
(Assignment (Ident zero_length) "=" (TextLiteral #() 0))]),
(r#"unclosed = ""#, block![(Assignment (Ident unclosed) "=" (TextLiteral #() 0))]),
(Assignment (Ident zero_length) "=" (TextLiteral #()))]),
(r#""type""#, block![(TextLiteral #((Section "type")))]),
(r#"unclosed = ""#, block![(Assignment (Ident unclosed) "=" (TextLiteral #()))]),
(r#"unclosed = "a"#, block![
(Assignment (Ident unclosed) "=" (TextLiteral #((Section "a")) 0))]),
(r#"'Other quote type'"#, block![(TextLiteral #((Section "Other quote type")) 0)]),
(r#""Non-escape: \n""#, block![(TextLiteral #((Section "Non-escape: \\n")) 0)]),
(Assignment (Ident unclosed) "=" (TextLiteral #((Section "a"))))]),
(r#"'Other quote type'"#, block![(TextLiteral #((Section "Other quote type")))]),
(r#""Non-escape: \n""#, block![(TextLiteral #((Section "Non-escape: \\n")))]),
(r#""String with \" escape""#, block![
(TextLiteral
#((Section "String with ") (EscapeChar "\"") (Section " escape"))
0)]),
#((Section "String with ") (Escape '\"') (Section " escape")))]),
(r#"'\u0915\u094D\u0937\u093F'"#, block![(TextLiteral #(
(Escape '\u{0915}') (Escape '\u{094D}') (Escape '\u{0937}') (Escape '\u{093F}')))]),
(r#"('\n')"#, block![(Group (TextLiteral #((Escape '\n'))))]),
(r#"`"#, block![(Invalid)]),
(r#"(")")"#, block![(Group (TextLiteral #((Section ")"))))]),
];
cases.into_iter().for_each(|(code, expected)| test(code, expected));
}
#[test]
fn multiline_text_literals() {
test("'''", block![(TextLiteral #() 0)]);
const CODE: &str = r#"'''
test("'''", block![(TextLiteral #())]);
let code = r#""""
part of the string
3-spaces indented line, part of the Text Block
this does not end the string -> '''
also part of the string
`also` part of the string
x"#;
#[rustfmt::skip]
let expected = block![
(TextLiteral
#((Section "\n") (Section "part of the string")
(Section "\n") (Section "3-spaces indented line, part of the Text Block")
(Section "\n") (Section "this does not end the string -> '''")
#((Section "part of the string\n")
(Section " 3-spaces indented line, part of the Text Block\n")
(Section "this does not end the string -> '''\n")
(Section "\n")
(Section "\n") (Section "also part of the string")
(Section "\n"))
4)
(Section "`also` part of the string\n")))
(Ident x)
];
test(CODE, expected);
test(code, expected);
let code = r#""""
multiline string that doesn't end in a newline
x"#;
#[rustfmt::skip]
let expected = block![
(TextLiteral #((Section "multiline string that doesn't end in a newline")))
(Ident x)
];
test(code, expected);
}
#[test]
fn interpolated_literals_in_inline_text() {
#[rustfmt::skip]
let cases = [
(r#"'Simple case.'"#, block![(TextLiteral #((Section "Simple case.")) 0)]),
(r#"'Simple case.'"#, block![(TextLiteral #((Section "Simple case.")))]),
(r#"'With a `splice`.'"#, block![(TextLiteral
#((Section "With a ")
(Splice "`" (Ident splice) "`")
(Section "."))
0)]),
(Splice (Ident splice))
(Section ".")))]),
(r#"'` SpliceWithLeadingWhitespace`'"#, block![(TextLiteral
#((Splice (Ident SpliceWithLeadingWhitespace))))]),
(r#"'String with \n escape'"#, block![
(TextLiteral
#((Section "String with ") (EscapeChar "n") (Section " escape"))
0)]),
#((Section "String with ") (Escape '\n') (Section " escape")))]),
(r#"'\x0Aescape'"#, block![
(TextLiteral #((EscapeSequence "0A") (Section "escape")) 0)]),
(TextLiteral #((Escape '\n') (Section "escape")))]),
(r#"'\u000Aescape'"#, block![
(TextLiteral #((EscapeSequence "000A") (Section "escape")) 0)]),
(TextLiteral #((Escape '\n') (Section "escape")))]),
(r#"'\u{0000A}escape'"#, block![
(TextLiteral #((EscapeSequence "0000A") (Section "escape")) 0)]),
(TextLiteral #((Escape '\n') (Section "escape")))]),
(r#"'\U0000000Aescape'"#, block![
(TextLiteral #((EscapeSequence "0000000A") (Section "escape")) 0)]),
(TextLiteral #((Escape '\n') (Section "escape")))]),
];
cases.into_iter().for_each(|(code, expected)| test(code, expected));
}
#[test]
fn interpolated_literals_in_multiline_text() {
const CODE: &str = r#"'''
let code = r#"'''
`splice` at start"#;
#[rustfmt::skip]
let expected = block![
(TextLiteral #((Splice (Ident splice)) (Section " at start")))];
test(code, expected);
let code = r#"'''
text with a `splice`
and some \u000Aescapes\'"#;
#[rustfmt::skip]
let expected = block![
(TextLiteral
#((Section "\n") (Section "text with a ") (Splice "`" (Ident splice) "`")
(Section "\n") (Section "and some ")
(EscapeSequence "000A")
(Section "escapes")
(EscapeChar "'")) 4)];
test(CODE, expected);
#((Section "text with a ") (Splice (Ident splice)) (Section "\n")
(Section "and some ") (Escape '\n') (Section "escapes") (Escape '\'')))];
test(code, expected);
}
// === Lambdas ===
#[test]
fn lambdas() {
fn new_lambdas() {
let cases = [
("\\v -> v", block![(Lambda "\\" (Arrow #((Ident v)) "->" (Ident v)))]),
("\\a b -> x", block![(Lambda "\\" (Arrow #((Ident a) (Ident b)) "->" (Ident x)))]),
(r#"\v -> v"#, block![(Lambda "\\" (OprApp (Ident v) (Ok "->") (Ident v)))]),
(r#"\a b -> x"#, block![
(Lambda "\\" (OprApp (App (Ident a) (Ident b)) (Ok "->") (Ident x)))]),
];
cases.into_iter().for_each(|(code, expected)| test(code, expected));
}
#[test]
fn old_lambdas() {
let cases = [("v -> v", block![(OprApp (Ident v) (Ok "->") (Ident v))])];
cases.into_iter().for_each(|(code, expected)| test(code, expected));
}
// === Pattern Matching ===
@ -746,9 +889,7 @@ fn pattern_irrefutable() {
let expected = block![(Assignment (App (Ident Point) (Ident x_val)) "=" (Ident my_point))];
test(code, expected);
let code = "Point _ = my_point";
let expected = block![(Assignment (App (Ident Point) (Wildcard)) "=" (Ident my_point))];
test(code, expected);
test("Vector _ = x", block![(Assignment (App (Ident Vector) (Wildcard -1)) "=" (Ident x))]);
}
#[test]
@ -761,9 +902,9 @@ fn case_expression() {
];
#[rustfmt::skip]
let expected = block![
(Case (Ident a) () #(
(Arrow #((Ident Some)) "->" (Ident x))
(Arrow #((Ident Int)) "->" ())))
(CaseOf (Ident a) #(
(((Ident Some) "->" (Ident x)))
(((Ident Int) "->" ()))))
];
test(&code.join("\n"), expected);
@ -774,8 +915,8 @@ fn case_expression() {
];
#[rustfmt::skip]
let expected = block![
(Case (Ident a) () #((Arrow #((Ident Vector_2d) (Ident x) (Ident y)) "->" (Ident x))))
];
(CaseOf (Ident a) #(
(((App (App (Ident Vector_2d) (Ident x)) (Ident y)) "->" (Ident x)))))];
test(&code.join("\n"), expected);
#[rustfmt::skip]
@ -786,10 +927,24 @@ fn case_expression() {
];
#[rustfmt::skip]
let expected = block![
(Case (Ident self) () #(
(Arrow #((Ident Vector_2d)) "->" (Ident x))
(Arrow #((Wildcard)) "->" (Ident x))))
(CaseOf (Ident self) #(
(((Ident Vector_2d) "->" (Ident x)))
(((Wildcard -1) "->" (Ident x)))))];
test(&code.join("\n"), expected);
#[rustfmt::skip]
let code = [
"case foo of",
" v:My_Type -> x",
" v:(My_Type _ _) -> x",
];
#[rustfmt::skip]
let expected = block![
(CaseOf (Ident foo) #(
(((TypeAnnotated (Ident v) ":" (Ident My_Type)) "->" (Ident x)))
(((TypeAnnotated (Ident v) ":"
(Group (App (App (Ident My_Type) (Wildcard -1)) (Wildcard -1))))
"->" (Ident x)))))];
test(&code.join("\n"), expected);
}
@ -802,8 +957,7 @@ fn pattern_match_auto_scope() {
];
#[rustfmt::skip]
let expected = block![
(Case (Ident self) () #((Arrow #((Ident Vector_2d) (AutoScope)) "->" (Ident x))))
];
(CaseOf (Ident self) #((((App (Ident Vector_2d) (AutoScope)) "->" (Ident x)))))];
test(&code.join("\n"), expected);
}
@ -813,9 +967,12 @@ fn pattern_match_auto_scope() {
#[test]
fn array_literals() {
let cases = [
("[]", block![(Array "[" () #() "]")]),
("[x]", block![(Array "[" (Ident x) #() "]")]),
("[x, y]", block![(Array "[" (Ident x) #(("," (Ident y))) "]")]),
("[]", block![(Array () #())]),
("[x]", block![(Array (Ident x) #())]),
("[x, y]", block![(Array (Ident x) #(("," (Ident y))))]),
("[x, y, z]", block![(Array (Ident x) #(("," (Ident y)) ("," (Ident z))))]),
("[ x , y ]", block![(Array (Ident x) #(("," (Ident y))))]),
("[ x , y , z ]", block![(Array (Ident x) #(("," (Ident y)) ("," (Ident z))))]),
];
cases.into_iter().for_each(|(code, expected)| test(code, expected));
}
@ -823,9 +980,9 @@ fn array_literals() {
#[test]
fn tuple_literals() {
let cases = [
("{}", block![(Tuple "{" () #() "}")]),
("{x}", block![(Tuple "{" (Ident x) #() "}")]),
("{x, y}", block![(Tuple "{" (Ident x) #(("," (Ident y))) "}")]),
("{}", block![(Tuple () #())]),
("{x}", block![(Tuple (Ident x) #())]),
("{x, y}", block![(Tuple (Ident x) #(("," (Ident y))))]),
];
cases.into_iter().for_each(|(code, expected)| test(code, expected));
}
@ -838,9 +995,26 @@ fn numbers() {
let cases = [
("100_000", block![(Number () "100_000" ())]),
("10_000.99", block![(Number () "10_000" ("." "99"))]),
("1 . 0", block![(OprApp (Number () "1" ()) (Ok ".") (Number () "0" ()))]),
("1 .0", block![(App (Number () "1" ()) (OprSectionBoundary 1 (OprApp () (Ok ".") (Number () "0" ()))))]),
("1. 0", block![(OprSectionBoundary 1 (App (OprApp (Number () "1" ()) (Ok ".") ()) (Number () "0" ())))]),
("0b10101010", block![(Number "0b" "10101010" ())]),
("0o122137", block![(Number "0o" "122137" ())]),
("0xAE2F14", block![(Number "0x" "AE2F14" ())]),
("pi = 3.14", block![(Assignment (Ident pi) "=" (Number () "3" ("." "14")))])
];
cases.into_iter().for_each(|(code, expected)| test(code, expected));
}
// === Whitespace ===
#[test]
fn trailing_whitespace() {
let cases = [
("a ", block![(Ident a) ()]),
("a \n", block![(Ident a) ()]),
("a = \n x", block![(Function a #() "=" (BodyBlock #((Ident x))))]),
];
cases.into_iter().for_each(|(code, expected)| test(code, expected));
}
@ -896,22 +1070,15 @@ where T: serde::Serialize + Reflect {
to_s_expr.mapper(ast_ty, strip_hidden_fields);
let ident_token = rust_to_meta[&token::variant::Ident::reflect().id];
let operator_token = rust_to_meta[&token::variant::Operator::reflect().id];
let symbol_token = rust_to_meta[&token::variant::Symbol::reflect().id];
let open_symbol_token = rust_to_meta[&token::variant::OpenSymbol::reflect().id];
let close_symbol_token = rust_to_meta[&token::variant::CloseSymbol::reflect().id];
let number_token = rust_to_meta[&token::variant::Digits::reflect().id];
let number_base_token = rust_to_meta[&token::variant::NumberBase::reflect().id];
let newline_token = rust_to_meta[&token::variant::Newline::reflect().id];
let text_start_token = rust_to_meta[&token::variant::TextStart::reflect().id];
let text_end_token = rust_to_meta[&token::variant::TextEnd::reflect().id];
let text_escape_symbol_token = rust_to_meta[&token::variant::TextEscapeSymbol::reflect().id];
let text_escape_char_token = rust_to_meta[&token::variant::TextEscapeChar::reflect().id];
let text_escape_leader_token = rust_to_meta[&token::variant::TextEscapeLeader::reflect().id];
let text_escape_hex_digits_token =
rust_to_meta[&token::variant::TextEscapeHexDigits::reflect().id];
let text_escape_sequence_start_token =
rust_to_meta[&token::variant::TextEscapeSequenceStart::reflect().id];
let text_escape_sequence_end_token =
rust_to_meta[&token::variant::TextEscapeSequenceEnd::reflect().id];
let text_section_token = rust_to_meta[&token::variant::TextSection::reflect().id];
let text_escape_token = rust_to_meta[&token::variant::TextEscape::reflect().id];
let wildcard_token = rust_to_meta[&token::variant::Wildcard::reflect().id];
let autoscope_token = rust_to_meta[&token::variant::AutoScope::reflect().id];
// TODO: Implement `#[reflect(flag = "enso::concrete")]`, which just attaches user data to the
@ -925,13 +1092,6 @@ where T: serde::Serialize + Reflect {
let token_to_str_ = token_to_str.clone();
to_s_expr.mapper(operator_token, move |token| Value::string(token_to_str_(token)));
let token_to_str_ = token_to_str.clone();
to_s_expr.mapper(symbol_token, move |token| Value::string(token_to_str_(token)));
let token_to_str_ = token_to_str.clone();
to_s_expr.mapper(text_escape_char_token, move |token| Value::string(token_to_str_(token)));
let token_to_str_ = token_to_str.clone();
to_s_expr
.mapper(text_escape_hex_digits_token, move |token| Value::string(token_to_str_(token)));
let token_to_str_ = token_to_str.clone();
to_s_expr.mapper(text_section_token, move |token| Value::string(token_to_str_(token)));
let token_to_str_ = token_to_str.clone();
to_s_expr.mapper(number_token, move |token| Value::string(token_to_str_(token)));
@ -957,21 +1117,36 @@ where T: serde::Serialize + Reflect {
};
Value::cons(expression, list)
};
let simplify_escape = |mut list| {
let mut last = None;
while let Value::Cons(cons) = list {
let (car, cdr) = cons.into_pair();
last = Some(car);
list = cdr;
}
last.unwrap()
};
let strip_invalid = |list| {
let Value::Cons(cons) = list else { unreachable!() };
let (car, _) = cons.into_pair();
Value::cons(car, Value::Null)
};
let line = rust_to_meta[&tree::block::Line::reflect().id];
let operator_line = rust_to_meta[&tree::block::OperatorLine::reflect().id];
let case = rust_to_meta[&tree::Case::reflect().id];
let case = rust_to_meta[&tree::CaseOf::reflect().id];
let invalid = rust_to_meta[&tree::Invalid::reflect().id];
to_s_expr.mapper(line, into_car);
to_s_expr.mapper(operator_line, into_car);
to_s_expr.mapper(case, simplify_case);
to_s_expr.mapper(invalid, strip_invalid);
to_s_expr.mapper(text_escape_token, simplify_escape);
to_s_expr.skip(newline_token);
to_s_expr.skip(wildcard_token);
to_s_expr.skip(autoscope_token);
to_s_expr.skip(text_escape_symbol_token);
to_s_expr.skip(text_escape_leader_token);
to_s_expr.skip(text_escape_sequence_start_token);
to_s_expr.skip(text_escape_sequence_end_token);
to_s_expr.skip(text_start_token);
to_s_expr.skip(text_end_token);
to_s_expr.skip(open_symbol_token);
to_s_expr.skip(close_symbol_token);
tuplify(to_s_expr.value(ast_ty, &value))
}

2
lib/rust/reflect/src/lib.rs
View File

@ -270,6 +270,8 @@ macro_rules! reflect_primitive {
reflect_primitive!(bool, Primitive::Bool);
reflect_primitive!(usize, Primitive::Usize);
reflect_primitive!(u32, Primitive::U32);
reflect_primitive!(i32, Primitive::I32);
reflect_primitive!(char, Primitive::Char);
reflect_primitive!(String, Primitive::String);

2
project/FrgaalJavaCompiler.scala
View File

@ -23,7 +23,7 @@ import scala.sys.process.Process
object FrgaalJavaCompiler {
val frgaal = "org.frgaal" % "compiler" % "19.0.0-RC1" % "provided"
val frgaal = "org.frgaal" % "compiler" % "19.0.0" % "provided"
def compilers(classpath: sbt.Keys.Classpath, sbtCompilers: xsbti.compile.Compilers, javaVersion: String) = {
// Enable Java 11+ features by invoking Frgaal instead of regular javac