enso-org/enso
1
1
Fork 0
mirror of https://github.com/enso-org/enso.git synced 2024-11-22 03:32:23 +03:00
Code Issues Projects Releases Wiki Activity

Enable Scalajs For Syntax (#329)

Browse Source
This commit is contained in:
Josef 2019-11-26 14:02:50 +01:00 committed by GitHub
parent 05877a7d0c
commit 9665150c5d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 172 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.jvmopts
View File

@ -1,2 +1,3 @@
-Xss8M
-Xmx3072M
-Xss16M
-Xmx3G
-XX:+UseCompressedOops

1
.scalafmt.conf
View File

@ -13,6 +13,7 @@ align.tokens = [
{code = "=>", owner = "Case"}
{code = "%", owner = "Term.ApplyInfix"}
{code = "%%", owner = "Term.ApplyInfix"}
{code = "%%%", owner = "Term.ApplyInfix"}
{code = "="}
{code = "<-"}
{code = "->"}

14
azure-pipelines.yml
View File

@ -38,6 +38,10 @@ jobs:
tar -x -z -C ~ -f sbt.tgz
echo "##vso[task.setvariable variable=PATH]~/sbt/bin/:$PATH"
displayName: Install SBT
- script: |
sbt -no-colors syntaxJS/test
displayName: Test JS
continueOnError: true
- script: |
sbt -no-colors test
displayName: Test All
@ -88,6 +92,10 @@ jobs:
tar -x -z -C ~ -f sbt.tgz
echo "##vso[task.setvariable variable=PATH]~/sbt/bin/:$PATH"
displayName: Install SBT
- script: |
sbt -no-colors syntaxJS/test
displayName: Test JS
continueOnError: true
- script: |
sbt -no-colors test
displayName: Test All
@ -128,13 +136,9 @@ jobs:
echo ##vso[task.setvariable variable=PATH]%USERPROFILE%\sbt\bin\;%PATH%
displayName: Adjust Environment Variables
- script: |
sbt.bat test
sbt test
continueOnError: true
displayName: Test All
- script: |
sbt syntax/bench
displayName: Benchmark the Parser
continueOnError: true
- script: |
sbt runtime/Benchmark/compile
displayName: Check Runtime Benchmarks Compile

149
build.sbt
View File

@ -3,6 +3,7 @@ import scala.sys.process._
import org.enso.build.BenchTasks._
import org.enso.build.WithDebugCommand
import sbtassembly.AssemblyPlugin.defaultUniversalScript
import sbtcrossproject.CrossPlugin.autoImport.{crossProject, CrossType}
//////////////////////////////
//// Global Configuration ////
@ -10,7 +11,7 @@ import sbtassembly.AssemblyPlugin.defaultUniversalScript
val scalacVersion = "2.12.10"
val graalVersion = "19.3.0"
val circeVersion = "0.11.1"
val circeVersion = "0.12.3"
organization in ThisBuild := "org.enso"
scalaVersion in ThisBuild := scalacVersion
@ -92,16 +93,16 @@ lazy val buildNativeImage =
lazy val enso = (project in file("."))
.settings(version := "0.1")
.aggregate(
file_manager,
flexer,
graph,
unused.jvm,
flexer.jvm,
syntax_definition.jvm,
syntax.jvm,
pkg,
project_manager,
runtime,
parser_service,
syntax,
syntax_definition,
unused
file_manager,
project_manager
)
.settings(Global / concurrentRestrictions += Tags.exclusive(Exclusive))
@ -162,37 +163,115 @@ val silencerVersion = "1.4.4"
//// Internal Libraries ////
////////////////////////////
lazy val logger = (project in file("common/scala/logger"))
lazy val logger = crossProject(JVMPlatform, JSPlatform)
.withoutSuffixFor(JVMPlatform)
.crossType(CrossType.Pure)
.in(file("common/scala/logger"))
.dependsOn(unused)
.settings(
version := "0.1",
libraryDependencies ++= scala_compiler
)
.jsSettings(testFrameworks := Nil)
lazy val flexer = (project in file("common/scala/flexer"))
lazy val flexer = crossProject(JVMPlatform, JSPlatform)
.withoutSuffixFor(JVMPlatform)
.crossType(CrossType.Pure)
.in(file("common/scala/flexer"))
.dependsOn(logger)
.settings(
version := "0.1",
scalacOptions -= "-deprecation", // FIXME
resolvers += Resolver.sonatypeRepo("releases"),
libraryDependencies ++= scala_compiler ++ Seq(
"org.feijoas" %% "mango" % "0.14"
"org.feijoas" %% "mango" % "0.14",
"org.typelevel" %%% "cats-core" % "2.0.0-RC1",
"org.typelevel" %%% "kittens" % "2.0.0"
)
)
.jsSettings(testFrameworks := Nil)
lazy val unused = (project in file("common/scala/unused"))
lazy val unused = crossProject(JVMPlatform, JSPlatform)
.withoutSuffixFor(JVMPlatform)
.crossType(CrossType.Pure)
.in(file("common/scala/unused"))
.settings(version := "0.1", scalacOptions += "-nowarn")
.jsSettings(testFrameworks := Nil)
lazy val syntax_definition = (project in file("common/scala/syntax/definition"))
lazy val syntax_definition = crossProject(JVMPlatform, JSPlatform)
.withoutSuffixFor(JVMPlatform)
.crossType(CrossType.Pure)
.in(file("common/scala/syntax/definition"))
.dependsOn(logger, flexer)
.settings(
libraryDependencies ++= monocle ++ cats ++ circe ++ scala_compiler ++ Seq(
"com.lihaoyi" %% "scalatags" % "0.7.0"
libraryDependencies ++= monocle ++ scala_compiler ++ Seq(
"org.typelevel" %%% "cats-core" % "2.0.0-RC1",
"org.typelevel" %%% "kittens" % "2.0.0",
"com.lihaoyi" %%% "scalatags" % "0.7.0",
"io.circe" %%% "circe-core" % circeVersion,
"io.circe" %%% "circe-generic" % circeVersion,
"io.circe" %%% "circe-parser" % circeVersion
)
)
.jsSettings(testFrameworks := Nil)
lazy val syntax = crossProject(JVMPlatform, JSPlatform)
.withoutSuffixFor(JVMPlatform)
.crossType(CrossType.Pure)
.in(file("common/scala/syntax/specialization"))
.dependsOn(logger, flexer, syntax_definition)
.configs(Test)
.configs(Benchmark)
.settings(
testFrameworks := Nil,
mainClass in (Compile, run) := Some("org.enso.syntax.text.Main"),
version := "0.1",
logBuffered := false,
libraryDependencies ++= Seq(
"org.scalatest" %%% "scalatest" % "3.0.5" % Test,
"com.lihaoyi" %%% "pprint" % "0.5.3",
"io.circe" %%% "circe-core" % circeVersion,
"io.circe" %%% "circe-generic" % circeVersion,
"io.circe" %%% "circe-parser" % circeVersion
),
compile := (Compile / compile)
.dependsOn(Def.taskDyn {
val parserCompile =
(syntax_definition.jvm / Compile / compileIncremental).value
if (parserCompile.hasModified) {
Def.task {
streams.value.log.info("Parser changed, forcing recompilation.")
clean.value
}
} else Def.task {}
})
.value
)
.jvmSettings(
inConfig(Benchmark)(Defaults.testSettings),
unmanagedSourceDirectories in Benchmark +=
baseDirectory.value.getParentFile / "src/bench/scala",
libraryDependencies += "com.storm-enroute" %% "scalameter" % "0.17" % "bench",
testFrameworks := List(
new TestFramework("org.scalatest.tools.Framework"),
new TestFramework("org.scalameter.ScalaMeterFramework")
),
bench := (test in Benchmark).tag(Exclusive).value
)
.jsSettings(
scalaJSUseMainModuleInitializer := true,
testFrameworks := List(new TestFramework("org.scalatest.tools.Framework"))
)
lazy val parser_service = (project in file("common/scala/parser-service"))
.dependsOn(syntax.jvm)
.settings(
libraryDependencies ++= akka,
mainClass := Some("org.enso.ParserServiceMain")
)
lazy val graph = (project in file("common/scala/graph/"))
.dependsOn(logger)
.dependsOn(logger.jvm)
.configs(Test)
.settings(
version := "0.1",
@ -219,44 +298,6 @@ lazy val graph = (project in file("common/scala/graph/"))
)
)
lazy val syntax = (project in file("common/scala/syntax/specialization"))
.dependsOn(logger, flexer, syntax_definition)
.configs(Test)
.configs(Benchmark)
.settings(
mainClass in (Compile, run) := Some("org.enso.syntax.text.Main"),
version := "0.1",
testFrameworks += new TestFramework("org.scalameter.ScalaMeterFramework"),
logBuffered := false,
inConfig(Benchmark)(Defaults.testSettings),
bench := (test in Benchmark).tag(Exclusive).value,
parallelExecution in Benchmark := false,
libraryDependencies ++= circe ++ Seq(
"com.storm-enroute" %% "scalameter" % "0.17" % "bench",
"org.scalatest" %% "scalatest" % "3.0.5" % Test,
"com.lihaoyi" %% "pprint" % "0.5.3"
),
compile := (Compile / compile)
.dependsOn(Def.taskDyn {
val parserCompile =
(syntax_definition / Compile / compileIncremental).value
if (parserCompile.hasModified) {
Def.task {
streams.value.log.info("Parser changed, forcing recompilation.")
clean.value
}
} else Def.task {}
})
.value
)
lazy val parser_service = (project in file("common/scala/parser-service"))
.dependsOn(syntax)
.settings(
libraryDependencies ++= akka,
mainClass := Some("org.enso.ParserServiceMain")
)
lazy val pkg = (project in file("common/scala/pkg"))
.settings(
mainClass in (Compile, run) := Some("org.enso.pkg.Main"),
@ -365,7 +406,7 @@ lazy val runtime = (project in file("engine/runtime"))
parallelExecution in Benchmark := false
)
.dependsOn(pkg)
.dependsOn(syntax)
.dependsOn(syntax.jvm)
lazy val language_server = project
.in(file("engine/language-server"))

2
common/scala/flexer/src/main/scala/org/enso/flexer/ADT.scala
View File

@ -8,7 +8,7 @@ object ADT {
def constructorsImpl[T: c.WeakTypeTag](c: Context): c.Expr[Set[T]] = {
import c.universe._
val subs = weakTypeTag[T].tpe.typeSymbol.asClass.knownDirectSubclasses.map {
val subs = c.weakTypeTag[T].tpe.typeSymbol.asClass.knownDirectSubclasses.map {
symbol =>
q"${c.mirror.staticModule(symbol.fullName)}"
}

6
common/scala/flexer/src/main/scala/org/enso/flexer/spec/Macro.scala
View File

@ -22,7 +22,7 @@ object Macro {
c.macroApplication match {
case Apply(Select(lhs, _), _) => q"$lhs.run(${showCode(tree)})"
case x => throw new Error("Unsupported shape")
case _ => throw new Error("Unsupported shape")
}
}
@ -35,8 +35,8 @@ object Macro {
val expr = q"$tree"
val parser = c.eval(c.Expr[Parser[T]](c.untypecheck(expr.duplicate)))
val groups = q"..${parser.state.registry.map(_.generate(c))}"
val (superClassName, tree2) = tree match {
case Apply(Select(tree2 @ Select(_, name), _), _) => (name, tree2)
val tree2 = tree match {
case Apply(Select(tree2 @ Select(_, _), _), _) => tree2
case _ =>
throw new Error(
s""" ERROR: Wrong shape

32
common/scala/syntax/definition/src/main/scala/org/enso/data/Pool.scala
View File

@ -1,32 +0,0 @@
package org.enso.data
import scala.reflect.internal.util.WeakHashSet
/** Thread safe pool for objects with 1-1 hashcode-object mapping.
*
* Useful for not having lots of duplicate objects in memory.
*
* As an example it can be used to pool small strings:
* `pool.get("Hi"); pool.get("Hi"); pool.get("Hi")`
* will always return pointer to the first instace "Hi".
*
* Once nobody except [[Pool]] holds references to the first
* "Hi" instance, it will get automatically removed from
* it on the next garbage collection call.
*
* The current usecase is pooling of AST nodes,
* achieving up to 30% memory savings for nontrivial input programs.
*/
final class Pool[T <: AnyRef] {
private val astPool = WeakHashSet[T]()
/** Returns object from pool such that `object == t`, or puts t into pool
* and returns it, if no such object is found.
*
* The asymptotic complexity is almost identical to the of HashSet.get (logN)
* The function is thread safe and throws an error when t is null
*/
def get(t: T): T = synchronized(astPool.findEntryOrUpdate(t))
}

31
common/scala/syntax/definition/src/main/scala/org/enso/syntax/text/AST.scala
View File

@ -13,7 +13,6 @@ import io.circe.generic.auto._
import org.enso.data.List1._
import org.enso.data.Index
import org.enso.data.List1
import org.enso.data.Pool
import org.enso.data.Shifted
import org.enso.data.Size
import org.enso.data.Span
@ -489,29 +488,25 @@ object AST {
def apply(): Blank = blank
}
object Var {
private val pool = new Pool[VarOf[AST]]()
val any = UnapplyByType[Var]
def unapply(t: AST) = Unapply[Var].run(_.name)(t)
def apply(name: String): Var = pool.get(VarOf[AST](name))
val any = UnapplyByType[Var]
def unapply(t: AST) = Unapply[Var].run(_.name)(t)
def apply(name: String): Var = VarOf[AST](name)
}
object Cons {
private val pool = new Pool[ConsOf[AST]]()
val any = UnapplyByType[Cons]
def unapply(t: AST) = Unapply[Cons].run(_.name)(t)
def apply(name: String): Cons = pool.get(ConsOf[AST](name))
val any = UnapplyByType[Cons]
def unapply(t: AST) = Unapply[Cons].run(_.name)(t)
def apply(name: String): Cons = ConsOf[AST](name)
}
object Mod {
private val pool = new Pool[ModOf[AST]]()
val any = UnapplyByType[Mod]
def unapply(t: AST) = Unapply[Mod].run(_.name)(t)
def apply(name: String): Mod = pool.get(ModOf[AST](name))
val any = UnapplyByType[Mod]
def unapply(t: AST) = Unapply[Mod].run(_.name)(t)
def apply(name: String): Mod = ModOf[AST](name)
}
object Opr {
private val pool = new Pool[OprOf[AST]]()
val app = Opr(" ")
val any = UnapplyByType[Opr]
def unapply(t: AST) = Unapply[Opr].run(_.name)(t)
def apply(name: String): Opr = pool.get(OprOf[AST](name))
val app = Opr(" ")
val any = UnapplyByType[Opr]
def unapply(t: AST) = Unapply[Opr].run(_.name)(t)
def apply(name: String): Opr = OprOf[AST](name)
}
///////////////////////

26
common/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/meta/Builder.scala
View File

@ -7,6 +7,7 @@ import org.enso.syntax.text.AST
import org.enso.syntax.text.AST.Ident
import org.enso.syntax.text.AST.Macro
import Pattern.streamShift
import cats.data.NonEmptyList
import scala.annotation.tailrec
@ -14,7 +15,7 @@ import scala.annotation.tailrec
//// Builder ////
/////////////////
class Builder(
final class Builder(
head: Ident,
offset: Int = 0,
lineBegin: Boolean = false,
@ -61,7 +62,7 @@ class Builder(
case Some(mdef) =>
val revSegPats = mdef.fwdPats.reverse
val revSegsOuts = revSegBldrs.zipWith(revSegPats)(_.build(_))
val revSegsOuts = zipWith(revSegBldrs, revSegPats)(_.build(_))
val revSegs = revSegsOuts.map(_._1)
val revSegStreams = revSegsOuts.map(_._2)
val tailStream = revSegStreams.head
@ -100,6 +101,23 @@ class Builder(
}
}
// FIXME This is here because of bug in scalajs https://github.com/scala-js/scala-js/issues/3885
private def zipWith[A, B, C](a: NonEmptyList[A], b: NonEmptyList[B])(
f: (A, B) => C
): NonEmptyList[C] = {
@tailrec
def zwRev(as: List[A], bs: List[B], acc: List[C]): List[C] =
(as, bs) match {
case (Nil, Nil) => acc // without this we get match error
case (Nil, _) => acc
case (_, Nil) => acc
case (x :: xs, y :: ys) => zwRev(xs, ys, f(x, y) :: acc)
}
NonEmptyList(f(a.head, b.head), zwRev(a.tail, b.tail, Nil).reverse)
}
if (isModuleBuilder)
macroDef = Some(
Macro.Definition((AST.Blank(): AST) -> Pattern.Expr()) { ctx =>
@ -135,7 +153,7 @@ object Builder {
case class Context(tree: Registry.Tree, parent: Option[Context]) {
def lookup(t: AST): Option[Registry.Tree] = tree.get(t)
def isEmpty: Boolean = tree.isLeaf
def isEmpty: Boolean = tree.isLeaf
@tailrec
final def parentLookup(t: AST): Boolean = {
@ -150,7 +168,7 @@ object Builder {
}
}
object Context {
def apply(): Context = Context(data.Tree(), None)
def apply(): Context = Context(data.Tree(), None)
def apply(tree: Registry.Tree): Context = Context(tree, None)
}

34
common/scala/syntax/definition/src/main/scala/org/enso/syntax/text/prec/Macro.scala
View File

@ -18,10 +18,15 @@ object Macro {
def run(module: AST.Module): AST.Module =
module.map(transform)
private def transform(t: AST): AST = {
private def transform(t: AST): AST =
new Transformer(t).run(AST.tokenize(t).toList())
final private class Transformer(t: AST) {
val root = Builder.Context(Builtin.registry.tree)
var builder: Builder = Builder.moduleBuilder()
var builderStack: List[Builder] = Nil
var isLineBegin: Boolean = true
def pushBuilder(name: AST.Ident, off: Int, lineBegin: Boolean): Unit =
logger.trace {
@ -40,22 +45,21 @@ object Macro {
}
}
var isLineBegin: Boolean = true
@tailrec
def finalize(): AST = {
def finish(): AST = {
popBuilder() match {
case Some(bldr) =>
logger.log("End of input (in stack)")
builder.merge(bldr)
finalize()
finish()
case None =>
logger.log("End of input (not in stack)")
builder.buildAsModule()
}
}
@tailrec
def go(input: AST.Stream): AST = {
def run(input: AST.Stream): AST = {
input match {
case Nil =>
val builders = builder :: builderStack
@ -78,7 +82,7 @@ object Macro {
builder = newBuilders.head
builderStack = newBuilders.tail
finalize()
finish()
case (t1 @ Shifted(off, AST.Ident.any(el1))) :: t2_ =>
logger.log(s"Token $t1")
logger.beginGroup()
@ -92,7 +96,7 @@ object Macro {
tr.value.map(Some(_)).getOrElse(builder.macroDef)
builder.context = builder.context.copy(tree = tr)
logger.endGroup()
go(t2_)
run(t2_)
case None =>
root.lookup(el1) match {
@ -103,7 +107,7 @@ object Macro {
builder.macroDef = tr.value
builder.context = Builder.Context(tr, Some(context))
logger.endGroup()
go(t2_)
run(t2_)
case _ =>
val currentClosed = builder.context.isEmpty
@ -120,27 +124,25 @@ object Macro {
popBuilder()
builder.merge(subBuilder)
logger.endGroup()
go(input)
run(input)
case false =>
logger.log("Add token")
builder.current.revStream +:= t1
logger.endGroup()
go(t2_)
run(t2_)
}
}
}
case (Shifted(off, AST.Block.any(el1))) :: t2_ =>
case Shifted(off, AST.Block.any(el1)) :: t2_ =>
val nt1 = Shifted(off, el1.map(transform))
builder.current.revStream +:= nt1
go(t2_)
run(t2_)
case t1 :: t2_ =>
builder.current.revStream +:= t1
go(t2_)
run(t2_)
}
}
val stream = AST.tokenize(t).toList()
go(stream)
}
}

2
common/scala/syntax/specialization/src/bench/scala/org/enso/syntax/ParserBenchmark.scala
View File

@ -70,7 +70,7 @@ object ParserBenchmark extends Bench.OfflineRegressionReport {
)
val filename = "common/scala/syntax/specialization/target/bench-input.txt"
val filename = "common/scala/syntax/specialization/.jvm/target/bench-input.txt"
if (!new File(filename).exists()) {
val file = new PrintWriter(new File(filename))

7
common/scala/syntax/specialization/src/test/scala/org/enso/syntax/text/DocParserTests.scala
View File

@ -5,6 +5,7 @@ import org.enso.syntax.text.ast.Doc._
import org.enso.syntax.text.ast.Doc.Elem._
import org.enso.Logger
import org.enso.flexer.Parser.Result
import org.enso.flexer.Reader
import org.scalatest.FlatSpec
import org.scalatest.Matchers
import org.scalatest.Assertion
@ -17,7 +18,7 @@ class DocParserTests extends FlatSpec with Matchers {
output match {
case Result(_, Result.Success(value)) =>
assert(value == result)
assert(value.show() == input)
assert(value.show() == new Reader(input).toString())
case _ =>
fail(s"Parsing documentation failed, consumed ${output.offset} chars")
}
@ -31,9 +32,7 @@ class DocParserTests extends FlatSpec with Matchers {
private val testBase = it should parseDocumentation(input)
def ?=(out: Doc): Unit = testBase in {
assertExpr(input, out)
}
def ?=(out: Doc): Unit = testBase in { assertExpr(input, out) }
}
//////////////////////////////////////////////////////////////////////////////

4
project/plugins.sbt
View File

@ -1,2 +1,4 @@
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.10")
addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "1.3.4")
addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "1.3.4")
addSbtPlugin("org.portable-scala" % "sbt-scalajs-crossproject" % "0.6.1")
addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.29")