Editions Integration (#1832)

.github/CODEOWNERS

@@ -2,7 +2,7 @@
 * @iamrecursion @kustosz @radeusgd @4e6
 
 # Distribution
-/distribution/std-lib @iamrecursion @kustosz @radeusgd @4e6
+/distribution/lib @iamrecursion @kustosz @radeusgd @4e6
 
 # Scala Libraries
 /lib/scala/cli @iamrecursion @radeusgd

.github/workflows/nightly.yml

@@ -263,10 +263,10 @@ jobs:
           sbt "enso/verifyGeneratedPackage engine ${{ env.ENGINE_DIST_DIR }}/THIRD-PARTY"
           sbt "enso/verifyGeneratedPackage launcher ${{ env.LAUNCHER_DIST_DIR }}/THIRD-PARTY"
           sbt "enso/verifyGeneratedPackage project-manager ${{ env.PROJECTMANAGER_DIST_DIR }}/THIRD-PARTY"
-          sbt "enso/verifyGeneratedPackage Base ${{ env.ENGINE_DIST_DIR }}/std-lib/Base/THIRD-PARTY"
-          sbt "enso/verifyGeneratedPackage Table ${{ env.ENGINE_DIST_DIR }}/std-lib/Table/THIRD-PARTY"
-          sbt "enso/verifyGeneratedPackage Image ${{ env.ENGINE_DIST_DIR }}/std-lib/Image/THIRD-PARTY"
-          sbt "enso/verifyGeneratedPackage Database ${{ env.ENGINE_DIST_DIR }}/std-lib/Database/THIRD-PARTY"
+          sbt "enso/verifyGeneratedPackage Base ${{ env.ENGINE_DIST_DIR }}/lib/Standard/Base/0.1.0/THIRD-PARTY"
+          sbt "enso/verifyGeneratedPackage Table ${{ env.ENGINE_DIST_DIR }}/lib/Standard/Table/0.1.0/THIRD-PARTY"
+          sbt "enso/verifyGeneratedPackage Image ${{ env.ENGINE_DIST_DIR }}/lib/Standard/Image/0.1.0/THIRD-PARTY"
+          sbt "enso/verifyGeneratedPackage Database ${{ env.ENGINE_DIST_DIR }}/lib/Standard/Database/0.1.0/THIRD-PARTY"
 
       # Publish
       - name: Upload the Engine Artifact

.github/workflows/release.yml

@@ -223,7 +223,10 @@ jobs:
           sbt "enso/verifyGeneratedPackage engine ${{ env.ENGINE_DIST_DIR }}/THIRD-PARTY"
           sbt "enso/verifyGeneratedPackage launcher ${{ env.LAUNCHER_DIST_DIR }}/THIRD-PARTY"
           sbt "enso/verifyGeneratedPackage project-manager ${{ env.PROJECTMANAGER_DIST_DIR }}/THIRD-PARTY"
-          sbt "enso/verifyGeneratedPackage Standard ${{ env.ENGINE_DIST_DIR }}/std-lib/Standard/THIRD-PARTY"
+          sbt "enso/verifyGeneratedPackage Base ${{ env.ENGINE_DIST_DIR }}/lib/Standard/Base/0.1.0/THIRD-PARTY"
+          sbt "enso/verifyGeneratedPackage Table ${{ env.ENGINE_DIST_DIR }}/lib/Standard/Table/0.1.0/THIRD-PARTY"
+          sbt "enso/verifyGeneratedPackage Image ${{ env.ENGINE_DIST_DIR }}/lib/Standard/Image/0.1.0/THIRD-PARTY"
+          sbt "enso/verifyGeneratedPackage Database ${{ env.ENGINE_DIST_DIR }}/lib/Standard/Database/0.1.0/THIRD-PARTY"
 
       # Publish
       - name: Upload the Engine Artifact

.github/workflows/scala.yml

@@ -314,7 +314,7 @@ jobs:
       #        uses: andstor/copycat-action@v3
       #        with:
       #          personal_token: ${{ secrets.CI_PAT }}
-      #          src_path: ./distribution/std-lib/docs-js/
+      #          src_path: ./distribution/docs-js/
       #          dst_path: /docs/reference/.
       #          dst_branch: stdlib-update
       #          dst_owner: enso-org

.gitignore

@@ -70,8 +70,8 @@ package-lock.json
 
 javadoc/
 scaladoc/
-distribution/std-lib/docs
-distribution/std-lib/docs-js
+distribution/docs
+distribution/docs-js
 
 #######################
 ## Benchmark Reports ##
@@ -118,6 +118,6 @@ build-cache/
 ## Enso-Specific ##
 ###################
 
-distribution/std-lib/Examples/data/scratch_file
-distribution/std-lib/Examples/data/image.png
-
+distribution/lib/Standard/Examples/0.1.0/data/scratch_file
+distribution/lib/Standard/Examples/0.1.0/data/image.png
+distribution/editions

.prettierignore

@@ -6,6 +6,8 @@ distribution/launcher/THIRD-PARTY
 distribution/engine/THIRD-PARTY
 distribution/project-manager/THIRD-PARTY
 tools/legal-review
-distribution/std-lib/Base/THIRD-PARTY
-distribution/std-lib/Table/THIRD-PARTY
+distribution/lib/Standard/Base/0.1.0/THIRD-PARTY
+distribution/lib/Standard/Table/0.1.0/THIRD-PARTY
+distribution/lib/Standard/Image/0.1.0/THIRD-PARTY
+distribution/lib/Standard/Database/0.1.0/THIRD-PARTY
 built-distribution/

RELEASES.md

@@ -11,6 +11,8 @@
   ([#1826](https://github.com/enso-org/enso/pull/1826)). Currently, it only
   supports the loading of local libraries, but will be integrated with the
   editions system soon.
+- Integrated the library loading mechanism with the editions system
+  ([#1832](https://github.com/enso-org/enso/pull/1832)).
 
 ## Tooling
 

build.sbt

@@ -3,6 +3,10 @@ import org.enso.build.WithDebugCommand
 import sbt.Keys.{libraryDependencies, scalacOptions}
 import sbt.addCompilerPlugin
 import sbtcrossproject.CrossPlugin.autoImport.{crossProject, CrossType}
+import src.main.scala.licenses.{
+  DistributionDescription,
+  SBTDistributionComponent
+}
 
 import java.io.File
 
@@ -10,11 +14,13 @@ import java.io.File
 // === Global Configuration ===================================================
 // ============================================================================
 
-val scalacVersion = "2.13.6"
-val rustVersion   = "1.54.0-nightly"
-val graalVersion  = "21.1.0"
-val javaVersion   = "11"
-val ensoVersion   = "0.2.13-SNAPSHOT" // Note [Engine And Launcher Version]
+val scalacVersion  = "2.13.6"
+val rustVersion    = "1.54.0-nightly"
+val graalVersion   = "21.1.0"
+val javaVersion    = "11"
+val ensoVersion    = "0.2.13-SNAPSHOT" // Note [Engine And Launcher Version]
+val currentEdition = "2021.1"          // Note [Default Editions]
+val stdLibVersion  = "0.1.0"           // Note [Standard Library Version]
 
 /* Note [Engine And Launcher Version]
  * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -24,6 +30,31 @@ val ensoVersion   = "0.2.13-SNAPSHOT" // Note [Engine And Launcher Version]
  * scripts at .github/workflows accordingly.
  */
 
+/* Note [Default Editions]
+ * ~~~~~~~~~~~~~~~~~~~~~~~
+ * Currently, the default edition to use is inferred based on the engine
+ * version. Each Enso version has an associated default edition name and the
+ * `currentEdition` field specifies the default edition name for the upcoming
+ * release.
+ *
+ * Thus the `library-manager` needs to depend on the `version-output` to get
+ * this defaults from the build metadata.
+ *
+ * In the future we may automate generating this edition number when cutting a
+ * release.
+ */
+
+/* Note [Standard Library Version]
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ * Currently the Standard library version is fixed at 0.1.0.
+ *
+ * Once the library repository is up and we start releasing the libraries, this
+ * version will need to be bumped, so for now the best way to achieve that will
+ * be to keep it in-sync with the engine version. However this will require
+ * creating a tool which will bump these version numbers in all standard library
+ * packages.
+ */
+
 ThisBuild / organization := "org.enso"
 ThisBuild / scalaVersion := scalacVersion
 
@@ -45,6 +76,16 @@ lazy val verifyGeneratedPackage =
   )
 verifyGeneratedPackage := GatherLicenses.verifyGeneratedPackage.evaluated
 
+def makeStdLibDistribution(
+  name: String,
+  components: Seq[SBTDistributionComponent]
+): DistributionDescription =
+  Distribution(
+    name,
+    file(s"distribution/lib/Standard/$name/$stdLibVersion/THIRD-PARTY"),
+    components
+  )
+
 GatherLicenses.distributions := Seq(
   Distribution(
     "launcher",
@@ -65,27 +106,12 @@ GatherLicenses.distributions := Seq(
     file("distribution/project-manager/THIRD-PARTY"),
     Distribution.sbtProjects(`project-manager`)
   ),
-  Distribution(
-    "Base",
-    file("distribution/std-lib/Base/THIRD-PARTY"),
-    Distribution.sbtProjects(`std-base`)
-  ),
-  Distribution(
-    "Table",
-    file("distribution/std-lib/Table/THIRD-PARTY"),
-    Distribution.sbtProjects(`std-table`)
-  ),
-  Distribution(
-    "Database",
-    file("distribution/std-lib/Database/THIRD-PARTY"),
-    Distribution.sbtProjects(`std-database`)
-  ),
-  Distribution(
-    "Image",
-    file("distribution/std-lib/Image/THIRD-PARTY"),
-    Distribution.sbtProjects(`std-image`)
-  )
+  makeStdLibDistribution("Base", Distribution.sbtProjects(`std-base`)),
+  makeStdLibDistribution("Table", Distribution.sbtProjects(`std-table`)),
+  makeStdLibDistribution("Database", Distribution.sbtProjects(`std-database`)),
+  makeStdLibDistribution("Image", Distribution.sbtProjects(`std-image`))
 )
+
 GatherLicenses.licenseConfigurations := Set("compile")
 GatherLicenses.configurationRoot := file("tools/legal-review")
 
@@ -696,11 +722,12 @@ lazy val `version-output` = (project in file("lib/scala/version-output"))
       val file = (Compile / sourceManaged).value / "buildinfo" / "Info.scala"
       BuildInfo
         .writeBuildInfoFile(
-          file,
-          state.value.log,
-          ensoVersion,
-          scalacVersion,
-          graalVersion
+          file           = file,
+          log            = state.value.log,
+          ensoVersion    = ensoVersion,
+          scalacVersion  = scalacVersion,
+          graalVersion   = graalVersion,
+          currentEdition = currentEdition
         )
     }.taskValue
   )
@@ -901,6 +928,7 @@ lazy val `polyglot-api` = project
   .in(file("engine/polyglot-api"))
   .settings(
     Test / fork := true,
+    Test / envVars ++= distributionEnvironmentOverrides,
     Test / javaOptions ++= {
       // Note [Classpath Separation]
       val runtimeClasspath =
@@ -1001,6 +1029,24 @@ lazy val cleanInstruments = taskKey[Unit](
   "Cleans fragile class files to force a full recompilation and preserve" +
   "consistency of instrumentation configuration."
 )
+
+/** Overrides for the environment variables related to the distribution, so that
+  * a local installation does not interfere with runtime tests.
+  */
+val distributionEnvironmentOverrides = {
+  val fakeDir = file("target/fake_dir").getAbsolutePath
+  Map(
+    "ENSO_DATA_DIRECTORY"           -> fakeDir,
+    "ENSO_CONFIG_DIRECTORY"         -> fakeDir,
+    "ENSO_RUNTIME_DIRECTORY"        -> file("target/run").getAbsolutePath,
+    "ENSO_LOG_DIRECTORY"            -> file("target/logs").getAbsolutePath,
+    "ENSO_HOME"                     -> fakeDir,
+    "ENSO_EDITION_PATH"             -> "",
+    "ENSO_LIBRARY_PATH"             -> "",
+    "ENSO_AUXILIARY_LIBRARY_CACHES" -> ""
+  )
+}
+
 lazy val runtime = (project in file("engine/runtime"))
   .configs(Benchmark)
   .settings(
@@ -1045,6 +1091,8 @@ lazy val runtime = (project in file("engine/runtime"))
       "-Dgraalvm.locatorDisabled=true",
       s"--upgrade-module-path=${file("engine/runtime/build-cache/truffle-api.jar").absolutePath}"
     ),
+    Test / fork := true,
+    Test / envVars ++= distributionEnvironmentOverrides,
     bootstrap := CopyTruffleJAR.bootstrapJARs.value,
     Global / onLoad := EnvironmentCheck.addVersionCheck(
       graalVersion,
@@ -1101,6 +1149,22 @@ lazy val runtime = (project in file("engine/runtime"))
       case _ => MergeStrategy.first
     }
   )
+  .settings(
+    (Compile / compile) := (Compile / compile)
+      .dependsOn(
+        Def.task {
+          Editions.writeEditionConfig(
+            ensoVersion = ensoVersion,
+            editionName = currentEdition,
+            libraryVersion =
+              "0.1.0", // TODO [RW] Once we start releasing the standard libraries, this will be synced with engine version.
+            log = streams.value.log
+          )
+        }
+      )
+      .value,
+    cleanFiles += baseDirectory.value / ".." / ".." / "distribution" / "editions"
+  )
   .dependsOn(pkg)
   .dependsOn(`interpreter-dsl`)
   .dependsOn(syntax.jvm)
@@ -1177,6 +1241,7 @@ lazy val `engine-runner` = project
   )
   .dependsOn(`version-output`)
   .dependsOn(pkg)
+  .dependsOn(`library-manager`)
   .dependsOn(`language-server`)
   .dependsOn(`polyglot-api`)
   .dependsOn(`logging-service`)
@@ -1276,6 +1341,7 @@ lazy val `library-manager` = project
       "org.scalatest"              %% "scalatest"     % scalatestVersion % Test
     )
   )
+  .dependsOn(`version-output`) // Note [Default Editions]
   .dependsOn(editions)
   .dependsOn(cli)
   .dependsOn(`distribution-manager`)
@@ -1330,12 +1396,14 @@ lazy val `locking-test-helper` = project
     assembly / assemblyOutputPath := file("locking-test-helper.jar")
   )
 
-val `std-lib-root`           = file("distribution/std-lib/")
-val `standard-polyglot-root` = `std-lib-root` / "Standard" / "polyglot" / "java"
-val `base-polyglot-root`     = `std-lib-root` / "Base" / "polyglot" / "java"
-val `table-polyglot-root`    = `std-lib-root` / "Table" / "polyglot" / "java"
-val `image-polyglot-root`    = `std-lib-root` / "Image" / "polyglot" / "java"
-val `database-polyglot-root` = `std-lib-root` / "Database" / "polyglot" / "java"
+val `std-lib-root` = file("distribution/lib/Standard/")
+def stdLibComponentRoot(name: String): File =
+  `std-lib-root` / name / stdLibVersion
+val `base-polyglot-root`  = stdLibComponentRoot("Base") / "polyglot" / "java"
+val `table-polyglot-root` = stdLibComponentRoot("Table") / "polyglot" / "java"
+val `image-polyglot-root` = stdLibComponentRoot("Image") / "polyglot" / "java"
+val `database-polyglot-root` =
+  stdLibComponentRoot("Database") / "polyglot" / "java"
 
 lazy val `std-base` = project
   .in(file("std-bits") / "base")

distribution/engine/THIRD-PARTY/com.google.auto.service.auto-service-1.0-rc7/NOTICES

@@ -1,19 +1,3 @@
-/*
- * Copyright 2008 Google LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
 /*
  * Copyright 2013 Google LLC
  *
@@ -43,3 +27,19 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+/*
+ * Copyright 2008 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */

distribution/engine/THIRD-PARTY/com.google.errorprone.error_prone_annotations-2.3.4/NOTICES

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014 The Error Prone Authors.
+ * Copyright 2016 The Error Prone Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@
  */
 
 /*
- * Copyright 2016 The Error Prone Authors.
+ * Copyright 2015 The Error Prone Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -47,7 +47,7 @@
  */
 
 /*
- * Copyright 2015 The Error Prone Authors.
+ * Copyright 2014 The Error Prone Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

distribution/engine/THIRD-PARTY/com.typesafe.akka.akka-http-spray-json_2.13-10.2.0-RC1/NOTICES

@@ -1,7 +1,7 @@
-/*
- * Copyright (C) 2009-2020 Lightbend Inc. <https://www.lightbend.com>
- */
-
 /*
  * Copyright (C) 2017-2020 Lightbend Inc. <https://www.lightbend.com>
  */
+
+/*
+ * Copyright (C) 2009-2020 Lightbend Inc. <https://www.lightbend.com>
+ */

distribution/engine/THIRD-PARTY/dev.zio.zio_2.13-1.0.1/NOTICES

@@ -15,8 +15,6 @@
  * limitations under the License.
  */
 
-Copyright 2017-2019 John A. De Goes and the ZIO Contributors
-
 /*
  * Copyright 2017-2020 John A. De Goes and the ZIO Contributors
  * Copyright 2017-2018 Łukasz Biały, Paul Chiusano, Michael Pilquist,
@@ -36,3 +34,5 @@ Copyright 2017-2019 John A. De Goes and the ZIO Contributors
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+Copyright 2017-2019 John A. De Goes and the ZIO Contributors

distribution/engine/THIRD-PARTY/io.spray.spray-json_2.13-1.3.5/NOTICES

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2011,2012 Mathias Doenitz, Johannes Rudolph
+ * Copyright (C) 2011 Mathias Doenitz
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@
  */
 
 /*
- * Copyright (C) 2011 Mathias Doenitz
+ * Copyright (C) 2011,2012 Mathias Doenitz, Johannes Rudolph
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

distribution/lib/Standard/Examples/0.1.0/src/Main.enso

@@ -140,8 +140,8 @@ uri = Enso_Uri.parse "http://user:pass@example.com/foo/bar?key=val"
    ! Downloading
      Calling this method will result in the image file being downloaded from
      the internet if it is not already present on your disk. If you do not want
-     this to happen, please place the image in the `std-lib/Standard/data`
-     folder for your Enso distribution.
+     this to happen, please place the image in the
+     `lib/Standard/Examples/0.1.0/data` folder for your Enso distribution.
 image_file : File
 image_file =
     url = "https://upload.wikimedia.org/wikipedia/commons/thumb/e/e9/Hue_alpha_falloff.png/320px-Hue_alpha_falloff.png"
@@ -155,8 +155,8 @@ image_file =
    ! Downloading
      Calling this method will result in the image file being downloaded from
      the internet if it is not already present on your disk. If you do not want
-     this to happen, please place the image in the `std-lib/Standard/data`
-     folder for your Enso distribution.
+     this to happen, please place the image in the
+     `lib/Standard/Examples/0.1.0/data` folder for your Enso distribution.
 image : Enso_Image.Image
 image = Enso_Image.read here.image_file [Codecs.Read_Alpha_Channel]